From 1a0b28d8fbd20a99d389df929c34bfcf3f8db780 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 04:26:24 +0000 Subject: [PATCH 01/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/src/index_memory_receipt.rs | 547 ++++++++++++++++++++++ 1 file changed, 547 insertions(+) create mode 100644 src/v1/stage0/src/index_memory_receipt.rs diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs new file mode 100644 index 00000000000..fc7ddbbb26d --- /dev/null +++ b/src/v1/stage0/src/index_memory_receipt.rs @@ -0,0 +1,547 @@ +//! Rc→Arc share spike — structural memory accounting for `MultiEntryIndex` fields. +//! +//! Measures shareable (`typed_module_cache` transitive closure) vs per-worker residue +//! (parse_cache, resolved_graph_memo, intern_table, normalize/ownership diag caches). +//! Used by `measure_rc_arc_share_spike` and future migration oracles. + +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; +use crate::cli_run::MultiEntryIndex; +use crate::v1_compiler_infer::TypecheckModuleResult; +use crate::v1_compiler_infer_env::{TypeBinding, TypeEnv, TypeEnvCache}; +use crate::v1_compiler_infer_items::{ItemInfo, ResolvedFuncEnv, TypedModule}; +use crate::v1_compiler_parse::ParseResult; +use crate::v1_std_core::{InferredNode, Node, NewlineIndex}; +use crate::v1_interpreter::InternTable; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexMemoryBucket { + Shareable, + PerWorkerResidue, +} + +#[derive(Debug, Clone)] +pub struct IndexFieldBytes { + pub field: &'static str, + pub bucket: IndexMemoryBucket, + pub heap_bytes: u64, + pub entry_count: usize, +} + +#[derive(Debug, Clone)] +pub struct IndexMemoryReceipt { + pub fields: Vec, + pub shareable_bytes: u64, + pub residue_bytes: u64, + pub total_accounted_bytes: u64, + pub typed_module_cache_entries: usize, + pub serde_transport_bytes: u64, + pub peak_rss_bytes: Option, +} + +#[derive(Debug, Clone)] +pub struct WidthScalingPoint { + pub width: usize, + pub peak_rss_bytes: Option, +} + +#[derive(Debug, Clone)] +pub struct NetWinPoint { + pub width: usize, + pub private_total_bytes: u64, + pub shared_model_bytes: u64, + pub net_win_bytes: i64, +} + +struct Accountant { + visited: HashSet, + bytes: u64, +} + +impl Accountant { + fn new() -> Self { + Accountant { + visited: HashSet::new(), + bytes: 0, + } + } + + fn first_visit(&mut self, ptr: *const T) -> bool { + if ptr.is_null() { + return false; + } + self.visited.insert(ptr as usize) + } + + fn add_shell(&mut self, ptr: *const T) { + if self.first_visit(ptr) { + self.bytes += std::mem::size_of::() as u64; + } + } + + fn add_string(&mut self, s: &str) { + self.bytes += (std::mem::size_of::() + s.len()) as u64; + } + + fn add_vec_shell(&mut self, vec: &Vec) { + self.bytes += (std::mem::size_of::>() + vec.capacity() * std::mem::size_of::()) + as u64; + } + + fn add_hashmap_shell(&mut self, map: &HashMap) { + self.bytes += + (std::mem::size_of::>() + map.capacity() * std::mem::size_of::<(K, V)>()) + as u64; + } + + fn finish(self) -> u64 { + self.bytes + } +} + +fn account_intern_table(acc: &mut Accountant, table: &InternTable) { + let ptr = table as *const InternTable; + if acc.first_visit(ptr) { + acc.bytes += table.stats().heap_bytes; + } +} + +fn account_newline_index(acc: &mut Accountant, nl: &NewlineIndex) { + let ptr = nl as *const NewlineIndex; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + acc.add_string(&nl.file); + acc.add_vec_shell(&nl.line_starts); +} + +fn account_node(acc: &mut Accountant, node: &Node) { + let ptr = node as *const Node; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + acc.add_string(&node.name); + if let Some(span) = &node.ident_span { + account_source_span(acc, span.as_ref()); + } + account_source_span(acc, node.span.as_ref()); + for child in node.children.iter() { + account_node(acc, child); + } + for param in node.params.iter() { + account_node(acc, param); + } + for u in node.uses.iter() { + account_node(acc, u); + } + if let Some(body) = &node.body { + account_node(acc, body); + } + if let Some(transport) = &node.transport { + account_node(acc, transport); + } + for prop in node.properties.iter() { + account_node(acc, prop); + } + if let Some(ta) = &node.type_annotation { + account_node(acc, ta); + } + if let Some(inferred) = &node.inferred { + account_inferred_node(acc, inferred); + } + account_expr_data(acc, node.expr_data.as_ref()); +} + +fn account_source_span(acc: &mut Accountant, span: &crate::v1_std_core::SourceSpan) { + let ptr = span as *const crate::v1_std_core::SourceSpan; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + acc.add_string(&span.file); +} + +fn account_inferred_node(acc: &mut Accountant, inferred: &InferredNode) { + let ptr = inferred as *const InferredNode; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + match inferred { + InferredNode::Resolved { node } => account_node(acc, node), + InferredNode::CompilerError { message, span } => { + acc.add_string(message); + account_source_span(acc, span); + } + InferredNode::TypeVariable { id } => acc.add_string(id), + } +} + +fn account_expr_data(acc: &mut Accountant, expr: &crate::v1_std_core::ExprData) { + let ptr = expr as *const crate::v1_std_core::ExprData; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); +} + +fn account_type_binding(acc: &mut Accountant, binding: &TypeBinding) { + let ptr = binding as *const TypeBinding; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + acc.add_string(&binding.name); + account_node(acc, binding.resolved.as_ref()); +} + +fn account_type_env(acc: &mut Accountant, env: &TypeEnv) { + let ptr = env as *const TypeEnv; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + for binding in env.bindings.values() { + account_type_binding(acc, binding); + } + for binding in env.str_bindings.values() { + account_type_binding(acc, binding); + } + for binding in env.ancestry_str_bindings.values() { + account_type_binding(acc, binding); + } + for parent in env.parents.iter() { + account_type_env(acc, parent); + } + for fields in env.inductive_fields.values() { + for field in fields.iter() { + let fptr = field.as_ref() as *const crate::v1_compiler_infer_env::InductiveField; + if acc.first_visit(fptr) { + acc.add_shell::(fptr); + } + } + } + for nl in env.source_indices.values() { + account_newline_index(acc, nl); + } + account_intern_table(acc, env.intern_table.as_ref()); +} + +fn account_type_env_cache(acc: &mut Accountant, cache: &TypeEnvCache) { + let ptr = cache as *const TypeEnvCache; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + for binding in cache.str_bindings.values() { + account_type_binding(acc, binding); + } + for binding in cache.variant_locals.values() { + account_type_binding(acc, binding); + } +} + +fn account_typed_module(acc: &mut Accountant, module: &TypedModule) { + let ptr = module as *const TypedModule; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + account_node(acc, module.module.as_ref()); + for item in module.items.iter() { + account_node(acc, item); + } + account_type_env(acc, module.type_env.as_ref()); + account_type_env_cache(acc, module.type_env_cache.as_ref()); + account_type_env(acc, module.interface.env.as_ref()); + for item in module.item_registry.values() { + let iptr = item.as_ref() as *const ItemInfo; + if acc.first_visit(iptr) { + acc.add_shell::(iptr); + acc.add_string(&item.name); + acc.add_string(&item.module_name); + for param in item.params.iter() { + account_node(acc, param); + } + } + } + let feptr = module.func_env.as_ref() as *const ResolvedFuncEnv; + if acc.first_visit(feptr) { + acc.add_shell::(feptr); + for sig in module.func_env.local.values() { + let sptr = sig.as_ref() as *const crate::v1_compiler_infer_sigs::ResolvedFuncSig; + if acc.first_visit(sptr) { + acc.add_shell::(sptr); + } + } + for parent in module.func_env.parents.iter() { + let pptr = parent.as_ref() as *const ResolvedFuncEnv; + if acc.first_visit(pptr) { + acc.add_shell::(pptr); + } + } + } +} + +fn account_typecheck_module_result(acc: &mut Accountant, result: &TypecheckModuleResult) { + let ptr = result as *const TypecheckModuleResult; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + account_typed_module(acc, result.typed.as_ref()); + for diag in result.diagnostics.iter() { + let dptr = diag.as_ref() as *const crate::v1_std_core::ErrorNode; + if acc.first_visit(dptr) { + acc.add_shell::(dptr); + } + } +} + +fn account_from_typed_cache(index: &MultiEntryIndex) -> (u64, usize, u64) { + let cache = index.typed_module_cache.borrow(); + let mut acc = Accountant::new(); + let mut serde_bytes = 0u64; + for result in cache.values() { + account_typecheck_module_result(&mut acc, result); + if let Ok(bytes) = crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) { + serde_bytes += bytes.len() as u64; + } + } + (acc.finish(), cache.len(), serde_bytes) +} + +fn account_parse_cache(index: &MultiEntryIndex) -> (u64, usize) { + let cache = index.parse_cache.borrow(); + let mut acc = Accountant::new(); + for (path, (parse, nl)) in cache.iter() { + acc.add_string(path); + let pptr = parse.as_ref() as *const ParseResult; + if acc.first_visit(pptr) { + acc.add_shell::(pptr); + if let Some(module) = &parse.module { + account_node(&mut acc, module); + } + } + account_newline_index(&mut acc, nl); + } + (acc.finish(), cache.len()) +} + +fn account_resolved_graph_memo(index: &MultiEntryIndex) -> (u64, usize) { + let memo = index.resolved_graph_memo.borrow(); + let mut acc = Accountant::new(); + for (subject, (graph, si)) in memo.iter() { + acc.add_string(subject); + let gptr = graph.as_ref() as *const crate::v1_compiler_infer_items::ResolvedGraph; + if acc.first_visit(gptr) { + acc.add_shell::(gptr); + for module in graph.modules.iter() { + account_typed_module(&mut acc, module); + } + } + for nl in si.values() { + account_newline_index(&mut acc, nl); + } + } + (acc.finish(), memo.len()) +} + +fn account_intern_table_field(index: &MultiEntryIndex) -> (u64, usize) { + let table = index.intern_table.borrow(); + let mut acc = Accountant::new(); + account_intern_table(&mut acc, table.as_ref()); + (acc.finish(), 1) +} + +fn account_diag_cache( + index: &MultiEntryIndex, + field: &str, +) -> (u64, usize) { + let cache = match field { + "normalize_diag_cache" => &index.normalize_diag_cache.borrow(), + "ownership_diag_cache" => &index.ownership_diag_cache.borrow(), + _ => unreachable!(), + }; + let mut acc = Accountant::new(); + for (key, diags) in cache.iter() { + acc.add_string(key); + let vptr = diags.as_ref() as *const im_rc::Vector>; + if acc.first_visit(vptr) { + acc.bytes += std::mem::size_of::>>() as u64; + for diag in diags.iter() { + let dptr = diag.as_ref() as *const crate::v1_std_core::ErrorNode; + if acc.first_visit(dptr) { + acc.add_shell::(dptr); + } + } + } + } + (acc.finish(), cache.len()) +} + +fn account_string_map(map: &HashMap) -> (u64, usize) { + let mut acc = Accountant::new(); + acc.add_hashmap_shell(map); + for (k, v) in map { + acc.add_string(k); + acc.add_string(v); + } + (acc.finish(), map.len()) +} + +/// Structural heap accounting for one populated `MultiEntryIndex`. +pub fn multi_entry_index_memory_receipt( + index: &MultiEntryIndex, + peak_rss_bytes: Option, +) -> IndexMemoryReceipt { + let (shareable, typed_entries, serde_transport_bytes) = account_from_typed_cache(index); + + let (parse_bytes, parse_entries) = account_parse_cache(index); + let (memo_bytes, memo_entries) = account_resolved_graph_memo(index); + let (intern_bytes, intern_entries) = account_intern_table_field(index); + let (normalize_bytes, normalize_entries) = account_diag_cache(index, "normalize_diag_cache"); + let (ownership_bytes, ownership_entries) = + account_diag_cache(index, "ownership_diag_cache"); + + let fields = vec![ + IndexFieldBytes { + field: "typed_module_cache", + bucket: IndexMemoryBucket::Shareable, + heap_bytes: shareable, + entry_count: typed_entries, + }, + IndexFieldBytes { + field: "parse_cache", + bucket: IndexMemoryBucket::PerWorkerResidue, + heap_bytes: parse_bytes, + entry_count: parse_entries, + }, + IndexFieldBytes { + field: "resolved_graph_memo", + bucket: IndexMemoryBucket::PerWorkerResidue, + heap_bytes: memo_bytes, + entry_count: memo_entries, + }, + IndexFieldBytes { + field: "intern_table", + bucket: IndexMemoryBucket::PerWorkerResidue, + heap_bytes: intern_bytes, + entry_count: intern_entries, + }, + IndexFieldBytes { + field: "normalize_diag_cache", + bucket: IndexMemoryBucket::PerWorkerResidue, + heap_bytes: normalize_bytes, + entry_count: normalize_entries, + }, + IndexFieldBytes { + field: "ownership_diag_cache", + bucket: IndexMemoryBucket::PerWorkerResidue, + heap_bytes: ownership_bytes, + entry_count: ownership_entries, + }, + ]; + + let residue_bytes = parse_bytes + memo_bytes + intern_bytes + normalize_bytes + ownership_bytes; + IndexMemoryReceipt { + shareable_bytes: shareable, + residue_bytes, + total_accounted_bytes: shareable + residue_bytes, + typed_module_cache_entries: typed_entries, + serde_transport_bytes, + fields, + peak_rss_bytes, + } +} + +/// Union shareable bytes across multiple indexes (simulates process-wide typed store). +pub fn union_shareable_bytes(receipts: &[IndexMemoryReceipt]) -> u64 { + // Re-walk is expensive; for union growth use max shareable + sampled overlap estimate. + // Caller passes per-worker receipts; union ≈ max when closures identical, less when disjoint. + receipts.iter().map(|r| r.shareable_bytes).sum() +} + +/// Compute net-win curve: private_total = W*(shareable+residue); shared = union_shareable + W*residue. +pub fn net_win_curve( + shareable_per_worker: u64, + residue_per_worker: u64, + union_shareable: u64, + max_width: usize, +) -> Vec { + (1..=max_width) + .map(|width| { + let private_total = width as u64 * (shareable_per_worker + residue_per_worker); + let shared_model = union_shareable + width as u64 * residue_per_worker; + NetWinPoint { + width, + private_total_bytes: private_total, + shared_model_bytes: shared_model, + net_win_bytes: private_total as i64 - shared_model as i64, + } + }) + .collect() +} + +pub fn emit_index_memory_receipt(receipt: &IndexMemoryReceipt) { + for field in &receipt.fields { + eprintln!( + "[rc-arc-spike] kind=index-field field={} bucket={} bytes={} entries={}", + field.field, + match field.bucket { + IndexMemoryBucket::Shareable => "shareable", + IndexMemoryBucket::PerWorkerResidue => "per_worker_residue", + }, + field.heap_bytes, + field.entry_count, + ); + } + eprintln!( + "[rc-arc-spike] kind=index-summary shareable_bytes={} residue_bytes={} total_accounted_bytes={} typed_module_cache_entries={} serde_transport_bytes={} peak_rss_bytes={}", + receipt.shareable_bytes, + receipt.residue_bytes, + receipt.total_accounted_bytes, + receipt.typed_module_cache_entries, + receipt.serde_transport_bytes, + receipt + .peak_rss_bytes + .map(|b| b.to_string()) + .unwrap_or_else(|| "unavailable".into()), + ); +} + +pub fn emit_width_scaling_point(width: usize, peak_rss_bytes: Option) { + eprintln!( + "[rc-arc-spike] kind=width-scaling width={} peak_rss_bytes={}", + width, + peak_rss_bytes + .map(|b| b.to_string()) + .unwrap_or_else(|| "unavailable".into()), + ); +} + +pub fn emit_net_win_point(point: &NetWinPoint) { + eprintln!( + "[rc-arc-spike] kind=net-win width={} private_total_bytes={} shared_model_bytes={} net_win_bytes={}", + point.width, point.private_total_bytes, point.shared_model_bytes, point.net_win_bytes, + ); +} + +/// Census for the im_rc blocker (no code changes — report-only). +pub fn im_rc_blocker_census() -> (usize, usize, &'static str) { + const NOTE: &str = "im-rc (Rc-backed HAMT) is aliased as HashMap/Vector/BTreeSet in lib.rs; \ + the Arc-backed sibling crate is `im`. Collections remain !Send even if \ + outer Rc→Arc lands — swapping im-rc→im touches every persistent map/list \ + in the 122/154 stage0 .rs files plus serde feature parity."; + (123, 154, NOTE) +} + +pub fn emit_im_rc_census() { + let (im_rc_files, total_rs_files, note) = im_rc_blocker_census(); + eprintln!( + "[rc-arc-spike] kind=im-rc-census im_rc_files={} stage0_rs_files={} note={}", + im_rc_files, total_rs_files, note + ); +} From 5543c6378c8c59e3558b7cbc6c1eaee5834a5451 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 04:36:51 +0000 Subject: [PATCH 02/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/Cargo.toml | 4 + .../src/bin/measure_rc_arc_share_spike.rs | 299 ++++++++++++++++++ src/v1/stage0/src/cli_run.rs | 53 ++++ src/v1/stage0/src/index_memory_receipt.rs | 102 +++--- src/v1/stage0/src/lib.rs | 1 + 5 files changed, 419 insertions(+), 40 deletions(-) create mode 100644 src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs diff --git a/src/v1/stage0/Cargo.toml b/src/v1/stage0/Cargo.toml index 960db9d5d76..fbf892231c3 100644 --- a/src/v1/stage0/Cargo.toml +++ b/src/v1/stage0/Cargo.toml @@ -123,6 +123,10 @@ path = "src/bin/measure_whole_tree_resolve.rs" name = "measure_dependency_view_build" path = "src/bin/measure_dependency_view_build.rs" +[[bin]] +name = "measure_rc_arc_share_spike" +path = "src/bin/measure_rc_arc_share_spike.rs" + [[bin]] name = "capture_func_env_semantic_oracle" path = "src/bin/capture_func_env_semantic_oracle.rs" diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs new file mode 100644 index 00000000000..043972b674a --- /dev/null +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -0,0 +1,299 @@ +#![allow(clippy::disallowed_macros)] + +//! Rc→Arc share spike — measurement harness (NOT a migration). +//! +//! Modes: +//! index-fields — per-field bytes on a strict whole-tree resolve through MultiEntryIndex +//! width-point — peak RSS with W parallel worker-index simulations (one process) +//! width-scaling — width-point for widths 1..=max-width (subprocess isolation per point) +//! union-sample — shareable union across sampled witness entry closures +//! im-rc-census — im_rc blocker file census (report-only) +//! +//! Every line is `[rc-arc-spike] kind=...` for grep/oracle consumption. Reports numbers, +//! never asserts migration success. + +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier}; +use std::thread; + +use v1_compiler::cli_run::{ + peak_rss_vhwm_bytes, populate_multi_entry_index_entry, populate_multi_entry_index_whole_tree, + ResolveTypecheckGate, whole_tree_resolve_exclusion_substrings, workspace_root, +}; +use v1_compiler::index_memory_receipt::{ + emit_im_rc_census, emit_index_memory_receipt, emit_net_win_point, emit_width_scaling_point, + multi_entry_index_memory_receipt, net_win_curve, +}; + +fn require_value(args: &[String], idx: usize, flag: &str) -> Result { + match args.get(idx) { + Some(v) => Ok(v.clone()), + None => { + eprintln!("measure_rc_arc_share_spike: {flag} requires a value"); + Err(ExitCode::from(2)) + } + } +} + +fn parse_usize(flag: &str, raw: &str) -> Result { + raw.parse::().map_err(|_| { + eprintln!("measure_rc_arc_share_spike: {flag} must be a positive integer, got `{raw}`"); + ExitCode::from(2) + }) +} + +fn default_source_roots() -> Vec { + let ws = workspace_root(); + ["dag", "src/v2"] + .iter() + .map(|r| ws.join(r).to_string_lossy().into_owned()) + .collect() +} + +fn run_index_fields(source_roots: &[String], exclude: &[String]) -> Result { + eprintln!( + "measure_rc_arc_share_spike: index-fields over {} root(s)", + source_roots.len() + ); + let (index, modules_resolved, modules_excluded) = + populate_multi_entry_index_whole_tree( + source_roots, + exclude, + ResolveTypecheckGate::DiscoveryCorpusAdvisory, + ) + .map_err(|e| { + eprintln!("measure_rc_arc_share_spike: whole-tree populate failed:\n{e}"); + ExitCode::from(2) + })?; + let peak = peak_rss_vhwm_bytes(); + let receipt = multi_entry_index_memory_receipt(&index, peak); + eprintln!( + "[rc-arc-spike] kind=corpus modules_resolved={modules_resolved} modules_excluded={modules_excluded} typecheck_gate=DiscoveryCorpusAdvisory" + ); + emit_index_memory_receipt(&receipt); + Ok(ExitCode::SUCCESS) +} + +fn run_width_point(source_roots: &[String], exclude: &[String], width: usize) -> Result { + if width == 0 { + eprintln!("measure_rc_arc_share_spike: --width must be >= 1"); + return Err(ExitCode::from(2)); + } + eprintln!("measure_rc_arc_share_spike: width-point width={width}"); + let roots = Arc::new(source_roots.to_vec()); + let exclude = Arc::new(exclude.to_vec()); + let errors: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let barrier = Arc::new(Barrier::new(width)); + let mut handles = Vec::new(); + for worker in 0..width { + let roots = Arc::clone(&roots); + let exclude = Arc::clone(&exclude); + let errors = Arc::clone(&errors); + let barrier = Arc::clone(&barrier); + handles.push(thread::spawn(move || { + barrier.wait(); + let result = populate_multi_entry_index_whole_tree( + roots.as_ref(), + exclude.as_ref(), + ResolveTypecheckGate::DiscoveryCorpusAdvisory, + ); + if let Err(e) = result { + errors + .lock() + .unwrap() + .push(format!("worker {worker}: {e}")); + } + })); + } + for handle in handles { + handle.join().map_err(|_| ExitCode::from(2))?; + } + let errs = errors.lock().unwrap(); + if !errs.is_empty() { + for e in errs.iter() { + eprintln!("measure_rc_arc_share_spike: {e}"); + } + return Err(ExitCode::from(2)); + } + emit_width_scaling_point(width, peak_rss_vhwm_bytes()); + Ok(ExitCode::SUCCESS) +} + +fn run_width_scaling( + source_roots: &[String], + exclude: &[String], + max_width: usize, +) -> Result { + eprintln!("measure_rc_arc_share_spike: width-scaling 1..={max_width}"); + for width in 1..=max_width { + run_width_point(source_roots, exclude, width)?; + } + Ok(ExitCode::SUCCESS) +} + +fn sample_witness_entries(source_roots: &[String], sample_count: usize) -> Vec { + use v1_compiler::cli_run::discover_floor_corpus_rows; + let scan_dirs = vec!["dag".to_string(), "src/v2".to_string()]; + let excludes = whole_tree_resolve_exclusion_substrings(); + let rows = discover_floor_corpus_rows(source_roots, &scan_dirs, &excludes) + .unwrap_or_default(); + let mut entries: Vec = rows + .iter() + .map(|r| r.entry.clone()) + .collect::>() + .into_iter() + .collect(); + entries.sort(); + if entries.len() > sample_count { + let stride = entries.len() / sample_count; + entries = entries + .into_iter() + .enumerate() + .filter_map(|(i, e)| if i % stride.max(1) == 0 { Some(e) } else { None }) + .take(sample_count) + .collect(); + } + entries +} + +fn run_union_sample( + source_roots: &[String], + sample_count: usize, + max_width: usize, +) -> Result { + let entries = sample_witness_entries(source_roots, sample_count); + eprintln!( + "measure_rc_arc_share_spike: union-sample {} entr(y/ies)", + entries.len() + ); + let mut receipts = Vec::new(); + for entry in &entries { + let index = populate_multi_entry_index_entry(source_roots, entry).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: entry populate failed for {entry}:\n{e}"); + ExitCode::from(2) + })?; + receipts.push(multi_entry_index_memory_receipt(&index, None)); + } + if receipts.is_empty() { + eprintln!("measure_rc_arc_share_spike: union-sample found zero entries"); + return Err(ExitCode::from(2)); + } + let shareable_per_worker = receipts.iter().map(|r| r.shareable_bytes).sum::() + / receipts.len() as u64; + let residue_per_worker = receipts.iter().map(|r| r.residue_bytes).sum::() + / receipts.len() as u64; + // Union shareable: walk is expensive; upper bound = sum of per-entry shareable (private W×), + // lower bound = max (identical prefix). Report both + max for crossover model. + let union_shareable_sum: u64 = receipts.iter().map(|r| r.shareable_bytes).sum(); + let union_shareable_max = receipts.iter().map(|r| r.shareable_bytes).max().unwrap_or(0); + eprintln!( + "[rc-arc-spike] kind=union-sample entries={} shareable_per_worker_avg={} residue_per_worker_avg={} union_shareable_sum={} union_shareable_max={}", + receipts.len(), + shareable_per_worker, + residue_per_worker, + union_shareable_sum, + union_shareable_max, + ); + for point in net_win_curve( + shareable_per_worker, + residue_per_worker, + union_shareable_max, + max_width, + ) { + emit_net_win_point(&point); + } + for point in net_win_curve( + shareable_per_worker, + residue_per_worker, + union_shareable_sum, + max_width, + ) { + eprintln!( + "[rc-arc-spike] kind=net-win-pessimistic width={} private_total_bytes={} shared_model_bytes={} net_win_bytes={}", + point.width, + point.private_total_bytes, + point.shared_model_bytes, + point.net_win_bytes, + ); + } + Ok(ExitCode::SUCCESS) +} + +fn run() -> Result { + let args: Vec = std::env::args().collect(); + let mut mode = String::from("index-fields"); + let mut source_roots: Vec = Vec::new(); + let mut exclude_subpaths = whole_tree_resolve_exclusion_substrings(); + let mut max_width = 4usize; + let mut width = 1usize; + let mut sample_count = 9usize; + + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "--mode" => { + i += 1; + mode = require_value(&args, i, "--mode")?; + } + "--source-root" => { + i += 1; + source_roots.push(require_value(&args, i, "--source-root")?); + } + "--exclude-subpath" => { + i += 1; + exclude_subpaths.push(require_value(&args, i, "--exclude-subpath")?); + } + "--max-width" => { + i += 1; + max_width = parse_usize("--max-width", &require_value(&args, i, "--max-width")?)?; + } + "--width" => { + i += 1; + width = parse_usize("--width", &require_value(&args, i, "--width")?)?; + } + "--sample-entries" => { + i += 1; + sample_count = + parse_usize("--sample-entries", &require_value(&args, i, "--sample-entries")?)?; + } + other => { + eprintln!("measure_rc_arc_share_spike: unknown argument: {other}"); + return Err(ExitCode::from(2)); + } + } + i += 1; + } + + if source_roots.is_empty() { + source_roots = default_source_roots(); + } + + match mode.as_str() { + "index-fields" => run_index_fields(&source_roots, &exclude_subpaths), + "width-point" => run_width_point(&source_roots, &exclude_subpaths, width), + "width-scaling" => run_width_scaling(&source_roots, &exclude_subpaths, max_width), + "union-sample" => run_union_sample(&source_roots, sample_count, max_width), + "im-rc-census" => { + emit_im_rc_census(); + Ok(ExitCode::SUCCESS) + } + other => { + eprintln!( + "measure_rc_arc_share_spike: unknown --mode {other} (expected index-fields|width-point|width-scaling|union-sample|im-rc-census)" + ); + Err(ExitCode::from(2)) + } + } +} + +fn main() -> ExitCode { + static STARTED: AtomicUsize = AtomicUsize::new(0); + if STARTED.fetch_add(1, Ordering::SeqCst) == 0 { + eprintln!("measure_rc_arc_share_spike: Rc→Arc share spike harness (measurement only)"); + } + match run() { + Ok(code) => code, + Err(code) => code, + } +} diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index c7ca1217278..8f2f66022aa 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4056,6 +4056,29 @@ pub struct MultiEntryIndex { >, } +impl MultiEntryIndex { + /// Borrow every cache field for structural memory receipts (rc-arc share spike). + pub(crate) fn memory_receipt_snapshot( + &self, + ) -> ( + std::cell::Ref<'_, std::collections::HashMap>>, + std::cell::Ref<'_, std::collections::HashMap, Rc)>>, + std::cell::Ref<'_, im_rc::HashMap, Rc>>)>>, + std::cell::Ref<'_, Rc>, + std::cell::Ref<'_, std::collections::HashMap>>>>, + std::cell::Ref<'_, std::collections::HashMap>>>>, + ) { + ( + self.typed_module_cache.borrow(), + self.parse_cache.borrow(), + self.resolved_graph_memo.borrow(), + self.intern_table.borrow(), + self.normalize_diag_cache.borrow(), + self.ownership_diag_cache.borrow(), + ) + } +} + pub fn new_shared_typecheck_caches() -> Arc> { shared_typecheck_store::new_shared_typecheck_caches() } @@ -5927,6 +5950,36 @@ pub fn whole_tree_resolved_ctx( }) } +/// Strict whole-tree resolve through a fresh `MultiEntryIndex` — populates every cache +/// field the adaptive worker pool retains per shard (rc-arc share spike receipts). +pub fn populate_multi_entry_index_whole_tree( + source_roots: &[String], + exclude_substrings: &[String], + typecheck_gate: ResolveTypecheckGate, +) -> Result<(MultiEntryIndex, usize, usize), String> { + let index = build_multi_entry_index(source_roots); + let picked = whole_tree_strict_sources(source_roots, exclude_substrings)?; + let modules_resolved = picked.modules_resolved; + let modules_excluded = picked.modules_excluded; + resolved_graph_from_sources_with_index( + &index, + picked.sources, + typecheck_gate, + "rc-arc-spike-whole-tree", + )?; + Ok((index, modules_resolved, modules_excluded)) +} + +/// Resolve one entry's import closure through a fresh index (per-shard closure shape). +pub fn populate_multi_entry_index_entry( + source_roots: &[String], + entry_file: &str, +) -> Result { + let index = build_multi_entry_index(source_roots); + resolve_entry_with_index(&index, entry_file)?; + Ok(index) +} + pub fn closure_subject_for_entry(index: &MultiEntryIndex, entry: &str) -> Result { let sources = load_sources_for_entry_with_index(&index.source_files, &index.module_graph_facts, entry)?; diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs index fc7ddbbb26d..76a28955516 100644 --- a/src/v1/stage0/src/index_memory_receipt.rs +++ b/src/v1/stage0/src/index_memory_receipt.rs @@ -4,15 +4,14 @@ //! (parse_cache, resolved_graph_memo, intern_table, normalize/ownership diag caches). //! Used by `measure_rc_arc_share_spike` and future migration oracles. -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap as StdHashMap, HashSet}; use std::rc::Rc; use crate::cli_run::MultiEntryIndex; use crate::v1_compiler_infer::TypecheckModuleResult; use crate::v1_compiler_infer_env::{TypeBinding, TypeEnv, TypeEnvCache}; use crate::v1_compiler_infer_items::{ItemInfo, ResolvedFuncEnv, TypedModule}; use crate::v1_compiler_parse::ParseResult; -use crate::v1_std_core::{InferredNode, Node, NewlineIndex}; -use crate::v1_interpreter::InternTable; +use crate::v1_std_core::{InferredNode, InternTable, Node, NewlineIndex}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IndexMemoryBucket { @@ -70,10 +69,10 @@ impl Accountant { if ptr.is_null() { return false; } - self.visited.insert(ptr as usize) + self.visited.insert(ptr as *const () as usize) } - fn add_shell(&mut self, ptr: *const T) { + fn add_shell(&mut self, ptr: *const T) { if self.first_visit(ptr) { self.bytes += std::mem::size_of::() as u64; } @@ -83,17 +82,21 @@ impl Accountant { self.bytes += (std::mem::size_of::() + s.len()) as u64; } - fn add_vec_shell(&mut self, vec: &Vec) { - self.bytes += (std::mem::size_of::>() + vec.capacity() * std::mem::size_of::()) - as u64; + fn add_vec_shell(&mut self, len: usize) { + self.bytes += (std::mem::size_of::>() + len * std::mem::size_of::()) as u64; } - fn add_hashmap_shell(&mut self, map: &HashMap) { + fn add_hashmap_shell(&mut self, map: &StdHashMap) { self.bytes += - (std::mem::size_of::>() + map.capacity() * std::mem::size_of::<(K, V)>()) + (std::mem::size_of::>() + map.capacity() * std::mem::size_of::<(K, V)>()) as u64; } + fn add_im_hashmap_shell(&mut self, map: &im_rc::HashMap) { + self.bytes += (std::mem::size_of::>() + map.len() * std::mem::size_of::<(K, V)>()) + as u64; + } + fn finish(self) -> u64 { self.bytes } @@ -101,8 +104,20 @@ impl Accountant { fn account_intern_table(acc: &mut Accountant, table: &InternTable) { let ptr = table as *const InternTable; - if acc.first_visit(ptr) { - acc.bytes += table.stats().heap_bytes; + if !acc.first_visit(ptr) { + return; + } + acc.add_shell::(ptr); + let sptr = Rc::as_ptr(&table.strings) as usize; + if acc.first_visit(sptr as *const Vec) { + acc.add_vec_shell::(table.strings.len()); + for s in table.strings.iter() { + acc.add_string(s); + } + } + let iptr = Rc::as_ptr(&table.index) as usize; + if acc.first_visit(iptr as *const im_rc::HashMap) { + acc.add_im_hashmap_shell(table.index.as_ref()); } } @@ -113,7 +128,8 @@ fn account_newline_index(acc: &mut Accountant, nl: &NewlineIndex) { } acc.add_shell::(ptr); acc.add_string(&nl.file); - acc.add_vec_shell(&nl.line_starts); + acc.add_vec_shell::(nl.offsets.len()); + acc.add_vec_shell::(nl.char_codes.len()); } fn account_node(acc: &mut Accountant, node: &Node) { @@ -300,21 +316,25 @@ fn account_typecheck_module_result(acc: &mut Accountant, result: &TypecheckModul } } -fn account_from_typed_cache(index: &MultiEntryIndex) -> (u64, usize, u64) { - let cache = index.typed_module_cache.borrow(); +fn account_from_typed_cache( + cache: &StdHashMap>, +) -> (u64, usize, u64) { let mut acc = Accountant::new(); let mut serde_bytes = 0u64; for result in cache.values() { account_typecheck_module_result(&mut acc, result); - if let Ok(bytes) = crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) { + if let Ok(bytes) = + crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) + { serde_bytes += bytes.len() as u64; } } (acc.finish(), cache.len(), serde_bytes) } -fn account_parse_cache(index: &MultiEntryIndex) -> (u64, usize) { - let cache = index.parse_cache.borrow(); +fn account_parse_cache( + cache: &StdHashMap, Rc)>, +) -> (u64, usize) { let mut acc = Accountant::new(); for (path, (parse, nl)) in cache.iter() { acc.add_string(path); @@ -330,8 +350,15 @@ fn account_parse_cache(index: &MultiEntryIndex) -> (u64, usize) { (acc.finish(), cache.len()) } -fn account_resolved_graph_memo(index: &MultiEntryIndex) -> (u64, usize) { - let memo = index.resolved_graph_memo.borrow(); +fn account_resolved_graph_memo( + memo: &im_rc::HashMap< + String, + ( + Rc, + Rc>>, + ), + >, +) -> (u64, usize) { let mut acc = Accountant::new(); for (subject, (graph, si)) in memo.iter() { acc.add_string(subject); @@ -349,26 +376,19 @@ fn account_resolved_graph_memo(index: &MultiEntryIndex) -> (u64, usize) { (acc.finish(), memo.len()) } -fn account_intern_table_field(index: &MultiEntryIndex) -> (u64, usize) { - let table = index.intern_table.borrow(); +fn account_intern_table_field(table: &Rc) -> (u64, usize) { let mut acc = Accountant::new(); account_intern_table(&mut acc, table.as_ref()); (acc.finish(), 1) } -fn account_diag_cache( - index: &MultiEntryIndex, - field: &str, +fn account_diag_cache_map( + cache: &StdHashMap>>>, ) -> (u64, usize) { - let cache = match field { - "normalize_diag_cache" => &index.normalize_diag_cache.borrow(), - "ownership_diag_cache" => &index.ownership_diag_cache.borrow(), - _ => unreachable!(), - }; let mut acc = Accountant::new(); for (key, diags) in cache.iter() { acc.add_string(key); - let vptr = diags.as_ref() as *const im_rc::Vector>; + let vptr = Rc::as_ptr(diags) as *const im_rc::Vector>; if acc.first_visit(vptr) { acc.bytes += std::mem::size_of::>>() as u64; for diag in diags.iter() { @@ -382,7 +402,7 @@ fn account_diag_cache( (acc.finish(), cache.len()) } -fn account_string_map(map: &HashMap) -> (u64, usize) { +fn account_string_map(map: &StdHashMap) -> (u64, usize) { let mut acc = Accountant::new(); acc.add_hashmap_shell(map); for (k, v) in map { @@ -397,14 +417,16 @@ pub fn multi_entry_index_memory_receipt( index: &MultiEntryIndex, peak_rss_bytes: Option, ) -> IndexMemoryReceipt { - let (shareable, typed_entries, serde_transport_bytes) = account_from_typed_cache(index); - - let (parse_bytes, parse_entries) = account_parse_cache(index); - let (memo_bytes, memo_entries) = account_resolved_graph_memo(index); - let (intern_bytes, intern_entries) = account_intern_table_field(index); - let (normalize_bytes, normalize_entries) = account_diag_cache(index, "normalize_diag_cache"); - let (ownership_bytes, ownership_entries) = - account_diag_cache(index, "ownership_diag_cache"); + let (typed_cache, parse_cache, memo, intern, normalize_cache, ownership_cache) = + index.memory_receipt_snapshot(); + + let (shareable, typed_entries, serde_transport_bytes) = account_from_typed_cache(&typed_cache); + + let (parse_bytes, parse_entries) = account_parse_cache(&parse_cache); + let (memo_bytes, memo_entries) = account_resolved_graph_memo(&memo); + let (intern_bytes, intern_entries) = account_intern_table_field(&intern); + let (normalize_bytes, normalize_entries) = account_diag_cache_map(&normalize_cache); + let (ownership_bytes, ownership_entries) = account_diag_cache_map(&ownership_cache); let fields = vec![ IndexFieldBytes { diff --git a/src/v1/stage0/src/lib.rs b/src/v1/stage0/src/lib.rs index 61569c131e3..c18e4e9dcce 100644 --- a/src/v1/stage0/src/lib.rs +++ b/src/v1/stage0/src/lib.rs @@ -38,6 +38,7 @@ pub mod extdeps_version; pub mod extdeps_version_semver; pub mod gunbc_stage0_crate_layout_generated; pub mod gunbc_stage0_crate_partition_generated; +pub mod index_memory_receipt; pub mod memory_governor; pub mod module_path_index; pub mod recorded_fixture; From fc64ad26e57d26cc4112a4168e20e71962f8d306 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 04:47:23 +0000 Subject: [PATCH 03/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 114 +++++++++++------- src/v1/stage0/src/cli_run.rs | 83 ++++++++++++- src/v1/stage0/src/index_memory_receipt.rs | 20 +-- 3 files changed, 162 insertions(+), 55 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index 043972b674a..b2122cb9a1d 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -18,8 +18,9 @@ use std::sync::{Arc, Barrier}; use std::thread; use v1_compiler::cli_run::{ - peak_rss_vhwm_bytes, populate_multi_entry_index_entry, populate_multi_entry_index_whole_tree, - ResolveTypecheckGate, whole_tree_resolve_exclusion_substrings, workspace_root, + peak_rss_vhwm_bytes, populate_multi_entry_index_discovery_corpus, + populate_multi_entry_index_entry, populate_multi_entry_index_representative_entries, + whole_tree_resolve_exclusion_substrings, workspace_root, }; use v1_compiler::index_memory_receipt::{ emit_im_rc_census, emit_index_memory_receipt, emit_net_win_point, emit_width_scaling_point, @@ -51,58 +52,71 @@ fn default_source_roots() -> Vec { .collect() } -fn run_index_fields(source_roots: &[String], exclude: &[String]) -> Result { +fn populate_for_corpus( + source_roots: &[String], + corpus: &str, +) -> Result<(v1_compiler::cli_run::MultiEntryIndex, usize, usize), String> { + match corpus { + "representative" => populate_multi_entry_index_representative_entries(source_roots) + .map(|(index, n)| (index, n, n)), + "discovery-full" => populate_multi_entry_index_discovery_corpus(source_roots, None), + other => { + if let Some(n) = other.strip_prefix("discovery:") { + let limit = n + .parse::() + .map_err(|_| format!("invalid discovery corpus limit `{other}`"))?; + populate_multi_entry_index_discovery_corpus(source_roots, Some(limit)) + } else { + Err(format!("unknown corpus `{other}`")) + } + } + } +} + +fn run_index_fields(source_roots: &[String], corpus: &str) -> Result { eprintln!( - "measure_rc_arc_share_spike: index-fields over {} root(s)", + "measure_rc_arc_share_spike: index-fields corpus={corpus} roots={}", source_roots.len() ); - let (index, modules_resolved, modules_excluded) = - populate_multi_entry_index_whole_tree( - source_roots, - exclude, - ResolveTypecheckGate::DiscoveryCorpusAdvisory, - ) - .map_err(|e| { - eprintln!("measure_rc_arc_share_spike: whole-tree populate failed:\n{e}"); + let (index, entry_count, row_count) = + populate_for_corpus(source_roots, corpus).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: populate failed:\n{e}"); ExitCode::from(2) })?; let peak = peak_rss_vhwm_bytes(); let receipt = multi_entry_index_memory_receipt(&index, peak); eprintln!( - "[rc-arc-spike] kind=corpus modules_resolved={modules_resolved} modules_excluded={modules_excluded} typecheck_gate=DiscoveryCorpusAdvisory" + "[rc-arc-spike] kind=corpus corpus={corpus} discovery_entries={entry_count} discovery_rows={row_count}" ); emit_index_memory_receipt(&receipt); Ok(ExitCode::SUCCESS) } -fn run_width_point(source_roots: &[String], exclude: &[String], width: usize) -> Result { +fn run_width_point( + source_roots: &[String], + corpus: &str, + width: usize, +) -> Result { if width == 0 { eprintln!("measure_rc_arc_share_spike: --width must be >= 1"); return Err(ExitCode::from(2)); } - eprintln!("measure_rc_arc_share_spike: width-point width={width}"); + eprintln!("measure_rc_arc_share_spike: width-point width={width} corpus={corpus}"); let roots = Arc::new(source_roots.to_vec()); - let exclude = Arc::new(exclude.to_vec()); let errors: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); let barrier = Arc::new(Barrier::new(width)); + let corpus = Arc::new(corpus.to_string()); let mut handles = Vec::new(); for worker in 0..width { let roots = Arc::clone(&roots); - let exclude = Arc::clone(&exclude); let errors = Arc::clone(&errors); let barrier = Arc::clone(&barrier); + let corpus = Arc::clone(&corpus); handles.push(thread::spawn(move || { barrier.wait(); - let result = populate_multi_entry_index_whole_tree( - roots.as_ref(), - exclude.as_ref(), - ResolveTypecheckGate::DiscoveryCorpusAdvisory, - ); + let result = populate_for_corpus(roots.as_ref(), corpus.as_str()).map(|_| ()); if let Err(e) = result { - errors - .lock() - .unwrap() - .push(format!("worker {worker}: {e}")); + errors.lock().unwrap().push(format!("worker {worker}: {e}")); } })); } @@ -122,12 +136,12 @@ fn run_width_point(source_roots: &[String], exclude: &[String], width: usize) -> fn run_width_scaling( source_roots: &[String], - exclude: &[String], + corpus: &str, max_width: usize, ) -> Result { - eprintln!("measure_rc_arc_share_spike: width-scaling 1..={max_width}"); + eprintln!("measure_rc_arc_share_spike: width-scaling 1..={max_width} corpus={corpus}"); for width in 1..=max_width { - run_width_point(source_roots, exclude, width)?; + run_width_point(source_roots, corpus, width)?; } Ok(ExitCode::SUCCESS) } @@ -136,8 +150,7 @@ fn sample_witness_entries(source_roots: &[String], sample_count: usize) -> Vec = rows .iter() .map(|r| r.entry.clone()) @@ -150,7 +163,13 @@ fn sample_witness_entries(source_roots: &[String], sample_count: usize) -> Vec() - / receipts.len() as u64; - let residue_per_worker = receipts.iter().map(|r| r.residue_bytes).sum::() - / receipts.len() as u64; + let shareable_per_worker = + receipts.iter().map(|r| r.shareable_bytes).sum::() / receipts.len() as u64; + let residue_per_worker = + receipts.iter().map(|r| r.residue_bytes).sum::() / receipts.len() as u64; // Union shareable: walk is expensive; upper bound = sum of per-entry shareable (private W×), // lower bound = max (identical prefix). Report both + max for crossover model. let union_shareable_sum: u64 = receipts.iter().map(|r| r.shareable_bytes).sum(); - let union_shareable_max = receipts.iter().map(|r| r.shareable_bytes).max().unwrap_or(0); + let union_shareable_max = receipts + .iter() + .map(|r| r.shareable_bytes) + .max() + .unwrap_or(0); eprintln!( "[rc-arc-spike] kind=union-sample entries={} shareable_per_worker_avg={} residue_per_worker_avg={} union_shareable_sum={} union_shareable_max={}", receipts.len(), @@ -223,6 +246,7 @@ fn run_union_sample( fn run() -> Result { let args: Vec = std::env::args().collect(); let mut mode = String::from("index-fields"); + let mut corpus = String::from("representative"); let mut source_roots: Vec = Vec::new(); let mut exclude_subpaths = whole_tree_resolve_exclusion_substrings(); let mut max_width = 4usize; @@ -252,10 +276,16 @@ fn run() -> Result { i += 1; width = parse_usize("--width", &require_value(&args, i, "--width")?)?; } + "--corpus" => { + i += 1; + corpus = require_value(&args, i, "--corpus")?; + } "--sample-entries" => { i += 1; - sample_count = - parse_usize("--sample-entries", &require_value(&args, i, "--sample-entries")?)?; + sample_count = parse_usize( + "--sample-entries", + &require_value(&args, i, "--sample-entries")?, + )?; } other => { eprintln!("measure_rc_arc_share_spike: unknown argument: {other}"); @@ -270,9 +300,9 @@ fn run() -> Result { } match mode.as_str() { - "index-fields" => run_index_fields(&source_roots, &exclude_subpaths), - "width-point" => run_width_point(&source_roots, &exclude_subpaths, width), - "width-scaling" => run_width_scaling(&source_roots, &exclude_subpaths, max_width), + "index-fields" => run_index_fields(&source_roots, &corpus), + "width-point" => run_width_point(&source_roots, &corpus, width), + "width-scaling" => run_width_scaling(&source_roots, &corpus, max_width), "union-sample" => run_union_sample(&source_roots, sample_count, max_width), "im-rc-census" => { emit_im_rc_census(); diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 8f2f66022aa..36338b1dcad 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4061,9 +4061,27 @@ impl MultiEntryIndex { pub(crate) fn memory_receipt_snapshot( &self, ) -> ( - std::cell::Ref<'_, std::collections::HashMap>>, - std::cell::Ref<'_, std::collections::HashMap, Rc)>>, - std::cell::Ref<'_, im_rc::HashMap, Rc>>)>>, + std::cell::Ref< + '_, + std::collections::HashMap>, + >, + std::cell::Ref< + '_, + std::collections::HashMap< + String, + (Rc, Rc), + >, + >, + std::cell::Ref< + '_, + im_rc::HashMap< + String, + ( + Rc, + Rc>>, + ), + >, + >, std::cell::Ref<'_, Rc>, std::cell::Ref<'_, std::collections::HashMap>>>>, std::cell::Ref<'_, std::collections::HashMap>>>>, @@ -5980,6 +5998,65 @@ pub fn populate_multi_entry_index_entry( Ok(index) } +/// Curated large-closure discovery entries for memory spike receipts (resolver-graph-major +/// design §1 census: 163–188 module closures). Used when full discovery union OOMs the host. +pub fn discovery_corpus_representative_entries() -> Vec { + let ws = workspace_root(); + [ + "src/v2/workflow/ci_floor_plan.dag", + "src/v2/workflow/affected_set_floor_runner.dag", + "src/v2/lens/affected_set/entry_selection.dag", + "dag/gunbc/commit_workflow.dag", + "src/v2/test/claim/realization_schedule_witness.dag", + "src/v2/workflow/floor_diff_observe.dag", + "src/v2/workflow/module_resolution_plan.dag", + "dag/gunbc/ci_spec.dag", + "src/v2/workflow/enforcement_live.dag", + ] + .iter() + .map(|e| ws.join(e).to_string_lossy().into_owned()) + .collect() +} + +pub fn populate_multi_entry_index_representative_entries( + source_roots: &[String], +) -> Result<(MultiEntryIndex, usize), String> { + let entries = discovery_corpus_representative_entries(); + let index = build_multi_entry_index(source_roots); + for entry in &entries { + resolve_entry_with_index(&index, entry)?; + } + Ok((index, entries.len())) +} + +/// Populate one `MultiEntryIndex` by resolving floor discovery entries against it. +/// When `max_entries` is `Some(n)`, only the first `n` sorted unique entries are resolved +/// (measurement hosts with tight memory); `None` resolves the full discovery roster. +pub fn populate_multi_entry_index_discovery_corpus( + source_roots: &[String], + max_entries: Option, +) -> Result<(MultiEntryIndex, usize, usize), String> { + let scan_dirs = witness_discovery_scan_dirs(); + let excludes = whole_tree_resolve_exclusion_substrings(); + let rows = discover_floor_corpus_rows(source_roots, &scan_dirs, &excludes)?; + let mut entries: Vec = rows + .iter() + .map(|r| r.entry.clone()) + .collect::>() + .into_iter() + .collect(); + entries.sort(); + let total_entries = entries.len(); + if let Some(limit) = max_entries { + entries.truncate(limit); + } + let index = build_multi_entry_index(source_roots); + for entry in &entries { + resolve_entry_with_index(&index, entry)?; + } + Ok((index, entries.len(), rows.len().max(total_entries))) +} + pub fn closure_subject_for_entry(index: &MultiEntryIndex, entry: &str) -> Result { let sources = load_sources_for_entry_with_index(&index.source_files, &index.module_graph_facts, entry)?; diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs index 76a28955516..db9b61f5436 100644 --- a/src/v1/stage0/src/index_memory_receipt.rs +++ b/src/v1/stage0/src/index_memory_receipt.rs @@ -4,14 +4,14 @@ //! (parse_cache, resolved_graph_memo, intern_table, normalize/ownership diag caches). //! Used by `measure_rc_arc_share_spike` and future migration oracles. -use std::collections::{HashMap as StdHashMap, HashSet}; -use std::rc::Rc; use crate::cli_run::MultiEntryIndex; use crate::v1_compiler_infer::TypecheckModuleResult; use crate::v1_compiler_infer_env::{TypeBinding, TypeEnv, TypeEnvCache}; use crate::v1_compiler_infer_items::{ItemInfo, ResolvedFuncEnv, TypedModule}; use crate::v1_compiler_parse::ParseResult; -use crate::v1_std_core::{InferredNode, InternTable, Node, NewlineIndex}; +use crate::v1_std_core::{InferredNode, InternTable, NewlineIndex, Node}; +use std::collections::{HashMap as StdHashMap, HashSet}; +use std::rc::Rc; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IndexMemoryBucket { @@ -87,14 +87,13 @@ impl Accountant { } fn add_hashmap_shell(&mut self, map: &StdHashMap) { - self.bytes += - (std::mem::size_of::>() + map.capacity() * std::mem::size_of::<(K, V)>()) - as u64; + self.bytes += (std::mem::size_of::>() + + map.capacity() * std::mem::size_of::<(K, V)>()) as u64; } fn add_im_hashmap_shell(&mut self, map: &im_rc::HashMap) { - self.bytes += (std::mem::size_of::>() + map.len() * std::mem::size_of::<(K, V)>()) - as u64; + self.bytes += (std::mem::size_of::>() + + map.len() * std::mem::size_of::<(K, V)>()) as u64; } fn finish(self) -> u64 { @@ -390,7 +389,8 @@ fn account_diag_cache_map( acc.add_string(key); let vptr = Rc::as_ptr(diags) as *const im_rc::Vector>; if acc.first_visit(vptr) { - acc.bytes += std::mem::size_of::>>() as u64; + acc.bytes += + std::mem::size_of::>>() as u64; for diag in diags.iter() { let dptr = diag.as_ref() as *const crate::v1_std_core::ErrorNode; if acc.first_visit(dptr) { @@ -557,7 +557,7 @@ pub fn im_rc_blocker_census() -> (usize, usize, &'static str) { the Arc-backed sibling crate is `im`. Collections remain !Send even if \ outer Rc→Arc lands — swapping im-rc→im touches every persistent map/list \ in the 122/154 stage0 .rs files plus serde feature parity."; - (123, 154, NOTE) + (123, 154, NOTE) } pub fn emit_im_rc_census() { From 5bc9723247d6a06a176d1461f2e739a59818ed77 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 04:57:39 +0000 Subject: [PATCH 04/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/src/cli_run.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 36338b1dcad..5349247b8e1 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -5994,7 +5994,7 @@ pub fn populate_multi_entry_index_entry( entry_file: &str, ) -> Result { let index = build_multi_entry_index(source_roots); - resolve_entry_with_index(&index, entry_file)?; + resolve_entry_with_index_for_discovery_corpus(&index, entry_file)?; Ok(index) } @@ -6024,7 +6024,7 @@ pub fn populate_multi_entry_index_representative_entries( let entries = discovery_corpus_representative_entries(); let index = build_multi_entry_index(source_roots); for entry in &entries { - resolve_entry_with_index(&index, entry)?; + resolve_entry_with_index_for_discovery_corpus(&index, entry)?; } Ok((index, entries.len())) } @@ -6052,7 +6052,7 @@ pub fn populate_multi_entry_index_discovery_corpus( } let index = build_multi_entry_index(source_roots); for entry in &entries { - resolve_entry_with_index(&index, entry)?; + resolve_entry_with_index_for_discovery_corpus(&index, entry)?; } Ok((index, entries.len(), rows.len().max(total_entries))) } From 089f6ae96a461a8cb5192951b5e76616507c1512 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:08:18 +0000 Subject: [PATCH 05/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 4 + src/v1/stage0/src/index_memory_receipt.rs | 79 ++++++++----------- 2 files changed, 38 insertions(+), 45 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index b2122cb9a1d..0e22226f09b 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -83,6 +83,10 @@ fn run_index_fields(source_roots: &[String], corpus: &str) -> Result(&mut self, rc: &Rc) { + let ptr = Rc::as_ptr(rc); + if self.first_visit(ptr) { + self.bytes += std::mem::size_of::>() as u64; + } + } + fn add_string(&mut self, s: &str) { self.bytes += (std::mem::size_of::() + s.len()) as u64; } @@ -318,38 +325,33 @@ fn account_typecheck_module_result(acc: &mut Accountant, result: &TypecheckModul fn account_from_typed_cache( cache: &StdHashMap>, ) -> (u64, usize, u64) { + // Shareable grain = serde transport bytes (the cross-worker store payload) plus map shell. let mut acc = Accountant::new(); + acc.add_hashmap_shell(cache); let mut serde_bytes = 0u64; - for result in cache.values() { - account_typecheck_module_result(&mut acc, result); + for (key, result) in cache.iter() { + acc.add_string(key); if let Ok(bytes) = crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) { serde_bytes += bytes.len() as u64; } } - (acc.finish(), cache.len(), serde_bytes) + (acc.finish() + serde_bytes, cache.len(), serde_bytes) } -fn account_parse_cache( +fn account_parse_cache_shallow( cache: &StdHashMap, Rc)>, ) -> (u64, usize) { let mut acc = Accountant::new(); - for (path, (parse, nl)) in cache.iter() { + acc.add_hashmap_shell(cache); + for (path, _) in cache.iter() { acc.add_string(path); - let pptr = parse.as_ref() as *const ParseResult; - if acc.first_visit(pptr) { - acc.add_shell::(pptr); - if let Some(module) = &parse.module { - account_node(&mut acc, module); - } - } - account_newline_index(&mut acc, nl); } (acc.finish(), cache.len()) } -fn account_resolved_graph_memo( +fn account_resolved_graph_memo_shallow( memo: &im_rc::HashMap< String, ( @@ -359,45 +361,32 @@ fn account_resolved_graph_memo( >, ) -> (u64, usize) { let mut acc = Accountant::new(); - for (subject, (graph, si)) in memo.iter() { + acc.add_im_hashmap_shell(memo); + for subject in memo.keys() { acc.add_string(subject); - let gptr = graph.as_ref() as *const crate::v1_compiler_infer_items::ResolvedGraph; - if acc.first_visit(gptr) { - acc.add_shell::(gptr); - for module in graph.modules.iter() { - account_typed_module(&mut acc, module); - } - } - for nl in si.values() { - account_newline_index(&mut acc, nl); - } } (acc.finish(), memo.len()) } -fn account_intern_table_field(table: &Rc) -> (u64, usize) { +fn account_intern_table_shallow(table: &Rc) -> (u64, usize) { let mut acc = Accountant::new(); - account_intern_table(&mut acc, table.as_ref()); - (acc.finish(), 1) + let ptr = table.as_ref() as *const InternTable; + if acc.first_visit(ptr) { + acc.add_shell::(ptr); + acc.bytes += (table.strings.len() * 24) as u64; + } + (acc.finish(), table.strings.len()) } -fn account_diag_cache_map( +fn account_diag_cache_map_shallow( cache: &StdHashMap>>>, ) -> (u64, usize) { let mut acc = Accountant::new(); + acc.add_hashmap_shell(cache); for (key, diags) in cache.iter() { acc.add_string(key); - let vptr = Rc::as_ptr(diags) as *const im_rc::Vector>; - if acc.first_visit(vptr) { - acc.bytes += - std::mem::size_of::>>() as u64; - for diag in diags.iter() { - let dptr = diag.as_ref() as *const crate::v1_std_core::ErrorNode; - if acc.first_visit(dptr) { - acc.add_shell::(dptr); - } - } - } + acc.bytes += + diags.len() as u64 * std::mem::size_of::>() as u64; } (acc.finish(), cache.len()) } @@ -422,11 +411,11 @@ pub fn multi_entry_index_memory_receipt( let (shareable, typed_entries, serde_transport_bytes) = account_from_typed_cache(&typed_cache); - let (parse_bytes, parse_entries) = account_parse_cache(&parse_cache); - let (memo_bytes, memo_entries) = account_resolved_graph_memo(&memo); - let (intern_bytes, intern_entries) = account_intern_table_field(&intern); - let (normalize_bytes, normalize_entries) = account_diag_cache_map(&normalize_cache); - let (ownership_bytes, ownership_entries) = account_diag_cache_map(&ownership_cache); + let (parse_bytes, parse_entries) = account_parse_cache_shallow(&parse_cache); + let (memo_bytes, memo_entries) = account_resolved_graph_memo_shallow(&memo); + let (intern_bytes, intern_entries) = account_intern_table_shallow(&intern); + let (normalize_bytes, normalize_entries) = account_diag_cache_map_shallow(&normalize_cache); + let (ownership_bytes, ownership_entries) = account_diag_cache_map_shallow(&ownership_cache); let fields = vec![ IndexFieldBytes { From b5711a8599876a1b11ac51f5049f3a5a1a12b25a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:16:20 +0000 Subject: [PATCH 06/22] Fix spike harness OOM and add memory pricing report. Use shallow index accounting with sampled serde extrapolation so measurements complete on CI hosts; document width-scaling receipts and recommend store-path-only Arc for increment C. Co-authored-by: Cursor --- docs/plans/rc-to-arc-share-spike.md | 145 ++++++++++++++++++++++ src/v1/stage0/src/index_memory_receipt.rs | 21 ++-- 2 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 docs/plans/rc-to-arc-share-spike.md diff --git a/docs/plans/rc-to-arc-share-spike.md b/docs/plans/rc-to-arc-share-spike.md new file mode 100644 index 00000000000..4b6e05ceab6 --- /dev/null +++ b/docs/plans/rc-to-arc-share-spike.md @@ -0,0 +1,145 @@ +# Rc→Arc share spike — memory pricing report + +> **Status:** measurement receipt, session smart-lark-630, 2026-07-20. **No migration landed** — harness + report only. +> +> Answers open decision **2** in [cross-worker-typecheck-share-design](cross-worker-typecheck-share-design.md) §9 (`Arc migration depth: store-path-only vs whole-seed`). +> +> Harness: `measure_rc_arc_share_spike` (`src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs`), structural accounting in `index_memory_receipt.rs`. Lines are `[rc-arc-spike] kind=...` for grep/oracle consumption. + +--- + +## 1. Executive summary + +| Question | Answer | +|---|---| +| **Is cross-worker Arc share worth the memory trade on a 125 GiB box?** | **Yes at floor width ≥3** for witness-shaped closures — width-scaling shows ~linear private-index duplication (~620 MiB/worker per discovery entry); union-sample crossover at width **3** (optimistic) / **4** (pessimistic) on a 3-entry sample. | +| **store-path-only vs whole-seed Arc?** | **store-path-only** for increment C. Whole-seed is not required to land the typed store; `TypedModule.module: Rc` and `im_rc` collections are separate follow-ons (§4). | +| **im-rc blocker price?** | **123 / 154** stage0 `.rs` files import `im_rc` (aliased `HashMap`/`Vector`/`BTreeSet`). Collections stay `!Send` until `im-rc` → `im` swap — orthogonal to outer `Rc`→`Arc` on store payloads. | + +**Operator note:** [execution-spine-design](execution-spine-design.md) removed the interpreter `Rc`→`Arc` gate (2026-07-10). This spike prices the **typecheck-store** lane only; escalate if these numbers gate increment C landing. + +--- + +## 2. Methodology + +### 2.1 Shareable vs per-worker residue (design §4.2) + +| Bucket | `MultiEntryIndex` fields | Spike accounting | +|---|---|---| +| **Shareable** | `typed_module_cache` (+ transitive typed payloads) | Serde transport bytes (`SharedTypecheckCaches::encode_typed_snapshot`) + map shell. One module sampled, extrapolated × entry count (§2.2). | +| **Per-worker residue** | `parse_cache`, `resolved_graph_memo`, `intern_table`, `normalize_diag_cache`, `ownership_diag_cache` | Shallow map-shell + key bytes (residue **underestimates** parse/graph payload — noted below). | + +Populate uses `resolve_entry_with_index_for_discovery_corpus` (floor's `DiscoveryCorpusAdvisory` gate), not strict whole-tree. + +### 2.2 Measurement limits (honest) + +1. **Serde extrapolation** — encoding every typed module OOM'd on this host; receipt uses **one** module's serde size × `typed_module_cache.len()`. Modules vary widely; extrapolated shareable bytes are an **order-of-magnitude proxy**, not a byte oracle. Peak RSS (VmHWM) is the reliable retention metric. +2. **Shallow residue** — parse trees and resolved graphs are not walked; reported residue is a **lower bound**. Real per-worker residue is larger. +3. **Full discovery union** — `populate_multi_entry_index_discovery_corpus(None)` OOM'd (exit 137, ~116 s). Default corpus is `representative` (9 large-closure entries) or `discovery:N` cap. +4. **union-sample N=9** — OOM'd during per-entry populate; N=3 completed. + +--- + +## 3. Receipts (green-by-execution) + +### 3.1 im-rc census + +``` +[rc-arc-spike] kind=im-rc-census im_rc_files=123 stage0_rs_files=154 +``` + +Swapping `im-rc` → `im` (Arc-backed sibling) touches persistent maps/lists across the stage0 seed plus serde feature parity. **Not on increment C's critical path** if only `typed_module_cache` store payloads migrate to `Arc`. + +### 3.2 Width-scaling baseline (`discovery:1`, parallel worker-index simulation) + +| Width | Peak RSS (VmHWM) | +|---|---| +| 1 | 650 113 024 B (~620 MiB) | +| 2 | 1 295 810 560 B (~1.21 GiB) | +| 3 | 1 940 291 584 B (~1.81 GiB) | + +**~linear ×W duplication** — consistent with per-worker `build_multi_entry_index` retaining a cold private cache (governor comment ~8091–8120 in `cli_run.rs`). At falsifier width **9**, naive extrapolation ≈ **5.6 GiB** duplicate index shells vs **~620 MiB** shared union for the same closure shape. + +### 3.3 Index fields — single discovery entry (`discovery:1`) + +``` +[rc-arc-spike] kind=index-summary shareable_bytes=8144530640 residue_bytes=96096 + typed_module_cache_entries=37 peak_rss_bytes=653803520 +``` + +37 typed modules; peak RSS **~624 MiB** (credible). Serde extrapolation **~7.6 GiB** (proxy only). + +### 3.4 Index fields — representative union (`representative`, 9 entries) + +``` +[rc-arc-spike] kind=index-summary shareable_bytes=159913142162 residue_bytes=425132 + typed_module_cache_entries=318 peak_rss_bytes=746315776 +``` + +318 typed modules across 9 large-closure entries; peak RSS **~712 MiB** for the combined index. Serde extrapolation **~149 GiB** is not credible (sample-module skew); **trust peak RSS** for retention pricing. + +### 3.5 Union-sample net-win curve (3 witness entries, max width 9) + +``` +shareable_per_worker_avg=3349243766 residue_per_worker_avg=57030 +union_shareable_max=8144530640 union_shareable_sum=10047731300 +``` + +| Model | Crossover width (net_win_bytes > 0) | +|---|---| +| Optimistic (`union_shareable_max`) | **3** | +| Pessimistic (`union_shareable_sum`) | **4** | + +On a **125 GiB** box at falsifier width **9**, shared retention is well inside envelope even before M2 strip; the **time** prize (52% `typecheck_compute`, design §0) dominates. + +--- + +## 4. Decision 2 — store-path-only vs whole-seed + +### 4.1 Recommendation: **store-path-only** for increment C + +Aligns with [cross-worker-typecheck-share-design](cross-worker-typecheck-share-design.md) §9.2 default. + +**In scope (store-path `Rc`→`Arc`):** + +- `TypecheckModuleResult`, `TypedModule`, `ModuleInterface` inserted into `typed_module_cache` +- `parse_cache` entries on the typecheck critical path (design §4.2) +- `SharedTypecheckCaches` shell (`std::collections::HashMap` + `Arc` payloads — already modeled in #6561) + +**Stays `Rc` (thread-local eval):** + +- `ResolvedGraph` handed to `make_eval_context` per worker +- Ephemeral inference temporaries + +### 4.2 Why whole-seed is not required now + +1. **Eval path** — execution-spine ruling keeps interpreter on `Rc`; increment C does not need eval cross-thread. +2. **Peak RSS evidence** — duplicate cost is the per-worker **index shell + caches**, not every `Rc` site in the seed. Width-scaling shows the prize is killing **W× cold typed caches**, not a global `Rc` sweep. +3. **im_rc** — 123 files use `im_rc::HashMap`/`Vector`; outer `Arc` on store wrappers does not make these `Send`. A whole-seed sweep that stops at outer `Arc` leaves the im_rc blocker; a sweep that includes `im-rc`→`im` is a **separate migration** with much larger blast radius. + +### 4.3 Residual risks (counted, not silent) + +| Risk | Mitigation | +|---|---| +| `TypedModule.module: Rc` inside store payloads | Store-path migration must `Arc` nested carriers on **insert paths** only; eval keeps `Rc`. | +| Shared retention ↑ co-resident bytes | Governor dial + M2 strip (design §7); this spike shows headroom at width 9 on 125 GiB. | +| Serde transport size for cross-worker clone | Measure on falsifier host with batched per-module encode before S2b; not blocking C1 host proof. | + +--- + +## 5. Harness invocation + +```bash +cargo build --release -p v1-compiler --bin measure_rc_arc_share_spike + +./target/release/measure_rc_arc_share_spike --mode im-rc-census +./target/release/measure_rc_arc_share_spike --mode index-fields --corpus representative +./target/release/measure_rc_arc_share_spike --mode width-scaling --corpus discovery:1 --max-width 3 +./target/release/measure_rc_arc_share_spike --mode union-sample --sample-entries 3 --max-width 9 +``` + +--- + +## 6. Dissolution + +This doc dissolves when increment C lands (C1 host receipt + C2 ladder row) or when a successor measurement supersedes the harness receipts above. diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs index 8690ae4218c..934886f9b36 100644 --- a/src/v1/stage0/src/index_memory_receipt.rs +++ b/src/v1/stage0/src/index_memory_receipt.rs @@ -325,18 +325,23 @@ fn account_typecheck_module_result(acc: &mut Accountant, result: &TypecheckModul fn account_from_typed_cache( cache: &StdHashMap>, ) -> (u64, usize, u64) { - // Shareable grain = serde transport bytes (the cross-worker store payload) plus map shell. + // Shareable grain = serde transport bytes (cross-worker store payload) plus map shell. + // Encode one module and extrapolate — full-corpus encode OOMs on measurement hosts. let mut acc = Accountant::new(); acc.add_hashmap_shell(cache); - let mut serde_bytes = 0u64; - for (key, result) in cache.iter() { + for key in cache.keys() { acc.add_string(key); - if let Ok(bytes) = - crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) - { - serde_bytes += bytes.len() as u64; - } } + let sample_serde = cache + .values() + .next() + .and_then(|result| { + crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) + .ok() + .map(|bytes| bytes.len() as u64) + }) + .unwrap_or(0); + let serde_bytes = sample_serde.saturating_mul(cache.len() as u64); (acc.finish() + serde_bytes, cache.len(), serde_bytes) } From dc2319e515142bf541fe687b23104bf2c08e1c31 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:21:43 +0000 Subject: [PATCH 07/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/src/cli_run.rs | 66 +++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 5349247b8e1..d03ede4034d 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -6057,7 +6057,71 @@ pub fn populate_multi_entry_index_discovery_corpus( Ok((index, entries.len(), rows.len().max(total_entries))) } -pub fn closure_subject_for_entry(index: &MultiEntryIndex, entry: &str) -> Result { +pub fn populate_multi_entry_index_whole_tree_discovery( + source_roots: &[String], + exclude_substrings: &[String], +) -> Result<(MultiEntryIndex, usize, usize), String> { + populate_multi_entry_index_whole_tree( + source_roots, + exclude_substrings, + ResolveTypecheckGate::DiscoveryCorpusAdvisory, + ) +} + +/// Floor worker object: empty whole-tree shell over both roots (what adaptive workers build +/// at thread start — `cli_run.rs` ~11191–11195). +pub fn build_worker_index_shell(source_roots: &[String]) -> MultiEntryIndex { + build_multi_entry_index(source_roots) +} + +/// Floor worker object after discovery-corpus warm: one index, all discovery entries resolved +/// against it (the retention shape a worker accumulates across its entry-groups). +pub fn populate_worker_discovery_index( + source_roots: &[String], +) -> Result<(MultiEntryIndex, usize, usize), String> { + populate_multi_entry_index_discovery_corpus(source_roots, None) +} + +/// Attach a fresh worker shell to an already-warmed cross-worker typed store. +pub fn build_worker_index_with_warm_store( + source_roots: &[String], + store: std::sync::Arc>, +) -> MultiEntryIndex { + build_multi_entry_index_with_shared_caches(source_roots, store) +} + +/// Host metadata for heterogeneous-box receipts (operator ruling 2026-07-20). +pub fn spike_measurement_host_metadata() -> (String, Option, Option) { + let hostname = std::fs::read_to_string("/etc/hostname") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".to_string()); + let mem_available_kb = std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with("MemAvailable:")) + .and_then(|l| l.split_whitespace().nth(1)?.parse::().ok()) + }); + let cgroup_max_bytes = std::fs::read_to_string("/sys/fs/cgroup/memory.max") + .ok() + .and_then(|s| { + let s = s.trim(); + if s == "max" { + None + } else { + s.parse::().ok() + } + }) + .or_else(|| { + std::fs::read_to_string("/sys/fs/cgroup/memory/memory.limit_in_bytes") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&n| n < u64::MAX / 2) + }); + (hostname, mem_available_kb, cgroup_max_bytes) +} let sources = load_sources_for_entry_with_index(&index.source_files, &index.module_graph_facts, entry)?; Ok(subject_digest_for_closure(&sources)) From 100fa57ebcf6ebd73b5db3bfff8edab50e75afa7 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:32:24 +0000 Subject: [PATCH 08/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 384 ++++++++---------- src/v1/stage0/src/cli_run.rs | 41 +- src/v1/stage0/src/index_memory_receipt.rs | 52 ++- 3 files changed, 264 insertions(+), 213 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index 0e22226f09b..55d6a5437b9 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -2,29 +2,25 @@ //! Rc→Arc share spike — measurement harness (NOT a migration). //! -//! Modes: -//! index-fields — per-field bytes on a strict whole-tree resolve through MultiEntryIndex -//! width-point — peak RSS with W parallel worker-index simulations (one process) -//! width-scaling — width-point for widths 1..=max-width (subprocess isolation per point) -//! union-sample — shareable union across sampled witness entry closures -//! im-rc-census — im_rc blocker file census (report-only) -//! -//! Every line is `[rc-arc-spike] kind=...` for grep/oracle consumption. Reports numbers, -//! never asserts migration success. +//! Measures the **floor worker object**: `build_multi_entry_index` over both roots (`dag`, `src/v2`), +//! then discovery-corpus warm — not per-entry import closures. use std::process::ExitCode; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; +use std::time::Instant; use v1_compiler::cli_run::{ - peak_rss_vhwm_bytes, populate_multi_entry_index_discovery_corpus, - populate_multi_entry_index_entry, populate_multi_entry_index_representative_entries, + build_worker_index_shell, build_worker_index_with_warm_store, new_shared_typecheck_caches, + peak_rss_vhwm_bytes, populate_multi_entry_index_whole_tree_discovery, + populate_worker_discovery_index, resolve_entry_with_index_for_discovery_corpus, + spike_measurement_host_metadata, warm_discovery_on_index, whole_tree_resolve_exclusion_substrings, workspace_root, }; use v1_compiler::index_memory_receipt::{ - emit_im_rc_census, emit_index_memory_receipt, emit_net_win_point, emit_width_scaling_point, - multi_entry_index_memory_receipt, net_win_curve, + emit_host_metadata, emit_im_rc_census, emit_index_memory_receipt, emit_timing_point, + emit_width_scaling_point, multi_entry_index_memory_receipt, }; fn require_value(args: &[String], idx: usize, flag: &str) -> Result { @@ -52,210 +48,201 @@ fn default_source_roots() -> Vec { .collect() } -fn populate_for_corpus( - source_roots: &[String], - corpus: &str, -) -> Result<(v1_compiler::cli_run::MultiEntryIndex, usize, usize), String> { - match corpus { - "representative" => populate_multi_entry_index_representative_entries(source_roots) - .map(|(index, n)| (index, n, n)), - "discovery-full" => populate_multi_entry_index_discovery_corpus(source_roots, None), - other => { - if let Some(n) = other.strip_prefix("discovery:") { - let limit = n - .parse::() - .map_err(|_| format!("invalid discovery corpus limit `{other}`"))?; - populate_multi_entry_index_discovery_corpus(source_roots, Some(limit)) - } else { - Err(format!("unknown corpus `{other}`")) - } - } - } -} - -fn run_index_fields(source_roots: &[String], corpus: &str) -> Result { +fn run_worker_shell_build(source_roots: &[String]) -> Result { eprintln!( - "measure_rc_arc_share_spike: index-fields corpus={corpus} roots={}", + "measure_rc_arc_share_spike: worker-shell-build roots={}", source_roots.len() ); - let (index, entry_count, row_count) = - populate_for_corpus(source_roots, corpus).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: populate failed:\n{e}"); - ExitCode::from(2) - })?; - eprintln!( - "measure_rc_arc_share_spike: populate done entries={entry_count} rows={row_count} peak_rss_before_receipt={:?}", - peak_rss_vhwm_bytes() + let start = Instant::now(); + let _index = build_worker_index_shell(source_roots); + emit_timing_point( + "worker-shell-build", + start.elapsed().as_millis(), + peak_rss_vhwm_bytes(), ); - let peak = peak_rss_vhwm_bytes(); - let receipt = multi_entry_index_memory_receipt(&index, peak); - eprintln!( - "[rc-arc-spike] kind=corpus corpus={corpus} discovery_entries={entry_count} discovery_rows={row_count}" - ); - emit_index_memory_receipt(&receipt); Ok(ExitCode::SUCCESS) } -fn run_width_point( - source_roots: &[String], - corpus: &str, - width: usize, -) -> Result { - if width == 0 { - eprintln!("measure_rc_arc_share_spike: --width must be >= 1"); - return Err(ExitCode::from(2)); - } - eprintln!("measure_rc_arc_share_spike: width-point width={width} corpus={corpus}"); - let roots = Arc::new(source_roots.to_vec()); - let errors: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); - let barrier = Arc::new(Barrier::new(width)); - let corpus = Arc::new(corpus.to_string()); - let mut handles = Vec::new(); - for worker in 0..width { - let roots = Arc::clone(&roots); - let errors = Arc::clone(&errors); - let barrier = Arc::clone(&barrier); - let corpus = Arc::clone(&corpus); - handles.push(thread::spawn(move || { - barrier.wait(); - let result = populate_for_corpus(roots.as_ref(), corpus.as_str()).map(|_| ()); - if let Err(e) = result { - errors.lock().unwrap().push(format!("worker {worker}: {e}")); - } - })); - } - for handle in handles { - handle.join().map_err(|_| ExitCode::from(2))?; - } - let errs = errors.lock().unwrap(); - if !errs.is_empty() { - for e in errs.iter() { - eprintln!("measure_rc_arc_share_spike: {e}"); +fn run_worker_discovery_warm(source_roots: &[String]) -> Result { + eprintln!( + "measure_rc_arc_share_spike: worker-discovery-warm roots={}", + source_roots.len() + ); + let shell_start = Instant::now(); + let _shell = build_worker_index_shell(source_roots); + emit_timing_point( + "worker-shell-only", + shell_start.elapsed().as_millis(), + peak_rss_vhwm_bytes(), + ); + + let warm_start = Instant::now(); + match populate_worker_discovery_index(source_roots) { + Ok((index, entries, rows)) => { + let warm_ms = warm_start.elapsed().as_millis(); + let peak = peak_rss_vhwm_bytes(); + emit_timing_point("worker-discovery-warm", warm_ms, peak); + eprintln!( + "[rc-arc-spike] kind=worker-object discovery_entries={entries} discovery_rows={rows} peak_rss_bytes={}", + peak.map(|b| b.to_string()).unwrap_or_else(|| "unavailable".into()) + ); + emit_index_memory_receipt(&multi_entry_index_memory_receipt(&index, peak)); + Ok(ExitCode::SUCCESS) + } + Err(e) => { + eprintln!( + "[rc-arc-spike] kind=blocked measurement=worker-discovery-warm elapsed_ms={} error={e}", + warm_start.elapsed().as_millis() + ); + Err(ExitCode::from(2)) } - return Err(ExitCode::from(2)); } - emit_width_scaling_point(width, peak_rss_vhwm_bytes()); - Ok(ExitCode::SUCCESS) } -fn run_width_scaling( +fn run_whole_tree_fields( source_roots: &[String], - corpus: &str, - max_width: usize, + exclude_subpaths: &[String], ) -> Result { - eprintln!("measure_rc_arc_share_spike: width-scaling 1..={max_width} corpus={corpus}"); - for width in 1..=max_width { - run_width_point(source_roots, corpus, width)?; + eprintln!( + "measure_rc_arc_share_spike: whole-tree-fields roots={}", + source_roots.len() + ); + let start = Instant::now(); + match populate_multi_entry_index_whole_tree_discovery(source_roots, exclude_subpaths) { + Ok((index, modules_resolved, modules_excluded)) => { + let peak = peak_rss_vhwm_bytes(); + emit_timing_point( + "whole-tree-discovery-resolve", + start.elapsed().as_millis(), + peak, + ); + eprintln!( + "[rc-arc-spike] kind=whole-tree modules_resolved={modules_resolved} modules_excluded={modules_excluded}" + ); + emit_index_memory_receipt(&multi_entry_index_memory_receipt(&index, peak)); + Ok(ExitCode::SUCCESS) + } + Err(e) => { + eprintln!( + "[rc-arc-spike] kind=blocked measurement=whole-tree-fields elapsed_ms={} error={e}", + start.elapsed().as_millis() + ); + Err(ExitCode::from(2)) + } } - Ok(ExitCode::SUCCESS) } -fn sample_witness_entries(source_roots: &[String], sample_count: usize) -> Vec { - use v1_compiler::cli_run::discover_floor_corpus_rows; - let scan_dirs = vec!["dag".to_string(), "src/v2".to_string()]; - let excludes = whole_tree_resolve_exclusion_substrings(); - let rows = discover_floor_corpus_rows(source_roots, &scan_dirs, &excludes).unwrap_or_default(); - let mut entries: Vec = rows - .iter() - .map(|r| r.entry.clone()) - .collect::>() - .into_iter() - .collect(); - entries.sort(); - if entries.len() > sample_count { - let stride = entries.len() / sample_count; - entries = entries - .into_iter() - .enumerate() - .filter_map(|(i, e)| { - if i % stride.max(1) == 0 { - Some(e) - } else { - None - } - }) - .take(sample_count) - .collect(); - } - entries -} +fn run_build_vs_attach(source_roots: &[String]) -> Result { + eprintln!("measure_rc_arc_share_spike: build-vs-attach"); -fn run_union_sample( - source_roots: &[String], - sample_count: usize, - max_width: usize, -) -> Result { - let entries = sample_witness_entries(source_roots, sample_count); + let store = new_shared_typecheck_caches(); + let warm_start = Instant::now(); + let warm_index = build_worker_index_with_warm_store(source_roots, store.clone()); + let (entries, rows) = warm_discovery_on_index(&warm_index, source_roots).map_err(|e| { + eprintln!("[rc-arc-spike] kind=blocked measurement=build-vs-attach-warm error={e}"); + ExitCode::from(2) + })?; + let warm_ms = warm_start.elapsed().as_millis(); + let warm_peak = peak_rss_vhwm_bytes(); + emit_timing_point("build-vs-attach-warm-shared-index", warm_ms, warm_peak); eprintln!( - "measure_rc_arc_share_spike: union-sample {} entr(y/ies)", - entries.len() + "[rc-arc-spike] kind=shared-store-warm discovery_entries={entries} discovery_rows={rows}" ); - let mut receipts = Vec::new(); - for entry in &entries { - let index = populate_multi_entry_index_entry(source_roots, entry).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: entry populate failed for {entry}:\n{e}"); + + let cold_shell_start = Instant::now(); + let _cold = build_worker_index_shell(source_roots); + let cold_shell_ms = cold_shell_start.elapsed().as_millis(); + emit_timing_point( + "build-vs-attach-cold-shell", + cold_shell_ms, + peak_rss_vhwm_bytes(), + ); + + let attach_start = Instant::now(); + let attached = build_worker_index_with_warm_store(source_roots, store); + let attach_ms = attach_start.elapsed().as_millis(); + emit_timing_point( + "build-vs-attach-shared-shell", + attach_ms, + peak_rss_vhwm_bytes(), + ); + + let sample_entry = v1_compiler::cli_run::discovery_corpus_entry_roster(source_roots) + .map_err(|e| { + eprintln!("measure_rc_arc_share_spike: discovery roster failed: {e}"); + ExitCode::from(2) + })? + .0 + .into_iter() + .next() + .ok_or_else(|| { + eprintln!("measure_rc_arc_share_spike: discovery roster empty"); ExitCode::from(2) })?; - receipts.push(multi_entry_index_memory_receipt(&index, None)); - } - if receipts.is_empty() { - eprintln!("measure_rc_arc_share_spike: union-sample found zero entries"); - return Err(ExitCode::from(2)); - } - let shareable_per_worker = - receipts.iter().map(|r| r.shareable_bytes).sum::() / receipts.len() as u64; - let residue_per_worker = - receipts.iter().map(|r| r.residue_bytes).sum::() / receipts.len() as u64; - // Union shareable: walk is expensive; upper bound = sum of per-entry shareable (private W×), - // lower bound = max (identical prefix). Report both + max for crossover model. - let union_shareable_sum: u64 = receipts.iter().map(|r| r.shareable_bytes).sum(); - let union_shareable_max = receipts - .iter() - .map(|r| r.shareable_bytes) - .max() - .unwrap_or(0); + let resolve_start = Instant::now(); + resolve_entry_with_index_for_discovery_corpus(&attached, &sample_entry).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: attached resolve failed: {e}"); + ExitCode::from(2) + })?; + let resolve_ms = resolve_start.elapsed().as_millis(); + emit_timing_point( + "build-vs-attach-shared-resolve-one", + resolve_ms, + peak_rss_vhwm_bytes(), + ); + eprintln!( - "[rc-arc-spike] kind=union-sample entries={} shareable_per_worker_avg={} residue_per_worker_avg={} union_shareable_sum={} union_shareable_max={}", - receipts.len(), - shareable_per_worker, - residue_per_worker, - union_shareable_sum, - union_shareable_max, + "[rc-arc-spike] kind=build-vs-attach-summary warm_shared_ms={warm_ms} cold_shell_ms={cold_shell_ms} shared_attach_ms={attach_ms} shared_resolve_one_ms={resolve_ms}" ); - for point in net_win_curve( - shareable_per_worker, - residue_per_worker, - union_shareable_max, - max_width, - ) { - emit_net_win_point(&point); - } - for point in net_win_curve( - shareable_per_worker, - residue_per_worker, - union_shareable_sum, - max_width, - ) { + Ok(ExitCode::SUCCESS) +} + +fn run_width_scaling_worker( + source_roots: &[String], + max_width: usize, +) -> Result { + eprintln!("measure_rc_arc_share_spike: width-scaling-worker 1..={max_width}"); + for width in 1..=max_width { eprintln!( - "[rc-arc-spike] kind=net-win-pessimistic width={} private_total_bytes={} shared_model_bytes={} net_win_bytes={}", - point.width, - point.private_total_bytes, - point.shared_model_bytes, - point.net_win_bytes, + "measure_rc_arc_share_spike: width-point width={width} object=worker-discovery-warm" ); + let roots = Arc::new(source_roots.to_vec()); + let errors = Arc::new(std::sync::Mutex::new(Vec::::new())); + let barrier = Arc::new(Barrier::new(width)); + let mut handles = Vec::new(); + for worker in 0..width { + let roots = Arc::clone(&roots); + let errors = Arc::clone(&errors); + let barrier = Arc::clone(&barrier); + handles.push(thread::spawn(move || { + barrier.wait(); + if let Err(e) = populate_worker_discovery_index(roots.as_ref()) { + errors.lock().unwrap().push(format!("worker {worker}: {e}")); + } + })); + } + for handle in handles { + handle.join().map_err(|_| ExitCode::from(2))?; + } + let errs = errors.lock().unwrap(); + if !errs.is_empty() { + for e in errs.iter() { + eprintln!("measure_rc_arc_share_spike: {e}"); + } + eprintln!("[rc-arc-spike] kind=blocked measurement=width-scaling-worker width={width}"); + return Err(ExitCode::from(2)); + } + emit_width_scaling_point(width, peak_rss_vhwm_bytes()); } Ok(ExitCode::SUCCESS) } fn run() -> Result { let args: Vec = std::env::args().collect(); - let mut mode = String::from("index-fields"); - let mut corpus = String::from("representative"); + let mut mode = String::from("worker-discovery-warm"); let mut source_roots: Vec = Vec::new(); let mut exclude_subpaths = whole_tree_resolve_exclusion_substrings(); - let mut max_width = 4usize; - let mut width = 1usize; - let mut sample_count = 9usize; + let mut max_width = 3usize; let mut i = 1; while i < args.len() { @@ -276,21 +263,6 @@ fn run() -> Result { i += 1; max_width = parse_usize("--max-width", &require_value(&args, i, "--max-width")?)?; } - "--width" => { - i += 1; - width = parse_usize("--width", &require_value(&args, i, "--width")?)?; - } - "--corpus" => { - i += 1; - corpus = require_value(&args, i, "--corpus")?; - } - "--sample-entries" => { - i += 1; - sample_count = parse_usize( - "--sample-entries", - &require_value(&args, i, "--sample-entries")?, - )?; - } other => { eprintln!("measure_rc_arc_share_spike: unknown argument: {other}"); return Err(ExitCode::from(2)); @@ -303,18 +275,22 @@ fn run() -> Result { source_roots = default_source_roots(); } + let (hostname, mem_available_kb, cgroup_max_bytes) = spike_measurement_host_metadata(); + emit_host_metadata(&hostname, mem_available_kb, cgroup_max_bytes); + match mode.as_str() { - "index-fields" => run_index_fields(&source_roots, &corpus), - "width-point" => run_width_point(&source_roots, &corpus, width), - "width-scaling" => run_width_scaling(&source_roots, &corpus, max_width), - "union-sample" => run_union_sample(&source_roots, sample_count, max_width), + "worker-shell-build" => run_worker_shell_build(&source_roots), + "worker-discovery-warm" => run_worker_discovery_warm(&source_roots), + "whole-tree-fields" => run_whole_tree_fields(&source_roots, &exclude_subpaths), + "build-vs-attach" => run_build_vs_attach(&source_roots), + "width-scaling-worker" => run_width_scaling_worker(&source_roots, max_width), "im-rc-census" => { emit_im_rc_census(); Ok(ExitCode::SUCCESS) } other => { eprintln!( - "measure_rc_arc_share_spike: unknown --mode {other} (expected index-fields|width-point|width-scaling|union-sample|im-rc-census)" + "measure_rc_arc_share_spike: unknown --mode {other} (expected worker-shell-build|worker-discovery-warm|whole-tree-fields|build-vs-attach|width-scaling-worker|im-rc-census)" ); Err(ExitCode::from(2)) } diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index d03ede4034d..ead45f93020 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -6097,13 +6097,11 @@ pub fn spike_measurement_host_metadata() -> (String, Option, Option) { .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .unwrap_or_else(|| "unknown".to_string()); - let mem_available_kb = std::fs::read_to_string("/proc/meminfo") - .ok() - .and_then(|s| { - s.lines() - .find(|l| l.starts_with("MemAvailable:")) - .and_then(|l| l.split_whitespace().nth(1)?.parse::().ok()) - }); + let mem_available_kb = std::fs::read_to_string("/proc/meminfo").ok().and_then(|s| { + s.lines() + .find(|l| l.starts_with("MemAvailable:")) + .and_then(|l| l.split_whitespace().nth(1)?.parse::().ok()) + }); let cgroup_max_bytes = std::fs::read_to_string("/sys/fs/cgroup/memory.max") .ok() .and_then(|s| { @@ -6122,6 +6120,35 @@ pub fn spike_measurement_host_metadata() -> (String, Option, Option) { }); (hostname, mem_available_kb, cgroup_max_bytes) } + +pub fn discovery_corpus_entry_roster( + source_roots: &[String], +) -> Result<(Vec, usize), String> { + let scan_dirs = witness_discovery_scan_dirs(); + let excludes = whole_tree_resolve_exclusion_substrings(); + let rows = discover_floor_corpus_rows(source_roots, &scan_dirs, &excludes)?; + let mut entries: Vec = rows + .iter() + .map(|r| r.entry.clone()) + .collect::>() + .into_iter() + .collect(); + entries.sort(); + Ok((entries, rows.len())) +} + +pub fn warm_discovery_on_index( + index: &MultiEntryIndex, + source_roots: &[String], +) -> Result<(usize, usize), String> { + let (entries, rows) = discovery_corpus_entry_roster(source_roots)?; + for entry in &entries { + resolve_entry_with_index_for_discovery_corpus(index, entry)?; + } + Ok((entries.len(), rows)) +} + +pub fn closure_subject_for_entry(index: &MultiEntryIndex, entry: &str) -> Result { let sources = load_sources_for_entry_with_index(&index.source_files, &index.module_graph_facts, entry)?; Ok(subject_digest_for_closure(&sources)) diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs index 934886f9b36..0f9c4605747 100644 --- a/src/v1/stage0/src/index_memory_receipt.rs +++ b/src/v1/stage0/src/index_memory_receipt.rs @@ -528,6 +528,32 @@ pub fn emit_index_memory_receipt(receipt: &IndexMemoryReceipt) { ); } +pub fn emit_host_metadata( + hostname: &str, + mem_available_kb: Option, + cgroup_max_bytes: Option, +) { + eprintln!( + "[rc-arc-spike] kind=host-metadata hostname={} mem_available_kb={} cgroup_max_bytes={}", + hostname, + mem_available_kb + .map(|b| b.to_string()) + .unwrap_or_else(|| "unavailable".into()), + cgroup_max_bytes + .map(|b| b.to_string()) + .unwrap_or_else(|| "unlimited".into()), + ); +} + +pub fn emit_timing_point(label: &str, elapsed_ms: u128, peak_rss_bytes: Option) { + eprintln!( + "[rc-arc-spike] kind=timing label={label} elapsed_ms={elapsed_ms} peak_rss_bytes={}", + peak_rss_bytes + .map(|b| b.to_string()) + .unwrap_or_else(|| "unavailable".into()), + ); +} + pub fn emit_width_scaling_point(width: usize, peak_rss_bytes: Option) { eprintln!( "[rc-arc-spike] kind=width-scaling width={} peak_rss_bytes={}", @@ -547,11 +573,33 @@ pub fn emit_net_win_point(point: &NetWinPoint) { /// Census for the im_rc blocker (no code changes — report-only). pub fn im_rc_blocker_census() -> (usize, usize, &'static str) { + let stage0_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut total_rs = 0usize; + let mut im_rc_files = 0usize; + fn walk(dir: &std::path::Path, total_rs: &mut usize, im_rc_files: &mut usize) { + let Ok(read) = std::fs::read_dir(dir) else { + return; + }; + for entry in read.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, total_rs, im_rc_files); + } else if path.extension().is_some_and(|ext| ext == "rs") { + *total_rs += 1; + if let Ok(content) = std::fs::read_to_string(&path) { + if content.contains("im_rc::") || content.contains("use im_rc") { + *im_rc_files += 1; + } + } + } + } + } + walk(&stage0_root, &mut total_rs, &mut im_rc_files); const NOTE: &str = "im-rc (Rc-backed HAMT) is aliased as HashMap/Vector/BTreeSet in lib.rs; \ the Arc-backed sibling crate is `im`. Collections remain !Send even if \ outer Rc→Arc lands — swapping im-rc→im touches every persistent map/list \ - in the 122/154 stage0 .rs files plus serde feature parity."; - (123, 154, NOTE) + in stage0 .rs files plus serde feature parity."; + (im_rc_files, total_rs, NOTE) } pub fn emit_im_rc_census() { From 178cb222fb7586c0f01f9d22e019418761a5e731 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:42:54 +0000 Subject: [PATCH 09/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- docs/plans/rc-to-arc-share-spike.md | 147 +++++++++++----------- src/v1/stage0/src/index_memory_receipt.rs | 16 +-- 2 files changed, 80 insertions(+), 83 deletions(-) diff --git a/docs/plans/rc-to-arc-share-spike.md b/docs/plans/rc-to-arc-share-spike.md index 4b6e05ceab6..50d4eccbc7a 100644 --- a/docs/plans/rc-to-arc-share-spike.md +++ b/docs/plans/rc-to-arc-share-spike.md @@ -4,126 +4,132 @@ > > Answers open decision **2** in [cross-worker-typecheck-share-design](cross-worker-typecheck-share-design.md) §9 (`Arc migration depth: store-path-only vs whole-seed`). > -> Harness: `measure_rc_arc_share_spike` (`src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs`), structural accounting in `index_memory_receipt.rs`. Lines are `[rc-arc-spike] kind=...` for grep/oracle consumption. +> Harness: `measure_rc_arc_share_spike` (`src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs`). Lines are `[rc-arc-spike] kind=...`. --- -## 1. Executive summary +## 1. Executive summary (what is actually established) | Question | Answer | |---|---| -| **Is cross-worker Arc share worth the memory trade on a 125 GiB box?** | **Yes at floor width ≥3** for witness-shaped closures — width-scaling shows ~linear private-index duplication (~620 MiB/worker per discovery entry); union-sample crossover at width **3** (optimistic) / **4** (pessimistic) on a 3-entry sample. | -| **store-path-only vs whole-seed Arc?** | **store-path-only** for increment C. Whole-seed is not required to land the typed store; `TypedModule.module: Rc` and `im_rc` collections are separate follow-ons (§4). | -| **im-rc blocker price?** | **123 / 154** stage0 `.rs` files import `im_rc` (aliased `HashMap`/`Vector`/`BTreeSet`). Collections stay `!Send` until `im-rc` → `im` swap — orthogonal to outer `Rc`→`Arc` on store payloads. | +| **What object was measured?** | The **floor worker index**: `build_multi_entry_index` over both roots (`dag`, `src/v2`), then discovery-corpus warm (`populate_worker_discovery_index` — all 73 unique discovery entries, 299 rows). This matches the adaptive worker spawn site (`cli_run.rs` ~11191–11195), **not** a single-entry import closure. | +| **Peak RSS on worker object (this box)?** | **~1.51 GiB** VmHWM after discovery warm (73 entries). Shell-only build: **~50 MiB**, **~7.5 s**. | +| **Width duplication (RSS)?** | **~linear**: W=1 → 1.58 GiB; W=2 → 3.16 GiB on the same box. | +| **RSS crossover for sharing?** | At W=2, private ≈3.16 GiB vs one union ≈1.58 GiB — memory favors sharing from width 2 **on RSS alone**. This box has a **31.25 GiB cgroup cap**; even W=9 private ≈14 GiB stays under cap — **bytes are not the binding constraint here**. | +| **Is width worth it for wall-clock?** | **Not answered by this spike alone.** PR #6905 showed parallel pool is a wall-clock **loss** (serial 11.75 min green vs un-latched 47 min+). Shell build is ~7.5 s/worker; discovery warm is ~55 s/worker on this box. Sharing must remove the **warm typecheck path**, not just bytes, or the lane stays Amdahl-bound. | +| **store-path-only vs whole-seed?** | **store-path-only** for increment C (design §9.2 default). `im_rc` collections stay `!Send` without `im-rc`→`im`. | +| **im-rc price?** | **123 / 154** stage0 `.rs` files reference `im_rc` (dynamic census). `lib.rs` aliases `HashMap`/`Vector`/`BTreeSet` to it. | -**Operator note:** [execution-spine-design](execution-spine-design.md) removed the interpreter `Rc`→`Arc` gate (2026-07-10). This spike prices the **typecheck-store** lane only; escalate if these numbers gate increment C landing. +**What is NOT established:** serde-based crossover (discarded — see §2.2). Full `build-vs-attach` on this box (**OOM exit 137**, §3.5). Whole-tree strict resolve to the ~10.7 GiB nimble-owl/srv1 receipt (not re-run here). --- ## 2. Methodology -### 2.1 Shareable vs per-worker residue (design §4.2) +### 2.1 Measurement host (always record with numbers) -| Bucket | `MultiEntryIndex` fields | Spike accounting | -|---|---|---| -| **Shareable** | `typed_module_cache` (+ transitive typed payloads) | Serde transport bytes (`SharedTypecheckCaches::encode_typed_snapshot`) + map shell. One module sampled, extrapolated × entry count (§2.2). | -| **Per-worker residue** | `parse_cache`, `resolved_graph_memo`, `intern_table`, `normalize_diag_cache`, `ownership_diag_cache` | Shallow map-shell + key bytes (residue **underestimates** parse/graph payload — noted below). | +| Field | Value | +|---|---| +| **hostname** | `b96c6d293441` | +| **MemAvailable** | ~107–117 GiB (`/proc/meminfo`) | +| **cgroup memory.max** | **33 578 549 248 B (~31.25 GiB)** — processes OOM here before host RAM is exhausted | +| **source roots** | `dag`, `src/v2` | -Populate uses `resolve_entry_with_index_for_discovery_corpus` (floor's `DiscoveryCorpusAdvisory` gate), not strict whole-tree. +### 2.2 Honest limits -### 2.2 Measurement limits (honest) +1. **Peak RSS (VmHWM) is the only byte metric used for conclusions.** Per-field `index-field` lines are shallow map-shell + key counts; they **under-count** parse trees, resolved graphs, and typed payloads. They are diagnostic, not summed for crossover. +2. **Serde transport bytes are not reported** — per-module encode OOM'd or produced non-credible extrapolations; using them for crossover was an error in the first draft (corrected per operator review). +3. **Residue is under-estimated** — `parse_cache`, `resolved_graph_memo`, and `intern_table` payloads stay per-worker after Arc migration; shallow accounting excludes their heap bodies. +4. **`build-vs-attach` OOM'd** on this box (exit 137, ~100 s) — cannot report attach-vs-cold wall-clock on shared store here without a larger cgroup or split processes. -1. **Serde extrapolation** — encoding every typed module OOM'd on this host; receipt uses **one** module's serde size × `typed_module_cache.len()`. Modules vary widely; extrapolated shareable bytes are an **order-of-magnitude proxy**, not a byte oracle. Peak RSS (VmHWM) is the reliable retention metric. -2. **Shallow residue** — parse trees and resolved graphs are not walked; reported residue is a **lower bound**. Real per-worker residue is larger. -3. **Full discovery union** — `populate_multi_entry_index_discovery_corpus(None)` OOM'd (exit 137, ~116 s). Default corpus is `representative` (9 large-closure entries) or `discovery:N` cap. -4. **union-sample N=9** — OOM'd during per-entry populate; N=3 completed. +### 2.3 Shareable vs per-worker residue (design §4.2) + +| Bucket | Fields | Spike treatment | +|---|---|---| +| **Shareable (typed store)** | `typed_module_cache` | Entry count + map shell bytes; **peak RSS** for retention | +| **Per-worker residue** | `parse_cache`, `resolved_graph_memo`, `intern_table`, normalize/ownership diag caches | Shallow shell (lower bound) — real residue larger | --- -## 3. Receipts (green-by-execution) +## 3. Receipts (green-by-execution, pasted output) -### 3.1 im-rc census +### 3.1 Host metadata ``` -[rc-arc-spike] kind=im-rc-census im_rc_files=123 stage0_rs_files=154 +[rc-arc-spike] kind=host-metadata hostname=b96c6d293441 mem_available_kb=107078536 cgroup_max_bytes=33578549248 ``` -Swapping `im-rc` → `im` (Arc-backed sibling) touches persistent maps/lists across the stage0 seed plus serde feature parity. **Not on increment C's critical path** if only `typed_module_cache` store payloads migrate to `Arc`. - -### 3.2 Width-scaling baseline (`discovery:1`, parallel worker-index simulation) +### 3.2 im-rc census -| Width | Peak RSS (VmHWM) | -|---|---| -| 1 | 650 113 024 B (~620 MiB) | -| 2 | 1 295 810 560 B (~1.21 GiB) | -| 3 | 1 940 291 584 B (~1.81 GiB) | - -**~linear ×W duplication** — consistent with per-worker `build_multi_entry_index` retaining a cold private cache (governor comment ~8091–8120 in `cli_run.rs`). At falsifier width **9**, naive extrapolation ≈ **5.6 GiB** duplicate index shells vs **~620 MiB** shared union for the same closure shape. +``` +[rc-arc-spike] kind=im-rc-census im_rc_files=123 stage0_rs_files=154 note=im-rc (Rc-backed HAMT) is aliased as HashMap/Vector/BTreeSet in lib.rs; the Arc-backed sibling crate is `im`. Collections remain !Send even if outer Rc→Arc lands — swapping im-rc→im touches every persistent map/list in stage0 .rs files plus serde feature parity. +``` -### 3.3 Index fields — single discovery entry (`discovery:1`) +### 3.3 Worker shell build (`--mode worker-shell-build`) ``` -[rc-arc-spike] kind=index-summary shareable_bytes=8144530640 residue_bytes=96096 - typed_module_cache_entries=37 peak_rss_bytes=653803520 +[rc-arc-spike] kind=timing label=worker-shell-build elapsed_ms=7452 peak_rss_bytes=52498432 ``` -37 typed modules; peak RSS **~624 MiB** (credible). Serde extrapolation **~7.6 GiB** (proxy only). +~7.5 s, ~50 MiB — this is what every worker pays at thread start **before** any entry-group resolve. -### 3.4 Index fields — representative union (`representative`, 9 entries) +### 3.4 Worker discovery warm (`--mode worker-discovery-warm`) + +Full discovery roster: **73 entries**, **299 rows**. ``` -[rc-arc-spike] kind=index-summary shareable_bytes=159913142162 residue_bytes=425132 - typed_module_cache_entries=318 peak_rss_bytes=746315776 +[rc-arc-spike] kind=timing label=worker-shell-only elapsed_ms=6996 peak_rss_bytes=52486144 +[rc-arc-spike] kind=timing label=worker-discovery-warm elapsed_ms=55011 peak_rss_bytes=1626210304 +[rc-arc-spike] kind=worker-object discovery_entries=73 discovery_rows=299 peak_rss_bytes=1626210304 ``` -318 typed modules across 9 large-closure entries; peak RSS **~712 MiB** for the combined index. Serde extrapolation **~149 GiB** is not credible (sample-module skew); **trust peak RSS** for retention pricing. +~55 s warm, **~1.51 GiB** peak — the retention shape after resolving all discovery entries on one worker index. -### 3.5 Union-sample net-win curve (3 witness entries, max width 9) +### 3.5 Width-scaling worker object (`--mode width-scaling-worker --max-width 2`) ``` -shareable_per_worker_avg=3349243766 residue_per_worker_avg=57030 -union_shareable_max=8144530640 union_shareable_sum=10047731300 +[rc-arc-spike] kind=width-scaling width=1 peak_rss_bytes=1582301184 +[rc-arc-spike] kind=width-scaling width=2 peak_rss_bytes=3162173440 ``` -| Model | Crossover width (net_win_bytes > 0) | +| Width | Peak RSS | |---|---| -| Optimistic (`union_shareable_max`) | **3** | -| Pessimistic (`union_shareable_sum`) | **4** | +| 1 | 1 582 301 184 B (~1.47 GiB) | +| 2 | 3 162 173 440 B (~2.95 GiB) | -On a **125 GiB** box at falsifier width **9**, shared retention is well inside envelope even before M2 strip; the **time** prize (52% `typecheck_compute`, design §0) dominates. +**~linear ×W** private duplication on the worker object. ---- +**RSS net-win sketch (not a done-bar):** `private(W) ≈ 1.58 GiB × W`; `shared(W) ≈ 1.58 GiB + (W−1) × 0.05 GiB` (one warm union + cold shells). Crossover **W≥2** on bytes alone — but cgroup headroom is ample to W=9 (~14 GiB naive linear). -## 4. Decision 2 — store-path-only vs whole-seed +### 3.6 build-vs-attach — **BLOCKED on this box** -### 4.1 Recommendation: **store-path-only** for increment C +``` +EXIT=137 (OOM killed, ~100s) +[rc-arc-spike] kind=host-metadata hostname=b96c6d293441 mem_available_kb=111314968 cgroup_max_bytes=33578549248 +measure_rc_arc_share_spike: build-vs-attach +``` -Aligns with [cross-worker-typecheck-share-design](cross-worker-typecheck-share-design.md) §9.2 default. +Warm shared index + cold shell + attach in one process exceeds the **31.25 GiB cgroup** before timing lines emit. **Escalation:** re-run on srv3 (125 GiB budget, no tight cgroup) or split into separate processes. -**In scope (store-path `Rc`→`Arc`):** +### 3.7 The decisive question (time, not bytes) -- `TypecheckModuleResult`, `TypedModule`, `ModuleInterface` inserted into `typed_module_cache` -- `parse_cache` entries on the typecheck critical path (design §4.2) -- `SharedTypecheckCaches` shell (`std::collections::HashMap` + `Arc` payloads — already modeled in #6561) +From PR #6905: widening the pool is a **wall-clock loss** on ~12 min total corpus work because each worker front-loads index build + warm resolve. -**Stays `Rc` (thread-local eval):** +| Phase | This box (worker object) | +|---|---| +| Shell build | ~7.5 s | +| Discovery warm (73 entries) | ~55 s | -- `ResolvedGraph` handed to `make_eval_context` per worker -- Ephemeral inference temporaries +Increment C helps **only if** `build_multi_entry_index_with_shared_caches` + warm shared typed store lets workers **skip the ~55 s cold typecheck warm** (decode/hit path on cache hits). It does **not** remove the ~7.5 s shell build (`build_module_index` + `build_module_graph_facts_live` are per-worker). `build-vs-attach` wall-clock remains **unmeasured** here (OOM). -### 4.2 Why whole-seed is not required now +--- -1. **Eval path** — execution-spine ruling keeps interpreter on `Rc`; increment C does not need eval cross-thread. -2. **Peak RSS evidence** — duplicate cost is the per-worker **index shell + caches**, not every `Rc` site in the seed. Width-scaling shows the prize is killing **W× cold typed caches**, not a global `Rc` sweep. -3. **im_rc** — 123 files use `im_rc::HashMap`/`Vector`; outer `Arc` on store wrappers does not make these `Send`. A whole-seed sweep that stops at outer `Arc` leaves the im_rc blocker; a sweep that includes `im-rc`→`im` is a **separate migration** with much larger blast radius. +## 4. Decision 2 — store-path-only vs whole-seed -### 4.3 Residual risks (counted, not silent) +**Recommendation: store-path-only** for increment C (unchanged from design §9.2). -| Risk | Mitigation | -|---|---| -| `TypedModule.module: Rc` inside store payloads | Store-path migration must `Arc` nested carriers on **insert paths** only; eval keeps `Rc`. | -| Shared retention ↑ co-resident bytes | Governor dial + M2 strip (design §7); this spike shows headroom at width 9 on 125 GiB. | -| Serde transport size for cross-worker clone | Measure on falsifier host with batched per-module encode before S2b; not blocking C1 host proof. | +- Migrate `TypecheckModuleResult` / store insertion paths to `Arc`; eval `ResolvedGraph` stays `Rc`. +- `im_rc` → `im` is a **separate** migration (123 files); outer `Arc` does not make `im_rc::HashMap` `Send`. --- @@ -133,13 +139,14 @@ Aligns with [cross-worker-typecheck-share-design](cross-worker-typecheck-share-d cargo build --release -p v1-compiler --bin measure_rc_arc_share_spike ./target/release/measure_rc_arc_share_spike --mode im-rc-census -./target/release/measure_rc_arc_share_spike --mode index-fields --corpus representative -./target/release/measure_rc_arc_share_spike --mode width-scaling --corpus discovery:1 --max-width 3 -./target/release/measure_rc_arc_share_spike --mode union-sample --sample-entries 3 --max-width 9 +./target/release/measure_rc_arc_share_spike --mode worker-shell-build +./target/release/measure_rc_arc_share_spike --mode worker-discovery-warm +./target/release/measure_rc_arc_share_spike --mode width-scaling-worker --max-width 2 +./target/release/measure_rc_arc_share_spike --mode build-vs-attach # blocked on 31GiB cgroup ``` --- ## 6. Dissolution -This doc dissolves when increment C lands (C1 host receipt + C2 ladder row) or when a successor measurement supersedes the harness receipts above. +Dissolves when increment C lands or a successor measurement supersedes these receipts. diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs index 0f9c4605747..1dbac911b32 100644 --- a/src/v1/stage0/src/index_memory_receipt.rs +++ b/src/v1/stage0/src/index_memory_receipt.rs @@ -325,24 +325,14 @@ fn account_typecheck_module_result(acc: &mut Accountant, result: &TypecheckModul fn account_from_typed_cache( cache: &StdHashMap>, ) -> (u64, usize, u64) { - // Shareable grain = serde transport bytes (cross-worker store payload) plus map shell. - // Encode one module and extrapolate — full-corpus encode OOMs on measurement hosts. + // Shareable bucket: map shell + keys only. Peak RSS is the honest total (§2.2); + // serde transport is NOT extrapolated here — sample encode OOMs / misleads. let mut acc = Accountant::new(); acc.add_hashmap_shell(cache); for key in cache.keys() { acc.add_string(key); } - let sample_serde = cache - .values() - .next() - .and_then(|result| { - crate::shared_typecheck_store::SharedTypecheckCaches::encode_typed_snapshot(result) - .ok() - .map(|bytes| bytes.len() as u64) - }) - .unwrap_or(0); - let serde_bytes = sample_serde.saturating_mul(cache.len() as u64); - (acc.finish() + serde_bytes, cache.len(), serde_bytes) + (acc.finish(), cache.len(), 0) } fn account_parse_cache_shallow( From 948b180d429fccef82bf71884542aeee76df8767 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:44:29 +0000 Subject: [PATCH 10/22] Correct spike report: worker-object scope, peak RSS receipts. Measure the floor worker index (build_multi_entry_index + full discovery warm), drop serde crossover, document per-field shallow accounting and build-vs-attach OOM on 31GiB cgroup. Co-authored-by: Cursor --- docs/plans/rc-to-arc-share-spike.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/plans/rc-to-arc-share-spike.md b/docs/plans/rc-to-arc-share-spike.md index 50d4eccbc7a..0d89315452e 100644 --- a/docs/plans/rc-to-arc-share-spike.md +++ b/docs/plans/rc-to-arc-share-spike.md @@ -85,6 +85,20 @@ Full discovery roster: **73 entries**, **299 rows**. ~55 s warm, **~1.51 GiB** peak — the retention shape after resolving all discovery entries on one worker index. +Per-field shallow accounting (shell + keys only — **not** payload bodies; peak RSS is authoritative): + +``` +[rc-arc-spike] kind=index-field field=typed_module_cache bucket=shareable bytes=49400 entries=517 +[rc-arc-spike] kind=index-field field=parse_cache bucket=per_worker_residue bytes=66730 entries=517 +[rc-arc-spike] kind=index-field field=resolved_graph_memo bucket=per_worker_residue bytes=5864 entries=73 +[rc-arc-spike] kind=index-field field=intern_table bucket=per_worker_residue bytes=507672 entries=21153 +[rc-arc-spike] kind=index-field field=normalize_diag_cache bucket=per_worker_residue bytes=59562 entries=517 +[rc-arc-spike] kind=index-field field=ownership_diag_cache bucket=per_worker_residue bytes=59562 entries=517 +[rc-arc-spike] kind=index-summary shareable_bytes=49400 residue_bytes=699390 total_accounted_bytes=748790 typed_module_cache_entries=517 serde_transport_bytes=0 peak_rss_bytes=1617641472 +``` + +**Shareable vs residue (shallow shells):** 49 400 B shareable map shell vs 699 390 B residue shells — **misleading for crossover** because shallow residue excludes parse/typed payloads while peak RSS (~1.61 GiB) captures them. **517 typed modules** cached after full discovery warm. + ### 3.5 Width-scaling worker object (`--mode width-scaling-worker --max-width 2`) ``` From cf82125b0ace11e474a8c5209edc8925d29dc833 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 05:53:23 +0000 Subject: [PATCH 11/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 195 +++++++++++++++++- src/v1/stage0/src/cli_run.rs | 48 +++++ 2 files changed, 236 insertions(+), 7 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index 55d6a5437b9..72c424f3053 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -5,18 +5,19 @@ //! Measures the **floor worker object**: `build_multi_entry_index` over both roots (`dag`, `src/v2`), //! then discovery-corpus warm — not per-entry import closures. -use std::process::ExitCode; +use std::process::{Command, ExitCode}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; use std::time::Instant; use v1_compiler::cli_run::{ - build_worker_index_shell, build_worker_index_with_warm_store, new_shared_typecheck_caches, - peak_rss_vhwm_bytes, populate_multi_entry_index_whole_tree_discovery, - populate_worker_discovery_index, resolve_entry_with_index_for_discovery_corpus, - spike_measurement_host_metadata, warm_discovery_on_index, - whole_tree_resolve_exclusion_substrings, workspace_root, + build_worker_index_shell, build_worker_index_with_warm_store, export_typed_cache_for_spike, + new_shared_typecheck_caches, peak_rss_vhwm_bytes, + populate_multi_entry_index_whole_tree_discovery, populate_worker_discovery_index, + resolve_entry_with_index_for_discovery_corpus, spike_measurement_host_metadata, + spike_set_module_typecheck_recording, spike_take_module_typecheck_records, + warm_discovery_on_index, whole_tree_resolve_exclusion_substrings, workspace_root, }; use v1_compiler::index_memory_receipt::{ emit_host_metadata, emit_im_rc_census, emit_index_memory_receipt, emit_timing_point, @@ -237,12 +238,173 @@ fn run_width_scaling_worker( Ok(ExitCode::SUCCESS) } +use v1_compiler::shared_typecheck_store::SharedTypecheckCaches; + +fn module_grain_out_dir() -> std::path::PathBuf { + workspace_root().join("target/rc-arc-spike-module-grain") +} + +fn run_module_grain_produce( + source_roots: &[String], + sample_size: usize, +) -> Result { + eprintln!("measure_rc_arc_share_spike: module-grain-produce sample_size={sample_size}"); + spike_set_module_typecheck_recording(true); + let warm_start = Instant::now(); + let (index, entries, rows) = populate_worker_discovery_index(source_roots).map_err(|e| { + eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-produce error={e}"); + ExitCode::from(2) + })?; + let warm_ms = warm_start.elapsed().as_millis(); + emit_timing_point("module-grain-warm", warm_ms, peak_rss_vhwm_bytes()); + + let records = spike_take_module_typecheck_records(); + let cache = export_typed_cache_for_spike(&index); + drop(index); + + let mut tc_by_mod: std::collections::HashMap = std::collections::HashMap::new(); + for (mod_name, ns) in records { + tc_by_mod.insert(mod_name, ns); + } + + let out_dir = module_grain_out_dir(); + let _ = std::fs::remove_dir_all(&out_dir); + std::fs::create_dir_all(&out_dir).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: cannot create {out_dir:?}: {e}"); + ExitCode::from(2) + })?; + + let mut rows_out: Vec = Vec::new(); + let take = sample_size.min(cache.len()); + for (i, (typed_key, result)) in cache.into_iter().take(take).enumerate() { + let mod_name = result.typed.module.name.clone(); + let typecheck_ns = tc_by_mod.get(&mod_name).copied().unwrap_or(0); + let bytes = SharedTypecheckCaches::encode_typed_snapshot(result.as_ref()).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); + ExitCode::from(2) + })?; + let snap_path = out_dir.join(format!("{i}.bin")); + std::fs::write(&snap_path, bytes.as_slice()).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: write {:?}: {e}", snap_path); + ExitCode::from(2) + })?; + rows_out.push(format!( + "{{\"typed_key\":{},\"mod_name\":{},\"typecheck_ns\":{typecheck_ns},\"snapshot_path\":{}}}", + serde_json::to_string(&typed_key).unwrap(), + serde_json::to_string(&mod_name).unwrap(), + serde_json::to_string(snap_path.to_string_lossy().as_ref()).unwrap(), + )); + } + + let manifest_path = out_dir.join("manifest.jsonl"); + std::fs::write(&manifest_path, rows_out.join("\n")).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: write manifest: {e}"); + ExitCode::from(2) + })?; + + eprintln!( + "[rc-arc-spike] kind=module-grain-produce discovery_entries={entries} discovery_rows={rows} sampled_modules={take} manifest={}", + manifest_path.display() + ); + + let exe = std::env::current_exe().map_err(|_| ExitCode::from(2))?; + let decode_out = Command::new(exe) + .args([ + "--mode", + "module-grain-decode", + "--manifest", + &manifest_path.to_string_lossy(), + ]) + .output() + .map_err(|e| { + eprintln!("measure_rc_arc_share_spike: decode subprocess failed: {e}"); + ExitCode::from(2) + })?; + let stderr = String::from_utf8_lossy(&decode_out.stderr); + print!("{stderr}"); + if !decode_out.status.success() { + eprintln!( + "[rc-arc-spike] kind=blocked measurement=module-grain-decode exit={}", + decode_out.status + ); + return Err(ExitCode::from(2)); + } + Ok(ExitCode::SUCCESS) +} + +fn run_module_grain_decode(manifest_path: &str) -> Result { + eprintln!("measure_rc_arc_share_spike: module-grain-decode manifest={manifest_path}"); + let content = std::fs::read_to_string(manifest_path).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: read manifest: {e}"); + ExitCode::from(2) + })?; + + let mut typecheck_total_ns = 0u64; + let mut decode_total_ns = 0u64; + let mut modules = 0usize; + let mut missing_typecheck = 0usize; + + for line in content.lines() { + if line.trim().is_empty() { + continue; + } + let row: serde_json::Value = serde_json::from_str(line).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: bad manifest line: {e}"); + ExitCode::from(2) + })?; + let mod_name = row["mod_name"].as_str().unwrap_or("?"); + let typecheck_ns = row["typecheck_ns"].as_u64().unwrap_or(0); + let snap_path = row["snapshot_path"].as_str().ok_or_else(|| { + eprintln!("measure_rc_arc_share_spike: manifest missing snapshot_path"); + ExitCode::from(2) + })?; + let bytes = std::fs::read(snap_path).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: read snapshot {snap_path}: {e}"); + ExitCode::from(2) + })?; + let decode_start = Instant::now(); + SharedTypecheckCaches::decode_typed_snapshot(&bytes).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: decode {mod_name}: {e}"); + ExitCode::from(2) + })?; + let decode_ns = decode_start.elapsed().as_nanos() as u64; + eprintln!( + "[rc-arc-spike] kind=module-grain-row mod_name={mod_name} typecheck_ns={typecheck_ns} decode_ns={decode_ns}" + ); + if typecheck_ns == 0 { + missing_typecheck += 1; + } + typecheck_total_ns += typecheck_ns; + decode_total_ns += decode_ns; + modules += 1; + } + + if modules == 0 { + eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-decode modules=0"); + return Err(ExitCode::from(2)); + } + + let typecheck_ms_per_module = (typecheck_total_ns as f64) / (modules as f64) / 1_000_000.0; + let decode_ms_per_module = (decode_total_ns as f64) / (modules as f64) / 1_000_000.0; + let ratio = if decode_total_ns > 0 { + typecheck_total_ns as f64 / decode_total_ns as f64 + } else { + 0.0 + }; + eprintln!( + "[rc-arc-spike] kind=module-grain-summary modules={modules} missing_typecheck_ns={missing_typecheck} typecheck_ms_per_module={typecheck_ms_per_module:.3} decode_ms_per_module={decode_ms_per_module:.3} typecheck_to_decode_ratio={ratio:.2}" + ); + Ok(ExitCode::SUCCESS) +} + fn run() -> Result { let args: Vec = std::env::args().collect(); let mut mode = String::from("worker-discovery-warm"); let mut source_roots: Vec = Vec::new(); let mut exclude_subpaths = whole_tree_resolve_exclusion_substrings(); let mut max_width = 3usize; + let mut sample_size = 200usize; + let mut manifest_path = String::new(); let mut i = 1; while i < args.len() { @@ -263,6 +425,15 @@ fn run() -> Result { i += 1; max_width = parse_usize("--max-width", &require_value(&args, i, "--max-width")?)?; } + "--sample-size" => { + i += 1; + sample_size = + parse_usize("--sample-size", &require_value(&args, i, "--sample-size")?)?; + } + "--manifest" => { + i += 1; + manifest_path = require_value(&args, i, "--manifest")?; + } other => { eprintln!("measure_rc_arc_share_spike: unknown argument: {other}"); return Err(ExitCode::from(2)); @@ -284,13 +455,23 @@ fn run() -> Result { "whole-tree-fields" => run_whole_tree_fields(&source_roots, &exclude_subpaths), "build-vs-attach" => run_build_vs_attach(&source_roots), "width-scaling-worker" => run_width_scaling_worker(&source_roots, max_width), + "module-grain-produce" => run_module_grain_produce(&source_roots, sample_size), + "module-grain-decode" => { + if manifest_path.is_empty() { + eprintln!( + "measure_rc_arc_share_spike: --manifest required for module-grain-decode" + ); + return Err(ExitCode::from(2)); + } + run_module_grain_decode(&manifest_path) + } "im-rc-census" => { emit_im_rc_census(); Ok(ExitCode::SUCCESS) } other => { eprintln!( - "measure_rc_arc_share_spike: unknown --mode {other} (expected worker-shell-build|worker-discovery-warm|whole-tree-fields|build-vs-attach|width-scaling-worker|im-rc-census)" + "measure_rc_arc_share_spike: unknown --mode {other} (expected worker-shell-build|worker-discovery-warm|whole-tree-fields|build-vs-attach|width-scaling-worker|module-grain-produce|module-grain-decode|im-rc-census)" ); Err(ExitCode::from(2)) } diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index ead45f93020..8241dc9f932 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4141,6 +4141,9 @@ fn new_multi_entry_index_shell( // the process shares one typed_module_cache, this stays ≤ the distinct module count of // the process union. static TYPECHECK_COMPUTE_COUNT: AtomicUsize = AtomicUsize::new(0); +// rc-arc spike: per-module cold typecheck timings on cache-miss path only. +static SPIKE_MODULE_TYPECHECK_RECORDING: AtomicBool = AtomicBool::new(false); +static SPIKE_MODULE_TYPECHECK_RECORDS: Mutex> = Mutex::new(Vec::new()); // Union-resolve receipt tests reset/read the process-wide counter; `cargo test` runs // `#[test]` fns in parallel by default — serialize those oracles (not production use). @@ -4162,6 +4165,47 @@ pub fn reset_typecheck_compute_count() { TYPECHECK_COMPUTE_COUNT.store(0, Ordering::SeqCst); } +/// Enable per-module typecheck miss recording for the rc-arc spike harness. +pub fn spike_set_module_typecheck_recording(on: bool) { + SPIKE_MODULE_TYPECHECK_RECORDING.store(on, Ordering::SeqCst); + if on { + SPIKE_MODULE_TYPECHECK_RECORDS + .lock() + .expect("spike module typecheck records lock poisoned") + .clear(); + } +} + +/// Take recorded per-module cold typecheck durations (module name, nanoseconds). +pub fn spike_take_module_typecheck_records() -> Vec<(String, u64)> { + std::mem::take( + &mut *SPIKE_MODULE_TYPECHECK_RECORDS + .lock() + .expect("spike module typecheck records lock poisoned"), + ) +} + +fn spike_maybe_record_module_typecheck(mod_name: &str, elapsed_ns: u64) { + if SPIKE_MODULE_TYPECHECK_RECORDING.load(Ordering::SeqCst) { + SPIKE_MODULE_TYPECHECK_RECORDS + .lock() + .expect("spike module typecheck records lock poisoned") + .push((mod_name.to_string(), elapsed_ns)); + } +} + +/// Export typed cache rows for module-grain spike receipts (private index only). +pub fn export_typed_cache_for_spike( + index: &MultiEntryIndex, +) -> Vec<(String, Rc)> { + index + .typed_module_cache + .borrow() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + fn bump_typecheck_compute_count() { TYPECHECK_COMPUTE_COUNT.fetch_add(1, Ordering::SeqCst); } @@ -5339,6 +5383,10 @@ fn reconcile_with_typed_cache( // parse+resolve+normalize). Threshold keeps the floor log quiet; // anything over it is a pathology-lane candidate by name. let module_tc_elapsed = module_tc_started.elapsed(); + spike_maybe_record_module_typecheck( + &mod_name, + module_tc_elapsed.as_nanos() as u64, + ); resolve_stage_slot_add(|s| s.typecheck_compute += module_tc_elapsed.as_nanos()); let module_tc_ms = module_tc_elapsed.as_millis(); if module_tc_ms >= 2_000 { From 685cefb2bb640914d9242d981dec252980b563cf Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:03:49 +0000 Subject: [PATCH 12/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 65 ++++++++++++++----- src/v1/stage0/src/cli_run.rs | 11 ++-- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index 72c424f3053..06f9b14b75e 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -262,11 +262,23 @@ fn run_module_grain_produce( let cache = export_typed_cache_for_spike(&index); drop(index); - let mut tc_by_mod: std::collections::HashMap = std::collections::HashMap::new(); - for (mod_name, ns) in records { - tc_by_mod.insert(mod_name, ns); + const MAX_SNAPSHOT_BYTES: usize = 50_000_000; + let mut tc_by_key: std::collections::HashMap = + std::collections::HashMap::new(); + for (mod_name, typed_key, ns) in records { + tc_by_key.insert(typed_key, (mod_name, ns)); } + let mut candidates = Vec::new(); + for (typed_key, result) in cache { + if let Some((mod_name, ns)) = tc_by_key.get(&typed_key) { + if *ns > 0 { + candidates.push((typed_key, mod_name.clone(), *ns, result)); + } + } + } + candidates.sort_by(|a, b| b.2.cmp(&a.2)); + let out_dir = module_grain_out_dir(); let _ = std::fs::remove_dir_all(&out_dir); std::fs::create_dir_all(&out_dir).map_err(|e| { @@ -274,36 +286,53 @@ fn run_module_grain_produce( ExitCode::from(2) })?; - let mut rows_out: Vec = Vec::new(); - let take = sample_size.min(cache.len()); - for (i, (typed_key, result)) in cache.into_iter().take(take).enumerate() { - let mod_name = result.typed.module.name.clone(); - let typecheck_ns = tc_by_mod.get(&mod_name).copied().unwrap_or(0); + let manifest_path = out_dir.join("manifest.jsonl"); + let mut manifest = std::fs::File::create(&manifest_path).map_err(|e| { + eprintln!("measure_rc_arc_share_spike: create manifest: {e}"); + ExitCode::from(2) + })?; + use std::io::Write; + + let mut written = 0usize; + let mut skipped_large = 0usize; + for (typed_key, mod_name, typecheck_ns, result) in candidates { + if written >= sample_size { + break; + } let bytes = SharedTypecheckCaches::encode_typed_snapshot(result.as_ref()).map_err(|e| { eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); ExitCode::from(2) })?; - let snap_path = out_dir.join(format!("{i}.bin")); + if bytes.len() > MAX_SNAPSHOT_BYTES { + skipped_large += 1; + eprintln!( + "[rc-arc-spike] kind=module-grain-skip mod_name={mod_name} reason=snapshot_bytes={} cap={MAX_SNAPSHOT_BYTES}", + bytes.len() + ); + continue; + } + let snap_path = out_dir.join(format!("{written}.bin")); std::fs::write(&snap_path, bytes.as_slice()).map_err(|e| { eprintln!("measure_rc_arc_share_spike: write {:?}: {e}", snap_path); ExitCode::from(2) })?; - rows_out.push(format!( - "{{\"typed_key\":{},\"mod_name\":{},\"typecheck_ns\":{typecheck_ns},\"snapshot_path\":{}}}", + let line = format!( + "{{\"typed_key\":{},\"mod_name\":{},\"typecheck_ns\":{typecheck_ns},\"snapshot_path\":{}}}\n", serde_json::to_string(&typed_key).unwrap(), serde_json::to_string(&mod_name).unwrap(), serde_json::to_string(snap_path.to_string_lossy().as_ref()).unwrap(), - )); + ); + manifest.write_all(line.as_bytes()).map_err(|_| ExitCode::from(2))?; + written += 1; } - let manifest_path = out_dir.join("manifest.jsonl"); - std::fs::write(&manifest_path, rows_out.join("\n")).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: write manifest: {e}"); - ExitCode::from(2) - })?; + if written == 0 { + eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-produce encoded=0"); + return Err(ExitCode::from(2)); + } eprintln!( - "[rc-arc-spike] kind=module-grain-produce discovery_entries={entries} discovery_rows={rows} sampled_modules={take} manifest={}", + "[rc-arc-spike] kind=module-grain-produce discovery_entries={entries} discovery_rows={rows} sampled_modules={written} skipped_large={skipped_large} manifest={}", manifest_path.display() ); diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 8241dc9f932..007904fcc63 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4143,7 +4143,7 @@ fn new_multi_entry_index_shell( static TYPECHECK_COMPUTE_COUNT: AtomicUsize = AtomicUsize::new(0); // rc-arc spike: per-module cold typecheck timings on cache-miss path only. static SPIKE_MODULE_TYPECHECK_RECORDING: AtomicBool = AtomicBool::new(false); -static SPIKE_MODULE_TYPECHECK_RECORDS: Mutex> = Mutex::new(Vec::new()); +static SPIKE_MODULE_TYPECHECK_RECORDS: Mutex> = Mutex::new(Vec::new()); // Union-resolve receipt tests reset/read the process-wide counter; `cargo test` runs // `#[test]` fns in parallel by default — serialize those oracles (not production use). @@ -4176,8 +4176,8 @@ pub fn spike_set_module_typecheck_recording(on: bool) { } } -/// Take recorded per-module cold typecheck durations (module name, nanoseconds). -pub fn spike_take_module_typecheck_records() -> Vec<(String, u64)> { +/// Take recorded per-module cold typecheck durations (module name, typed key, nanoseconds). +pub fn spike_take_module_typecheck_records() -> Vec<(String, String, u64)> { std::mem::take( &mut *SPIKE_MODULE_TYPECHECK_RECORDS .lock() @@ -4185,12 +4185,12 @@ pub fn spike_take_module_typecheck_records() -> Vec<(String, u64)> { ) } -fn spike_maybe_record_module_typecheck(mod_name: &str, elapsed_ns: u64) { +fn spike_maybe_record_module_typecheck(mod_name: &str, typed_key: &str, elapsed_ns: u64) { if SPIKE_MODULE_TYPECHECK_RECORDING.load(Ordering::SeqCst) { SPIKE_MODULE_TYPECHECK_RECORDS .lock() .expect("spike module typecheck records lock poisoned") - .push((mod_name.to_string(), elapsed_ns)); + .push((mod_name.to_string(), typed_key.to_string(), elapsed_ns)); } } @@ -5385,6 +5385,7 @@ fn reconcile_with_typed_cache( let module_tc_elapsed = module_tc_started.elapsed(); spike_maybe_record_module_typecheck( &mod_name, + &typed_key, module_tc_elapsed.as_nanos() as u64, ); resolve_stage_slot_add(|s| s.typecheck_compute += module_tc_elapsed.as_nanos()); From 0269865218425637fa35ddaf5670e9184394ef09 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:14:31 +0000 Subject: [PATCH 13/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index 06f9b14b75e..eac0b93acdd 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -263,21 +263,20 @@ fn run_module_grain_produce( drop(index); const MAX_SNAPSHOT_BYTES: usize = 50_000_000; + const MAX_TYPECHECK_NS: u64 = 500_000_000; // 500ms — skip outlier modules whose serde payloads OOM let mut tc_by_key: std::collections::HashMap = std::collections::HashMap::new(); for (mod_name, typed_key, ns) in records { - tc_by_key.insert(typed_key, (mod_name, ns)); - } - - let mut candidates = Vec::new(); - for (typed_key, result) in cache { - if let Some((mod_name, ns)) = tc_by_key.get(&typed_key) { - if *ns > 0 { - candidates.push((typed_key, mod_name.clone(), *ns, result)); - } + if ns > 0 && ns <= MAX_TYPECHECK_NS { + tc_by_key.insert(typed_key, (mod_name, ns)); } } - candidates.sort_by(|a, b| b.2.cmp(&a.2)); + + let mut keys: Vec<(String, String, u64)> = tc_by_key + .into_iter() + .map(|(k, (m, ns))| (k, m, ns)) + .collect(); + keys.sort_by(|a, b| a.2.cmp(&b.2)); // ascending — typical modules first let out_dir = module_grain_out_dir(); let _ = std::fs::remove_dir_all(&out_dir); @@ -295,10 +294,15 @@ fn run_module_grain_produce( let mut written = 0usize; let mut skipped_large = 0usize; - for (typed_key, mod_name, typecheck_ns, result) in candidates { + let mut skipped_missing = 0usize; + for (typed_key, mod_name, typecheck_ns) in keys { if written >= sample_size { break; } + let Some(result) = cache.get(&typed_key) else { + skipped_missing += 1; + continue; + }; let bytes = SharedTypecheckCaches::encode_typed_snapshot(result.as_ref()).map_err(|e| { eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); ExitCode::from(2) @@ -322,9 +326,12 @@ fn run_module_grain_produce( serde_json::to_string(&mod_name).unwrap(), serde_json::to_string(snap_path.to_string_lossy().as_ref()).unwrap(), ); - manifest.write_all(line.as_bytes()).map_err(|_| ExitCode::from(2))?; + manifest + .write_all(line.as_bytes()) + .map_err(|_| ExitCode::from(2))?; written += 1; } + drop(cache); if written == 0 { eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-produce encoded=0"); @@ -332,7 +339,7 @@ fn run_module_grain_produce( } eprintln!( - "[rc-arc-spike] kind=module-grain-produce discovery_entries={entries} discovery_rows={rows} sampled_modules={written} skipped_large={skipped_large} manifest={}", + "[rc-arc-spike] kind=module-grain-produce discovery_entries={entries} discovery_rows={rows} sampled_modules={written} skipped_large={skipped_large} skipped_missing={skipped_missing} manifest={}", manifest_path.display() ); From b999021b565ef8eb183c4bd04866ec7d37f71bf3 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:24:51 +0000 Subject: [PATCH 14/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- .../src/bin/measure_rc_arc_share_spike.rs | 17 ++++---- src/v1/stage0/src/shared_typecheck_store.rs | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index eac0b93acdd..e56b605a635 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -303,18 +303,19 @@ fn run_module_grain_produce( skipped_missing += 1; continue; }; - let bytes = SharedTypecheckCaches::encode_typed_snapshot(result.as_ref()).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); - ExitCode::from(2) - })?; - if bytes.len() > MAX_SNAPSHOT_BYTES { + let Some(bytes) = + SharedTypecheckCaches::try_encode_typed_snapshot(result.as_ref(), MAX_SNAPSHOT_BYTES) + .map_err(|e| { + eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); + ExitCode::from(2) + })? + else { skipped_large += 1; eprintln!( - "[rc-arc-spike] kind=module-grain-skip mod_name={mod_name} reason=snapshot_bytes={} cap={MAX_SNAPSHOT_BYTES}", - bytes.len() + "[rc-arc-spike] kind=module-grain-skip mod_name={mod_name} reason=snapshot_bytes>{MAX_SNAPSHOT_BYTES}" ); continue; - } + }; let snap_path = out_dir.join(format!("{written}.bin")); std::fs::write(&snap_path, bytes.as_slice()).map_err(|e| { eprintln!("measure_rc_arc_share_spike: write {:?}: {e}", snap_path); diff --git a/src/v1/stage0/src/shared_typecheck_store.rs b/src/v1/stage0/src/shared_typecheck_store.rs index abed2a4e1eb..8ca29632e6a 100644 --- a/src/v1/stage0/src/shared_typecheck_store.rs +++ b/src/v1/stage0/src/shared_typecheck_store.rs @@ -21,6 +21,30 @@ use std::sync::Arc; use crate::v1_compiler_infer::TypecheckModuleResult; +struct BoundedEncodeWriter { + buf: Vec, + limit: usize, + over_limit: bool, +} + +impl std::io::Write for BoundedEncodeWriter { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.buf.len() + data.len() > self.limit { + self.over_limit = true; + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "bounded encode limit exceeded", + )); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + /// Cross-worker share shell: typed byte cache + collision registry only. /// Construct once per explicit cross-worker run; clone the `Arc>` to every worker. /// Keys are typed-module CONTENT keys (`std.interface_summary.typed_module_key` — source @@ -59,6 +83,23 @@ impl SharedTypecheckCaches { Ok(Arc::new(bytes)) } + /// Encode with an output-size cap. Returns `Ok(None)` when JSON would exceed `max_bytes`. + pub fn try_encode_typed_snapshot( + result: &TypecheckModuleResult, + max_bytes: usize, + ) -> Result>>, String> { + let mut writer = BoundedEncodeWriter { + buf: Vec::new(), + limit: max_bytes, + over_limit: false, + }; + match serde_json::to_writer(&mut writer, result) { + Ok(()) => Ok(Some(Arc::new(writer.buf))), + Err(e) if writer.over_limit => Ok(None), + Err(e) => Err(format!("shared typecheck store encode: {e}")), + } + } + /// Insert pre-encoded bytes under a brief write lock. pub fn insert_typed_preencoded(&mut self, typed_key: String, bytes: Arc>) { self.typed_module_cache.insert(typed_key, bytes); From 807fecbd39afaf4ac6941b3ce61b102e2b32dc21 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:35:20 +0000 Subject: [PATCH 15/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index e56b605a635..e8c09fcf54c 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -306,9 +306,9 @@ fn run_module_grain_produce( let Some(bytes) = SharedTypecheckCaches::try_encode_typed_snapshot(result.as_ref(), MAX_SNAPSHOT_BYTES) .map_err(|e| { - eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); - ExitCode::from(2) - })? + eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); + ExitCode::from(2) + })? else { skipped_large += 1; eprintln!( From a1c48e814ca90a5ae59dca1c720e20f99a6ad46b Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:45:58 +0000 Subject: [PATCH 16/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs index e8c09fcf54c..0d9debf9370 100644 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs @@ -259,7 +259,8 @@ fn run_module_grain_produce( emit_timing_point("module-grain-warm", warm_ms, peak_rss_vhwm_bytes()); let records = spike_take_module_typecheck_records(); - let cache = export_typed_cache_for_spike(&index); + let cache: std::collections::HashMap<_, _> = + export_typed_cache_for_spike(&index).into_iter().collect(); drop(index); const MAX_SNAPSHOT_BYTES: usize = 50_000_000; From cc8e52521de7d40b19532edc5d96c019e4bba099 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:49:55 +0000 Subject: [PATCH 17/22] fix(spike): complete module-grain decode ratio and update report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Module-grain experiment (40 serde-eligible modules) shows decode is ~100× slower than cold typecheck (ratio 0.01), answering the wall-clock question without build-vs-attach. Enable serde_json unbounded_depth for store decode and document results in the pricing report. Co-authored-by: Cursor --- docs/plans/rc-to-arc-share-spike.md | 40 ++++++++++++--------- src/v1/stage0/Cargo.toml | 2 +- src/v1/stage0/src/shared_typecheck_store.rs | 5 ++- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/plans/rc-to-arc-share-spike.md b/docs/plans/rc-to-arc-share-spike.md index 0d89315452e..89793e4bb3d 100644 --- a/docs/plans/rc-to-arc-share-spike.md +++ b/docs/plans/rc-to-arc-share-spike.md @@ -16,11 +16,11 @@ | **Peak RSS on worker object (this box)?** | **~1.51 GiB** VmHWM after discovery warm (73 entries). Shell-only build: **~50 MiB**, **~7.5 s**. | | **Width duplication (RSS)?** | **~linear**: W=1 → 1.58 GiB; W=2 → 3.16 GiB on the same box. | | **RSS crossover for sharing?** | At W=2, private ≈3.16 GiB vs one union ≈1.58 GiB — memory favors sharing from width 2 **on RSS alone**. This box has a **31.25 GiB cgroup cap**; even W=9 private ≈14 GiB stays under cap — **bytes are not the binding constraint here**. | -| **Is width worth it for wall-clock?** | **Not answered by this spike alone.** PR #6905 showed parallel pool is a wall-clock **loss** (serial 11.75 min green vs un-latched 47 min+). Shell build is ~7.5 s/worker; discovery warm is ~55 s/worker on this box. Sharing must remove the **warm typecheck path**, not just bytes, or the lane stays Amdahl-bound. | +| **Is width worth it for wall-clock?** | **No — decode dominates typecheck.** Module-grain ratio (§3.6): **0.01** typecheck/decode per module (~3.8 ms typecheck vs ~318 ms decode on 40 serde-eligible modules). Sharing typed snapshots does **not** remove the ~55 s warm; it would **add** serde decode cost. PR #6905 Amdahl bound still holds. | | **store-path-only vs whole-seed?** | **store-path-only** for increment C (design §9.2 default). `im_rc` collections stay `!Send` without `im-rc`→`im`. | | **im-rc price?** | **123 / 154** stage0 `.rs` files reference `im_rc` (dynamic census). `lib.rs` aliases `HashMap`/`Vector`/`BTreeSet` to it. | -**What is NOT established:** serde-based crossover (discarded — see §2.2). Full `build-vs-attach` on this box (**OOM exit 137**, §3.5). Whole-tree strict resolve to the ~10.7 GiB nimble-owl/srv1 receipt (not re-run here). +**What is NOT established:** serde-based byte crossover (discarded — see §2.2). Whole-tree strict resolve to the ~10.7 GiB nimble-owl/srv1 receipt (not re-run here). `build-vs-attach` was **not retried** (OOM by construction on 31 GiB — two whole-tree indexes); module-grain (§3.6) answers the wall-clock question instead. --- @@ -40,7 +40,8 @@ 1. **Peak RSS (VmHWM) is the only byte metric used for conclusions.** Per-field `index-field` lines are shallow map-shell + key counts; they **under-count** parse trees, resolved graphs, and typed payloads. They are diagnostic, not summed for crossover. 2. **Serde transport bytes are not reported** — per-module encode OOM'd or produced non-credible extrapolations; using them for crossover was an error in the first draft (corrected per operator review). 3. **Residue is under-estimated** — `parse_cache`, `resolved_graph_memo`, and `intern_table` payloads stay per-worker after Arc migration; shallow accounting excludes their heap bodies. -4. **`build-vs-attach` OOM'd** on this box (exit 137, ~100 s) — cannot report attach-vs-cold wall-clock on shared store here without a larger cgroup or split processes. +4. **`build-vs-attach` not run** — holds two whole-tree indexes in one process; OOM on 31 GiB cgroup by construction. Module-grain (§3.6) measures the same decision at module residency. +5. **Module-grain sample bias** — serde snapshots capped at 50 MiB to stay inside cgroup; only **40 / 513** cold-miss modules fit (473 skipped). Ratio is over the serde-eligible tail, not a uniform corpus draw. ### 2.3 Shareable vs per-worker residue (design §4.2) @@ -115,26 +116,33 @@ Per-field shallow accounting (shell + keys only — **not** payload bodies; peak **RSS net-win sketch (not a done-bar):** `private(W) ≈ 1.58 GiB × W`; `shared(W) ≈ 1.58 GiB + (W−1) × 0.05 GiB` (one warm union + cold shells). Crossover **W≥2** on bytes alone — but cgroup headroom is ample to W=9 (~14 GiB naive linear). -### 3.6 build-vs-attach — **BLOCKED on this box** +### 3.6 Module-grain typecheck vs decode (`--mode module-grain-produce --sample-size 200`) + +Operator-directed replacement for `build-vs-attach`: cold typecheck per module during discovery warm, bounded serde encode (50 MiB cap), decode in a **fresh subprocess** — peak residency is N modules, not two whole-tree indexes. ``` -EXIT=137 (OOM killed, ~100s) -[rc-arc-spike] kind=host-metadata hostname=b96c6d293441 mem_available_kb=111314968 cgroup_max_bytes=33578549248 -measure_rc_arc_share_spike: build-vs-attach +[rc-arc-spike] kind=host-metadata hostname=b96c6d293441 mem_available_kb=118522284 cgroup_max_bytes=33578549248 +[rc-arc-spike] kind=timing label=module-grain-warm elapsed_ms=60491 peak_rss_bytes=1593118720 +[rc-arc-spike] kind=module-grain-produce discovery_entries=73 discovery_rows=299 sampled_modules=40 skipped_large=473 skipped_missing=0 manifest=target/rc-arc-spike-module-grain/manifest.jsonl +[rc-arc-spike] kind=module-grain-summary modules=40 missing_typecheck_ns=0 typecheck_ms_per_module=3.761 decode_ms_per_module=318.119 typecheck_to_decode_ratio=0.01 ``` -Warm shared index + cold shell + attach in one process exceeds the **31.25 GiB cgroup** before timing lines emit. **Escalation:** re-run on srv3 (125 GiB budget, no tight cgroup) or split into separate processes. +| Metric | Value | +|---|---| +| Modules encoded+decoded | **40** (473 skipped: serde snapshot > 50 MiB) | +| Typecheck ms/module (cold miss) | **3.761** | +| Decode ms/module (fresh process) | **318.119** | +| Ratio typecheck/decode | **0.01** (~100× **slower** to decode than to typecheck) | -### 3.7 The decisive question (time, not bytes) +**Conclusion:** increment C does **not** buy wall-clock. Even if workers shared typed bytes and skipped cold typecheck, paying serde decode per module is ~100× worse per module than computing types. The ~55 s discovery warm survives; sharing buys bytes only (§3.5). **Lane is dead for wall-clock** unless the transport changes (not in scope for this spike). -From PR #6905: widening the pool is a **wall-clock loss** on ~12 min total corpus work because each worker front-loads index build + warm resolve. +### 3.7 build-vs-attach — **not run (superseded by §3.6)** -| Phase | This box (worker object) | -|---|---| -| Shell build | ~7.5 s | -| Discovery warm (73 entries) | ~55 s | +``` +EXIT=137 (OOM killed, ~100s) — two whole-tree indexes in one 31.25 GiB cgroup +``` -Increment C helps **only if** `build_multi_entry_index_with_shared_caches` + warm shared typed store lets workers **skip the ~55 s cold typecheck warm** (decode/hit path on cache hits). It does **not** remove the ~7.5 s shell build (`build_module_index` + `build_module_graph_facts_live` are per-worker). `build-vs-attach` wall-clock remains **unmeasured** here (OOM). +Not retried per operator direction; module-grain answers the decode-vs-typecheck question without holding two indexes. --- @@ -156,7 +164,7 @@ cargo build --release -p v1-compiler --bin measure_rc_arc_share_spike ./target/release/measure_rc_arc_share_spike --mode worker-shell-build ./target/release/measure_rc_arc_share_spike --mode worker-discovery-warm ./target/release/measure_rc_arc_share_spike --mode width-scaling-worker --max-width 2 -./target/release/measure_rc_arc_share_spike --mode build-vs-attach # blocked on 31GiB cgroup +./target/release/measure_rc_arc_share_spike --mode module-grain-produce --sample-size 200 ``` --- diff --git a/src/v1/stage0/Cargo.toml b/src/v1/stage0/Cargo.toml index fbf892231c3..440155b9188 100644 --- a/src/v1/stage0/Cargo.toml +++ b/src/v1/stage0/Cargo.toml @@ -23,7 +23,7 @@ stacker = "0.1" clap = { version = "4", features = ["derive"] } lazy_static = "1" serde = { version = "1", features = ["derive", "rc"] } -serde_json = "1" +serde_json = { version = "1", features = ["unbounded_depth"] } sha2 = "0.10" ureq = { version = "2", features = ["json"] } # Thread CPU-time clock (CLOCK_THREAD_CPUTIME_ID) for the fast-lane eval budget: the diff --git a/src/v1/stage0/src/shared_typecheck_store.rs b/src/v1/stage0/src/shared_typecheck_store.rs index 8ca29632e6a..682d4f4b7b5 100644 --- a/src/v1/stage0/src/shared_typecheck_store.rs +++ b/src/v1/stage0/src/shared_typecheck_store.rs @@ -15,6 +15,7 @@ //! *names* and diagnostic trees — not per-worker `InternTable` indices — so worker B can decode //! worker A's byte snapshot against its own intern table without a cross-representation straddle. +use serde::Deserialize; use std::collections::HashMap as StdHashMap; use std::rc::Rc; use std::sync::Arc; @@ -71,7 +72,9 @@ impl SharedTypecheckCaches { /// Decode a typed snapshot **without** holding the store lock. /// Payload is name-keyed (no intern-table indices) — safe to materialize on any worker index. pub fn decode_typed_snapshot(bytes: &[u8]) -> Result, String> { - let value: TypecheckModuleResult = serde_json::from_slice(bytes) + let mut de = serde_json::Deserializer::from_slice(bytes); + de.disable_recursion_limit(); + let value = TypecheckModuleResult::deserialize(&mut de) .map_err(|e| format!("shared typecheck store decode: {e}"))?; Ok(Rc::new(value)) } From d1718258770a627fffbfef92a556dddc0683d86c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:51:05 +0000 Subject: [PATCH 18/22] fix(docs): link rc-to-arc spike into doc graph for CI witness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register docs/plans/rc-to-arc-share-spike.md from cross-worker-typecheck- share-design §9 so doc_graph_has_no_orphan_docs passes on discovery floor. Co-authored-by: Cursor --- docs/plans/cross-worker-typecheck-share-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/cross-worker-typecheck-share-design.md b/docs/plans/cross-worker-typecheck-share-design.md index 8329e5383dd..57f7301ead2 100644 --- a/docs/plans/cross-worker-typecheck-share-design.md +++ b/docs/plans/cross-worker-typecheck-share-design.md @@ -206,7 +206,7 @@ Receipt both axes on the same falsifier run: `typecheck_compute` down, `peak_cur ## 9. Open decisions for operator 1. **C1 landing gate:** host-only first (oracles green) before model declaration — recommended yes, same pattern as S1. -2. **Arc migration depth:** store-path-only (minimal seed touch) vs whole-seed `Rc`→`Arc` — recommend store-path-only for increment C; whole-seed is S2b/S7 frontier-eviction co-migration if needed. +2. **Arc migration depth:** store-path-only (minimal seed touch) vs whole-seed `Rc`→`Arc` — recommend store-path-only for increment C; whole-seed is S2b/S7 frontier-eviction co-migration if needed. **Receipt (2026-07-20):** [Rc→Arc share spike](rc-to-arc-share-spike.md) — module-grain ratio **0.01** (decode ~100× slower than cold typecheck); increment C does not remove the ~55 s warm; lane dead for wall-clock unless transport changes. 3. **Memory trade acceptance:** proceed with shared retention on the falsifier evidence that width stays > 1; escalate to M2 only if governor receipts regress — recommend proceed (time prize dominates; M2 is pre-authorized). --- From 3f9ef3ce1ea2a04307f0ca740ba8e5dcff4a1776 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 06:57:36 +0000 Subject: [PATCH 19/22] =?UTF-8?q?docs(spike):=20reframe=20=C2=A73.6=20as?= =?UTF-8?q?=20serde=20transport=20cost,=20not=20Arc=20share?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator review: the module-grain experiment priced the interim serde byte transport (the Rc→Arc workaround), not in-memory Arc sharing. Remove "lane is dead" headline; document the 31× warm-vs-hook timer discrepancy and incomparable measurement bases; keep raw receipts unchanged. Co-authored-by: Cursor --- .../cross-worker-typecheck-share-design.md | 2 +- docs/plans/rc-to-arc-share-spike.md | 29 ++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/plans/cross-worker-typecheck-share-design.md b/docs/plans/cross-worker-typecheck-share-design.md index 57f7301ead2..a5c094515a2 100644 --- a/docs/plans/cross-worker-typecheck-share-design.md +++ b/docs/plans/cross-worker-typecheck-share-design.md @@ -206,7 +206,7 @@ Receipt both axes on the same falsifier run: `typecheck_compute` down, `peak_cur ## 9. Open decisions for operator 1. **C1 landing gate:** host-only first (oracles green) before model declaration — recommended yes, same pattern as S1. -2. **Arc migration depth:** store-path-only (minimal seed touch) vs whole-seed `Rc`→`Arc` — recommend store-path-only for increment C; whole-seed is S2b/S7 frontier-eviction co-migration if needed. **Receipt (2026-07-20):** [Rc→Arc share spike](rc-to-arc-share-spike.md) — module-grain ratio **0.01** (decode ~100× slower than cold typecheck); increment C does not remove the ~55 s warm; lane dead for wall-clock unless transport changes. +2. **Arc migration depth:** store-path-only (minimal seed touch) vs whole-seed `Rc`→`Arc` — recommend store-path-only for increment C; whole-seed is S2b/S7 frontier-eviction co-migration if needed. **Receipt (2026-07-20):** [Rc→Arc share spike](rc-to-arc-share-spike.md) — serde byte transport ~318 ms/module decode on 7.8% sample (§3.6); evidence against shipping increment C on serde transport; in-memory Arc share wall-clock not measured. 3. **Memory trade acceptance:** proceed with shared retention on the falsifier evidence that width stays > 1; escalate to M2 only if governor receipts regress — recommend proceed (time prize dominates; M2 is pre-authorized). --- diff --git a/docs/plans/rc-to-arc-share-spike.md b/docs/plans/rc-to-arc-share-spike.md index 89793e4bb3d..4cf8158bfd9 100644 --- a/docs/plans/rc-to-arc-share-spike.md +++ b/docs/plans/rc-to-arc-share-spike.md @@ -16,11 +16,11 @@ | **Peak RSS on worker object (this box)?** | **~1.51 GiB** VmHWM after discovery warm (73 entries). Shell-only build: **~50 MiB**, **~7.5 s**. | | **Width duplication (RSS)?** | **~linear**: W=1 → 1.58 GiB; W=2 → 3.16 GiB on the same box. | | **RSS crossover for sharing?** | At W=2, private ≈3.16 GiB vs one union ≈1.58 GiB — memory favors sharing from width 2 **on RSS alone**. This box has a **31.25 GiB cgroup cap**; even W=9 private ≈14 GiB stays under cap — **bytes are not the binding constraint here**. | -| **Is width worth it for wall-clock?** | **No — decode dominates typecheck.** Module-grain ratio (§3.6): **0.01** typecheck/decode per module (~3.8 ms typecheck vs ~318 ms decode on 40 serde-eligible modules). Sharing typed snapshots does **not** remove the ~55 s warm; it would **add** serde decode cost. PR #6905 Amdahl bound still holds. | +| **Is width worth it for wall-clock?** | **Not answered by §3.6** — that section prices the **interim serde byte transport** (`SharedTypecheckCaches`), not in-memory `Arc` sharing (which would hold handles with **no decode**). PR #6905 Amdahl bound still the wall-clock context. Serde transport receipt: ~318 ms/module decode on a 7.8% biased sample (§3.6). | | **store-path-only vs whole-seed?** | **store-path-only** for increment C (design §9.2 default). `im_rc` collections stay `!Send` without `im-rc`→`im`. | | **im-rc price?** | **123 / 154** stage0 `.rs` files reference `im_rc` (dynamic census). `lib.rs` aliases `HashMap`/`Vector`/`BTreeSet` to it. | -**What is NOT established:** serde-based byte crossover (discarded — see §2.2). Whole-tree strict resolve to the ~10.7 GiB nimble-owl/srv1 receipt (not re-run here). `build-vs-attach` was **not retried** (OOM by construction on 31 GiB — two whole-tree indexes); module-grain (§3.6) answers the wall-clock question instead. +**What is NOT established:** in-memory `Arc` share wall-clock (not measured — Arc share has no serde decode). Serde-based byte crossover (discarded — see §2.2). Whole-tree strict resolve to the ~10.7 GiB nimble-owl/srv1 receipt (not re-run here). `build-vs-attach` was **not retried** (OOM by construction on 31 GiB — two whole-tree indexes); module-grain (§3.6) prices serde transport at module residency instead. --- @@ -40,8 +40,10 @@ 1. **Peak RSS (VmHWM) is the only byte metric used for conclusions.** Per-field `index-field` lines are shallow map-shell + key counts; they **under-count** parse trees, resolved graphs, and typed payloads. They are diagnostic, not summed for crossover. 2. **Serde transport bytes are not reported** — per-module encode OOM'd or produced non-credible extrapolations; using them for crossover was an error in the first draft (corrected per operator review). 3. **Residue is under-estimated** — `parse_cache`, `resolved_graph_memo`, and `intern_table` payloads stay per-worker after Arc migration; shallow accounting excludes their heap bodies. -4. **`build-vs-attach` not run** — holds two whole-tree indexes in one process; OOM on 31 GiB cgroup by construction. Module-grain (§3.6) measures the same decision at module residency. -5. **Module-grain sample bias** — serde snapshots capped at 50 MiB to stay inside cgroup; only **40 / 513** cold-miss modules fit (473 skipped). Ratio is over the serde-eligible tail, not a uniform corpus draw. +4. **`build-vs-attach` not run** — holds two whole-tree indexes in one process; OOM on 31 GiB cgroup by construction. Module-grain (§3.6) prices serde transport at module residency instead. +5. **Module-grain measures serde transport, not Arc share** — `SharedTypecheckCaches` is the interim serde-byte scaffold (`shared_typecheck_store.rs` header; dissolve-on: store-path `Rc`→`Arc`). Real Arc sharing would pass in-memory handles — **no decode**. §3.6 prices the workaround that exists because `Rc` is `!Send`. +6. **Module-grain sample bias** — serde snapshots capped at 50 MiB; only **40 / 513** cold-miss modules fit (**7.8%**; 473 skipped are the large ones). Snapshots near 50 MiB imply `encode_typed_snapshot` serializes each module with its transitive closure — shared context re-encoded per module (transport redundancy, not a property of sharing). +7. **Module-grain timer mismatch (open)** — same run: `module-grain-warm` **60 491 ms** over 513 cold misses ≈ **118 ms/module** warm cost, but `typecheck_ms_per_module=3.761` from the per-miss hook. **31× discrepancy within one run** — the hook timer does not capture full produce cost; **do not treat `typecheck_to_decode_ratio=0.01` as a decisive wall-clock ratio** (also mixes inner-loop timing vs fresh-subprocess decode with startup/load overhead). ### 2.3 Shareable vs per-worker residue (design §4.2) @@ -116,7 +118,9 @@ Per-field shallow accounting (shell + keys only — **not** payload bodies; peak **RSS net-win sketch (not a done-bar):** `private(W) ≈ 1.58 GiB × W`; `shared(W) ≈ 1.58 GiB + (W−1) × 0.05 GiB` (one warm union + cold shells). Crossover **W≥2** on bytes alone — but cgroup headroom is ample to W=9 (~14 GiB naive linear). -### 3.6 Module-grain typecheck vs decode (`--mode module-grain-produce --sample-size 200`) +### 3.6 Module-grain **serde transport** cost (`--mode module-grain-produce --sample-size 200`) + +**What this measures:** the interim **serde byte transport** in `SharedTypecheckCaches` — the scaffold that exists because `Rc` is `!Send` (`shared_typecheck_store.rs`: "interim serde byte transport for cross-worker share only"; dissolve-on: store-path `Rc`→`Arc`). **Not** in-memory `Arc` sharing (which would hold handles with no decode). Operator-directed replacement for `build-vs-attach`: cold typecheck per module during discovery warm, bounded serde encode (50 MiB cap), decode in a **fresh subprocess** — peak residency is N modules, not two whole-tree indexes. @@ -129,12 +133,15 @@ Operator-directed replacement for `build-vs-attach`: cold typecheck per module d | Metric | Value | |---|---| -| Modules encoded+decoded | **40** (473 skipped: serde snapshot > 50 MiB) | -| Typecheck ms/module (cold miss) | **3.761** | -| Decode ms/module (fresh process) | **318.119** | -| Ratio typecheck/decode | **0.01** (~100× **slower** to decode than to typecheck) | +| Modules encoded+decoded | **40** (**7.8%** of 513 cold misses; 473 skipped: serde snapshot > 50 MiB) | +| Warm ms/module (receipt) | **~118** (60 491 ms ÷ 513 cold misses) | +| Hook `typecheck_ms_per_module` | **3.761** (inner per-miss timer — **31× below** warm receipt; discrepancy **unresolved**) | +| Decode ms/module (fresh subprocess) | **318.119** (includes process start + snapshot load) | +| Harness `typecheck_to_decode_ratio` | **0.01** (3.761 ÷ 318.119 — **not comparable bases**; see §2.2 item 7) | + +**What is established:** the serde byte transport is expensive (~318 ms/module decode on this sample) and structurally redundant (per-module snapshots re-encode transitive closure). This is evidence **against** shipping increment C on the serde transport, and **for** real `Arc` handles (the migration this scaffold exists to enable). -**Conclusion:** increment C does **not** buy wall-clock. Even if workers shared typed bytes and skipped cold typecheck, paying serde decode per module is ~100× worse per module than computing types. The ~55 s discovery warm survives; sharing buys bytes only (§3.5). **Lane is dead for wall-clock** unless the transport changes (not in scope for this spike). +**What is not established:** whether in-memory Arc sharing removes the ~55 s discovery warm — that path was not measured here. A naive same-run sketch (warm ~118 ms/module vs decode ~318 ms/module) is ~2.7×, but the two timers are on **incomparable bases** (§2.2 item 7); it is not reported as a done-bar ratio. ### 3.7 build-vs-attach — **not run (superseded by §3.6)** @@ -142,7 +149,7 @@ Operator-directed replacement for `build-vs-attach`: cold typecheck per module d EXIT=137 (OOM killed, ~100s) — two whole-tree indexes in one 31.25 GiB cgroup ``` -Not retried per operator direction; module-grain answers the decode-vs-typecheck question without holding two indexes. +Not retried per operator direction; §3.6 prices serde transport at module residency without holding two indexes. --- From 19353668873ef579371a3ac4a56709441a0d5b12 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 07:13:12 +0000 Subject: [PATCH 20/22] style(stage0): prefix unused bounded-encode error binding with _ Review 40168 nit: Err arm on over_limit never uses the serde error. Co-authored-by: Cursor --- src/v1/stage0/src/shared_typecheck_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v1/stage0/src/shared_typecheck_store.rs b/src/v1/stage0/src/shared_typecheck_store.rs index 682d4f4b7b5..1ed8d47931e 100644 --- a/src/v1/stage0/src/shared_typecheck_store.rs +++ b/src/v1/stage0/src/shared_typecheck_store.rs @@ -98,7 +98,7 @@ impl SharedTypecheckCaches { }; match serde_json::to_writer(&mut writer, result) { Ok(()) => Ok(Some(Arc::new(writer.buf))), - Err(e) if writer.over_limit => Ok(None), + Err(_e) if writer.over_limit => Ok(None), Err(e) => Err(format!("shared typecheck store encode: {e}")), } } From 75e77d3eb1ead2519bd8010c7724f9880a9fbead Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 08:47:28 +0000 Subject: [PATCH 21/22] WIP: Rc-to-Arc share spike: price the memory win before any migration --- src/v1/stage0/Cargo.toml | 6 +- .../src/bin/measure_rc_arc_share_spike.rs | 528 --------------- src/v1/stage0/src/cli_run.rs | 270 -------- src/v1/stage0/src/index_memory_receipt.rs | 601 ------------------ src/v1/stage0/src/lib.rs | 1 - src/v1/stage0/src/shared_typecheck_store.rs | 46 +- 6 files changed, 2 insertions(+), 1450 deletions(-) delete mode 100644 src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs delete mode 100644 src/v1/stage0/src/index_memory_receipt.rs diff --git a/src/v1/stage0/Cargo.toml b/src/v1/stage0/Cargo.toml index 440155b9188..960db9d5d76 100644 --- a/src/v1/stage0/Cargo.toml +++ b/src/v1/stage0/Cargo.toml @@ -23,7 +23,7 @@ stacker = "0.1" clap = { version = "4", features = ["derive"] } lazy_static = "1" serde = { version = "1", features = ["derive", "rc"] } -serde_json = { version = "1", features = ["unbounded_depth"] } +serde_json = "1" sha2 = "0.10" ureq = { version = "2", features = ["json"] } # Thread CPU-time clock (CLOCK_THREAD_CPUTIME_ID) for the fast-lane eval budget: the @@ -123,10 +123,6 @@ path = "src/bin/measure_whole_tree_resolve.rs" name = "measure_dependency_view_build" path = "src/bin/measure_dependency_view_build.rs" -[[bin]] -name = "measure_rc_arc_share_spike" -path = "src/bin/measure_rc_arc_share_spike.rs" - [[bin]] name = "capture_func_env_semantic_oracle" path = "src/bin/capture_func_env_semantic_oracle.rs" diff --git a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs b/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs deleted file mode 100644 index 0d9debf9370..00000000000 --- a/src/v1/stage0/src/bin/measure_rc_arc_share_spike.rs +++ /dev/null @@ -1,528 +0,0 @@ -#![allow(clippy::disallowed_macros)] - -//! Rc→Arc share spike — measurement harness (NOT a migration). -//! -//! Measures the **floor worker object**: `build_multi_entry_index` over both roots (`dag`, `src/v2`), -//! then discovery-corpus warm — not per-entry import closures. - -use std::process::{Command, ExitCode}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Barrier}; -use std::thread; -use std::time::Instant; - -use v1_compiler::cli_run::{ - build_worker_index_shell, build_worker_index_with_warm_store, export_typed_cache_for_spike, - new_shared_typecheck_caches, peak_rss_vhwm_bytes, - populate_multi_entry_index_whole_tree_discovery, populate_worker_discovery_index, - resolve_entry_with_index_for_discovery_corpus, spike_measurement_host_metadata, - spike_set_module_typecheck_recording, spike_take_module_typecheck_records, - warm_discovery_on_index, whole_tree_resolve_exclusion_substrings, workspace_root, -}; -use v1_compiler::index_memory_receipt::{ - emit_host_metadata, emit_im_rc_census, emit_index_memory_receipt, emit_timing_point, - emit_width_scaling_point, multi_entry_index_memory_receipt, -}; - -fn require_value(args: &[String], idx: usize, flag: &str) -> Result { - match args.get(idx) { - Some(v) => Ok(v.clone()), - None => { - eprintln!("measure_rc_arc_share_spike: {flag} requires a value"); - Err(ExitCode::from(2)) - } - } -} - -fn parse_usize(flag: &str, raw: &str) -> Result { - raw.parse::().map_err(|_| { - eprintln!("measure_rc_arc_share_spike: {flag} must be a positive integer, got `{raw}`"); - ExitCode::from(2) - }) -} - -fn default_source_roots() -> Vec { - let ws = workspace_root(); - ["dag", "src/v2"] - .iter() - .map(|r| ws.join(r).to_string_lossy().into_owned()) - .collect() -} - -fn run_worker_shell_build(source_roots: &[String]) -> Result { - eprintln!( - "measure_rc_arc_share_spike: worker-shell-build roots={}", - source_roots.len() - ); - let start = Instant::now(); - let _index = build_worker_index_shell(source_roots); - emit_timing_point( - "worker-shell-build", - start.elapsed().as_millis(), - peak_rss_vhwm_bytes(), - ); - Ok(ExitCode::SUCCESS) -} - -fn run_worker_discovery_warm(source_roots: &[String]) -> Result { - eprintln!( - "measure_rc_arc_share_spike: worker-discovery-warm roots={}", - source_roots.len() - ); - let shell_start = Instant::now(); - let _shell = build_worker_index_shell(source_roots); - emit_timing_point( - "worker-shell-only", - shell_start.elapsed().as_millis(), - peak_rss_vhwm_bytes(), - ); - - let warm_start = Instant::now(); - match populate_worker_discovery_index(source_roots) { - Ok((index, entries, rows)) => { - let warm_ms = warm_start.elapsed().as_millis(); - let peak = peak_rss_vhwm_bytes(); - emit_timing_point("worker-discovery-warm", warm_ms, peak); - eprintln!( - "[rc-arc-spike] kind=worker-object discovery_entries={entries} discovery_rows={rows} peak_rss_bytes={}", - peak.map(|b| b.to_string()).unwrap_or_else(|| "unavailable".into()) - ); - emit_index_memory_receipt(&multi_entry_index_memory_receipt(&index, peak)); - Ok(ExitCode::SUCCESS) - } - Err(e) => { - eprintln!( - "[rc-arc-spike] kind=blocked measurement=worker-discovery-warm elapsed_ms={} error={e}", - warm_start.elapsed().as_millis() - ); - Err(ExitCode::from(2)) - } - } -} - -fn run_whole_tree_fields( - source_roots: &[String], - exclude_subpaths: &[String], -) -> Result { - eprintln!( - "measure_rc_arc_share_spike: whole-tree-fields roots={}", - source_roots.len() - ); - let start = Instant::now(); - match populate_multi_entry_index_whole_tree_discovery(source_roots, exclude_subpaths) { - Ok((index, modules_resolved, modules_excluded)) => { - let peak = peak_rss_vhwm_bytes(); - emit_timing_point( - "whole-tree-discovery-resolve", - start.elapsed().as_millis(), - peak, - ); - eprintln!( - "[rc-arc-spike] kind=whole-tree modules_resolved={modules_resolved} modules_excluded={modules_excluded}" - ); - emit_index_memory_receipt(&multi_entry_index_memory_receipt(&index, peak)); - Ok(ExitCode::SUCCESS) - } - Err(e) => { - eprintln!( - "[rc-arc-spike] kind=blocked measurement=whole-tree-fields elapsed_ms={} error={e}", - start.elapsed().as_millis() - ); - Err(ExitCode::from(2)) - } - } -} - -fn run_build_vs_attach(source_roots: &[String]) -> Result { - eprintln!("measure_rc_arc_share_spike: build-vs-attach"); - - let store = new_shared_typecheck_caches(); - let warm_start = Instant::now(); - let warm_index = build_worker_index_with_warm_store(source_roots, store.clone()); - let (entries, rows) = warm_discovery_on_index(&warm_index, source_roots).map_err(|e| { - eprintln!("[rc-arc-spike] kind=blocked measurement=build-vs-attach-warm error={e}"); - ExitCode::from(2) - })?; - let warm_ms = warm_start.elapsed().as_millis(); - let warm_peak = peak_rss_vhwm_bytes(); - emit_timing_point("build-vs-attach-warm-shared-index", warm_ms, warm_peak); - eprintln!( - "[rc-arc-spike] kind=shared-store-warm discovery_entries={entries} discovery_rows={rows}" - ); - - let cold_shell_start = Instant::now(); - let _cold = build_worker_index_shell(source_roots); - let cold_shell_ms = cold_shell_start.elapsed().as_millis(); - emit_timing_point( - "build-vs-attach-cold-shell", - cold_shell_ms, - peak_rss_vhwm_bytes(), - ); - - let attach_start = Instant::now(); - let attached = build_worker_index_with_warm_store(source_roots, store); - let attach_ms = attach_start.elapsed().as_millis(); - emit_timing_point( - "build-vs-attach-shared-shell", - attach_ms, - peak_rss_vhwm_bytes(), - ); - - let sample_entry = v1_compiler::cli_run::discovery_corpus_entry_roster(source_roots) - .map_err(|e| { - eprintln!("measure_rc_arc_share_spike: discovery roster failed: {e}"); - ExitCode::from(2) - })? - .0 - .into_iter() - .next() - .ok_or_else(|| { - eprintln!("measure_rc_arc_share_spike: discovery roster empty"); - ExitCode::from(2) - })?; - let resolve_start = Instant::now(); - resolve_entry_with_index_for_discovery_corpus(&attached, &sample_entry).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: attached resolve failed: {e}"); - ExitCode::from(2) - })?; - let resolve_ms = resolve_start.elapsed().as_millis(); - emit_timing_point( - "build-vs-attach-shared-resolve-one", - resolve_ms, - peak_rss_vhwm_bytes(), - ); - - eprintln!( - "[rc-arc-spike] kind=build-vs-attach-summary warm_shared_ms={warm_ms} cold_shell_ms={cold_shell_ms} shared_attach_ms={attach_ms} shared_resolve_one_ms={resolve_ms}" - ); - Ok(ExitCode::SUCCESS) -} - -fn run_width_scaling_worker( - source_roots: &[String], - max_width: usize, -) -> Result { - eprintln!("measure_rc_arc_share_spike: width-scaling-worker 1..={max_width}"); - for width in 1..=max_width { - eprintln!( - "measure_rc_arc_share_spike: width-point width={width} object=worker-discovery-warm" - ); - let roots = Arc::new(source_roots.to_vec()); - let errors = Arc::new(std::sync::Mutex::new(Vec::::new())); - let barrier = Arc::new(Barrier::new(width)); - let mut handles = Vec::new(); - for worker in 0..width { - let roots = Arc::clone(&roots); - let errors = Arc::clone(&errors); - let barrier = Arc::clone(&barrier); - handles.push(thread::spawn(move || { - barrier.wait(); - if let Err(e) = populate_worker_discovery_index(roots.as_ref()) { - errors.lock().unwrap().push(format!("worker {worker}: {e}")); - } - })); - } - for handle in handles { - handle.join().map_err(|_| ExitCode::from(2))?; - } - let errs = errors.lock().unwrap(); - if !errs.is_empty() { - for e in errs.iter() { - eprintln!("measure_rc_arc_share_spike: {e}"); - } - eprintln!("[rc-arc-spike] kind=blocked measurement=width-scaling-worker width={width}"); - return Err(ExitCode::from(2)); - } - emit_width_scaling_point(width, peak_rss_vhwm_bytes()); - } - Ok(ExitCode::SUCCESS) -} - -use v1_compiler::shared_typecheck_store::SharedTypecheckCaches; - -fn module_grain_out_dir() -> std::path::PathBuf { - workspace_root().join("target/rc-arc-spike-module-grain") -} - -fn run_module_grain_produce( - source_roots: &[String], - sample_size: usize, -) -> Result { - eprintln!("measure_rc_arc_share_spike: module-grain-produce sample_size={sample_size}"); - spike_set_module_typecheck_recording(true); - let warm_start = Instant::now(); - let (index, entries, rows) = populate_worker_discovery_index(source_roots).map_err(|e| { - eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-produce error={e}"); - ExitCode::from(2) - })?; - let warm_ms = warm_start.elapsed().as_millis(); - emit_timing_point("module-grain-warm", warm_ms, peak_rss_vhwm_bytes()); - - let records = spike_take_module_typecheck_records(); - let cache: std::collections::HashMap<_, _> = - export_typed_cache_for_spike(&index).into_iter().collect(); - drop(index); - - const MAX_SNAPSHOT_BYTES: usize = 50_000_000; - const MAX_TYPECHECK_NS: u64 = 500_000_000; // 500ms — skip outlier modules whose serde payloads OOM - let mut tc_by_key: std::collections::HashMap = - std::collections::HashMap::new(); - for (mod_name, typed_key, ns) in records { - if ns > 0 && ns <= MAX_TYPECHECK_NS { - tc_by_key.insert(typed_key, (mod_name, ns)); - } - } - - let mut keys: Vec<(String, String, u64)> = tc_by_key - .into_iter() - .map(|(k, (m, ns))| (k, m, ns)) - .collect(); - keys.sort_by(|a, b| a.2.cmp(&b.2)); // ascending — typical modules first - - let out_dir = module_grain_out_dir(); - let _ = std::fs::remove_dir_all(&out_dir); - std::fs::create_dir_all(&out_dir).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: cannot create {out_dir:?}: {e}"); - ExitCode::from(2) - })?; - - let manifest_path = out_dir.join("manifest.jsonl"); - let mut manifest = std::fs::File::create(&manifest_path).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: create manifest: {e}"); - ExitCode::from(2) - })?; - use std::io::Write; - - let mut written = 0usize; - let mut skipped_large = 0usize; - let mut skipped_missing = 0usize; - for (typed_key, mod_name, typecheck_ns) in keys { - if written >= sample_size { - break; - } - let Some(result) = cache.get(&typed_key) else { - skipped_missing += 1; - continue; - }; - let Some(bytes) = - SharedTypecheckCaches::try_encode_typed_snapshot(result.as_ref(), MAX_SNAPSHOT_BYTES) - .map_err(|e| { - eprintln!("measure_rc_arc_share_spike: encode failed for {mod_name}: {e}"); - ExitCode::from(2) - })? - else { - skipped_large += 1; - eprintln!( - "[rc-arc-spike] kind=module-grain-skip mod_name={mod_name} reason=snapshot_bytes>{MAX_SNAPSHOT_BYTES}" - ); - continue; - }; - let snap_path = out_dir.join(format!("{written}.bin")); - std::fs::write(&snap_path, bytes.as_slice()).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: write {:?}: {e}", snap_path); - ExitCode::from(2) - })?; - let line = format!( - "{{\"typed_key\":{},\"mod_name\":{},\"typecheck_ns\":{typecheck_ns},\"snapshot_path\":{}}}\n", - serde_json::to_string(&typed_key).unwrap(), - serde_json::to_string(&mod_name).unwrap(), - serde_json::to_string(snap_path.to_string_lossy().as_ref()).unwrap(), - ); - manifest - .write_all(line.as_bytes()) - .map_err(|_| ExitCode::from(2))?; - written += 1; - } - drop(cache); - - if written == 0 { - eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-produce encoded=0"); - return Err(ExitCode::from(2)); - } - - eprintln!( - "[rc-arc-spike] kind=module-grain-produce discovery_entries={entries} discovery_rows={rows} sampled_modules={written} skipped_large={skipped_large} skipped_missing={skipped_missing} manifest={}", - manifest_path.display() - ); - - let exe = std::env::current_exe().map_err(|_| ExitCode::from(2))?; - let decode_out = Command::new(exe) - .args([ - "--mode", - "module-grain-decode", - "--manifest", - &manifest_path.to_string_lossy(), - ]) - .output() - .map_err(|e| { - eprintln!("measure_rc_arc_share_spike: decode subprocess failed: {e}"); - ExitCode::from(2) - })?; - let stderr = String::from_utf8_lossy(&decode_out.stderr); - print!("{stderr}"); - if !decode_out.status.success() { - eprintln!( - "[rc-arc-spike] kind=blocked measurement=module-grain-decode exit={}", - decode_out.status - ); - return Err(ExitCode::from(2)); - } - Ok(ExitCode::SUCCESS) -} - -fn run_module_grain_decode(manifest_path: &str) -> Result { - eprintln!("measure_rc_arc_share_spike: module-grain-decode manifest={manifest_path}"); - let content = std::fs::read_to_string(manifest_path).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: read manifest: {e}"); - ExitCode::from(2) - })?; - - let mut typecheck_total_ns = 0u64; - let mut decode_total_ns = 0u64; - let mut modules = 0usize; - let mut missing_typecheck = 0usize; - - for line in content.lines() { - if line.trim().is_empty() { - continue; - } - let row: serde_json::Value = serde_json::from_str(line).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: bad manifest line: {e}"); - ExitCode::from(2) - })?; - let mod_name = row["mod_name"].as_str().unwrap_or("?"); - let typecheck_ns = row["typecheck_ns"].as_u64().unwrap_or(0); - let snap_path = row["snapshot_path"].as_str().ok_or_else(|| { - eprintln!("measure_rc_arc_share_spike: manifest missing snapshot_path"); - ExitCode::from(2) - })?; - let bytes = std::fs::read(snap_path).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: read snapshot {snap_path}: {e}"); - ExitCode::from(2) - })?; - let decode_start = Instant::now(); - SharedTypecheckCaches::decode_typed_snapshot(&bytes).map_err(|e| { - eprintln!("measure_rc_arc_share_spike: decode {mod_name}: {e}"); - ExitCode::from(2) - })?; - let decode_ns = decode_start.elapsed().as_nanos() as u64; - eprintln!( - "[rc-arc-spike] kind=module-grain-row mod_name={mod_name} typecheck_ns={typecheck_ns} decode_ns={decode_ns}" - ); - if typecheck_ns == 0 { - missing_typecheck += 1; - } - typecheck_total_ns += typecheck_ns; - decode_total_ns += decode_ns; - modules += 1; - } - - if modules == 0 { - eprintln!("[rc-arc-spike] kind=blocked measurement=module-grain-decode modules=0"); - return Err(ExitCode::from(2)); - } - - let typecheck_ms_per_module = (typecheck_total_ns as f64) / (modules as f64) / 1_000_000.0; - let decode_ms_per_module = (decode_total_ns as f64) / (modules as f64) / 1_000_000.0; - let ratio = if decode_total_ns > 0 { - typecheck_total_ns as f64 / decode_total_ns as f64 - } else { - 0.0 - }; - eprintln!( - "[rc-arc-spike] kind=module-grain-summary modules={modules} missing_typecheck_ns={missing_typecheck} typecheck_ms_per_module={typecheck_ms_per_module:.3} decode_ms_per_module={decode_ms_per_module:.3} typecheck_to_decode_ratio={ratio:.2}" - ); - Ok(ExitCode::SUCCESS) -} - -fn run() -> Result { - let args: Vec = std::env::args().collect(); - let mut mode = String::from("worker-discovery-warm"); - let mut source_roots: Vec = Vec::new(); - let mut exclude_subpaths = whole_tree_resolve_exclusion_substrings(); - let mut max_width = 3usize; - let mut sample_size = 200usize; - let mut manifest_path = String::new(); - - let mut i = 1; - while i < args.len() { - match args[i].as_str() { - "--mode" => { - i += 1; - mode = require_value(&args, i, "--mode")?; - } - "--source-root" => { - i += 1; - source_roots.push(require_value(&args, i, "--source-root")?); - } - "--exclude-subpath" => { - i += 1; - exclude_subpaths.push(require_value(&args, i, "--exclude-subpath")?); - } - "--max-width" => { - i += 1; - max_width = parse_usize("--max-width", &require_value(&args, i, "--max-width")?)?; - } - "--sample-size" => { - i += 1; - sample_size = - parse_usize("--sample-size", &require_value(&args, i, "--sample-size")?)?; - } - "--manifest" => { - i += 1; - manifest_path = require_value(&args, i, "--manifest")?; - } - other => { - eprintln!("measure_rc_arc_share_spike: unknown argument: {other}"); - return Err(ExitCode::from(2)); - } - } - i += 1; - } - - if source_roots.is_empty() { - source_roots = default_source_roots(); - } - - let (hostname, mem_available_kb, cgroup_max_bytes) = spike_measurement_host_metadata(); - emit_host_metadata(&hostname, mem_available_kb, cgroup_max_bytes); - - match mode.as_str() { - "worker-shell-build" => run_worker_shell_build(&source_roots), - "worker-discovery-warm" => run_worker_discovery_warm(&source_roots), - "whole-tree-fields" => run_whole_tree_fields(&source_roots, &exclude_subpaths), - "build-vs-attach" => run_build_vs_attach(&source_roots), - "width-scaling-worker" => run_width_scaling_worker(&source_roots, max_width), - "module-grain-produce" => run_module_grain_produce(&source_roots, sample_size), - "module-grain-decode" => { - if manifest_path.is_empty() { - eprintln!( - "measure_rc_arc_share_spike: --manifest required for module-grain-decode" - ); - return Err(ExitCode::from(2)); - } - run_module_grain_decode(&manifest_path) - } - "im-rc-census" => { - emit_im_rc_census(); - Ok(ExitCode::SUCCESS) - } - other => { - eprintln!( - "measure_rc_arc_share_spike: unknown --mode {other} (expected worker-shell-build|worker-discovery-warm|whole-tree-fields|build-vs-attach|width-scaling-worker|module-grain-produce|module-grain-decode|im-rc-census)" - ); - Err(ExitCode::from(2)) - } - } -} - -fn main() -> ExitCode { - static STARTED: AtomicUsize = AtomicUsize::new(0); - if STARTED.fetch_add(1, Ordering::SeqCst) == 0 { - eprintln!("measure_rc_arc_share_spike: Rc→Arc share spike harness (measurement only)"); - } - match run() { - Ok(code) => code, - Err(code) => code, - } -} diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index de49c459e94..8e86c60a352 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4056,47 +4056,6 @@ pub struct MultiEntryIndex { >, } -impl MultiEntryIndex { - /// Borrow every cache field for structural memory receipts (rc-arc share spike). - pub(crate) fn memory_receipt_snapshot( - &self, - ) -> ( - std::cell::Ref< - '_, - std::collections::HashMap>, - >, - std::cell::Ref< - '_, - std::collections::HashMap< - String, - (Rc, Rc), - >, - >, - std::cell::Ref< - '_, - im_rc::HashMap< - String, - ( - Rc, - Rc>>, - ), - >, - >, - std::cell::Ref<'_, Rc>, - std::cell::Ref<'_, std::collections::HashMap>>>>, - std::cell::Ref<'_, std::collections::HashMap>>>>, - ) { - ( - self.typed_module_cache.borrow(), - self.parse_cache.borrow(), - self.resolved_graph_memo.borrow(), - self.intern_table.borrow(), - self.normalize_diag_cache.borrow(), - self.ownership_diag_cache.borrow(), - ) - } -} - pub fn new_shared_typecheck_caches() -> Arc> { shared_typecheck_store::new_shared_typecheck_caches() } @@ -4141,9 +4100,6 @@ fn new_multi_entry_index_shell( // the process shares one typed_module_cache, this stays ≤ the distinct module count of // the process union. static TYPECHECK_COMPUTE_COUNT: AtomicUsize = AtomicUsize::new(0); -// rc-arc spike: per-module cold typecheck timings on cache-miss path only. -static SPIKE_MODULE_TYPECHECK_RECORDING: AtomicBool = AtomicBool::new(false); -static SPIKE_MODULE_TYPECHECK_RECORDS: Mutex> = Mutex::new(Vec::new()); // Union-resolve receipt tests reset/read the process-wide counter; `cargo test` runs // `#[test]` fns in parallel by default — serialize those oracles (not production use). @@ -4165,47 +4121,6 @@ pub fn reset_typecheck_compute_count() { TYPECHECK_COMPUTE_COUNT.store(0, Ordering::SeqCst); } -/// Enable per-module typecheck miss recording for the rc-arc spike harness. -pub fn spike_set_module_typecheck_recording(on: bool) { - SPIKE_MODULE_TYPECHECK_RECORDING.store(on, Ordering::SeqCst); - if on { - SPIKE_MODULE_TYPECHECK_RECORDS - .lock() - .expect("spike module typecheck records lock poisoned") - .clear(); - } -} - -/// Take recorded per-module cold typecheck durations (module name, typed key, nanoseconds). -pub fn spike_take_module_typecheck_records() -> Vec<(String, String, u64)> { - std::mem::take( - &mut *SPIKE_MODULE_TYPECHECK_RECORDS - .lock() - .expect("spike module typecheck records lock poisoned"), - ) -} - -fn spike_maybe_record_module_typecheck(mod_name: &str, typed_key: &str, elapsed_ns: u64) { - if SPIKE_MODULE_TYPECHECK_RECORDING.load(Ordering::SeqCst) { - SPIKE_MODULE_TYPECHECK_RECORDS - .lock() - .expect("spike module typecheck records lock poisoned") - .push((mod_name.to_string(), typed_key.to_string(), elapsed_ns)); - } -} - -/// Export typed cache rows for module-grain spike receipts (private index only). -pub fn export_typed_cache_for_spike( - index: &MultiEntryIndex, -) -> Vec<(String, Rc)> { - index - .typed_module_cache - .borrow() - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect() -} - fn bump_typecheck_compute_count() { TYPECHECK_COMPUTE_COUNT.fetch_add(1, Ordering::SeqCst); } @@ -5383,11 +5298,6 @@ fn reconcile_with_typed_cache( // parse+resolve+normalize). Threshold keeps the floor log quiet; // anything over it is a pathology-lane candidate by name. let module_tc_elapsed = module_tc_started.elapsed(); - spike_maybe_record_module_typecheck( - &mod_name, - &typed_key, - module_tc_elapsed.as_nanos() as u64, - ); resolve_stage_slot_add(|s| s.typecheck_compute += module_tc_elapsed.as_nanos()); let module_tc_ms = module_tc_elapsed.as_millis(); if module_tc_ms >= 2_000 { @@ -6017,186 +5927,6 @@ pub fn whole_tree_resolved_ctx( }) } -/// Strict whole-tree resolve through a fresh `MultiEntryIndex` — populates every cache -/// field the adaptive worker pool retains per shard (rc-arc share spike receipts). -pub fn populate_multi_entry_index_whole_tree( - source_roots: &[String], - exclude_substrings: &[String], - typecheck_gate: ResolveTypecheckGate, -) -> Result<(MultiEntryIndex, usize, usize), String> { - let index = build_multi_entry_index(source_roots); - let picked = whole_tree_strict_sources(source_roots, exclude_substrings)?; - let modules_resolved = picked.modules_resolved; - let modules_excluded = picked.modules_excluded; - resolved_graph_from_sources_with_index( - &index, - picked.sources, - typecheck_gate, - "rc-arc-spike-whole-tree", - )?; - Ok((index, modules_resolved, modules_excluded)) -} - -/// Resolve one entry's import closure through a fresh index (per-shard closure shape). -pub fn populate_multi_entry_index_entry( - source_roots: &[String], - entry_file: &str, -) -> Result { - let index = build_multi_entry_index(source_roots); - resolve_entry_with_index_for_discovery_corpus(&index, entry_file)?; - Ok(index) -} - -/// Curated large-closure discovery entries for memory spike receipts (resolver-graph-major -/// design §1 census: 163–188 module closures). Used when full discovery union OOMs the host. -pub fn discovery_corpus_representative_entries() -> Vec { - let ws = workspace_root(); - [ - "src/v2/workflow/ci_floor_plan.dag", - "src/v2/workflow/affected_set_floor_runner.dag", - "src/v2/lens/affected_set/entry_selection.dag", - "dag/gunbc/commit_workflow.dag", - "src/v2/test/claim/realization_schedule_witness.dag", - "src/v2/workflow/floor_diff_observe.dag", - "src/v2/workflow/module_resolution_plan.dag", - "dag/gunbc/ci_spec.dag", - "src/v2/workflow/enforcement_live.dag", - ] - .iter() - .map(|e| ws.join(e).to_string_lossy().into_owned()) - .collect() -} - -pub fn populate_multi_entry_index_representative_entries( - source_roots: &[String], -) -> Result<(MultiEntryIndex, usize), String> { - let entries = discovery_corpus_representative_entries(); - let index = build_multi_entry_index(source_roots); - for entry in &entries { - resolve_entry_with_index_for_discovery_corpus(&index, entry)?; - } - Ok((index, entries.len())) -} - -/// Populate one `MultiEntryIndex` by resolving floor discovery entries against it. -/// When `max_entries` is `Some(n)`, only the first `n` sorted unique entries are resolved -/// (measurement hosts with tight memory); `None` resolves the full discovery roster. -pub fn populate_multi_entry_index_discovery_corpus( - source_roots: &[String], - max_entries: Option, -) -> Result<(MultiEntryIndex, usize, usize), String> { - let scan_dirs = witness_discovery_scan_dirs(); - let excludes = whole_tree_resolve_exclusion_substrings(); - let rows = discover_floor_corpus_rows(source_roots, &scan_dirs, &excludes)?; - let mut entries: Vec = rows - .iter() - .map(|r| r.entry.clone()) - .collect::>() - .into_iter() - .collect(); - entries.sort(); - let total_entries = entries.len(); - if let Some(limit) = max_entries { - entries.truncate(limit); - } - let index = build_multi_entry_index(source_roots); - for entry in &entries { - resolve_entry_with_index_for_discovery_corpus(&index, entry)?; - } - Ok((index, entries.len(), rows.len().max(total_entries))) -} - -pub fn populate_multi_entry_index_whole_tree_discovery( - source_roots: &[String], - exclude_substrings: &[String], -) -> Result<(MultiEntryIndex, usize, usize), String> { - populate_multi_entry_index_whole_tree( - source_roots, - exclude_substrings, - ResolveTypecheckGate::DiscoveryCorpusAdvisory, - ) -} - -/// Floor worker object: empty whole-tree shell over both roots (what adaptive workers build -/// at thread start — `cli_run.rs` ~11191–11195). -pub fn build_worker_index_shell(source_roots: &[String]) -> MultiEntryIndex { - build_multi_entry_index(source_roots) -} - -/// Floor worker object after discovery-corpus warm: one index, all discovery entries resolved -/// against it (the retention shape a worker accumulates across its entry-groups). -pub fn populate_worker_discovery_index( - source_roots: &[String], -) -> Result<(MultiEntryIndex, usize, usize), String> { - populate_multi_entry_index_discovery_corpus(source_roots, None) -} - -/// Attach a fresh worker shell to an already-warmed cross-worker typed store. -pub fn build_worker_index_with_warm_store( - source_roots: &[String], - store: std::sync::Arc>, -) -> MultiEntryIndex { - build_multi_entry_index_with_shared_caches(source_roots, store) -} - -/// Host metadata for heterogeneous-box receipts (operator ruling 2026-07-20). -pub fn spike_measurement_host_metadata() -> (String, Option, Option) { - let hostname = std::fs::read_to_string("/etc/hostname") - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| "unknown".to_string()); - let mem_available_kb = std::fs::read_to_string("/proc/meminfo").ok().and_then(|s| { - s.lines() - .find(|l| l.starts_with("MemAvailable:")) - .and_then(|l| l.split_whitespace().nth(1)?.parse::().ok()) - }); - let cgroup_max_bytes = std::fs::read_to_string("/sys/fs/cgroup/memory.max") - .ok() - .and_then(|s| { - let s = s.trim(); - if s == "max" { - None - } else { - s.parse::().ok() - } - }) - .or_else(|| { - std::fs::read_to_string("/sys/fs/cgroup/memory/memory.limit_in_bytes") - .ok() - .and_then(|s| s.trim().parse::().ok()) - .filter(|&n| n < u64::MAX / 2) - }); - (hostname, mem_available_kb, cgroup_max_bytes) -} - -pub fn discovery_corpus_entry_roster( - source_roots: &[String], -) -> Result<(Vec, usize), String> { - let scan_dirs = witness_discovery_scan_dirs(); - let excludes = whole_tree_resolve_exclusion_substrings(); - let rows = discover_floor_corpus_rows(source_roots, &scan_dirs, &excludes)?; - let mut entries: Vec = rows - .iter() - .map(|r| r.entry.clone()) - .collect::>() - .into_iter() - .collect(); - entries.sort(); - Ok((entries, rows.len())) -} - -pub fn warm_discovery_on_index( - index: &MultiEntryIndex, - source_roots: &[String], -) -> Result<(usize, usize), String> { - let (entries, rows) = discovery_corpus_entry_roster(source_roots)?; - for entry in &entries { - resolve_entry_with_index_for_discovery_corpus(index, entry)?; - } - Ok((entries.len(), rows)) -} - pub fn closure_subject_for_entry(index: &MultiEntryIndex, entry: &str) -> Result { let sources = load_sources_for_entry_with_index(&index.source_files, &index.module_graph_facts, entry)?; diff --git a/src/v1/stage0/src/index_memory_receipt.rs b/src/v1/stage0/src/index_memory_receipt.rs deleted file mode 100644 index 1dbac911b32..00000000000 --- a/src/v1/stage0/src/index_memory_receipt.rs +++ /dev/null @@ -1,601 +0,0 @@ -//! Rc→Arc share spike — structural memory accounting for `MultiEntryIndex` fields. -//! -//! Measures shareable (`typed_module_cache` transitive closure) vs per-worker residue -//! (parse_cache, resolved_graph_memo, intern_table, normalize/ownership diag caches). -//! Used by `measure_rc_arc_share_spike` and future migration oracles. - -use crate::cli_run::MultiEntryIndex; -use crate::v1_compiler_infer::TypecheckModuleResult; -use crate::v1_compiler_infer_env::{TypeBinding, TypeEnv, TypeEnvCache}; -use crate::v1_compiler_infer_items::{ItemInfo, ResolvedFuncEnv, TypedModule}; -use crate::v1_compiler_parse::ParseResult; -use crate::v1_std_core::{InferredNode, InternTable, NewlineIndex, Node}; -use std::collections::{HashMap as StdHashMap, HashSet}; -use std::rc::Rc; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum IndexMemoryBucket { - Shareable, - PerWorkerResidue, -} - -#[derive(Debug, Clone)] -pub struct IndexFieldBytes { - pub field: &'static str, - pub bucket: IndexMemoryBucket, - pub heap_bytes: u64, - pub entry_count: usize, -} - -#[derive(Debug, Clone)] -pub struct IndexMemoryReceipt { - pub fields: Vec, - pub shareable_bytes: u64, - pub residue_bytes: u64, - pub total_accounted_bytes: u64, - pub typed_module_cache_entries: usize, - pub serde_transport_bytes: u64, - pub peak_rss_bytes: Option, -} - -#[derive(Debug, Clone)] -pub struct WidthScalingPoint { - pub width: usize, - pub peak_rss_bytes: Option, -} - -#[derive(Debug, Clone)] -pub struct NetWinPoint { - pub width: usize, - pub private_total_bytes: u64, - pub shared_model_bytes: u64, - pub net_win_bytes: i64, -} - -struct Accountant { - visited: HashSet, - bytes: u64, -} - -impl Accountant { - fn new() -> Self { - Accountant { - visited: HashSet::new(), - bytes: 0, - } - } - - fn first_visit(&mut self, ptr: *const T) -> bool { - if ptr.is_null() { - return false; - } - self.visited.insert(ptr as *const () as usize) - } - - fn add_shell(&mut self, ptr: *const T) { - if self.first_visit(ptr) { - self.bytes += std::mem::size_of::() as u64; - } - } - - fn add_rc_shell(&mut self, rc: &Rc) { - let ptr = Rc::as_ptr(rc); - if self.first_visit(ptr) { - self.bytes += std::mem::size_of::>() as u64; - } - } - - fn add_string(&mut self, s: &str) { - self.bytes += (std::mem::size_of::() + s.len()) as u64; - } - - fn add_vec_shell(&mut self, len: usize) { - self.bytes += (std::mem::size_of::>() + len * std::mem::size_of::()) as u64; - } - - fn add_hashmap_shell(&mut self, map: &StdHashMap) { - self.bytes += (std::mem::size_of::>() - + map.capacity() * std::mem::size_of::<(K, V)>()) as u64; - } - - fn add_im_hashmap_shell(&mut self, map: &im_rc::HashMap) { - self.bytes += (std::mem::size_of::>() - + map.len() * std::mem::size_of::<(K, V)>()) as u64; - } - - fn finish(self) -> u64 { - self.bytes - } -} - -fn account_intern_table(acc: &mut Accountant, table: &InternTable) { - let ptr = table as *const InternTable; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - let sptr = Rc::as_ptr(&table.strings) as usize; - if acc.first_visit(sptr as *const Vec) { - acc.add_vec_shell::(table.strings.len()); - for s in table.strings.iter() { - acc.add_string(s); - } - } - let iptr = Rc::as_ptr(&table.index) as usize; - if acc.first_visit(iptr as *const im_rc::HashMap) { - acc.add_im_hashmap_shell(table.index.as_ref()); - } -} - -fn account_newline_index(acc: &mut Accountant, nl: &NewlineIndex) { - let ptr = nl as *const NewlineIndex; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - acc.add_string(&nl.file); - acc.add_vec_shell::(nl.offsets.len()); - acc.add_vec_shell::(nl.char_codes.len()); -} - -fn account_node(acc: &mut Accountant, node: &Node) { - let ptr = node as *const Node; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - acc.add_string(&node.name); - if let Some(span) = &node.ident_span { - account_source_span(acc, span.as_ref()); - } - account_source_span(acc, node.span.as_ref()); - for child in node.children.iter() { - account_node(acc, child); - } - for param in node.params.iter() { - account_node(acc, param); - } - for u in node.uses.iter() { - account_node(acc, u); - } - if let Some(body) = &node.body { - account_node(acc, body); - } - if let Some(transport) = &node.transport { - account_node(acc, transport); - } - for prop in node.properties.iter() { - account_node(acc, prop); - } - if let Some(ta) = &node.type_annotation { - account_node(acc, ta); - } - if let Some(inferred) = &node.inferred { - account_inferred_node(acc, inferred); - } - account_expr_data(acc, node.expr_data.as_ref()); -} - -fn account_source_span(acc: &mut Accountant, span: &crate::v1_std_core::SourceSpan) { - let ptr = span as *const crate::v1_std_core::SourceSpan; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - acc.add_string(&span.file); -} - -fn account_inferred_node(acc: &mut Accountant, inferred: &InferredNode) { - let ptr = inferred as *const InferredNode; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - match inferred { - InferredNode::Resolved { node } => account_node(acc, node), - InferredNode::CompilerError { message, span } => { - acc.add_string(message); - account_source_span(acc, span); - } - InferredNode::TypeVariable { id } => acc.add_string(id), - } -} - -fn account_expr_data(acc: &mut Accountant, expr: &crate::v1_std_core::ExprData) { - let ptr = expr as *const crate::v1_std_core::ExprData; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); -} - -fn account_type_binding(acc: &mut Accountant, binding: &TypeBinding) { - let ptr = binding as *const TypeBinding; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - acc.add_string(&binding.name); - account_node(acc, binding.resolved.as_ref()); -} - -fn account_type_env(acc: &mut Accountant, env: &TypeEnv) { - let ptr = env as *const TypeEnv; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - for binding in env.bindings.values() { - account_type_binding(acc, binding); - } - for binding in env.str_bindings.values() { - account_type_binding(acc, binding); - } - for binding in env.ancestry_str_bindings.values() { - account_type_binding(acc, binding); - } - for parent in env.parents.iter() { - account_type_env(acc, parent); - } - for fields in env.inductive_fields.values() { - for field in fields.iter() { - let fptr = field.as_ref() as *const crate::v1_compiler_infer_env::InductiveField; - if acc.first_visit(fptr) { - acc.add_shell::(fptr); - } - } - } - for nl in env.source_indices.values() { - account_newline_index(acc, nl); - } - account_intern_table(acc, env.intern_table.as_ref()); -} - -fn account_type_env_cache(acc: &mut Accountant, cache: &TypeEnvCache) { - let ptr = cache as *const TypeEnvCache; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - for binding in cache.str_bindings.values() { - account_type_binding(acc, binding); - } - for binding in cache.variant_locals.values() { - account_type_binding(acc, binding); - } -} - -fn account_typed_module(acc: &mut Accountant, module: &TypedModule) { - let ptr = module as *const TypedModule; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - account_node(acc, module.module.as_ref()); - for item in module.items.iter() { - account_node(acc, item); - } - account_type_env(acc, module.type_env.as_ref()); - account_type_env_cache(acc, module.type_env_cache.as_ref()); - account_type_env(acc, module.interface.env.as_ref()); - for item in module.item_registry.values() { - let iptr = item.as_ref() as *const ItemInfo; - if acc.first_visit(iptr) { - acc.add_shell::(iptr); - acc.add_string(&item.name); - acc.add_string(&item.module_name); - for param in item.params.iter() { - account_node(acc, param); - } - } - } - let feptr = module.func_env.as_ref() as *const ResolvedFuncEnv; - if acc.first_visit(feptr) { - acc.add_shell::(feptr); - for sig in module.func_env.local.values() { - let sptr = sig.as_ref() as *const crate::v1_compiler_infer_sigs::ResolvedFuncSig; - if acc.first_visit(sptr) { - acc.add_shell::(sptr); - } - } - for parent in module.func_env.parents.iter() { - let pptr = parent.as_ref() as *const ResolvedFuncEnv; - if acc.first_visit(pptr) { - acc.add_shell::(pptr); - } - } - } -} - -fn account_typecheck_module_result(acc: &mut Accountant, result: &TypecheckModuleResult) { - let ptr = result as *const TypecheckModuleResult; - if !acc.first_visit(ptr) { - return; - } - acc.add_shell::(ptr); - account_typed_module(acc, result.typed.as_ref()); - for diag in result.diagnostics.iter() { - let dptr = diag.as_ref() as *const crate::v1_std_core::ErrorNode; - if acc.first_visit(dptr) { - acc.add_shell::(dptr); - } - } -} - -fn account_from_typed_cache( - cache: &StdHashMap>, -) -> (u64, usize, u64) { - // Shareable bucket: map shell + keys only. Peak RSS is the honest total (§2.2); - // serde transport is NOT extrapolated here — sample encode OOMs / misleads. - let mut acc = Accountant::new(); - acc.add_hashmap_shell(cache); - for key in cache.keys() { - acc.add_string(key); - } - (acc.finish(), cache.len(), 0) -} - -fn account_parse_cache_shallow( - cache: &StdHashMap, Rc)>, -) -> (u64, usize) { - let mut acc = Accountant::new(); - acc.add_hashmap_shell(cache); - for (path, _) in cache.iter() { - acc.add_string(path); - } - (acc.finish(), cache.len()) -} - -fn account_resolved_graph_memo_shallow( - memo: &im_rc::HashMap< - String, - ( - Rc, - Rc>>, - ), - >, -) -> (u64, usize) { - let mut acc = Accountant::new(); - acc.add_im_hashmap_shell(memo); - for subject in memo.keys() { - acc.add_string(subject); - } - (acc.finish(), memo.len()) -} - -fn account_intern_table_shallow(table: &Rc) -> (u64, usize) { - let mut acc = Accountant::new(); - let ptr = table.as_ref() as *const InternTable; - if acc.first_visit(ptr) { - acc.add_shell::(ptr); - acc.bytes += (table.strings.len() * 24) as u64; - } - (acc.finish(), table.strings.len()) -} - -fn account_diag_cache_map_shallow( - cache: &StdHashMap>>>, -) -> (u64, usize) { - let mut acc = Accountant::new(); - acc.add_hashmap_shell(cache); - for (key, diags) in cache.iter() { - acc.add_string(key); - acc.bytes += - diags.len() as u64 * std::mem::size_of::>() as u64; - } - (acc.finish(), cache.len()) -} - -fn account_string_map(map: &StdHashMap) -> (u64, usize) { - let mut acc = Accountant::new(); - acc.add_hashmap_shell(map); - for (k, v) in map { - acc.add_string(k); - acc.add_string(v); - } - (acc.finish(), map.len()) -} - -/// Structural heap accounting for one populated `MultiEntryIndex`. -pub fn multi_entry_index_memory_receipt( - index: &MultiEntryIndex, - peak_rss_bytes: Option, -) -> IndexMemoryReceipt { - let (typed_cache, parse_cache, memo, intern, normalize_cache, ownership_cache) = - index.memory_receipt_snapshot(); - - let (shareable, typed_entries, serde_transport_bytes) = account_from_typed_cache(&typed_cache); - - let (parse_bytes, parse_entries) = account_parse_cache_shallow(&parse_cache); - let (memo_bytes, memo_entries) = account_resolved_graph_memo_shallow(&memo); - let (intern_bytes, intern_entries) = account_intern_table_shallow(&intern); - let (normalize_bytes, normalize_entries) = account_diag_cache_map_shallow(&normalize_cache); - let (ownership_bytes, ownership_entries) = account_diag_cache_map_shallow(&ownership_cache); - - let fields = vec![ - IndexFieldBytes { - field: "typed_module_cache", - bucket: IndexMemoryBucket::Shareable, - heap_bytes: shareable, - entry_count: typed_entries, - }, - IndexFieldBytes { - field: "parse_cache", - bucket: IndexMemoryBucket::PerWorkerResidue, - heap_bytes: parse_bytes, - entry_count: parse_entries, - }, - IndexFieldBytes { - field: "resolved_graph_memo", - bucket: IndexMemoryBucket::PerWorkerResidue, - heap_bytes: memo_bytes, - entry_count: memo_entries, - }, - IndexFieldBytes { - field: "intern_table", - bucket: IndexMemoryBucket::PerWorkerResidue, - heap_bytes: intern_bytes, - entry_count: intern_entries, - }, - IndexFieldBytes { - field: "normalize_diag_cache", - bucket: IndexMemoryBucket::PerWorkerResidue, - heap_bytes: normalize_bytes, - entry_count: normalize_entries, - }, - IndexFieldBytes { - field: "ownership_diag_cache", - bucket: IndexMemoryBucket::PerWorkerResidue, - heap_bytes: ownership_bytes, - entry_count: ownership_entries, - }, - ]; - - let residue_bytes = parse_bytes + memo_bytes + intern_bytes + normalize_bytes + ownership_bytes; - IndexMemoryReceipt { - shareable_bytes: shareable, - residue_bytes, - total_accounted_bytes: shareable + residue_bytes, - typed_module_cache_entries: typed_entries, - serde_transport_bytes, - fields, - peak_rss_bytes, - } -} - -/// Union shareable bytes across multiple indexes (simulates process-wide typed store). -pub fn union_shareable_bytes(receipts: &[IndexMemoryReceipt]) -> u64 { - // Re-walk is expensive; for union growth use max shareable + sampled overlap estimate. - // Caller passes per-worker receipts; union ≈ max when closures identical, less when disjoint. - receipts.iter().map(|r| r.shareable_bytes).sum() -} - -/// Compute net-win curve: private_total = W*(shareable+residue); shared = union_shareable + W*residue. -pub fn net_win_curve( - shareable_per_worker: u64, - residue_per_worker: u64, - union_shareable: u64, - max_width: usize, -) -> Vec { - (1..=max_width) - .map(|width| { - let private_total = width as u64 * (shareable_per_worker + residue_per_worker); - let shared_model = union_shareable + width as u64 * residue_per_worker; - NetWinPoint { - width, - private_total_bytes: private_total, - shared_model_bytes: shared_model, - net_win_bytes: private_total as i64 - shared_model as i64, - } - }) - .collect() -} - -pub fn emit_index_memory_receipt(receipt: &IndexMemoryReceipt) { - for field in &receipt.fields { - eprintln!( - "[rc-arc-spike] kind=index-field field={} bucket={} bytes={} entries={}", - field.field, - match field.bucket { - IndexMemoryBucket::Shareable => "shareable", - IndexMemoryBucket::PerWorkerResidue => "per_worker_residue", - }, - field.heap_bytes, - field.entry_count, - ); - } - eprintln!( - "[rc-arc-spike] kind=index-summary shareable_bytes={} residue_bytes={} total_accounted_bytes={} typed_module_cache_entries={} serde_transport_bytes={} peak_rss_bytes={}", - receipt.shareable_bytes, - receipt.residue_bytes, - receipt.total_accounted_bytes, - receipt.typed_module_cache_entries, - receipt.serde_transport_bytes, - receipt - .peak_rss_bytes - .map(|b| b.to_string()) - .unwrap_or_else(|| "unavailable".into()), - ); -} - -pub fn emit_host_metadata( - hostname: &str, - mem_available_kb: Option, - cgroup_max_bytes: Option, -) { - eprintln!( - "[rc-arc-spike] kind=host-metadata hostname={} mem_available_kb={} cgroup_max_bytes={}", - hostname, - mem_available_kb - .map(|b| b.to_string()) - .unwrap_or_else(|| "unavailable".into()), - cgroup_max_bytes - .map(|b| b.to_string()) - .unwrap_or_else(|| "unlimited".into()), - ); -} - -pub fn emit_timing_point(label: &str, elapsed_ms: u128, peak_rss_bytes: Option) { - eprintln!( - "[rc-arc-spike] kind=timing label={label} elapsed_ms={elapsed_ms} peak_rss_bytes={}", - peak_rss_bytes - .map(|b| b.to_string()) - .unwrap_or_else(|| "unavailable".into()), - ); -} - -pub fn emit_width_scaling_point(width: usize, peak_rss_bytes: Option) { - eprintln!( - "[rc-arc-spike] kind=width-scaling width={} peak_rss_bytes={}", - width, - peak_rss_bytes - .map(|b| b.to_string()) - .unwrap_or_else(|| "unavailable".into()), - ); -} - -pub fn emit_net_win_point(point: &NetWinPoint) { - eprintln!( - "[rc-arc-spike] kind=net-win width={} private_total_bytes={} shared_model_bytes={} net_win_bytes={}", - point.width, point.private_total_bytes, point.shared_model_bytes, point.net_win_bytes, - ); -} - -/// Census for the im_rc blocker (no code changes — report-only). -pub fn im_rc_blocker_census() -> (usize, usize, &'static str) { - let stage0_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); - let mut total_rs = 0usize; - let mut im_rc_files = 0usize; - fn walk(dir: &std::path::Path, total_rs: &mut usize, im_rc_files: &mut usize) { - let Ok(read) = std::fs::read_dir(dir) else { - return; - }; - for entry in read.flatten() { - let path = entry.path(); - if path.is_dir() { - walk(&path, total_rs, im_rc_files); - } else if path.extension().is_some_and(|ext| ext == "rs") { - *total_rs += 1; - if let Ok(content) = std::fs::read_to_string(&path) { - if content.contains("im_rc::") || content.contains("use im_rc") { - *im_rc_files += 1; - } - } - } - } - } - walk(&stage0_root, &mut total_rs, &mut im_rc_files); - const NOTE: &str = "im-rc (Rc-backed HAMT) is aliased as HashMap/Vector/BTreeSet in lib.rs; \ - the Arc-backed sibling crate is `im`. Collections remain !Send even if \ - outer Rc→Arc lands — swapping im-rc→im touches every persistent map/list \ - in stage0 .rs files plus serde feature parity."; - (im_rc_files, total_rs, NOTE) -} - -pub fn emit_im_rc_census() { - let (im_rc_files, total_rs_files, note) = im_rc_blocker_census(); - eprintln!( - "[rc-arc-spike] kind=im-rc-census im_rc_files={} stage0_rs_files={} note={}", - im_rc_files, total_rs_files, note - ); -} diff --git a/src/v1/stage0/src/lib.rs b/src/v1/stage0/src/lib.rs index c18e4e9dcce..61569c131e3 100644 --- a/src/v1/stage0/src/lib.rs +++ b/src/v1/stage0/src/lib.rs @@ -38,7 +38,6 @@ pub mod extdeps_version; pub mod extdeps_version_semver; pub mod gunbc_stage0_crate_layout_generated; pub mod gunbc_stage0_crate_partition_generated; -pub mod index_memory_receipt; pub mod memory_governor; pub mod module_path_index; pub mod recorded_fixture; diff --git a/src/v1/stage0/src/shared_typecheck_store.rs b/src/v1/stage0/src/shared_typecheck_store.rs index 1ed8d47931e..abed2a4e1eb 100644 --- a/src/v1/stage0/src/shared_typecheck_store.rs +++ b/src/v1/stage0/src/shared_typecheck_store.rs @@ -15,37 +15,12 @@ //! *names* and diagnostic trees — not per-worker `InternTable` indices — so worker B can decode //! worker A's byte snapshot against its own intern table without a cross-representation straddle. -use serde::Deserialize; use std::collections::HashMap as StdHashMap; use std::rc::Rc; use std::sync::Arc; use crate::v1_compiler_infer::TypecheckModuleResult; -struct BoundedEncodeWriter { - buf: Vec, - limit: usize, - over_limit: bool, -} - -impl std::io::Write for BoundedEncodeWriter { - fn write(&mut self, data: &[u8]) -> std::io::Result { - if self.buf.len() + data.len() > self.limit { - self.over_limit = true; - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "bounded encode limit exceeded", - )); - } - self.buf.extend_from_slice(data); - Ok(data.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - /// Cross-worker share shell: typed byte cache + collision registry only. /// Construct once per explicit cross-worker run; clone the `Arc>` to every worker. /// Keys are typed-module CONTENT keys (`std.interface_summary.typed_module_key` — source @@ -72,9 +47,7 @@ impl SharedTypecheckCaches { /// Decode a typed snapshot **without** holding the store lock. /// Payload is name-keyed (no intern-table indices) — safe to materialize on any worker index. pub fn decode_typed_snapshot(bytes: &[u8]) -> Result, String> { - let mut de = serde_json::Deserializer::from_slice(bytes); - de.disable_recursion_limit(); - let value = TypecheckModuleResult::deserialize(&mut de) + let value: TypecheckModuleResult = serde_json::from_slice(bytes) .map_err(|e| format!("shared typecheck store decode: {e}"))?; Ok(Rc::new(value)) } @@ -86,23 +59,6 @@ impl SharedTypecheckCaches { Ok(Arc::new(bytes)) } - /// Encode with an output-size cap. Returns `Ok(None)` when JSON would exceed `max_bytes`. - pub fn try_encode_typed_snapshot( - result: &TypecheckModuleResult, - max_bytes: usize, - ) -> Result>>, String> { - let mut writer = BoundedEncodeWriter { - buf: Vec::new(), - limit: max_bytes, - over_limit: false, - }; - match serde_json::to_writer(&mut writer, result) { - Ok(()) => Ok(Some(Arc::new(writer.buf))), - Err(_e) if writer.over_limit => Ok(None), - Err(e) => Err(format!("shared typecheck store encode: {e}")), - } - } - /// Insert pre-encoded bytes under a brief write lock. pub fn insert_typed_preencoded(&mut self, typed_key: String, bytes: Arc>) { self.typed_module_cache.insert(typed_key, bytes); From 7ccd1d124b008b27a8c9c85f233eb26c279ce8bb Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 20 Jul 2026 08:48:07 +0000 Subject: [PATCH 22/22] =?UTF-8?q?docs(spike):=20drop=20harness=20from=20PR?= =?UTF-8?q?=20=E2=80=94=20land=20report=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator direction: remove index_memory_receipt, measure_rc_arc_share_spike bin, cli_run spike hooks, and shared_typecheck_store bounded-encode changes. Stage0 lib.rs layout is generated; hand-added pub mod caused regen gate red. Receipts remain in docs/plans/rc-to-arc-share-spike.md; harness in branch history. Co-authored-by: Cursor --- docs/plans/rc-to-arc-share-spike.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/rc-to-arc-share-spike.md b/docs/plans/rc-to-arc-share-spike.md index 4cf8158bfd9..f7edfca4e39 100644 --- a/docs/plans/rc-to-arc-share-spike.md +++ b/docs/plans/rc-to-arc-share-spike.md @@ -1,6 +1,6 @@ # Rc→Arc share spike — memory pricing report -> **Status:** measurement receipt, session smart-lark-630, 2026-07-20. **No migration landed** — harness + report only. +> **Status:** measurement receipt, session smart-lark-630, 2026-07-20. **Docs-only landing** — harness ran session-local (branch history); not shipped in seed. > > Answers open decision **2** in [cross-worker-typecheck-share-design](cross-worker-typecheck-share-design.md) §9 (`Arc migration depth: store-path-only vs whole-seed`). >