Whole-tree census that resolves per module: a third driver mode at occurrence grain (XL-2's third capability)

dag/gunbc/source_root_eval_driver_seed_growth.dag

@@ -21,7 +21,7 @@ data source_root_eval_driver_seed_growth_justification: SeedGrowthJustification
     }
   ],
 
-  reason: "One hand-authored declaration renders the SourceRootEvalDriver's main.rs as target-language text: roughly 500 lines of Rust spelled as string literals inside a .dag concat, not emitter logic over Nodes. It is a single declaration by the citable count and one program by the real one, and the gap between those two numbers is the point of this row. WHAT IS IRREDUCIBLY HOST here: argv, the source-root walk and byte reads, the monotonic and wall clocks, the /proc memory reading, stdout and stderr, and the process exit status. Nothing in .dag can perform those, and the driver exists to own exactly that boundary (std.compiler_entry SourceRootEvalDriver). WHAT IS EXPRESSIBLE IN .dag AND STILL RETAINED, each waiting on a named capability rather than on effort: the host-facts TSV decode (read_host_facts, host_fact, host_fact_int), which needs the ingest direction of the one grammar to produce NativeLaneHostFacts from bytes; the [native-cost-partition] JSON rendering, whose ROW SET and reconcile verdict are already a .dag authority this main calls (std.compiler_entry NativeDriverExclusiveRows, native_driver_cost_account) so that only the rendering is retained; and the two-mode orchestration (run_census, run_adjudication: phase order, refusal arms, terminal marker), which needs a .dag driver that can receive host effects as values. THE NET SEED MOTION OF THE CHANGE THAT ADDS THIS IS FAVOURABLE AND IS NOT AN EXCUSE: src/v1/stage0/src/cli_run/native_lane_runner.rs loses about 859 lines as the universe derivation, the fan-out and the admission move into the emitted compiler, and both v1-interpreter brackets are deleted at the root rather than retained as a fallback. A shrinking neighbour does not discharge this declaration; it only means the lane is moving the right way while this piece has not moved yet.",
+  reason: "One hand-authored declaration renders the SourceRootEvalDriver's main.rs as target-language text: roughly 500 lines of Rust spelled as string literals inside a .dag concat, not emitter logic over Nodes. It is a single declaration by the citable count and one program by the real one, and the gap between those two numbers is the point of this row. WHAT IS IRREDUCIBLY HOST here: argv, the source-root walk and byte reads, the monotonic and wall clocks, the /proc memory reading, stdout and stderr, and the process exit status. Nothing in .dag can perform those, and the driver exists to own exactly that boundary (std.compiler_entry SourceRootEvalDriver). WHAT IS EXPRESSIBLE IN .dag AND STILL RETAINED, each waiting on a named capability rather than on effort: the host-facts TSV decode (read_host_facts, host_fact, host_fact_int), which needs the ingest direction of the one grammar to produce NativeLaneHostFacts from bytes; the [native-cost-partition] JSON rendering, whose ROW SET and reconcile verdict are already a .dag authority this main calls (std.compiler_entry NativeDriverExclusiveRows, native_driver_cost_account) so that only the rendering is retained; and the THREE-mode orchestration (run_census, run_census_resolve, run_adjudication: phase order, refusal arms, terminal marker), which needs a .dag driver that can receive host effects as values. The third mode was added by the whole-tree census-resolve cut, and it is named here rather than left to be discovered because this row's reason is the standing enumeration of what is retained: a mode that is not in it is hand-authored Rust no row accounts for. What that cut did NOT do is grow the hand-authored DECISION surface -- the roster, the per-module verdict and the occurrence-grain rows are v2.compiler.compile folds (native_census_modules, native_census_module_resolution, native_census_residual_rows), and the mode is the same loop-and-print shape the other two already are, so the retained program grew by an orchestration arm and not by a second place where resolution is decided. THE NET SEED MOTION OF THE CHANGE THAT ADDS THIS IS FAVOURABLE AND IS NOT AN EXCUSE: src/v1/stage0/src/cli_run/native_lane_runner.rs loses about 859 lines as the universe derivation, the fan-out and the admission move into the emitted compiler, and both v1-interpreter brackets are deleted at the root rather than retained as a fallback. A shrinking neighbour does not discharge this declaration; it only means the lane is moving the right way while this piece has not moved yet.",
 
   owning_dissolution_lane: "v1-hand-queue-drain" as RoadmapNodeId,
 

src/v1/05_emit_rust.dag

@@ -17108,6 +17108,7 @@ fn emit_source_root_eval_driver_main_rs(crate_name: String, pipeline_module: Str
       "use ", crate_name, "::extdeps_communication_medium::\{DecodeFidelity, Medium\};", "\n",
       "use ", crate_name, "::gunbc_witness_v2_native_route::\{NativeRouteEmittedBuild, NativeRouteMalformedControl, NativeRouteOldRouteControl\};", "\n",
       "use ", crate_name, "::", pipeline_mod, "::\{", "\n",
+      "    native_census_module_resolution, native_census_modules, native_census_residual_rows,", "\n",
       "    native_lane_controls, native_lane_file_refusal_index, native_lane_identity_row,", "\n",
       "    native_lane_module_inference,", "\n",
       "    native_lane_module_resolution,", "\n",
@@ -17116,6 +17117,7 @@ fn emit_source_root_eval_driver_main_rs(crate_name: String, pipeline_module: Str
       "    native_lane_source_facts, native_lane_universe, native_test_context_absorb,", "\n",
       "    native_test_context_finish, native_test_context_from_ingest, native_test_context_state_empty,", "\n",
       "    native_test_front_end_prepare,", "\n",
+      "    NativeCensusModuleOutcome,", "\n",
       "    NativeLaneHostFacts, NativeLaneModulePreparation, NativeLaneModuleResolution,", "\n",
       "    NativeRouteMemberRow,", "\n",
       "\};", "\n",
@@ -17260,6 +17262,131 @@ fn emit_source_root_eval_driver_main_rs(crate_name: String, pipeline_module: Str
       "    println!(\"{}\", serde_json::json!(\{ \"_terminal\": \"complete\", \"mode\": \"census\", \"file_refusals\": context.file_refusals.len() \}));", "\n",
       "\}", "\n",
       "\n",
+      "// THE THIRD MODE: THE WHOLE TREE, RESOLVED PER MODULE.", "\n",
+      "//", "\n",
+      "// WHY IT IS NOT A FLAG ON `census`. That mode has a live consumer -- the malformed control,", "\n",
+      "// which the host runner spawns over a ONE-FILE scratch root and reads one per-file refusal", "\n",
+      "// from -- and teaching it to resolve would make that control pay a whole-tree resolve to", "\n",
+      "// learn a fact the front end already answered. Two modes because they are two subjects: a", "\n",
+      "// roster of front-end refusals, and a roster of resolve refusals over every ingested module.", "\n",
+      "//", "\n",
+      "// WHY IT IS NOT THE `adjudicate` LOOP WIDENED. That loop walks `universe.modules` -- the", "\n",
+      "// v2.test.* modules discovery enrolled rows for -- over a context folded from the import", "\n",
+      "// CLOSURE of those modules. Both narrowings are correct for what adjudication asks and wrong", "\n",
+      "// for a census: a residual reported over the closure of the test corpus is not a residual over", "\n",
+      "// the tree, and the absence of a refusal in a population that was never resolved establishes", "\n",
+      "// nothing about the dependents outside it.", "\n",
+      "//", "\n",
+      "// THE RESOLVED TREE IS NOT KEPT. NativeCensusModuleOutcome carries the diagnostics of a refusal", "\n",
+      "// and nothing on the accepted arm, so each module's tree dies with the call that made it. The", "\n",
+      "// adjudicating loop drops its tree explicitly because it must hold one to infer from; this loop", "\n",
+      "// never holds one, which is the property that makes the whole-tree grain affordable at all.", "\n",
+      "fn run_census_resolve(source_roots: &[String]) \{", "\n",
+      "    if source_roots.is_empty() \{", "\n",
+      "        eprintln!(\"REFUSED: no source roots given\");", "\n",
+      "        std::process::exit(2);", "\n",
+      "    \}", "\n",
+      "    let load_started = Instant::now();", "\n",
+      "    let reads = read_ingest(source_roots);", "\n",
+      "    let corpus_reads = reads.len();", "\n",
+      "    let ingest = Rc::new(im::vector::Vector::from(reads));", "\n",
+      "    let load_nanos = span_nanos(load_started);", "\n",
+      "    let context_started = Instant::now();", "\n",
+      "    let context = match &*native_test_context_from_ingest(ingest.clone()) \{", "\n",
+      "        Outcome::Accepted \{ value, .. \} => value.clone(),", "\n",
+      "        Outcome::Rejected \{ diagnostics \} => \{", "\n",
+      "            eprintln!(\"CONTEXT-REFUSED: \{diagnostics:#?\}\");", "\n",
+      "            std::process::exit(1);", "\n",
+      "        \}", "\n",
+      "    \};", "\n",
+      "    let context_nanos = span_nanos(context_started);", "\n",
+      "    // The front-end refusals are reported in the same shape `census` reports them, because they", "\n",
+      "    // are the same fact read from the same context: a module whose file the context refused is", "\n",
+      "    // attributed there and is NOT re-reported below as a resolve refusal it never reached.", "\n",
+      "    for refusal in context.file_refusals.iter() \{", "\n",
+      "        println!(\"{}\", serde_json::json!(\{ \"file_refusal\": refusal \}));", "\n",
+      "    \}", "\n",
+      "    let refusal_index = native_lane_file_refusal_index(context.file_refusals.clone());", "\n",
+      "    let roster_started = Instant::now();", "\n",
+      "    let modules = native_census_modules(ingest.clone());", "\n",
+      "    let roster_nanos = span_nanos(roster_started);", "\n",
+      "    let mut resolve_total_nanos: u128 = 0;", "\n",
+      "    let mut resolve_samples: Vec<u128> = Vec::new();", "\n",
+      "    let mut resolved_ok: u64 = 0;", "\n",
+      "    let mut file_refused: u64 = 0;", "\n",
+      "    let mut resolve_refused: u64 = 0;", "\n",
+      "    let mut residual_rows: u64 = 0;", "\n",
+      "    for entry in modules.iter() \{", "\n",
+      "        let resolve_started = Instant::now();", "\n",
+      "        let outcome = native_census_module_resolution(", "\n",
+      "            context.clone(),", "\n",
+      "            refusal_index.clone(),", "\n",
+      "            entry.clone(),", "\n",
+      "        );", "\n",
+      "        let module_resolve_nanos = span_nanos(resolve_started);", "\n",
+      "        let outcome_label = match &*outcome \{", "\n",
+      "            NativeCensusModuleOutcome::NativeCensusModuleFileRefused => \{", "\n",
+      "                file_refused += 1;", "\n",
+      "                \"file_refused\"", "\n",
+      "            \}", "\n",
+      "            NativeCensusModuleOutcome::NativeCensusModuleResolved => \{", "\n",
+      "                resolved_ok += 1;", "\n",
+      "                \"resolved\"", "\n",
+      "            \}", "\n",
+      "            NativeCensusModuleOutcome::NativeCensusModuleResolveRefused \{ diagnostics \} => \{", "\n",
+      "                resolve_refused += 1;", "\n",
+      "                // ONE ROW PER DIAGNOSTIC, WHICH IS ONE ROW PER REFUSED OCCURRENCE. The rows are", "\n",
+      "                // written as they are decided rather than accumulated: the census is over the", "\n",
+      "                // whole tree, and a population held to the end is a second corpus-sized", "\n",
+      "                // allocation beside the context that produced it.", "\n",
+      "                for row in native_census_residual_rows(entry.clone(), diagnostics.clone()).iter() \{", "\n",
+      "                    residual_rows += 1;", "\n",
+      "                    println!(\"{}\", serde_json::json!(\{ \"census_residual\": row \}));", "\n",
+      "                \}", "\n",
+      "                \"resolve_refused\"", "\n",
+      "            \}", "\n",
+      "        \};", "\n",
+      "        resolve_total_nanos += module_resolve_nanos;", "\n",
+      "        resolve_samples.push(module_resolve_nanos);", "\n",
+      "        eprintln!(", "\n",
+      "            \"[native-census-resolve] {}\",", "\n",
+      "            serde_json::json!(\{", "\n",
+      "                \"producer\": \"std.compiler_entry.SourceRootEvalDriver\",", "\n",
+      "                \"module\": entry.module.clone(),", "\n",
+      "                \"path\": entry.path.clone(),", "\n",
+      "                \"resolve_nanos\": module_resolve_nanos,", "\n",
+      "                \"outcome\": outcome_label,", "\n",
+      "            \}),", "\n",
+      "        );", "\n",
+      "    \}", "\n",
+      "    resolve_samples.sort_unstable();", "\n",
+      "    // THE TERMINAL LINE CARRIES THE COST AND THE POPULATION TOGETHER, because the reading this", "\n",
+      "    // mode exists to support is whether a whole-tree resolve is affordable at all, and a residual", "\n",
+      "    // count without the wall and the peak beside it cannot answer that.", "\n",
+      "    println!(", "\n",
+      "        \"{}\",", "\n",
+      "        serde_json::json!(\{", "\n",
+      "            \"_terminal\": \"complete\",", "\n",
+      "            \"mode\": \"census-resolve\",", "\n",
+      "            \"corpus_reads\": corpus_reads,", "\n",
+      "            \"modules\": modules.len(),", "\n",
+      "            \"file_refusals\": context.file_refusals.len(),", "\n",
+      "            \"resolved\": resolved_ok,", "\n",
+      "            \"file_refused\": file_refused,", "\n",
+      "            \"resolve_refused\": resolve_refused,", "\n",
+      "            \"residual_rows\": residual_rows,", "\n",
+      "            \"load_nanos\": load_nanos,", "\n",
+      "            \"context_nanos\": context_nanos,", "\n",
+      "            \"roster_nanos\": roster_nanos,", "\n",
+      "            \"resolve_total_nanos\": resolve_total_nanos,", "\n",
+      "            \"resolve_p50_nanos\": native_cost_quantile_nanos(&resolve_samples, 1, 2),", "\n",
+      "            \"resolve_p95_nanos\": native_cost_quantile_nanos(&resolve_samples, 19, 20),", "\n",
+      "            \"resolve_max_nanos\": resolve_samples.last().copied().unwrap_or(0),", "\n",
+      "            \"peak_rss_bytes\": proc_status_kib(\"VmHWM\").map(|kib| kib * 1024),", "\n",
+      "        \}),", "\n",
+      "    );", "\n",
+      "\}", "\n",
+      "\n",
       "fn native_cost_quantile_nanos(sorted: &[u128], numer: usize, denom: usize) -> u128 \{", "\n",
       "    if sorted.is_empty() \{ return 0; \}", "\n",
       "    let idx = (sorted.len() - 1) * numer / denom;", "\n",
@@ -17840,7 +17967,7 @@ fn emit_source_root_eval_driver_main_rs(crate_name: String, pipeline_module: Str
       "    let mode = match args.next() \{", "\n",
       "        Some(mode) => mode,", "\n",
       "        None => \{", "\n",
-      "            eprintln!(\"REFUSED: no mode given -- usage: ", crate_name, " adjudicate <host-facts.tsv> <source-root>... | ", crate_name, " census <source-root>...\");", "\n",
+      "            eprintln!(\"REFUSED: no mode given -- usage: ", crate_name, " adjudicate <host-facts.tsv> <source-root>... | ", crate_name, " census <source-root>... | ", crate_name, " census-resolve <source-root>...\");", "\n",
       "            std::process::exit(2);", "\n",
       "        \}", "\n",
       "    \};", "\n",
@@ -17849,6 +17976,11 @@ fn emit_source_root_eval_driver_main_rs(crate_name: String, pipeline_module: Str
       "        run_census(&source_roots);", "\n",
       "        return;", "\n",
       "    \}", "\n",
+      "    if mode == \"census-resolve\" \{", "\n",
+      "        let source_roots: Vec<String> = args.collect();", "\n",
+      "        run_census_resolve(&source_roots);", "\n",
+      "        return;", "\n",
+      "    \}", "\n",
       "    if mode == \"adjudicate\" \{", "\n",
       "        let facts_path = match args.next() \{", "\n",
       "            Some(path) => path,", "\n",
@@ -17861,7 +17993,7 @@ fn emit_source_root_eval_driver_main_rs(crate_name: String, pipeline_module: Str
       "        run_adjudication(&facts_path, &source_roots);", "\n",
       "        return;", "\n",
       "    \}", "\n",
-      "    eprintln!(\"REFUSED: unknown mode \{mode\} -- expected adjudicate or census\");", "\n",
+      "    eprintln!(\"REFUSED: unknown mode \{mode\} -- expected adjudicate, census or census-resolve\");", "\n",
       "    std::process::exit(2);", "\n",
       "\}", "\n",
       "\n")