ParseTable carrier: prepare grammar once above the assemble fold (validate-once)

src/v2/compiler/02_parse.dag

@@ -193,6 +193,17 @@ type ParseTableRealization {
 
 type ParseTable = ParseTableRealization
 
+type PreparedModeledGrammar {
+  validated: ValidatedGrammarRoot
+  analysis: GrammarFirstAnalysis
+}
+
+type PreparedGrammar
+  = PreparedVoidGrammar
+  | PreparedModeled { modeled: PreparedModeledGrammar }
+
+data prepared_grammar_carrier_note: String = "PreparedGrammar is the once-per-grammar carrier: well-formedness, the five grammar_validate_for_parse checks, and the FIRST/nullable analysis (GrammarFirstAnalysis) are pure functions of the grammar, so prepare_grammar runs them ONCE and parse_module_prepared reuses the result per module. Previously parse_module re-ran the whole 5-check validation AND compute_grammar_first_analysis was computed twice (once in the ambiguity check, once in parse_table_for_production) on EVERY parse. Hoisting prepare_grammar above the assemble fold makes a door call over a K-module closure validate once, not K times — correctness-by-construction (the prepare sits above fold_list), no timing gate. The cache-interface fields on ParseTableRealization (grammar_digest/provider_id/materialization) stay for the cross-call content-addressed memo, which is gated on wiring the currently Miss-stubbed cache backend."
+
 fn parse_grammar_not_well_formed_diagnostic(at: Locus) -> Diagnostic {
   Diagnostic {
     reason: ^parse_grammar_not_well_formed,
@@ -438,11 +449,11 @@ fn parse_token_stream_digest(all_tokens: List<Token>) -> Hash {
   )
 }
 
-fn parse_table_for_production(
+fn parse_table_for_production_analyzed(
   validated: ValidatedGrammarRoot,
+  analysis: GrammarFirstAnalysis,
   all_tokens: List<Token>
 ) -> ParseTable {
-  let analysis = compute_grammar_first_analysis(root: validated.root)
   ParseTableRealization {
     artifact_kind: parse_table_memo_artifact_kind(),
     provider_id: parse_table_memo_provider_id(),
@@ -458,6 +469,17 @@ fn parse_table_for_production(
   }
 }
 
+fn parse_table_for_production(
+  validated: ValidatedGrammarRoot,
+  all_tokens: List<Token>
+) -> ParseTable {
+  parse_table_for_production_analyzed(
+    validated: validated,
+    analysis: compute_grammar_first_analysis(root: validated.root),
+    all_tokens: all_tokens
+  )
+}
+
 fn parse_table_empty() -> ParseTable {
   ParseTableRealization {
     artifact_kind: parse_table_memo_artifact_kind(),
@@ -1599,7 +1621,7 @@ fn grammar_overlap_residue_diagnostics(at: Locus, roster: List<GrammarChoiceAmbi
   )
 }
 
-fn grammar_validate_for_parse(root: GrammarRoot, at: Locus) -> Outcome<ValidatedGrammarRoot> {
+fn grammar_validate_and_analyze(root: GrammarRoot, at: Locus) -> Outcome<PreparedModeledGrammar> {
   if grammar_has_undefined_nonterminals(root: root) {
     Rejected { diagnostics: diagnostics_singleton(d: parse_grammar_undefined_nonterminal_diagnostic(at: at)) }
   } else {
@@ -1609,15 +1631,19 @@ fn grammar_validate_for_parse(root: GrammarRoot, at: Locus) -> Outcome<Validated
       if grammar_has_left_recursion(root: root) {
         Rejected { diagnostics: diagnostics_singleton(d: parse_grammar_left_recursive_diagnostic(at: at)) }
       } else {
-        let ambiguity_roster = grammar_choice_ambiguity_roster(root: root)
+        let analysis = compute_grammar_first_analysis(root: root)
+        let ambiguity_roster = grammar_choice_ambiguity_roster_from_analysis(root: root, analysis: analysis)
         let nullable_rows = grammar_nullable_ambiguity_rows(roster: ambiguity_roster)
         if !is_empty(xs: nullable_rows) {
           Rejected {
             diagnostics: grammar_choice_ambiguity_rejection_diagnostics(at: at, roster: nullable_rows)
           }
         } else {
           Accepted {
-            value: ValidatedGrammarRoot { root: root },
+            value: PreparedModeledGrammar {
+              validated: ValidatedGrammarRoot { root: root },
+              analysis: analysis
+            },
             diagnostics: grammar_overlap_residue_diagnostics(at: at, roster: ambiguity_roster)
           }
         }
@@ -1626,6 +1652,14 @@ fn grammar_validate_for_parse(root: GrammarRoot, at: Locus) -> Outcome<Validated
   }
 }
 
+fn grammar_validate_for_parse(root: GrammarRoot, at: Locus) -> Outcome<ValidatedGrammarRoot> {
+  match grammar_validate_and_analyze(root: root, at: at) {
+    Rejected { diagnostics: d } => Rejected { diagnostics: d }
+    Accepted { value: modeled, diagnostics: residue } =>
+      Accepted { value: modeled.validated, diagnostics: residue }
+  }
+}
+
 fn parse_sync_step(acc: ParseSyncState, tok: Token, sync_tokens: Set<Symbol>) -> ParseSyncState {
   match acc.found_sync {
     true => ParseSyncState { found_sync: true, result: list_snoc_item(xs: acc.result, item: tok) }
@@ -2075,17 +2109,18 @@ fn parse_production_memo_stats(
   }
 }
 
-fn parse_production(
+fn parse_production_prepared(
   tokens: List<Token>,
   validated: ValidatedGrammarRoot,
+  analysis: GrammarFirstAnalysis,
   production_name: Symbol
 ) -> Outcome<ParseArtifact> {
   let root = validated.root
   let at = port_locus(port: ^parse_stage_locus_port)
   match grammar_lookup_production(root: root, name: production_name) {
     Holds { value: production } =>
       let prov = parse_prov_initial()
-      let table = parse_table_for_production(validated: validated, all_tokens: tokens)
+      let table = parse_table_for_production_analyzed(validated: validated, analysis: analysis, all_tokens: tokens)
       match parse_expr(
         expr: production.expression,
         tokens: tokens,
@@ -2121,6 +2156,19 @@ fn parse_production(
   }
 }
 
+fn parse_production(
+  tokens: List<Token>,
+  validated: ValidatedGrammarRoot,
+  production_name: Symbol
+) -> Outcome<ParseArtifact> {
+  parse_production_prepared(
+    tokens: tokens,
+    validated: validated,
+    analysis: compute_grammar_first_analysis(root: validated.root),
+    production_name: production_name
+  )
+}
+
 fn parse_production_tree(
   tokens: List<Token>,
   validated: ValidatedGrammarRoot,
@@ -2137,39 +2185,20 @@ fn parse_production_tree(
       Rejected { diagnostics: r }
   }
 }
-fn parse_module(tokens: TokenStream, grammar: ParseGrammar) -> Outcome<ParseArtifact> {
+fn prepare_grammar(grammar: ParseGrammar) -> Outcome<PreparedGrammar> {
   let at = port_locus(port: ^parse_stage_locus_port)
-  let ntok = token_stream_remaining_count(stream: tokens)
   let grammar_node = grammar_to_node(grammar: grammar)
   if well_formed(n: grammar_node) {
     match grammar {
       VoidGrammar =>
-        if ntok == 0 {
-          Accepted {
-            value: ParseArtifact { tree: grammar_node, span_index: span_index_empty() },
-            diagnostics: None
-          }
-        } else {
-          Rejected { diagnostics: diagnostics_singleton(d: parse_g0_tokens_remain_diagnostic(at: at)) }
-        }
+        Accepted { value: PreparedVoidGrammar, diagnostics: None }
       ModeledGrammar { root: root } =>
-        match grammar_validate_for_parse(root: root, at: at) {
+        match grammar_validate_and_analyze(root: root, at: at) {
           Rejected { diagnostics: diags } => Rejected { diagnostics: diags }
-          Accepted { value: validated, diagnostics: validation_residue } =>
-            match parse_production(
-              tokens: token_stream_remaining(stream: tokens),
-              validated: validated,
-              production_name: root.start
-            ) {
-              Accepted { value: artifact, diagnostics: parse_diags } =>
-                Accepted {
-                  value: artifact,
-                  diagnostics: diagnostics_merge(outer: validation_residue, inner: parse_diags)
-                }
-              Rejected { diagnostics: parse_rejected } =>
-                Rejected {
-                  diagnostics: rejected_with_pending(pending: validation_residue, rejected: parse_rejected)
-                }
+          Accepted { value: modeled, diagnostics: validation_residue } =>
+            Accepted {
+              value: PreparedModeled { modeled: modeled },
+              diagnostics: validation_residue
             }
         }
     }
@@ -2178,6 +2207,51 @@ fn parse_module(tokens: TokenStream, grammar: ParseGrammar) -> Outcome<ParseArti
   }
 }
 
+fn parse_module_prepared(
+  tokens: TokenStream,
+  prepared: PreparedGrammar,
+  validation_residue: Diagnostics
+) -> Outcome<ParseArtifact> {
+  let at = port_locus(port: ^parse_stage_locus_port)
+  let ntok = token_stream_remaining_count(stream: tokens)
+  match prepared {
+    PreparedVoidGrammar =>
+      if ntok == 0 {
+        Accepted {
+          value: ParseArtifact { tree: grammar_to_node(grammar: VoidGrammar), span_index: span_index_empty() },
+          diagnostics: validation_residue
+        }
+      } else {
+        Rejected { diagnostics: diagnostics_singleton(d: parse_g0_tokens_remain_diagnostic(at: at)) }
+      }
+    PreparedModeled { modeled: modeled } =>
+      match parse_production_prepared(
+        tokens: token_stream_remaining(stream: tokens),
+        validated: modeled.validated,
+        analysis: modeled.analysis,
+        production_name: modeled.validated.root.start
+      ) {
+        Accepted { value: artifact, diagnostics: parse_diags } =>
+          Accepted {
+            value: artifact,
+            diagnostics: diagnostics_merge(outer: validation_residue, inner: parse_diags)
+          }
+        Rejected { diagnostics: parse_rejected } =>
+          Rejected {
+            diagnostics: rejected_with_pending(pending: validation_residue, rejected: parse_rejected)
+          }
+      }
+  }
+}
+
+fn parse_module(tokens: TokenStream, grammar: ParseGrammar) -> Outcome<ParseArtifact> {
+  match prepare_grammar(grammar: grammar) {
+    Rejected { diagnostics: diags } => Rejected { diagnostics: diags }
+    Accepted { value: prepared, diagnostics: validation_residue } =>
+      parse_module_prepared(tokens: tokens, prepared: prepared, validation_residue: validation_residue)
+  }
+}
+
 fn parse(tokens: TokenStream, grammar: ParseGrammar) -> Outcome<ParseTree> {
   match parse_module(tokens: tokens, grammar: grammar) {
     Accepted { value: artifact, diagnostics: d } =>

src/v2/compiler/program_assembly.dag

@@ -7,7 +7,7 @@ import v2.compiler.name_resolve {
   validate_module_roots
 }
 import v2.compiler.normalize { normalize }
-import v2.compiler.parse { parse_module }
+import v2.compiler.parse { PreparedGrammar, parse_module, parse_module_prepared, prepare_grammar }
 import v2.compiler.source_authority {
   DagSourceReadWitness,
   SourceRootIngest,
@@ -27,7 +27,9 @@ import v2.std.algebra { fold_list, list_snoc_item }
 import v2.std.artifact { SourceFile }
 import v2.std.diagnostic {
   Accepted,
+  Diagnostics,
   NonEmptyDiagnostics,
+  None,
   Outcome,
   Rejected,
   bind_outcome,
@@ -49,9 +51,11 @@ fn program_assembly_empty_roots() -> FreeMonoid<Node> {
   Empty
 }
 
-fn program_assembly_read_to_normalized_root(
+fn program_assembly_read_to_normalized_root_prepared(
   read: DagSourceReadWitness,
-  lm: LanguageModel
+  lm: LanguageModel,
+  prepared: PreparedGrammar,
+  residue: Diagnostics
 ) -> Outcome<Node> {
   match read.artifact.kind {
     SourceFile =>
@@ -63,7 +67,7 @@ fn program_assembly_read_to_normalized_root(
         ),
         f: fn(token_stream) {
           bind_outcome(
-            o: parse_module(tokens: token_stream, grammar: lm.grammar),
+            o: parse_module_prepared(tokens: token_stream, prepared: prepared, validation_residue: residue),
             f: fn(artifact) {
               normalize(parse_tree: artifact.tree)
             }
@@ -77,16 +81,34 @@ fn program_assembly_read_to_normalized_root(
   }
 }
 
+fn program_assembly_read_to_normalized_root(
+  read: DagSourceReadWitness,
+  lm: LanguageModel
+) -> Outcome<Node> {
+  match prepare_grammar(grammar: lm.grammar) {
+    Rejected { diagnostics: d } => Rejected { diagnostics: d }
+    Accepted { value: prepared, diagnostics: residue } =>
+      program_assembly_read_to_normalized_root_prepared(
+        read: read,
+        lm: lm,
+        prepared: prepared,
+        residue: residue
+      )
+  }
+}
+
 fn program_assembly_fold_step(
   state: ProgramAssemblyFoldState,
   read: DagSourceReadWitness,
-  lm: LanguageModel
+  lm: LanguageModel,
+  prepared: PreparedGrammar,
+  residue: Diagnostics
 ) -> ProgramAssemblyFoldState {
   match state {
     ProgramAssemblyFoldFailed { diagnostics: d } =>
       ProgramAssemblyFoldFailed { diagnostics: d }
     ProgramAssemblyFoldOk { roots: acc, index: idx } =>
-      match program_assembly_read_to_normalized_root(read: read, lm: lm) {
+      match program_assembly_read_to_normalized_root_prepared(read: read, lm: lm, prepared: prepared, residue: residue) {
         Accepted { value: root, diagnostics: _ } =>
           match qualified_name_from_module_node(root: root) {
             Accepted { value: qn, diagnostics: _ } =>
@@ -106,20 +128,35 @@ fn program_assembly_fold_step(
   }
 }
 
+data program_assembly_prepare_once_note: String = "prepare_grammar is hoisted ABOVE fold_list here so the grammar is well-formed-checked, validated (5 checks), and FIRST/nullable-analyzed ONCE per assembly — then parse_module_prepared reuses the PreparedGrammar for every module in the ingest. A door call over a K-module import closure therefore validates once, not K times (the recompute-per-module cost this dissolves). Empty-ingest is guarded so it never pays the one-time prepare for zero modules. Fail-closed: an invalid grammar fails the whole assembly (ProgramAssemblyFoldFailed) before any module is parsed, carrying the validation diagnostic — never a per-module silent skip."
+
 fn program_assembly_fold_ingest(
   ingest: SourceRootIngest,
   lm: LanguageModel
 ) -> ProgramAssemblyFoldState {
-  fold_list(
-    xs: ingest,
-    empty: ProgramAssemblyFoldOk {
-      roots: program_assembly_empty_roots(),
-      index: source_root_index_empty()
-    },
-    cons: fn(state, read) {
-      program_assembly_fold_step(state: state, read: read, lm: lm)
-    }
-  )
+  match ingest {
+    Empty =>
+      ProgramAssemblyFoldOk {
+        roots: program_assembly_empty_roots(),
+        index: source_root_index_empty()
+      }
+    _ =>
+      match prepare_grammar(grammar: lm.grammar) {
+        Rejected { diagnostics: d } =>
+          ProgramAssemblyFoldFailed { diagnostics: d }
+        Accepted { value: prepared, diagnostics: residue } =>
+          fold_list(
+            xs: ingest,
+            empty: ProgramAssemblyFoldOk {
+              roots: program_assembly_empty_roots(),
+              index: source_root_index_empty()
+            },
+            cons: fn(state, read) {
+              program_assembly_fold_step(state: state, read: read, lm: lm, prepared: prepared, residue: residue)
+            }
+          )
+      }
+  }
 }
 
 fn module_roots_from_source_root_ingest(

src/v2/test/claim/long/parse_table_memo_amortization_test.dag

@@ -136,3 +136,15 @@ test fn parse_table_grammar_memo_multi_file_ingest_parses() -> Bool {
 test fn parse_table_memo_hit_path_content_hash_stable_holds() -> Bool {
   witness_parse_table_memo_hit_path_content_hash_stable()
 }
+
+data empty_ingest_guard_note: String = "Covers the empty-ingest branch of program_assembly_fold_ingest, added with the prepare-once hoist: an empty ingest must assemble to empty roots WITHOUT paying the one-time grammar preparation (well-formed + 5-check validation + FIRST analysis) for zero modules. Discriminating — RED if the guard rejected, fabricated a root, or (regression) fell through to prepare_grammar for no modules. Fast by construction (no prepare, no parse), unlike the multi-file amortization witnesses above which each pay exactly ONE preparation for the whole fold."
+
+test fn empty_ingest_assembles_to_empty_roots_without_prepare() -> Bool {
+  match module_roots_from_source_root_ingest(
+    ingest: Empty,
+    lm: dag_language_model()
+  ) {
+    Accepted { value: roots, diagnostics: _ } => length(xs: roots) == 0
+    Rejected { diagnostics: _ } => false
+  }
+}