The door's census names every file it could not read, not the first (stacked on #12218)

src/v2/cli/compile_cli.dag

@@ -5,6 +5,7 @@ import std.process { ExitFailure, ExitSuccess, ProcessExit, exit_failure }
 import extdeps.process.gnu_bash_exit { exit_code_misuse }
 import std.algebra { Cons, Empty, FreeMonoid }
 import v2.compiler.self_host.closure_emission {
+  closure_census_refusals_json,
   closure_emission_first_refusal,
   closure_emission_json,
   closure_emission_refused_count,
@@ -356,6 +357,7 @@ fn v2_cli_plan_source_roots(plan: CliPlan) -> FreeMonoid<String> {
 type CliRunOutcome
   = CliEmitted { text: String, closure_size: Int, refused_count: Int, refusal_detail: String }
   | CliRunRefused { reason: Symbol, detail: String }
+  | CliCensusRefused { reason: Symbol, detail: String, text: String }
   | CliUsageRefused { reason: Symbol, detail: String }
 
 // THE REFUSAL PRINTS ITS CAUSE, AND FOR ONE GENERATION IT DID NOT. The rejected arm already
@@ -422,9 +424,18 @@ fn v2_cli_run(plan: CliPlan, ingest: SourceRootIngest) -> CliRunOutcome {
                   }
                 }
               Rejected { diagnostics: d } =>
-                CliRunRefused {
-                  reason: diagnostics_fatal_reason(d: d),
-                  detail: v2_cli_refusal_detail(d: d, spans: emitted.spans)
+                match emitted.census_refusals {
+                  Empty =>
+                    CliRunRefused {
+                      reason: diagnostics_fatal_reason(d: d),
+                      detail: v2_cli_refusal_detail(d: d, spans: emitted.spans)
+                    }
+                  Cons { head: _, tail: _ } =>
+                    CliCensusRefused {
+                      reason: diagnostics_fatal_reason(d: d),
+                      detail: v2_cli_refusal_detail(d: d, spans: emitted.spans),
+                      text: closure_census_refusals_json(refusals: emitted.census_refusals, spans: emitted.spans)
+                    }
                 }
             }
           }
@@ -473,6 +484,7 @@ fn v2_cli_exit(outcome: CliRunOutcome) -> ProcessExit {
         exit_failure(reason: detail)
       }
     CliRunRefused { reason: _, detail: detail } => exit_failure(reason: detail)
+    CliCensusRefused { reason: _, detail: detail, text: _ } => exit_failure(reason: detail)
     CliUsageRefused { reason: _, detail: detail } =>
       ExitFailure { code: exit_code_misuse, reason: detail }
   }
@@ -482,6 +494,7 @@ fn v2_cli_outcome_text(outcome: CliRunOutcome) -> String {
   match outcome {
     CliEmitted { text: text, closure_size: _, refused_count: _, refusal_detail: _ } => text
     CliRunRefused { reason: _, detail: _ } => ""
+    CliCensusRefused { reason: _, detail: _, text: text } => text
     CliUsageRefused { reason: _, detail: _ } => ""
   }
 }

src/v2/compiler/self_host/closure_emission.dag

@@ -60,12 +60,15 @@ import v2.std.diagnostic {
   Rejected,
   Unavailable,
   bind_outcome,
+  diagnostics_fatal,
   diagnostics_fatal_reason,
+  non_empty_diagnostics_append,
   node_locus,
   outcome_accepted,
   outcome_rejected
 }
 import v2.std.language_model { LanguageModel }
+import v2.std.lens_verdict { lens_verdict_diagnostic_locus_text_in_spans }
 import v2.std.logic { Bool }
 import v2.std.node { Node, Symbol }
 import v2.std.program_segment { partition_boundary_fail_closed, program_segment_whole_program }
@@ -122,8 +125,18 @@ type ClosureIngest {
   index: SourceRootIndex
 }
 
+// ONE FILE THE CENSUS COULD NOT READ, carried whole: its path and its located chain. The census
+// continues past it so one run names every such file -- the file-grain blocking set -- and the
+// emission still refuses when any exists, because membership cannot be derived while a file's
+// declarations are unreadable (DESIGN section 5).
+type ClosureCensusRefusal {
+  path: String
+  diagnostics: NonEmptyDiagnostics
+}
+
 type ClosureIngestState {
-  ingest: Outcome<ClosureIngest>
+  ingest: ClosureIngest
+  refusals: FreeMonoid<ClosureCensusRefusal>
   alloc: OccurrenceIdAllocator
   spans: SpanIndex
 }
@@ -142,6 +155,7 @@ type ClosureEmission {
 // gives: a refusal needs its file exactly when the outcome carries no value.
 type ClosureEmissionLocated {
   spans: SpanIndex
+  census_refusals: FreeMonoid<ClosureCensusRefusal>
   outcome: Outcome<ClosureEmission>
 }
 
@@ -153,88 +167,118 @@ fn closure_emission_diagnostic(reason: Symbol, at: Node) -> Diagnostic {
   }
 }
 
+fn closure_ingest_refused(
+  state: ClosureIngestState,
+  read: DagSourceReadWitness,
+  d: NonEmptyDiagnostics,
+  alloc: OccurrenceIdAllocator,
+  spans: SpanIndex
+) -> ClosureIngestState {
+  ClosureIngestState {
+    ingest: state.ingest,
+    refusals: list_snoc_item(xs: state.refusals, item: ClosureCensusRefusal { path: read.artifact.file_path, diagnostics: d }),
+    alloc: alloc,
+    spans: spans
+  }
+}
+
 fn closure_ingest_step(
   state: ClosureIngestState,
   read: DagSourceReadWitness,
   lm: LanguageModel,
   prepared: PreparedGrammar,
   residue: Diagnostics
 ) -> ClosureIngestState {
-  match state.ingest {
-    Rejected { diagnostics: _ } => state
-    Accepted { value: acc, diagnostics: _ } =>
+  let acc = state.ingest
+  match bind_outcome(
+    o: program_assembly_phase_tokenize(read: read, lm: lm),
+    f: fn(tokens) {
+      parse_module_prepared_seeded(
+        tokens: tokens,
+        prepared: prepared,
+        validation_residue: residue,
+        seed: state.alloc
+      )
+    }
+  ) {
+    Rejected { diagnostics: d } =>
+      closure_ingest_refused(state: state, read: read, d: d, alloc: state.alloc, spans: state.spans)
+    Accepted { value: artifact, diagnostics: _ } =>
+      let spans = span_index_merge(base: state.spans, incoming: artifact.span_index)
       match bind_outcome(
-        o: program_assembly_phase_tokenize(read: read, lm: lm),
-        f: fn(tokens) {
-          parse_module_prepared_seeded(
-            tokens: tokens,
-            prepared: prepared,
-            validation_residue: residue,
-            seed: state.alloc
+        o: normalize_census(parse_tree: artifact.tree),
+        f: fn(census) {
+          bind_outcome(
+            o: qualified_name_from_module_node(root: census.root),
+            f: fn(module_qn) {
+              if closure_ingest_declares(files: acc.files, member_module: module_qn) {
+                outcome_rejected(d: closure_emission_diagnostic(
+                  reason: ^closure_census_duplicate_module,
+                  at: census.root
+                ))
+              } else {
+                outcome_accepted(value: ClosureIngest {
+                  files: list_snoc_item(xs: acc.files, item: ClosureIngestFile {
+                    module: module_qn,
+                    parse_tree: artifact.tree,
+                    census: census,
+                    source_root: read.source_root
+                  }),
+                  index: source_root_index_insert(index: acc.index, module: module_qn, root: read.source_root)
+                })
+              }
+            }
           )
         }
       ) {
         Rejected { diagnostics: d } =>
-          ClosureIngestState { ingest: Rejected { diagnostics: d }, alloc: state.alloc, spans: state.spans }
-        Accepted { value: artifact, diagnostics: _ } =>
-          let spans = span_index_merge(base: state.spans, incoming: artifact.span_index)
-          match bind_outcome(
-            o: normalize_census(parse_tree: artifact.tree),
-            f: fn(census) {
-              bind_outcome(
-                o: qualified_name_from_module_node(root: census.root),
-                f: fn(module_qn) {
-                  if closure_ingest_declares(files: acc.files, member_module: module_qn) {
-                    outcome_rejected(d: closure_emission_diagnostic(
-                      reason: ^closure_census_duplicate_module,
-                      at: census.root
-                    ))
-                  } else {
-                  outcome_accepted(value: ClosureIngest {
-                    files: list_snoc_item(xs: acc.files, item: ClosureIngestFile {
-                      module: module_qn,
-                      parse_tree: artifact.tree,
-                      census: census,
-                      source_root: read.source_root
-                    }),
-                    index: source_root_index_insert(index: acc.index, module: module_qn, root: read.source_root)
-                  })
-                  }
-                }
-              )
-            }
-          ) {
-            Rejected { diagnostics: d } =>
-              ClosureIngestState { ingest: Rejected { diagnostics: d }, alloc: artifact.alloc, spans: spans }
-            Accepted { value: next, diagnostics: _ } =>
-              ClosureIngestState { ingest: outcome_accepted(value: next), alloc: artifact.alloc, spans: spans }
-          }
+          closure_ingest_refused(state: state, read: read, d: d, alloc: artifact.alloc, spans: spans)
+        Accepted { value: next, diagnostics: _ } =>
+          ClosureIngestState { ingest: next, refusals: state.refusals, alloc: artifact.alloc, spans: spans }
       }
   }
 }
 
 // PHASE ONE: EVERY FILE PARSED AND CENSUSED, ONE OCCURRENCE ALLOCATOR THREADED ACROSS THE WHOLE
-// INGEST (the seeded front end), the first refusal stopping the census with its file's spans kept.
-fn closure_ingest(ingest: SourceRootIngest, lm: LanguageModel) -> ClosureIngestState {
+// INGEST (the seeded front end). A refused file is recorded and the census continues, so the
+// refusal set is complete. The grammar failing to prepare is NOT a file's refusal -- no file was
+// read -- so it is its own arm and refuses the emission before any census exists (review 71156).
+type ClosureCensus
+  = ClosureGrammarUnprepared { diagnostics: NonEmptyDiagnostics }
+  | ClosureCensusRead { state: ClosureIngestState }
+
+fn closure_ingest(ingest: SourceRootIngest, lm: LanguageModel) -> ClosureCensus {
   match prepare_grammar(grammar: lm.grammar) {
-    Rejected { diagnostics: d } =>
-      ClosureIngestState {
-        ingest: Rejected { diagnostics: d },
-        alloc: occurrence_id_allocator_initial(),
-        spans: span_index_empty()
-      }
+    Rejected { diagnostics: d } => ClosureGrammarUnprepared { diagnostics: d }
     Accepted { value: prepared, diagnostics: residue } =>
-      fold_list(
-        xs: ingest,
-        empty: ClosureIngestState {
-          ingest: outcome_accepted(value: ClosureIngest { files: Empty, index: source_root_index_empty() }),
-          alloc: occurrence_id_allocator_initial(),
-          spans: span_index_empty()
-        },
-        cons: fn(state, read) {
-          closure_ingest_step(state: state, read: read, lm: lm, prepared: prepared, residue: residue)
-        }
-      )
+      ClosureCensusRead {
+        state: fold_list(
+          xs: ingest,
+          empty: ClosureIngestState {
+            ingest: ClosureIngest { files: Empty, index: source_root_index_empty() },
+            refusals: Empty,
+            alloc: occurrence_id_allocator_initial(),
+            spans: span_index_empty()
+          },
+          cons: fn(state, read) {
+            closure_ingest_step(state: state, read: read, lm: lm, prepared: prepared, residue: residue)
+          }
+        )
+      }
+  }
+}
+
+// The census admits the ingest only when no file refused; otherwise the emission refuses with every
+// refused file's chain, in ingest order, joined into one located chain.
+fn closure_census_outcome(census: ClosureIngestState) -> Outcome<ClosureIngest> {
+  match census.refusals {
+    Empty => outcome_accepted(value: census.ingest)
+    Cons { head: first, tail: rest } =>
+      Rejected {
+        diagnostics: fold_list(xs: rest, empty: first.diagnostics, cons: fn(acc, r) {
+          non_empty_diagnostics_append(a: acc, b: r.diagnostics)
+        })
+      }
   }
 }
 
@@ -428,12 +472,28 @@ fn emit_closure_from_ingest_located(
   target: TargetModel
 ) -> ClosureEmissionLocated {
   let lm = dag_language_model()
-  let census = closure_ingest(ingest: ingest, lm: lm)
   let active_roots = source_root_set_from_ingest(ingest: ingest)
+  match closure_ingest(ingest: ingest, lm: lm) {
+    ClosureGrammarUnprepared { diagnostics: d } =>
+      ClosureEmissionLocated { spans: span_index_empty(), census_refusals: Empty, outcome: Rejected { diagnostics: d } }
+    ClosureCensusRead { state: census } =>
+      closure_emission_from_census(lm: lm, census: census, entry: entry, entry_locus: entry_locus, active_roots: active_roots, target: target)
+  }
+}
+
+fn closure_emission_from_census(
+  lm: LanguageModel,
+  census: ClosureIngestState,
+  entry: QualifiedName,
+  entry_locus: Node,
+  active_roots: FreeMonoid<SourceRootRef>,
+  target: TargetModel
+) -> ClosureEmissionLocated {
   ClosureEmissionLocated {
     spans: census.spans,
+    census_refusals: census.refusals,
     outcome: bind_outcome(
-      o: census.ingest,
+      o: closure_census_outcome(census: census),
       f: fn(ingested) {
         let roster = closure_ingest_roster(files: ingested.files)
         let census_index = closure_census_index(files: ingested.files)
@@ -531,3 +591,17 @@ fn closure_emission_first_refusal(emission: ClosureEmission) -> Optional<NonEmpt
     }
   })
 }
+
+// THE CENSUS REFUSAL SET THROUGH THE SAME CODEC: one row per refused file, its path, its fatal reason
+// and where that fatal sits. This is the file-grain blocking set a gen-two run reports.
+fn closure_census_refusals_json(refusals: FreeMonoid<ClosureCensusRefusal>, spans: SpanIndex) -> String {
+  serialize_json(v: json_object(members: [
+    json_kv(key: "census_refused", value: json_array(elements: fold_list(xs: refusals, empty: Empty, cons: fn(acc, r) {
+      list_snoc_item(xs: acc, item: json_object(members: [
+        json_kv(key: "file", value: json_string(s: r.path)),
+        json_kv(key: "reason", value: json_string(s: symbol_lexeme(sym: diagnostics_fatal_reason(d: r.diagnostics)))),
+        json_kv(key: "at", value: json_string(s: lens_verdict_diagnostic_locus_text_in_spans(d: diagnostics_fatal(d: r.diagnostics), index: spans)))
+      ]))
+    })))
+  ]))
+}

src/v2/test/claim/cli/v2_native_cli_test.dag

@@ -6,6 +6,7 @@ import extdeps.process.gnu_bash_exit { exit_code_misuse }
 import extdeps.process.posix_exit { exit_code_general_error }
 import v2.cli.compile_cli {
   CliEmitClosure,
+  CliCensusRefused,
   CliEmitted,
   CliPlan,
   CliPlanParsed,
@@ -74,6 +75,7 @@ fn plan_first_root(plan: CliPlan) -> String {
 fn outcome_refusal_reason(outcome: CliRunOutcome) -> Symbol {
   match outcome {
     CliRunRefused { reason: reason, detail: _ } => reason
+    CliCensusRefused { reason: reason, detail: _, text: _ } => reason
     CliUsageRefused { reason: reason, detail: _ } => reason
     CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } =>
       ^cli_outcome_was_not_refused
@@ -83,6 +85,7 @@ fn outcome_refusal_reason(outcome: CliRunOutcome) -> Symbol {
 fn outcome_emitted(outcome: CliRunOutcome) -> Bool {
   match outcome {
     CliRunRefused { reason: _, detail: _ } => false
+    CliCensusRefused { reason: _, detail: _, text: _ } => false
     CliUsageRefused { reason: _, detail: _ } => false
     CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } => true
   }
@@ -462,6 +465,9 @@ test fn a_trailing_token_refusal_locates_its_fatal_in_the_file_holds() -> Bool {
     CliRunRefused { reason: _, detail: detail } =>
       string_contains(s: detail, pattern: "FATAL AT cli_probe_trailing_cu bytes ")
         && string_contains(s: detail, pattern: "parse_g0_tokens_remain @ cli_probe_trailing_cu bytes ")
+    CliCensusRefused { reason: _, detail: detail, text: _ } =>
+      string_contains(s: detail, pattern: "FATAL AT cli_probe_trailing_cu bytes ")
+        && string_contains(s: detail, pattern: "parse_g0_tokens_remain @ cli_probe_trailing_cu bytes ")
     CliUsageRefused { reason: _, detail: _ } => false
     CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } => false
   }

src/v2/test/claim/provenance/occurrence_file_attribution_test.dag

@@ -16,6 +16,7 @@ import v2.std.artifact { Artifact, SourceFile }
 import v2.std.compilers.lexing { symbol_intern_lexeme, symbol_lexeme }
 import v2.cli.compile_cli {
   CliEmitClosure,
+  CliCensusRefused,
   CliEmitted,
   CliPlanParsed,
   CliRunRefused,
@@ -280,6 +281,7 @@ fn door_probe_cli_detail() -> String {
   ) {
     CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } => ""
     CliRunRefused { reason: _, detail: detail } => detail
+    CliCensusRefused { reason: _, detail: detail, text: _ } => detail
     CliUsageRefused { reason: _, detail: detail } => detail
   }
 }