census_identity_digest is a set identity: fold the phase census over a canonical order

dag/gunbc/design_ledgers.dag

@@ -30,6 +30,7 @@ import gunbc.recurring_failure_mode { recurring_failure_mode_roster }
 import gunbc.rung_drop { rung_drop_roster }
 import v2.workflow.floor_discovery { floor_discovery_tree_identity }
 import gunbc.self_host_compile_phase_frontier {
+  census_identity_domain_separator,
   CompilePhaseFrontierEmpty,
   CompilePhaseFrontierValidated,
   CompilePhaseFrontierInvalid,
@@ -91,7 +92,10 @@ fn compile_phase_frontier_blocks() -> List<MarkdownBlock> {
       p(text: concat("Milestones: ", render_phase_board(board: latest.board))),
       p(text: concat("Tree: `", concat(floor_discovery_tree_identity(tree: latest.subject.git_tree_object), "`"))),
       p(text: concat("Invocation: `", concat(latest.invocation as String, "`"))),
-      p(text: concat("Census identity digest: `", concat(latest.board.census_identity_digest.digest as String, "`"))),
+      p(text: concat(
+        concat("Census identity digest (", concat(census_identity_domain_separator(algorithm: latest.board.census_identity_digest.algorithm), "): `")),
+        concat(latest.board.census_identity_digest.digest.digest as String, "`")
+      )),
       p(text: concat(
         "Error identities: ",
         concat(

dag/gunbc/self_host_compile_phase_frontier.dag

@@ -58,7 +58,7 @@ type RustcPhaseBoard {
   rows: List<RustcPhaseBoardRow>
   furthest_phase_reached: RustcPhase
   furthest_phase_evidence: PhaseCountEvidence
-  census_identity_digest: Fnv1a64Structural
+  census_identity_digest: CensusIdentityDigest
   total_error_diagnostics: Int
   coded_error_diagnostics: Int
   codeless_error_diagnostics: Int
@@ -118,6 +118,17 @@ fn deduplicate_identities(identities: List<String>) -> List<String> {
   )
 }
 
+// Duplicate handling is a POPULATION fact, and the two populations answer it at different sites.
+// raw_identities is set-deduplicated by the live producer before the fold, and on the persisted
+// path a repeat makes the deduplicated total disagree with the raw count, which receipt coherence
+// refuses. parse_refused_files had neither wall: a path rustfmt refused twice would be counted
+// twice in the Parse row and kept twice in the digest, so the digest would answer about a multiset
+// while its name claims a set. Refusing is the right arm rather than deduplicating here, because
+// collapsing the repeat would hide the producer defect behind a well-formed-looking census.
+fn census_population_is_duplicate_free(identities: List<String>) -> Bool {
+  count(deduplicate_identities(identities: identities)) == count(identities)
+}
+
 fn phase_identities(ds: List<RustcCodedDiagnostic>, phase: RustcPhase) -> List<String> {
   deduplicate_identities(
     identities: ds |> filter(d => diagnostic_is_phase(d: d, phase: phase)) |> map(d => rustc_diagnostic_row(d: d))
@@ -152,16 +163,57 @@ fn first_failing_checked_phase(ds: List<RustcCodedDiagnostic>) -> RustcPhase {
   else { Borrowck }
 }
 
-fn phase_census_digest(parse_refused_files: List<String>, raw_identities: List<String>) -> Fnv1a64Structural {
-  content_hash_atom(
-    value: (concat(
-      ["self-host-compile-phase-census-v1"],
-      concat(
-        parse_refused_files |> map(path => concat("rustfmt-refused\t", path)),
-        raw_identities
-      )
-    ) |> join(separator: "\n")) as NonEmptyStr
-  )
+// The digest names the census as a SET, so it is folded over a canonical ordering of each
+// population rather than the encounter order cargo and rustfmt happened to emit. Without the sort
+// the same census reordered upstream produces a different digest, and receipt_population_coherent
+// refuses a receipt whose census is in fact identical. The two populations stay separate segments:
+// canonicalizing them jointly would let a refused path and an identity trade places.
+//
+// EmissionOrderedV1 is retired and NO PRODUCER CONSTRUCTS IT. Its justification is NOT that it
+// labels a live value -- it does not. docs/design-ledgers.md published an emission-order digest
+// until this change, and regenerating that projection replaces it, so after this commit no
+// artifact in the tree carries one and the retired value survives only in git history.
+//
+// It is kept because an algorithm identity with exactly ONE inhabitant distinguishes nothing. The
+// field exists to stop two incompatible digest meanings hiding behind a single unlabeled hash, and
+// that requires both meanings to be nameable. So the honest fork is not "drop the arm" but "drop
+// the arm AND the field": a one-inhabitant tag is decoration, and keeping the tag while deleting
+// its only contrast is the worse of the two shapes.
+//
+// It is read by production -- census_identity_domain_separator matches it exhaustively -- and by a
+// witness asserting the two separators differ, so it is not an inert carrier under
+// v2.lens.inert_carrier, whose scope is carriers read by no production code.
+type CensusIdentityAlgorithm
+  = EmissionOrderedV1
+  | CanonicalIdentitySetV1
+
+// The domain separator is DERIVED from the algorithm rather than authored beside it. A separate
+// version literal would be a second authority for the same fact, and the two would drift.
+fn census_identity_domain_separator(algorithm: CensusIdentityAlgorithm) -> String {
+  match algorithm {
+    EmissionOrderedV1 => "self-host-compile-phase-census-v1"
+    CanonicalIdentitySetV1 => "self-host-compile-phase-census-canonical-identity-set-v1"
+  }
+}
+
+type CensusIdentityDigest {
+  algorithm: CensusIdentityAlgorithm
+  digest: Fnv1a64Structural
+}
+
+fn phase_census_digest(parse_refused_files: List<String>, raw_identities: List<String>) -> CensusIdentityDigest {
+  CensusIdentityDigest {
+    algorithm: CanonicalIdentitySetV1,
+    digest: content_hash_atom(
+      value: (concat(
+        [census_identity_domain_separator(algorithm: CanonicalIdentitySetV1)],
+        concat(
+          parse_refused_files |> map(path => concat("rustfmt-refused\t", path)) |> sort_by(line => line),
+          raw_identities |> sort_by(identity => identity)
+        )
+      ) |> join(separator: "\n")) as NonEmptyStr
+    )
+  }
 }
 
 fn cargo_phase_board(parse_observation: RustfmtParseObservation, ds: List<RustcCodedDiagnostic>) -> RustcPhaseBoardRead {
@@ -463,7 +515,7 @@ fn parse_evidence_rows(evidence: ParsePhaseEvidence) -> List<String> {
 fn optional_reclassified_digest(board: RustcPhaseBoard?) -> String {
   match board {
     Absent => "not-reclassified"
-    Present { value: b } => concat("reclassified:", b.census_identity_digest.digest as String)
+    Present { value: b } => concat("reclassified:", b.census_identity_digest.digest.digest as String)
   }
 }
 
@@ -485,7 +537,8 @@ fn receipt_payload_identity(receipt: CompilePhaseFrontierReceipt) -> Fnv1a64Stru
       receipt.producer.module_path,
       receipt.producer.function,
       receipt.producer.entry,
-      receipt.board.census_identity_digest.digest as String,
+      census_identity_domain_separator(algorithm: receipt.board.census_identity_digest.algorithm),
+      receipt.board.census_identity_digest.digest.digest as String,
       rustc_phase_name(phase: receipt.board.furthest_phase_reached)
     ], "\n") as NonEmptyStr
   )
@@ -913,7 +966,9 @@ fn receipt_population_coherent(receipt: CompilePhaseFrontierReceipt) -> Bool {
       parse_observation: parse,
       raw_identities: receipt.raw_error_diagnostic_identities
       )
-      derived.board.census_identity_digest == receipt.board.census_identity_digest
+      census_population_is_duplicate_free(identities: parse.refused_files)
+        && census_population_is_duplicate_free(identities: receipt.raw_error_diagnostic_identities)
+        && derived.board.census_identity_digest == receipt.board.census_identity_digest
         && derived.board.furthest_phase_reached == receipt.board.furthest_phase_reached
         && derived.board.rows == receipt.board.rows
         && phase_board_error_population_reconciles(board: receipt.board)

dag/test/claim/self_host_compile_phase_frontier_witness_test.dag

@@ -39,6 +39,12 @@ import gunbc.self_host_compile_phase_frontier {
   compile_phase_frontier_standing,
   genesis_5006ddc6_parse_observation,
   genesis_5006ddc6_raw_error_diagnostic_identities,
+  phase_census_digest,
+  CensusIdentityDigest,
+  CanonicalIdentitySetV1,
+  EmissionOrderedV1,
+  census_identity_domain_separator,
+  census_population_is_duplicate_free,
   render_phase_board
 }
 import extdeps.cargo_diagnostic {
@@ -88,7 +94,7 @@ fn fixture_borrow_board(borrow_ids: List<String>) -> RustcPhaseBoard {
     ],
     furthest_phase_reached: Borrowck,
     furthest_phase_evidence: Observed,
-    census_identity_digest: content_hash_atom(value: "fixture-borrow-board" as NonEmptyStr),
+    census_identity_digest: CensusIdentityDigest { algorithm: CanonicalIdentitySetV1, digest: content_hash_atom(value: "fixture-borrow-board" as NonEmptyStr) },
     total_error_diagnostics: count(borrow_ids),
     coded_error_diagnostics: count(borrow_ids),
     codeless_error_diagnostics: 0,
@@ -107,7 +113,7 @@ test fn coded_and_codeless_error_identities_must_reconcile_to_total() -> Bool {
     ],
     furthest_phase_reached: Typeck,
     furthest_phase_evidence: Observed,
-    census_identity_digest: content_hash_atom(value: "malformed-error-population" as NonEmptyStr),
+    census_identity_digest: CensusIdentityDigest { algorithm: CanonicalIdentitySetV1, digest: content_hash_atom(value: "malformed-error-population" as NonEmptyStr) },
     total_error_diagnostics: 1,
     coded_error_diagnostics: 1,
     codeless_error_diagnostics: 1,
@@ -141,7 +147,7 @@ fn fixture_board(resolve_ids: List<String>, typeck_ids: List<String>) -> RustcPh
     ],
     furthest_phase_reached: if count(resolve_ids) > 0 { Resolve } else { Typeck },
     furthest_phase_evidence: Observed,
-    census_identity_digest: content_hash_atom(value: "fixture-board" as NonEmptyStr),
+    census_identity_digest: CensusIdentityDigest { algorithm: CanonicalIdentitySetV1, digest: content_hash_atom(value: "fixture-board" as NonEmptyStr) },
     total_error_diagnostics: count(resolve_ids) + count(typeck_ids),
     coded_error_diagnostics: count(resolve_ids) + count(typeck_ids),
     codeless_error_diagnostics: 0,
@@ -395,3 +401,90 @@ test fn the_published_frontier_standing_does_not_claim_typeck_or_borrowck_passed
     standing => false
   }
 }
+
+// The digest is named an IDENTITY of the census, and a census is a set: cargo's diagnostic stream
+// and rustfmt's refusal list carry no ordering guarantee, so a reordered-but-identical census must
+// fold to the same digest or receipt_population_coherent refuses a receipt that in fact agrees.
+test fn the_census_digest_is_invariant_under_reordering_of_either_population() -> Bool {
+  let forward = phase_census_digest(
+    parse_refused_files: ["src/a.rs", "src/b.rs"],
+    raw_identities: ["src/a.rs:1:1\tE0308\tmismatch", "src/b.rs:2:2\tE0425\tunresolved"]
+  )
+  let reversed = phase_census_digest(
+    parse_refused_files: ["src/b.rs", "src/a.rs"],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/a.rs:1:1\tE0308\tmismatch"]
+  )
+  forward == reversed
+}
+
+// Separation controls. A digest that separated on CARDINALITY alone would pass an added/removed
+// pair, so the load-bearing case is substitution at equal cardinality: one identity swapped, count
+// unchanged. Alongside it, a value moved between the two populations must still separate, which is
+// what makes the segments separate rather than one canonicalized pool.
+test fn the_census_digest_separates_added_removed_and_substituted_identities() -> Bool {
+  let base = phase_census_digest(
+    parse_refused_files: ["src/a.rs"],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/c.rs:3:3\tE0433\tunresolved"]
+  )
+  let added = phase_census_digest(
+    parse_refused_files: ["src/a.rs"],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/c.rs:3:3\tE0433\tunresolved", "src/d.rs:4:4\tE0599\tno method"]
+  )
+  let removed = phase_census_digest(
+    parse_refused_files: ["src/a.rs"],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved"]
+  )
+  let substituted = phase_census_digest(
+    parse_refused_files: ["src/a.rs"],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/c.rs:3:3\tE0308\tmismatched types"]
+  )
+  let populations_swapped = phase_census_digest(
+    parse_refused_files: ["src/b.rs:2:2\tE0425\tunresolved"],
+    raw_identities: ["src/a.rs", "src/c.rs:3:3\tE0433\tunresolved"]
+  )
+  base != added
+    && base != removed
+    && base != substituted
+    && base != populations_swapped
+    && added != removed
+}
+
+// Duplicate handling is EXPLICIT and the decision is typed refusal upstream, not silent set-dedup
+// inside the digest. Collapsing duplicates here would make a duplicated population indistinguishable
+// from a clean one, hiding a population defect behind a digest that still looked well-formed. The
+// digest therefore separates them, and phase_board_from_census_identities refuses the duplicated
+// population by folding a deduplicated total that no longer equals the raw count -- which is the
+// exact equality receipt_population_coherent checks.
+test fn a_duplicated_census_population_is_not_silently_collapsed_to_a_set() -> Bool {
+  let clean = phase_census_digest(
+    parse_refused_files: [],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved"]
+  )
+  let duplicated = phase_census_digest(
+    parse_refused_files: [],
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/b.rs:2:2\tE0425\tunresolved"]
+  )
+  let duplicated_fold = phase_board_from_census_identities(
+    parse_observation: clean_parse_observation(),
+    raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/b.rs:2:2\tE0425\tunresolved"]
+  )
+  clean != duplicated && duplicated_fold.board.total_error_diagnostics != 2
+}
+
+// The digest carries its own algorithm identity, so a value folded under the retired emission-order
+// procedure and one folded over the canonical set can never be compared as bare hashes.
+test fn the_census_digest_carries_the_algorithm_that_produced_it() -> Bool {
+  let produced = phase_census_digest(parse_refused_files: [], raw_identities: ["src/b.rs:2:2\tE0425\tunresolved"])
+  produced.algorithm == CanonicalIdentitySetV1
+    && census_identity_domain_separator(algorithm: EmissionOrderedV1)
+       != census_identity_domain_separator(algorithm: CanonicalIdentitySetV1)
+}
+
+// The refused-file population had no duplicate wall at all until this predicate was wired into
+// receipt_population_coherent: rustfmt refusing one path twice would double-count the Parse row and
+// carry the repeat into the digest. This is the discriminating pair for that wall.
+test fn a_repeated_population_entry_is_refused_rather_than_collapsed() -> Bool {
+  !census_population_is_duplicate_free(identities: ["src/a.rs", "src/a.rs"])
+    && census_population_is_duplicate_free(identities: ["src/a.rs", "src/b.rs"])
+    && census_population_is_duplicate_free(identities: [])
+}

docs/design-ledgers.md

@@ -56,7 +56,7 @@ Tree: `sha1:1c7c79fbc01aa488abb541513fc8ea44367406ad`
 
 Invocation: `buildbuddy-b779daaf-cefa-4379-ab27-dbea5a8d2919`
 
-Census identity digest: `ce69956ca7a9de96`
+Census identity digest (self-host-compile-phase-census-canonical-identity-set-v1): `6daa51e6ebaea2e3`
 
 Error identities: 233 total = 233 coded + 0 codeless