diff --git a/dag/gunbc/design_ledgers.dag b/dag/gunbc/design_ledgers.dag index 5ab51d45ca9..8ccf2d21d3b 100644 --- a/dag/gunbc/design_ledgers.dag +++ b/dag/gunbc/design_ledgers.dag @@ -30,6 +30,7 @@ import gunbc.recurring_failure_mode { recurring_failure_mode_roster } import gunbc.rung_drop { rung_drop_roster } import v2.workflow.floor_discovery { floor_discovery_tree_identity } import gunbc.self_host_compile_phase_frontier { + census_identity_domain_separator, CompilePhaseFrontierEmpty, CompilePhaseFrontierValidated, CompilePhaseFrontierInvalid, @@ -91,7 +92,10 @@ fn compile_phase_frontier_blocks() -> List { p(text: concat("Milestones: ", render_phase_board(board: latest.board))), p(text: concat("Tree: `", concat(floor_discovery_tree_identity(tree: latest.subject.git_tree_object), "`"))), p(text: concat("Invocation: `", concat(latest.invocation as String, "`"))), - p(text: concat("Census identity digest: `", concat(latest.board.census_identity_digest.digest as String, "`"))), + p(text: concat( + concat("Census identity digest (", concat(census_identity_domain_separator(algorithm: latest.board.census_identity_digest.algorithm), "): `")), + concat(latest.board.census_identity_digest.digest.digest as String, "`") + )), p(text: concat( "Error identities: ", concat( diff --git a/dag/gunbc/self_host_compile_phase_frontier.dag b/dag/gunbc/self_host_compile_phase_frontier.dag index 3b25ab0a95c..97de048aaea 100644 --- a/dag/gunbc/self_host_compile_phase_frontier.dag +++ b/dag/gunbc/self_host_compile_phase_frontier.dag @@ -58,7 +58,7 @@ type RustcPhaseBoard { rows: List furthest_phase_reached: RustcPhase furthest_phase_evidence: PhaseCountEvidence - census_identity_digest: Fnv1a64Structural + census_identity_digest: CensusIdentityDigest total_error_diagnostics: Int coded_error_diagnostics: Int codeless_error_diagnostics: Int @@ -118,6 +118,17 @@ fn deduplicate_identities(identities: List) -> List { ) } +// Duplicate handling is a POPULATION fact, and the two populations answer it at different sites. +// raw_identities is set-deduplicated by the live producer before the fold, and on the persisted +// path a repeat makes the deduplicated total disagree with the raw count, which receipt coherence +// refuses. parse_refused_files had neither wall: a path rustfmt refused twice would be counted +// twice in the Parse row and kept twice in the digest, so the digest would answer about a multiset +// while its name claims a set. Refusing is the right arm rather than deduplicating here, because +// collapsing the repeat would hide the producer defect behind a well-formed-looking census. +fn census_population_is_duplicate_free(identities: List) -> Bool { + count(deduplicate_identities(identities: identities)) == count(identities) +} + fn phase_identities(ds: List, phase: RustcPhase) -> List { deduplicate_identities( identities: ds |> filter(d => diagnostic_is_phase(d: d, phase: phase)) |> map(d => rustc_diagnostic_row(d: d)) @@ -152,16 +163,57 @@ fn first_failing_checked_phase(ds: List) -> RustcPhase { else { Borrowck } } -fn phase_census_digest(parse_refused_files: List, raw_identities: List) -> Fnv1a64Structural { - content_hash_atom( - value: (concat( - ["self-host-compile-phase-census-v1"], - concat( - parse_refused_files |> map(path => concat("rustfmt-refused\t", path)), - raw_identities - ) - ) |> join(separator: "\n")) as NonEmptyStr - ) +// The digest names the census as a SET, so it is folded over a canonical ordering of each +// population rather than the encounter order cargo and rustfmt happened to emit. Without the sort +// the same census reordered upstream produces a different digest, and receipt_population_coherent +// refuses a receipt whose census is in fact identical. The two populations stay separate segments: +// canonicalizing them jointly would let a refused path and an identity trade places. +// +// EmissionOrderedV1 is retired and NO PRODUCER CONSTRUCTS IT. Its justification is NOT that it +// labels a live value -- it does not. docs/design-ledgers.md published an emission-order digest +// until this change, and regenerating that projection replaces it, so after this commit no +// artifact in the tree carries one and the retired value survives only in git history. +// +// It is kept because an algorithm identity with exactly ONE inhabitant distinguishes nothing. The +// field exists to stop two incompatible digest meanings hiding behind a single unlabeled hash, and +// that requires both meanings to be nameable. So the honest fork is not "drop the arm" but "drop +// the arm AND the field": a one-inhabitant tag is decoration, and keeping the tag while deleting +// its only contrast is the worse of the two shapes. +// +// It is read by production -- census_identity_domain_separator matches it exhaustively -- and by a +// witness asserting the two separators differ, so it is not an inert carrier under +// v2.lens.inert_carrier, whose scope is carriers read by no production code. +type CensusIdentityAlgorithm + = EmissionOrderedV1 + | CanonicalIdentitySetV1 + +// The domain separator is DERIVED from the algorithm rather than authored beside it. A separate +// version literal would be a second authority for the same fact, and the two would drift. +fn census_identity_domain_separator(algorithm: CensusIdentityAlgorithm) -> String { + match algorithm { + EmissionOrderedV1 => "self-host-compile-phase-census-v1" + CanonicalIdentitySetV1 => "self-host-compile-phase-census-canonical-identity-set-v1" + } +} + +type CensusIdentityDigest { + algorithm: CensusIdentityAlgorithm + digest: Fnv1a64Structural +} + +fn phase_census_digest(parse_refused_files: List, raw_identities: List) -> CensusIdentityDigest { + CensusIdentityDigest { + algorithm: CanonicalIdentitySetV1, + digest: content_hash_atom( + value: (concat( + [census_identity_domain_separator(algorithm: CanonicalIdentitySetV1)], + concat( + parse_refused_files |> map(path => concat("rustfmt-refused\t", path)) |> sort_by(line => line), + raw_identities |> sort_by(identity => identity) + ) + ) |> join(separator: "\n")) as NonEmptyStr + ) + } } fn cargo_phase_board(parse_observation: RustfmtParseObservation, ds: List) -> RustcPhaseBoardRead { @@ -463,7 +515,7 @@ fn parse_evidence_rows(evidence: ParsePhaseEvidence) -> List { fn optional_reclassified_digest(board: RustcPhaseBoard?) -> String { match board { Absent => "not-reclassified" - Present { value: b } => concat("reclassified:", b.census_identity_digest.digest as String) + Present { value: b } => concat("reclassified:", b.census_identity_digest.digest.digest as String) } } @@ -485,7 +537,8 @@ fn receipt_payload_identity(receipt: CompilePhaseFrontierReceipt) -> Fnv1a64Stru receipt.producer.module_path, receipt.producer.function, receipt.producer.entry, - receipt.board.census_identity_digest.digest as String, + census_identity_domain_separator(algorithm: receipt.board.census_identity_digest.algorithm), + receipt.board.census_identity_digest.digest.digest as String, rustc_phase_name(phase: receipt.board.furthest_phase_reached) ], "\n") as NonEmptyStr ) @@ -913,7 +966,9 @@ fn receipt_population_coherent(receipt: CompilePhaseFrontierReceipt) -> Bool { parse_observation: parse, raw_identities: receipt.raw_error_diagnostic_identities ) - derived.board.census_identity_digest == receipt.board.census_identity_digest + census_population_is_duplicate_free(identities: parse.refused_files) + && census_population_is_duplicate_free(identities: receipt.raw_error_diagnostic_identities) + && derived.board.census_identity_digest == receipt.board.census_identity_digest && derived.board.furthest_phase_reached == receipt.board.furthest_phase_reached && derived.board.rows == receipt.board.rows && phase_board_error_population_reconciles(board: receipt.board) diff --git a/dag/test/claim/self_host_compile_phase_frontier_witness_test.dag b/dag/test/claim/self_host_compile_phase_frontier_witness_test.dag index 6acf3b4f7f2..e2982114940 100644 --- a/dag/test/claim/self_host_compile_phase_frontier_witness_test.dag +++ b/dag/test/claim/self_host_compile_phase_frontier_witness_test.dag @@ -39,6 +39,12 @@ import gunbc.self_host_compile_phase_frontier { compile_phase_frontier_standing, genesis_5006ddc6_parse_observation, genesis_5006ddc6_raw_error_diagnostic_identities, + phase_census_digest, + CensusIdentityDigest, + CanonicalIdentitySetV1, + EmissionOrderedV1, + census_identity_domain_separator, + census_population_is_duplicate_free, render_phase_board } import extdeps.cargo_diagnostic { @@ -88,7 +94,7 @@ fn fixture_borrow_board(borrow_ids: List) -> RustcPhaseBoard { ], furthest_phase_reached: Borrowck, furthest_phase_evidence: Observed, - census_identity_digest: content_hash_atom(value: "fixture-borrow-board" as NonEmptyStr), + census_identity_digest: CensusIdentityDigest { algorithm: CanonicalIdentitySetV1, digest: content_hash_atom(value: "fixture-borrow-board" as NonEmptyStr) }, total_error_diagnostics: count(borrow_ids), coded_error_diagnostics: count(borrow_ids), codeless_error_diagnostics: 0, @@ -107,7 +113,7 @@ test fn coded_and_codeless_error_identities_must_reconcile_to_total() -> Bool { ], furthest_phase_reached: Typeck, furthest_phase_evidence: Observed, - census_identity_digest: content_hash_atom(value: "malformed-error-population" as NonEmptyStr), + census_identity_digest: CensusIdentityDigest { algorithm: CanonicalIdentitySetV1, digest: content_hash_atom(value: "malformed-error-population" as NonEmptyStr) }, total_error_diagnostics: 1, coded_error_diagnostics: 1, codeless_error_diagnostics: 1, @@ -141,7 +147,7 @@ fn fixture_board(resolve_ids: List, typeck_ids: List) -> RustcPh ], furthest_phase_reached: if count(resolve_ids) > 0 { Resolve } else { Typeck }, furthest_phase_evidence: Observed, - census_identity_digest: content_hash_atom(value: "fixture-board" as NonEmptyStr), + census_identity_digest: CensusIdentityDigest { algorithm: CanonicalIdentitySetV1, digest: content_hash_atom(value: "fixture-board" as NonEmptyStr) }, total_error_diagnostics: count(resolve_ids) + count(typeck_ids), coded_error_diagnostics: count(resolve_ids) + count(typeck_ids), codeless_error_diagnostics: 0, @@ -395,3 +401,90 @@ test fn the_published_frontier_standing_does_not_claim_typeck_or_borrowck_passed standing => false } } + +// The digest is named an IDENTITY of the census, and a census is a set: cargo's diagnostic stream +// and rustfmt's refusal list carry no ordering guarantee, so a reordered-but-identical census must +// fold to the same digest or receipt_population_coherent refuses a receipt that in fact agrees. +test fn the_census_digest_is_invariant_under_reordering_of_either_population() -> Bool { + let forward = phase_census_digest( + parse_refused_files: ["src/a.rs", "src/b.rs"], + raw_identities: ["src/a.rs:1:1\tE0308\tmismatch", "src/b.rs:2:2\tE0425\tunresolved"] + ) + let reversed = phase_census_digest( + parse_refused_files: ["src/b.rs", "src/a.rs"], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/a.rs:1:1\tE0308\tmismatch"] + ) + forward == reversed +} + +// Separation controls. A digest that separated on CARDINALITY alone would pass an added/removed +// pair, so the load-bearing case is substitution at equal cardinality: one identity swapped, count +// unchanged. Alongside it, a value moved between the two populations must still separate, which is +// what makes the segments separate rather than one canonicalized pool. +test fn the_census_digest_separates_added_removed_and_substituted_identities() -> Bool { + let base = phase_census_digest( + parse_refused_files: ["src/a.rs"], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/c.rs:3:3\tE0433\tunresolved"] + ) + let added = phase_census_digest( + parse_refused_files: ["src/a.rs"], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/c.rs:3:3\tE0433\tunresolved", "src/d.rs:4:4\tE0599\tno method"] + ) + let removed = phase_census_digest( + parse_refused_files: ["src/a.rs"], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved"] + ) + let substituted = phase_census_digest( + parse_refused_files: ["src/a.rs"], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/c.rs:3:3\tE0308\tmismatched types"] + ) + let populations_swapped = phase_census_digest( + parse_refused_files: ["src/b.rs:2:2\tE0425\tunresolved"], + raw_identities: ["src/a.rs", "src/c.rs:3:3\tE0433\tunresolved"] + ) + base != added + && base != removed + && base != substituted + && base != populations_swapped + && added != removed +} + +// Duplicate handling is EXPLICIT and the decision is typed refusal upstream, not silent set-dedup +// inside the digest. Collapsing duplicates here would make a duplicated population indistinguishable +// from a clean one, hiding a population defect behind a digest that still looked well-formed. The +// digest therefore separates them, and phase_board_from_census_identities refuses the duplicated +// population by folding a deduplicated total that no longer equals the raw count -- which is the +// exact equality receipt_population_coherent checks. +test fn a_duplicated_census_population_is_not_silently_collapsed_to_a_set() -> Bool { + let clean = phase_census_digest( + parse_refused_files: [], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved"] + ) + let duplicated = phase_census_digest( + parse_refused_files: [], + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/b.rs:2:2\tE0425\tunresolved"] + ) + let duplicated_fold = phase_board_from_census_identities( + parse_observation: clean_parse_observation(), + raw_identities: ["src/b.rs:2:2\tE0425\tunresolved", "src/b.rs:2:2\tE0425\tunresolved"] + ) + clean != duplicated && duplicated_fold.board.total_error_diagnostics != 2 +} + +// The digest carries its own algorithm identity, so a value folded under the retired emission-order +// procedure and one folded over the canonical set can never be compared as bare hashes. +test fn the_census_digest_carries_the_algorithm_that_produced_it() -> Bool { + let produced = phase_census_digest(parse_refused_files: [], raw_identities: ["src/b.rs:2:2\tE0425\tunresolved"]) + produced.algorithm == CanonicalIdentitySetV1 + && census_identity_domain_separator(algorithm: EmissionOrderedV1) + != census_identity_domain_separator(algorithm: CanonicalIdentitySetV1) +} + +// The refused-file population had no duplicate wall at all until this predicate was wired into +// receipt_population_coherent: rustfmt refusing one path twice would double-count the Parse row and +// carry the repeat into the digest. This is the discriminating pair for that wall. +test fn a_repeated_population_entry_is_refused_rather_than_collapsed() -> Bool { + !census_population_is_duplicate_free(identities: ["src/a.rs", "src/a.rs"]) + && census_population_is_duplicate_free(identities: ["src/a.rs", "src/b.rs"]) + && census_population_is_duplicate_free(identities: []) +} diff --git a/docs/design-ledgers.md b/docs/design-ledgers.md index 55359854879..902146a9a9a 100644 --- a/docs/design-ledgers.md +++ b/docs/design-ledgers.md @@ -56,7 +56,7 @@ Tree: `sha1:1c7c79fbc01aa488abb541513fc8ea44367406ad` Invocation: `buildbuddy-b779daaf-cefa-4379-ab27-dbea5a8d2919` -Census identity digest: `ce69956ca7a9de96` +Census identity digest (self-host-compile-phase-census-canonical-identity-set-v1): `6daa51e6ebaea2e3` Error identities: 233 total = 233 coded + 0 codeless