diff --git a/dag/gunbc/recurring_failure_mode/octet_offset_consumed_by_a_scalar_indexed_text_operation.dag b/dag/gunbc/recurring_failure_mode/octet_offset_consumed_by_a_scalar_indexed_text_operation.dag new file mode 100644 index 00000000000..05900421b0c --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/octet_offset_consumed_by_a_scalar_indexed_text_operation.dag @@ -0,0 +1,22 @@ +module gunbc.recurring_failure_mode.octet_offset_consumed_by_a_scalar_indexed_text_operation + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data octet_offset_consumed_by_a_scalar_indexed_text_operation: RecurringFailureMode = RecurringFailureMode { + identity: "octet_offset_consumed_by_a_scalar_indexed_text_operation" as NonEmptyStr, + + receipts: [ + "INVALID STATE: an offset measured in UTF-8 OCTETS is handed to a text operation that indexes Unicode SCALARS. The v2 tokenizer counts every walk offset in octets (v2.compiler.tokenize lexeme_utf8_octet_count), so every v2.std.diagnostic ByteRange the span index resolves to is an octet range. `substring` and `string_length` index scalars: the emitted runtime walks chars(). Both are plain Int, so nothing stops an octet offset from reaching substring. On ASCII text the two agree, which is why the defect passes every ASCII fixture and is silent on the first multibyte character.", + "SPECIMEN, CAUGHT IN REVIEW BEFORE LANDING (gunbc#13051, calm-boar-904): v2.cli.if_arm_reader_differential derived each site's line as the count of line feeds in substring(source, 0, ByteRange.start). Every multibyte scalar before a site moved its reported line. The census COUNT was unaffected; only the rendered locations were wrong. Repaired by reporting the byte range alone. The corpus's one line-from-octet reader was private to a lens (v2.lens.text_string_importer_census newline_offsets), not a shared authority, and swift-lynx-592 is moving it into v2.std.source_position. When that lands it is the sanctioned conversion this row's trigger names. The control is a nested if after 2-, 3- and 4-octet scalars, located at octet 237, which is scalar 223.", + "POPULATION, read at the same revision over dag/ and src/v2/ (about 486 substring calls, each offset traced to its producer): no other site passes an octet offset to substring. Two neighbours look like the class but are not. src/v2/lens/text_string_importer_census.dag newline_offsets counts octets over utf8_encode_bytes and compares them with ByteRange.start: octets against octets. dag/std/import.dag import_strip_step slices with import-statement spans from the v1 tokenizer, which indexes its source_chars code-point list: scalars against scalars. One site agrees only by its input: dag/gunbc/runner/runner_microvm.dag jit_drive_content subtracts an octet count from a substring end over a string of line feeds only, where octets and scalars coincide.", + "RUNG FOUND AT: below the floor (a plausible, wrong location rendered with no refusal). RUNG NOW: no wall. The population is zero by a one-off read, not by any gate, so a new site can be written today. CEILING: structurally impossible. An octet offset and a scalar index are different quantities and can be different types. NEXT-RUNG TRIGGER: v2.std.diagnostic ByteRange carries a distinct octet-offset type that substring does not accept, with a sanctioned conversion that walks the UTF-8 table (std.unicode.types unicode_scalar_utf8_octet_count). A source-locus renderer that wants lines then derives them from octets through that one conversion.", + ], + + evidence: [ + DeclarationRef { module_path: "v2.test.claim.cli.if_arm_reader_differential", decl_name: "multibyte_source_before_a_site_is_located_in_octets_holds", field: WholeDeclaration }, + DeclarationRef { module_path: "v2.compiler.tokenize", decl_name: "lexeme_utf8_octet_count", field: WholeDeclaration }, + DeclarationRef { module_path: "v2.cli.if_arm_reader_differential", decl_name: "iard_location", field: WholeDeclaration }, + ], +} diff --git a/src/v2/cli/compile_cli.dag b/src/v2/cli/compile_cli.dag index 8e48e1096b8..6b45306a672 100644 --- a/src/v2/cli/compile_cli.dag +++ b/src/v2/cli/compile_cli.dag @@ -12,6 +12,13 @@ import v2.compiler.self_host.closure_emission { emit_closure_from_ingest_located } import v2.compiler.source_authority { SourceRootIngest, source_root_ingest_from_files } +import v2.cli.if_arm_reader_differential { + IardCensus, + IardCensusTaken, + IardGrammarUnprepared, + if_arm_reader_differential, + if_arm_reader_differential_text +} import gunbc.source_root_read { source_root_read, SourceRootFilesRead, SourceRootFilesRefused } import v2.extdeps.languages.c { c_target_model } import v2.extdeps.languages.rust { rust_target_model } @@ -135,6 +142,7 @@ fn cli_emit_target_roster_text() -> String { type CliVerb = CliEmitClosure { entry: String, source_roots: FreeMonoid, target: CliEmitTarget } + | CliIfArmReaderDifferential { source_roots: FreeMonoid } // A PLAN IS PARSED OR IT IS REFUSED, and the refusal carries its own reason symbol rather than a // rendered sentence, so the host renders it once and a witness can discriminate on the CAUSE. A @@ -156,6 +164,8 @@ type CliParseState | CliParseSeekRootValue { entry: String, roots: FreeMonoid, target: CliEmitTarget } | CliParseSeekEntryValue { entry: String, roots: FreeMonoid, target: CliEmitTarget } | CliParseSeekTargetValue { entry: String, roots: FreeMonoid } + | CliParseSeekDifferentialOption { roots: FreeMonoid } + | CliParseSeekDifferentialRootValue { roots: FreeMonoid } | CliParseFailed { reason: Symbol, detail: String } data cli_option_source_root: String = "--source-root" @@ -166,8 +176,15 @@ data cli_option_target: String = "--target" data cli_verb_emit: String = "emit" +// THE NESTED-IF ARM-READER DIFFERENTIAL (v2.cli.if_arm_reader_differential). Named for exactly what +// it measures so it does not read as the general census, which lives on the native driver +// (v2.compiler.compile). It takes source roots only, and is deleted with its module. +data cli_verb_if_arm_reader_differential: String = "if-arm-reader-differential" + data cli_usage: String = "usage: emit --entry --source-root [--source-root ]... [--target ]" +data cli_differential_usage: String = "usage: if-arm-reader-differential --source-root [--source-root ]..." + fn cli_parse_step(state: CliParseState, arg: String) -> CliParseState { match state { CliParseFailed { reason: reason, detail: detail } => @@ -175,12 +192,32 @@ fn cli_parse_step(state: CliParseState, arg: String) -> CliParseState { CliParseSeekVerb => if arg == cli_verb_emit { CliParseSeekOption { entry: "", roots: Empty, target: cli_default_emit_target } + } else if arg == cli_verb_if_arm_reader_differential { + CliParseSeekDifferentialOption { roots: Empty } } else { CliParseFailed { reason: ^cli_unknown_verb, detail: concat(concat("unknown verb ", arg), concat(" -- ", cli_usage)) } } + CliParseSeekDifferentialOption { roots: roots } => + if arg == cli_option_source_root { + CliParseSeekDifferentialRootValue { roots: roots } + } else { + CliParseFailed { + reason: ^cli_unknown_option, + detail: concat(concat("unknown option ", arg), concat(" -- ", cli_differential_usage)) + } + } + CliParseSeekDifferentialRootValue { roots: roots } => + if cli_arg_is_option(arg: arg) { + CliParseFailed { + reason: ^cli_source_root_missing_value, + detail: concat(cli_option_source_root, " takes a directory, and an option followed it") + } + } else { + CliParseSeekDifferentialOption { roots: Cons { head: arg, tail: roots } } + } CliParseSeekRootValue { entry: entry, roots: roots, target: target } => if cli_arg_is_option(arg: arg) { CliParseFailed { @@ -263,6 +300,20 @@ fn cli_parse_finish(state: CliParseState) -> CliPlan { reason: ^cli_target_missing_value, detail: concat(cli_option_target, " takes a target name, and none followed it") } + CliParseSeekDifferentialRootValue { roots: _ } => + CliPlanRefused { + reason: ^cli_source_root_missing_value, + detail: concat(cli_option_source_root, " takes a directory, and none followed it") + } + CliParseSeekDifferentialOption { roots: roots } => + if length(xs: roots) == 0 { + CliPlanRefused { + reason: ^cli_no_source_root, + detail: concat("if-arm-reader-differential needs at least one source root -- ", cli_differential_usage) + } + } else { + CliPlanParsed { verb: CliIfArmReaderDifferential { source_roots: list_reverse(xs: roots) } } + } CliParseSeekOption { entry: entry, roots: roots, target: target } => if length(xs: roots) == 0 { CliPlanRefused { @@ -300,6 +351,7 @@ fn v2_cli_plan_source_roots(plan: CliPlan) -> FreeMonoid { CliPlanParsed { verb: verb } => match verb { CliEmitClosure { entry: _, source_roots: roots, target: _ } => roots + CliIfArmReaderDifferential { source_roots: roots } => roots } } } @@ -309,6 +361,7 @@ type CliRunOutcome | CliRunRefused { reason: Symbol, detail: String } | CliCensusRefused { reason: Symbol, detail: String, text: String } | CliUsageRefused { reason: Symbol, detail: String } + | CliDifferentialReported { text: String, observed: Bool, detail: String } // THE REFUSAL PRINTS ITS CAUSE, AND FOR ONE GENERATION IT DID NOT. The rejected arm already // CARRIED `d.head.reason` (now the FATAL reason -- the head was the first advisory that happened @@ -349,6 +402,15 @@ fn v2_cli_run(plan: CliPlan, ingest: SourceRootIngest) -> CliRunOutcome { CliUsageRefused { reason: reason, detail: detail } CliPlanParsed { verb: verb } => match verb { + CliIfArmReaderDifferential { source_roots: _ } => + if length(xs: ingest) == 0 { + CliRunRefused { + reason: ^cli_empty_ingest, + detail: "the source roots walked to zero .dag files; a census over nothing is not a census of zero" + } + } else { + cli_if_arm_reader_differential_outcome(census: if_arm_reader_differential(ingest: ingest)) + } CliEmitClosure { entry: entry, source_roots: _, target: target } => if length(xs: ingest) == 0 { CliRunRefused { @@ -393,6 +455,27 @@ fn v2_cli_run(plan: CliPlan, ingest: SourceRootIngest) -> CliRunOutcome { } } +// THE DIFFERENTIAL'S STATUS IS WHETHER THE POPULATION WAS WHOLLY OBSERVED, NEVER HOW MANY SITES +// DIFFER: the differing sites are the finding, not a failure. A file the census could not parse, or a +// grammar that did not prepare, leaves the population incomplete, and that fails the process so a +// partial count cannot be read as the whole one. +fn cli_if_arm_reader_differential_outcome(census: IardCensus) -> CliRunOutcome { + match census { + IardGrammarUnprepared { reason: _ } => + CliDifferentialReported { + text: if_arm_reader_differential_text(c: census), + observed: false, + detail: "the grammar did not prepare; no site was observed" + } + IardCensusTaken { tally: t } => + CliDifferentialReported { + text: if_arm_reader_differential_text(c: census), + observed: t.files_refused == 0, + detail: "some files did not parse; their sites are unobserved (see file_refused rows)" + } + } +} + // ONE RENDERING OF A LOCATED REFUSAL, for a refused closure and for a refused member alike: the whole // ordered chain, then the fatal's file and extent. A member's refusal fails the process as a closure // refusal does, so its status line owes the same located cause and not a summary sentence. @@ -437,6 +520,8 @@ fn v2_cli_exit(outcome: CliRunOutcome) -> ProcessExit { CliCensusRefused { reason: _, detail: detail, text: _ } => exit_failure(reason: detail) CliUsageRefused { reason: _, detail: detail } => ExitFailure { code: exit_code_misuse, reason: detail } + CliDifferentialReported { text: _, observed: observed, detail: detail } => + if observed { ExitSuccess } else { exit_failure(reason: detail) } } } @@ -446,6 +531,7 @@ fn v2_cli_outcome_text(outcome: CliRunOutcome) -> String { CliRunRefused { reason: _, detail: _ } => "" CliCensusRefused { reason: _, detail: _, text: text } => text CliUsageRefused { reason: _, detail: _ } => "" + CliDifferentialReported { text: text, observed: _, detail: _ } => text } } diff --git a/src/v2/cli/if_arm_reader_differential.dag b/src/v2/cli/if_arm_reader_differential.dag new file mode 100644 index 00000000000..243dd232f63 --- /dev/null +++ b/src/v2/cli/if_arm_reader_differential.dag @@ -0,0 +1,532 @@ +module v2.cli.if_arm_reader_differential + +import std.dissolution { DissolutionCondition, unbound_dissolution } +import std.algebra { Cons, Empty, FreeMonoid } +import std.occurrence_identity { occurrence_id_eq } +import v2.compiler.body_lowering_fold { + IfRowArms, + body_lower_deep_unwrap_optional, + body_lower_else_part_if_expr_optional, + body_lower_find_captured, + body_lower_if_expr_capture_optional, + body_lower_if_row_arms_optional, + body_lower_match_spine_left, + body_lower_match_spine_right, + body_lower_unwrap_surface_shell +} +import v2.compiler.parse { PreparedGrammar, parse_module_prepared } +import v2.compiler.program_assembly { dag_prepared_grammar } +import v2.compiler.source_authority { DagSourceReadWitness, SourceRootIngest } +import v2.compiler.tokenize { tokenize } +import v2.extdeps.languages.dag { + ParseSubtreeAbsent, + ParseSubtreeFound, + dag_lex, + dag_surface_kw_then_ident_from_captured, + parse_production_captured_child_optional, + parse_production_emitted_identity_optional +} +import v2.std.algebra { fold_list, list_reverse } +import v2.std.diagnostic { Accepted, Diagnostics, NonEmptyDiagnostics, Rejected, Textual, NodeLocus, SourcePortLocus, InvariantPortLocus, DeclarationLocus, ByteRange, WholeFile, bind_outcome, diagnostics_fatal_reason } +import v2.std.grammar { node_atom_identity_optional } +import v2.std.integer { integer_int_to_decimal_string } +import v2.std.node { Node, Symbol, is_empty_conj_root, symbol_lexeme } +import v2.std.optional { Absent, Optional, Present, optional_absent, optional_present } +import v2.std.provenance { SpanIndex, node_occurrence_id_optional, span_index_textual_locus_optional } +import v2.std.text { String } + +// THE NESTED-IF ARM-READER DIFFERENTIAL: an OFFLINE ORACLE, NEVER A PRODUCTION ROUTE. +// +// Until gunbc#13029, v2.compiler.body_lowering_fold read an if's arms by SEARCHING its capture +// left-first. For an if nested in the THEN arm (or the condition) of another if, the else search +// entered the nested if and answered ITS else as the outer if's: the outer else was dropped, the +// nested else lowered twice, and normalize accepted (gunbc.recurring_failure_mode +// else_arm_lowered_as_an_if_nested_inside_it, capture side). #13029 replaced the searching readers +// with the positional body_lower_if_row_arms_optional. Two sites were loud; the silent population +// was unknown. This module measures it: for every if_expr capture in the ingest it asks both +// readers for each arm and reports every site where they answer a different occurrence. +// +// THE FROZEN READERS BELOW ARE main's body_lower_if_else_capture_optional, +// body_lower_if_fn_body_capture_optional, body_lower_if_then_arm_optional and +// body_lower_if_then_fn_body_optional as they stood before #13029, renamed and nothing else. Every +// helper they call is unchanged by #13029, so this is the old reader's logic exactly. They serve +// from history as a differential oracle (DESIGN section 3, replacement migrations): only this +// module's census calls them, and nothing on a lowering route reaches this module. They are +// deleted with it. +// +// WHERE IT RUNS (calm-boar-904's ruling, relayed by tidy-raven-393): as the +// `if-arm-reader-differential` verb of the native CLI door, v2.cli.compile_cli, because that door +// already acquires its plan's source roots natively. The interpreted parse of one large module took +// hours, which is why it is not a claim. +data if_arm_reader_differential_dissolve: DissolutionCondition = unbound_dissolution(description: "DISSOLVES WHEN every site this census reports is dispositioned (each named as harmless or repaired by #13029's positional reader) and the census has been run over the corpus; the module, its frozen readers and the CLI verb are then deleted together.") + +// --- the frozen pre-#13029 readers (offline oracle only) --- + +fn iard_old_fn_body_capture_optional(spine: Node) -> Optional { + match body_lower_find_captured(root: spine, emitted: ^dag_surface_fn_body) { + ParseSubtreeFound { captured: body } => Present { value: body } + ParseSubtreeAbsent => + if is_empty_conj_root(n: spine) { + Absent + } else { + match body_lower_match_spine_left(spine: spine) { + Present { value: left } => + match body_lower_match_spine_right(spine: spine) { + Present { value: right } => + match node_atom_identity_optional(node: body_lower_deep_unwrap_optional(node: left)) { + Present { value: id } => + match id == ^dag_token_kw_else { + true => iard_old_fn_body_capture_optional(spine: right) + false => + match iard_old_fn_body_capture_optional(spine: left) { + Present { value: found } => Present { value: found } + Absent => iard_old_fn_body_capture_optional(spine: right) + } + } + Absent => + match iard_old_fn_body_capture_optional(spine: left) { + Present { value: found } => Present { value: found } + Absent => iard_old_fn_body_capture_optional(spine: right) + } + } + Absent => Absent + } + Absent => Absent + } + } + } +} + +fn iard_old_then_fn_body_optional(captured: Node) -> Optional { + let spine = body_lower_unwrap_surface_shell(node: captured) + match iard_old_fn_body_capture_optional(spine: spine) { + Present { value: first } => + match body_lower_find_captured(root: captured, emitted: ^dag_surface_fn_body) { + ParseSubtreeFound { captured: only } => Present { value: only } + ParseSubtreeAbsent => Present { value: first } + } + Absent => Absent + } +} + +fn iard_old_else_capture_optional(captured: Node) -> Optional { + let spine = body_lower_unwrap_surface_shell(node: captured) + match is_empty_conj_root(n: spine) { + true => Absent + false => + match body_lower_match_spine_left(spine: spine) { + Present { value: left } => + match body_lower_match_spine_right(spine: spine) { + Present { value: right } => + match node_atom_identity_optional(node: body_lower_deep_unwrap_optional(node: left)) { + Present { value: id } => + match id == ^dag_token_kw_else { + true => + match body_lower_else_part_if_expr_optional(else_part: right) { + Present { value: nested } => Present { value: nested } + Absent => + match iard_old_fn_body_capture_optional(spine: right) { + Present { value: body } => Present { value: body } + Absent => + match body_lower_if_expr_capture_optional(spine: right) { + Present { value: expr } => Present { value: expr } + Absent => Absent + } + } + } + false => + match iard_old_else_capture_optional(captured: left) { + Present { value: found } => Present { value: found } + Absent => iard_old_else_capture_optional(captured: right) + } + } + Absent => + match iard_old_else_capture_optional(captured: left) { + Present { value: found } => Present { value: found } + Absent => iard_old_else_capture_optional(captured: right) + } + } + Absent => Absent + } + Absent => + fold(spine.children, init: Absent, f: fn(acc, e) { + match acc { + Present { value: _ } => acc + Absent => iard_old_else_capture_optional(captured: e.target) + } + }) + } + } +} + +fn iard_old_then_arm_optional(captured: Node) -> Optional { + let spine = body_lower_unwrap_surface_shell(node: captured) + match body_lower_find_captured(root: spine, emitted: ^dag_surface_fn_body) { + ParseSubtreeFound { captured: body } => Present { value: body } + ParseSubtreeAbsent => + match iard_old_then_fn_body_optional(captured: captured) { + Present { value: body } => Present { value: body } + Absent => + match body_lower_match_spine_left(spine: spine) { + Present { value: left } => + match body_lower_match_spine_right(spine: spine) { + Present { value: right } => + match node_atom_identity_optional(node: body_lower_deep_unwrap_optional(node: left)) { + Present { value: id } => + match id == ^dag_token_kw_then { + true => body_lower_if_expr_capture_optional(spine: right) + false => + match iard_old_then_arm_optional(captured: left) { + Present { value: found } => Present { value: found } + Absent => iard_old_then_arm_optional(captured: right) + } + } + Absent => + match iard_old_then_arm_optional(captured: left) { + Present { value: found } => Present { value: found } + Absent => iard_old_then_arm_optional(captured: right) + } + } + Absent => Absent + } + Absent => Absent + } + } + } +} + +// --- the comparison --- + +// HOW TWO READERS' ANSWERS FOR ONE ARM COMPARE. Agreement is OCCURRENCE IDENTITY, never content: +// `if a { if b { x } else { u } } else { u }` has two else arms with equal content and different +// occurrences, and the old reader answering the inner one is exactly the defect. An answer with no +// occurrence id cannot be compared, and is its own arm rather than being counted as agreement. +type IardArmComparison + = IardArmAgree + | IardArmDiffer { old: Node, new: Node } + | IardArmOldOnly { old: Node } + | IardArmNewOnly { new: Node } + | IardArmBothAbsent + | IardArmUnidentified + +fn iard_compare_arm(old: Optional, new: Optional) -> IardArmComparison { + match old { + Absent => + match new { + Absent => IardArmBothAbsent + Present { value: n } => IardArmNewOnly { new: n } + } + Present { value: o } => + match new { + Absent => IardArmOldOnly { old: o } + Present { value: n } => + match node_occurrence_id_optional(node: o) { + Absent => IardArmUnidentified + Present { value: oid } => + match node_occurrence_id_optional(node: n) { + Absent => IardArmUnidentified + Present { value: nid } => + if occurrence_id_eq(left: oid, right: nid) { IardArmAgree } else { IardArmDiffer { old: o, new: n } } + } + } + } + } +} + +fn iard_new_then(arms: Optional) -> Optional { + match arms { + Absent => optional_absent() + Present { value: a } => optional_present(value: a.then_arm) + } +} + +fn iard_new_else(arms: Optional) -> Optional { + match arms { + Absent => optional_absent() + Present { value: a } => optional_present(value: a.else_part) + } +} + +// ONE if_expr SITE: where it is, which declaration holds it, and how each arm compares. +type IardSite { + declaration: String + shell: Node + then_arm: IardArmComparison + else_arm: IardArmComparison +} + +fn iard_site(declaration: String, shell: Node, captured: Node) -> IardSite { + let arms = body_lower_if_row_arms_optional(captured: captured) + IardSite { + declaration: declaration, + shell: shell, + then_arm: iard_compare_arm(old: iard_old_then_arm_optional(captured: captured), new: iard_new_then(arms: arms)), + else_arm: iard_compare_arm(old: iard_old_else_capture_optional(captured: captured), new: iard_new_else(arms: arms)) + } +} + +// A module-scope declaration's name, read by the existing keyword-then-identifier reader; a +// declaration it cannot name (an import, a `test fn`) is reported by its production, never guessed. +fn iard_declaration_name(shell: Node, emitted: Symbol) -> String { + match parse_production_captured_child_optional(node: shell) { + Absent => concat("<", concat(symbol_lexeme(sym: emitted), ">")) + Present { value: captured } => + match dag_surface_kw_then_ident_from_captured(captured: captured) { + Present { value: name } => symbol_lexeme(sym: name) + Absent => concat("<", concat(symbol_lexeme(sym: emitted), ">")) + } + } +} + +fn iard_is_declaration(id: Symbol) -> Bool { + (id == ^dag_surface_fn_decl) + || (id == ^dag_surface_test_fn_decl) + || (id == ^dag_surface_data_decl) + || (id == ^dag_surface_type_decl) + || (id == ^dag_surface_alias_decl) + || (id == ^dag_surface_service_decl) + || (id == ^dag_surface_resource_decl) +} + +// EVERY if_expr IN THE TREE, IN PRE-ORDER, WITH ITS ENCLOSING DECLARATION. Nested ifs are visited: +// the walk does not stop at an if, because the defect's subject is exactly an if inside an if. +// Sites are prepended and the list reversed once (DESIGN section 6, bare minimum cost). +fn iard_collect(node: Node, declaration: String, acc: FreeMonoid) -> FreeMonoid { + let here = match parse_production_emitted_identity_optional(node: node) { + Absent => IardVisit { declaration: declaration, acc: acc } + Present { value: id } => + if id == ^dag_surface_if_expr { + match parse_production_captured_child_optional(node: node) { + Absent => IardVisit { declaration: declaration, acc: acc } + Present { value: captured } => + IardVisit { declaration: declaration, acc: Cons { head: iard_site(declaration: declaration, shell: node, captured: captured), tail: acc } } + } + } else if iard_is_declaration(id: id) { + IardVisit { declaration: iard_declaration_name(shell: node, emitted: id), acc: acc } + } else { + IardVisit { declaration: declaration, acc: acc } + } + } + fold(node.children, init: here.acc, f: fn(a, e) { + iard_collect(node: e.target, declaration: here.declaration, acc: a) + }) +} + +type IardVisit { + declaration: String + acc: FreeMonoid +} + +fn iard_sites(tree: Node) -> FreeMonoid { + list_reverse(xs: iard_collect(node: tree, declaration: "", acc: Empty)) +} + +// --- location --- + +// A NODE'S LOCATION, AS THE SPAN INDEX RESOLVES IT (v2.std.provenance is the one authority for +// following a derived occurrence to its source): the file's byte range, in the UTF-8 octets the +// tokenizer counts (v2.compiler.tokenize lexeme_utf8_octet_count). NO LINE IS DERIVED HERE. The one +// line-from-octet reader in the corpus is private to a lens (v2.lens.text_string_importer_census +// newline_offsets), and swift-lynx-592 is moving it into v2.std.source_position as the shared +// authority. Deriving a line with `substring` instead is wrong: the emitted substring indexes Unicode +// scalars, so every multibyte character before a site moved the line it reported (calm-boar-904, +// review of gunbc#13051). Until that authority lands, the byte range is the answer. A line, if it +// returns, comes from that authority and from no second reader. +fn iard_location(node: Node, spans: SpanIndex) -> String { + match node_occurrence_id_optional(node: node) { + Absent => "" + Present { value: id } => + match span_index_textual_locus_optional(index: spans, id: id) { + Absent => "" + Present { value: locus } => + match locus { + Textual { file: _, extent: e } => + match e { + ByteRange { start: s, end: en } => + concat("bytes=", concat(integer_int_to_decimal_string(value: s), concat("..", integer_int_to_decimal_string(value: en)))) + WholeFile => "" + } + NodeLocus { anchor: _ } => "" + SourcePortLocus { anchor: _, file: _, extent: _ } => "" + InvariantPortLocus { anchor: _, invariant: _ } => "" + DeclarationLocus { declaration: _ } => "" + } + } + } +} + +// --- the per-file and whole-ingest census --- + +type IardFileOutcome + = IardFileRead { path: String, spans: SpanIndex, sites: FreeMonoid } + | IardFileRefused { path: String, reason: Symbol } + +fn iard_file(prepared: PreparedGrammar, residue: Diagnostics, read: DagSourceReadWitness) -> IardFileOutcome { + let parsed = bind_outcome( + o: tokenize(text: read.source.carried, file: read.compilation_unit, rules: dag_lex()), + f: fn(stream) { parse_module_prepared(tokens: stream, prepared: prepared, validation_residue: residue) } + ) + match parsed { + Rejected { diagnostics: d } => IardFileRefused { path: read.artifact.file_path, reason: diagnostics_fatal_reason(d: d) } + Accepted { value: artifact, diagnostics: _ } => + IardFileRead { + path: read.artifact.file_path, + spans: artifact.span_index, + sites: iard_sites(tree: artifact.tree) + } + } +} + +fn iard_arm_differs(c: IardArmComparison) -> Bool { + match c { + IardArmAgree => false + IardArmBothAbsent => false + IardArmDiffer { old: _, new: _ } => true + IardArmOldOnly { old: _ } => true + IardArmNewOnly { new: _ } => true + IardArmUnidentified => true + } +} + +fn iard_arm_label(c: IardArmComparison) -> String { + match c { + IardArmAgree => "agree" + IardArmDiffer { old: _, new: _ } => "differ" + IardArmOldOnly { old: _ } => "old_only" + IardArmNewOnly { new: _ } => "new_only" + IardArmBothAbsent => "both_absent" + IardArmUnidentified => "unidentified" + } +} + +fn iard_arm_detail(c: IardArmComparison, spans: SpanIndex) -> String { + match c { + IardArmDiffer { old: o, new: n } => + concat(concat(" old_at=[", concat(iard_location(node: o, spans: spans), "]")), + concat(" new_at=[", concat(iard_location(node: n, spans: spans), "]"))) + IardArmOldOnly { old: o } => concat(" old_at=[", concat(iard_location(node: o, spans: spans), "]")) + IardArmNewOnly { new: n } => concat(" new_at=[", concat(iard_location(node: n, spans: spans), "]")) + IardArmAgree => "" + IardArmBothAbsent => "" + IardArmUnidentified => "" + } +} + +// ONE ROW PER (SITE, ARM) WHERE THE READERS DISAGREE. An arm where both agree, or both are absent, +// writes nothing; every other comparison -- including one that could not be identified -- is a row, +// because a census that dropped what it could not compare would undercount silently. +fn iard_arm_row(path: String, site: IardSite, arm: String, c: IardArmComparison, spans: SpanIndex) -> String { + concat( + concat("if_arm_differs file=", path), + concat( + concat(" decl=", site.declaration), + concat( + concat(" at=[", concat(iard_location(node: site.shell, spans: spans), "]")), + concat(concat(" arm=", arm), concat(concat(" ", iard_arm_label(c: c)), concat(iard_arm_detail(c: c, spans: spans), "\n"))) + ) + ) + ) +} + +type IardTally { + text: String + files_read: Int + files_refused: Int + sites: Int + else_differs: Int + then_differs: Int + sites_differing: Int +} + +fn iard_tally_site(t: IardTally, path: String, spans: SpanIndex, site: IardSite) -> IardTally { + let e = iard_arm_differs(c: site.else_arm) + let h = iard_arm_differs(c: site.then_arm) + let with_then = if h { concat(t.text, iard_arm_row(path: path, site: site, arm: "then", c: site.then_arm, spans: spans)) } else { t.text } + let with_else = if e { concat(with_then, iard_arm_row(path: path, site: site, arm: "else", c: site.else_arm, spans: spans)) } else { with_then } + IardTally { + text: with_else, + files_read: t.files_read, + files_refused: t.files_refused, + sites: t.sites + 1, + else_differs: if e { t.else_differs + 1 } else { t.else_differs }, + then_differs: if h { t.then_differs + 1 } else { t.then_differs }, + sites_differing: if e || h { t.sites_differing + 1 } else { t.sites_differing } + } +} + +fn iard_tally_file(t: IardTally, outcome: IardFileOutcome) -> IardTally { + match outcome { + IardFileRefused { path: p, reason: r } => + IardTally { + text: concat(t.text, concat(concat("file_refused file=", p), concat(concat(" reason=", symbol_lexeme(sym: r)), "\n"))), + files_read: t.files_read, + files_refused: t.files_refused + 1, + sites: t.sites, + else_differs: t.else_differs, + then_differs: t.then_differs, + sites_differing: t.sites_differing + } + IardFileRead { path: p, spans: spans, sites: sites } => + let counted = IardTally { + text: t.text, + files_read: t.files_read + 1, + files_refused: t.files_refused, + sites: t.sites, + else_differs: t.else_differs, + then_differs: t.then_differs, + sites_differing: t.sites_differing + } + fold_list(xs: sites, empty: counted, cons: fn(acc, s) { + iard_tally_site(t: acc, path: p, spans: spans, site: s) + }) + } +} + +// THE CENSUS'S ANSWER. The grammar failing to prepare is no observation at all, distinct from a +// file refusing: a refused file is counted and named, and the census goes on, so one run names +// every file it could not read. +type IardCensus + = IardGrammarUnprepared { reason: Symbol } + | IardCensusTaken { tally: IardTally } + +fn if_arm_reader_differential(ingest: SourceRootIngest) -> IardCensus { + match dag_prepared_grammar() { + Rejected { diagnostics: d } => IardGrammarUnprepared { reason: diagnostics_fatal_reason(d: d) } + Accepted { value: prepared, diagnostics: residue } => + IardCensusTaken { + tally: fold_list( + xs: ingest, + empty: IardTally { text: "", files_read: 0, files_refused: 0, sites: 0, else_differs: 0, then_differs: 0, sites_differing: 0 }, + cons: fn(acc, read) { iard_tally_file(t: acc, outcome: iard_file(prepared: prepared, residue: residue, read: read)) } + ) + } + } +} + +// THE TERMINAL LINE: the population and its denominators together, so a total of zero differing +// sites can be read against how many sites and files were actually observed. +fn if_arm_reader_differential_summary(t: IardTally) -> String { + concat( + concat("if_arm_reader_differential files_read=", integer_int_to_decimal_string(value: t.files_read)), + concat( + concat(" files_refused=", integer_int_to_decimal_string(value: t.files_refused)), + concat( + concat(" if_sites=", integer_int_to_decimal_string(value: t.sites)), + concat( + concat(" sites_differing=", integer_int_to_decimal_string(value: t.sites_differing)), + concat( + concat(" else_differs=", integer_int_to_decimal_string(value: t.else_differs)), + concat(concat(" then_differs=", integer_int_to_decimal_string(value: t.then_differs)), "\n") + ) + ) + ) + ) + ) +} + +fn if_arm_reader_differential_text(c: IardCensus) -> String { + match c { + IardGrammarUnprepared { reason: r } => concat("if_arm_reader_differential grammar_unprepared reason=", concat(symbol_lexeme(sym: r), "\n")) + IardCensusTaken { tally: t } => concat(t.text, if_arm_reader_differential_summary(t: t)) + } +} diff --git a/src/v2/test/claim/cli/if_arm_reader_differential_test.dag b/src/v2/test/claim/cli/if_arm_reader_differential_test.dag new file mode 100644 index 00000000000..a349787dd33 --- /dev/null +++ b/src/v2/test/claim/cli/if_arm_reader_differential_test.dag @@ -0,0 +1,132 @@ +module v2.test.claim.cli.if_arm_reader_differential + +import v2.cli.if_arm_reader_differential { + IardCensus, + IardCensusTaken, + IardGrammarUnprepared, + IardTally, + if_arm_reader_differential, + if_arm_reader_differential_text +} +import v2.compiler.source_authority { DagSourceReadWitness, SourceRootIngest } +import v2.std.artifact { Artifact, SourceFile } +import v2.std.cross_tree.import_model { DagTree } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import v2.std.logic { Bool } +import v2.std.node { Symbol } +import v2.std.text { String } +import std.algebra { Cons, Empty } +import extdeps.communication.medium { Lossless, Medium } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// THE DIFFERENTIAL'S COMPARATOR, DISCRIMINATED BEFORE IT IS TRUSTED WITH A POPULATION. +// +// THE SUBJECT is v2.cli.if_arm_reader_differential if_arm_reader_differential: for each if_expr +// capture it asks the frozen pre-gunbc#13029 readers and the positional +// body_lower_if_row_arms_optional for each arm and counts the sites where they answer a different +// occurrence. A census reporting zero is only evidence if its comparator can report one, so: +// +// THE POSITIVE CONTROL: `if a { if b { 1 } else { 2 } } else { 3 }`. The old else reader entered the +// then-arm and answered the NESTED else `{ 2 }` for the OUTER if; the positional reader answers +// `{ 3 }`. Exactly one site (the outer if) differs, on its else arm, and its row names the +// declaration and the byte range of the site and of each arm. +// THE NEGATIVE CONTROLS, where the old reader was right: an `else if` chain, and an if nested in the +// ELSE arm. Every site there agrees, so a comparator that reported everything as differing is red. +// A FILE THAT DOES NOT PARSE is counted as refused and named, never folded into zero sites. +data iard_nested_then_source: String = "module v2.test.iard_nested_then\n\nimport v2.std.logic { Bool }\nimport v2.std.integer { Int }\n\nfn iard_nested_then_probe(a: Bool, b: Bool) -> Int {\n if a {\n if b { 1 } else { 2 }\n } else {\n 3\n }\n}\n" + +data iard_agreeing_source: String = "module v2.test.iard_agreeing\n\nimport v2.std.logic { Bool }\nimport v2.std.integer { Int }\n\nfn iard_else_if_probe(a: Bool, b: Bool) -> Int {\n if a { 1 } else if b { 2 } else { 3 }\n}\n\nfn iard_nested_else_probe(a: Bool, b: Bool) -> Int {\n if a {\n 1\n } else {\n if b { 2 } else { 3 }\n }\n}\n" + +data iard_multibyte_source: String = "module v2.test.iard_multibyte\n\nimport v2.std.logic { Bool }\nimport v2.std.integer { Int }\nimport v2.std.text { String }\n\ndata iard_multibyte_label: String = \"é—日本語 ✓ 𝄞\"\n\nfn iard_multibyte_probe(a: Bool, b: Bool) -> Int {\n if a {\n if b { 1 } else { 2 }\n } else {\n 3\n }\n}\n" + +data iard_unparsable_source: String = "module v2.test.iard_unparsable\n\nfn iard_broken( -> {\n" + +fn iard_read(source: String, id: Symbol, unit: Symbol, path: String) -> DagSourceReadWitness { + DagSourceReadWitness { + source: Medium { carried: source, fidelity: Lossless }, + artifact: Artifact { kind: SourceFile, id: id, file_path: path }, + compilation_unit: unit, + source_root: DagTree + } +} + +data iard_nested_then_ingest: SourceRootIngest = Cons { + head: iard_read(source: iard_nested_then_source, id: ^iard_nested_then_artifact, unit: ^iard_nested_then_cu, path: "src/v2/test/fixture/cli/iard_nested_then.dag"), + tail: Empty +} + +data iard_multibyte_ingest: SourceRootIngest = Cons { + head: iard_read(source: iard_multibyte_source, id: ^iard_multibyte_artifact, unit: ^iard_multibyte_cu, path: "src/v2/test/fixture/cli/iard_multibyte.dag"), + tail: Empty +} + +data iard_agreeing_ingest: SourceRootIngest = Cons { + head: iard_read(source: iard_agreeing_source, id: ^iard_agreeing_artifact, unit: ^iard_agreeing_cu, path: "src/v2/test/fixture/cli/iard_agreeing.dag"), + tail: Cons { + head: iard_read(source: iard_unparsable_source, id: ^iard_unparsable_artifact, unit: ^iard_unparsable_cu, path: "src/v2/test/fixture/cli/iard_unparsable.dag"), + tail: Empty + } +} + +// ONE FRONT END PER INGEST, stored as the census's own portable answer (counts and rendered rows, +// no Node), enrolled WARM in v2.workflow.floor_pure_producer_share so each claim reads it. +type IardFixtureCensuses { + nested_then: IardCensus + multibyte: IardCensus + agreeing: IardCensus +} + +fn iard_fixture_censuses() -> IardFixtureCensuses { + IardFixtureCensuses { + nested_then: if_arm_reader_differential(ingest: iard_nested_then_ingest), + multibyte: if_arm_reader_differential(ingest: iard_multibyte_ingest), + agreeing: if_arm_reader_differential(ingest: iard_agreeing_ingest) + } +} + +fn iard_tally_holds(c: IardCensus, check: fn(IardTally) -> Bool) -> Bool { + match c { + IardGrammarUnprepared { reason: _ } => false + IardCensusTaken { tally: t } => check(t) + } +} + +test fn an_if_nested_in_the_then_arm_differs_on_the_outer_else_only_holds() -> Bool { + iard_tally_holds(c: iard_fixture_censuses().nested_then, check: fn(t) { + (t.files_read == 1) && (t.files_refused == 0) && (t.sites == 2) + && (t.sites_differing == 1) && (t.else_differs == 1) && (t.then_differs == 0) + }) +} + +// THE EXPECTED EXTENTS ARE THE SOURCE'S UTF-8 OCTETS: the outer `if` token, the nested else's `{` +// (what the old reader answered) and the outer else's `{` (the real arm). +test fn the_differing_row_names_the_declaration_the_arm_and_each_byte_range_holds() -> Bool { + string_contains( + s: if_arm_reader_differential_text(c: iard_fixture_censuses().nested_then), + pattern: "if_arm_differs file=src/v2/test/fixture/cli/iard_nested_then.dag decl=iard_nested_then_probe at=[bytes=148..150] arm=else differ old_at=[bytes=175..176] new_at=[bytes=190..191]\n" + ) +} + +// MULTIBYTE UTF-8 BEFORE THE SITE (calm-boar-904's objection to gunbc#13051): the label holds +// 2-, 3- and 4-octet scalars, so the outer `if` is at octet 237 but at scalar 223. A location +// counted in scalars anywhere on the route reports 223 and is red here. +test fn multibyte_source_before_a_site_is_located_in_octets_holds() -> Bool { + string_contains( + s: if_arm_reader_differential_text(c: iard_fixture_censuses().multibyte), + pattern: "if_arm_differs file=src/v2/test/fixture/cli/iard_multibyte.dag decl=iard_multibyte_probe at=[bytes=237..239] arm=else differ old_at=[bytes=264..265] new_at=[bytes=279..280]\n" + ) +} + +test fn an_else_if_chain_and_an_if_nested_in_the_else_arm_agree_holds() -> Bool { + iard_tally_holds(c: iard_fixture_censuses().agreeing, check: fn(t) { + (t.sites == 4) && (t.sites_differing == 0) && (t.else_differs == 0) && (t.then_differs == 0) + }) +} + +test fn a_file_that_does_not_parse_is_counted_refused_and_named_holds() -> Bool { + iard_tally_holds(c: iard_fixture_censuses().agreeing, check: fn(t) { + (t.files_read == 1) && (t.files_refused == 1) + }) + && string_contains(s: if_arm_reader_differential_text(c: iard_fixture_censuses().agreeing), pattern: "file_refused file=src/v2/test/fixture/cli/iard_unparsable.dag reason=") +} diff --git a/src/v2/test/claim/cli/v2_native_cli_test.dag b/src/v2/test/claim/cli/v2_native_cli_test.dag index 19a3e8f1ec2..1ee7e96ec1f 100644 --- a/src/v2/test/claim/cli/v2_native_cli_test.dag +++ b/src/v2/test/claim/cli/v2_native_cli_test.dag @@ -18,6 +18,8 @@ import v2.cli.compile_cli { CliTargetRust, CliTargetSwift, CliUsageRefused, + CliDifferentialReported, + CliIfArmReaderDifferential, cli_emit_target_spelling, v2_cli_exit, v2_cli_parse, @@ -68,6 +70,7 @@ fn plan_first_root(plan: CliPlan) -> String { HeadAbsent => "" HeadFound { value: first } => first } + CliIfArmReaderDifferential { source_roots: _ } => "" } } } @@ -79,6 +82,8 @@ fn outcome_refusal_reason(outcome: CliRunOutcome) -> Symbol { CliUsageRefused { reason: reason, detail: _ } => reason CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } => ^cli_outcome_was_not_refused + CliDifferentialReported { text: _, observed: _, detail: _ } => + ^cli_outcome_was_not_refused } } @@ -88,6 +93,7 @@ fn outcome_emitted(outcome: CliRunOutcome) -> Bool { CliCensusRefused { reason: _, detail: _, text: _ } => false CliUsageRefused { reason: _, detail: _ } => false CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } => true + CliDifferentialReported { text: _, observed: _, detail: _ } => false } } @@ -259,6 +265,7 @@ fn plan_is_target(plan: CliPlan, expected: CliEmitTarget) -> Bool { match verb { CliEmitClosure { entry: _, source_roots: _, target: target } => cli_emit_target_spelling(t: target) == cli_emit_target_spelling(t: expected) + CliIfArmReaderDifferential { source_roots: _ } => false } } } @@ -470,5 +477,6 @@ test fn a_trailing_token_refusal_locates_its_fatal_in_the_file_holds() -> Bool { && string_contains(s: detail, pattern: "parse_g0_tokens_remain @ cli_probe_trailing_cu bytes ") CliUsageRefused { reason: _, detail: _ } => false CliEmitted { text: _, closure_size: _, refused_count: _, refusal_detail: _ } => false + CliDifferentialReported { text: _, observed: _, detail: _ } => false } } diff --git a/src/v2/test/claim/provenance/occurrence_file_attribution_test.dag b/src/v2/test/claim/provenance/occurrence_file_attribution_test.dag index a8ec55bd083..c8b8b9f66be 100644 --- a/src/v2/test/claim/provenance/occurrence_file_attribution_test.dag +++ b/src/v2/test/claim/provenance/occurrence_file_attribution_test.dag @@ -21,6 +21,7 @@ import v2.cli.compile_cli { CliRunRefused, CliTargetRust, CliUsageRefused, + CliDifferentialReported, v2_cli_run } import v2.std.algebra { list_reverse, list_starts_with } @@ -282,6 +283,7 @@ fn door_probe_cli_detail() -> String { CliRunRefused { reason: _, detail: detail } => detail CliCensusRefused { reason: _, detail: detail, text: _ } => detail CliUsageRefused { reason: _, detail: detail } => detail + CliDifferentialReported { text: _, observed: _, detail: detail } => detail } } diff --git a/src/v2/workflow/floor_pure_producer_share.dag b/src/v2/workflow/floor_pure_producer_share.dag index 76d80a4b24a..b5d218451fb 100644 --- a/src/v2/workflow/floor_pure_producer_share.dag +++ b/src/v2/workflow/floor_pure_producer_share.dag @@ -652,8 +652,12 @@ import v2.std.collection { List } // THE XL-2 ELSE-ARM ROW IS THE SAME GROUND AT FIVE SPECIMENS: v2.test.claim.namespace_xl0.else_arm_nested_if // eam_outcomes drives one front end over five inline modules and stores one verdict arm per module. // THE IF-ARM-POSITION ROW IS THE SAME GROUND AT FOUR SPECIMENS: v2.test.claim.body_lowering.if_arm_position -// iap_verdicts tokenizes, parses and normalizes four inline modules and stores four Bools. Its first floor -// run (37073686904) refused all four claims over the enrolment margin, at 353k-687k eval steps each. +// iap_verdicts tokenizes, parses and normalizes four inline modules and stores four Bools. Unshared, +// each claim frame paid its own tokenize -> parse -> normalize, over the required floor's new-witness +// enrolment margin (v2.workflow.required_floor claim_eval_step_budget_for_identity). +// THE IF-ARM READER DIFFERENTIAL ROW IS THE SAME GROUND AT FOUR SPECIMENS: +// v2.test.claim.cli.if_arm_reader_differential iard_fixture_censuses tokenizes and parses four inline +// modules and stores three censuses (counts and rendered rows, no Node), portable for the same reason. // THE ARROW-LAMBDA BLOCK-BODY ROW IS THE SAME GROUND AT FIVE SPECIMENS: v2.test.claim.body_lowering.arrow_lambda_block_body // alb_verdicts normalizes five inline modules and stores five Bools; each would otherwise be over the floor's // new-witness enrolment margin (the required floor's claim_eval_step_budget_for_identity). @@ -907,6 +911,7 @@ data floor_cross_claim_pure_producers_warm: List = [ "v2.test.claim.namespace_xl0.wildcard_arm_resolve.wc_outcomes", "v2.test.claim.namespace_xl0.else_arm_nested_if.eam_outcomes", "v2.test.claim.body_lowering.if_arm_position.iap_verdicts", + "v2.test.claim.cli.if_arm_reader_differential.iard_fixture_censuses", "v2.test.claim.body_lowering.arrow_lambda_block_body.alb_verdicts", "v2.test.cli.v2_native_cli.cli_probe_trailing_outcome", "v2.test.emit.closure_emit_arrow_body_refusal.arrow_body_probe_verdict",