diff --git a/dag/gunbc/floor_cost_distribution.dag b/dag/gunbc/floor_cost_distribution.dag new file mode 100644 index 00000000000..f64d6df5e7a --- /dev/null +++ b/dag/gunbc/floor_cost_distribution.dag @@ -0,0 +1,226 @@ +module gunbc.floor_cost_distribution + +import std.types { Bool, Int, String, NonEmptyStr } +import std.measure { Millisecond, millisecond, millisecond_count } + +// THE DERIVATIONS BEHIND docs/plans/floor-cost-distribution-near-the-ceiling.md, SO THAT DOCUMENT +// CITES A PRODUCER INSTEAD OF CARRYING COPIED FIGURES. +// +// DESIGN section 6: name the instrument, never transcribe its output -- and if a measurement is +// worth re-deriving it is worth an entry point. The first version of that memo named +// `gunbc.witness_floor_workflow`, which produces the RAW ROWS and performs none of the analysis +// below; the ten-run intersection, the band histogram, the counterfactual replay and the inflation +// percentiles existed only as prose plus numbers. Review 58983 refused it on exactly that ground +// and was right to: a reader could check the arithmetic of nothing. +// +// EVERY FUNCTION HERE IS PURE AND TAKES ITS ROWS AS AN ARGUMENT. Reading the artifacts off disk is +// a separate fact and lives in `tools.floor_cost_distribution_instrument`, which is the §3 split +// between the interface (what the analysis IS) and the realization (where the bytes come from). +// The consequence that matters for evidence: every derivation below is witnessable on a hand-built +// fixture with no run artifacts present, which is what +// `test.claim.floor_cost_distribution_witness` does. +// +// THE INPUT IS THE `required-floor-claim-cost` ARTIFACT that every required-floor run uploads -- +// the COMPLETE executed population, not the 25-row `[over-cost]` preview the job log prints. + +type FloorCostRow { + identity: String + module: String + cpu_ms: Millisecond +} + +// One run's population, named by its run id so a finding can cite which runs it rests on. +type RunCost { + run: NonEmptyStr + rows: List +} + +// A TSV LINE IS EITHER A ROW OR IT IS NOT, AND "NOT" IS NEVER A FABRICATED ROW. The artifact +// carries a `#`-prefixed summary line and a header line before the data, and a short or +// unparseable line is refused rather than defaulted to zero -- a zero would sit silently at the +// bottom of every band and inflate the safe population. +fn parse_claim_cost_line(line: String) -> FloorCostRow? { + if starts_with(s: line, prefix: "#") { + none + } else { + let fields = split(s: line, delimiter: "\t") + if count(fields) < 6 { + none + } else if fields[0] == "identity" { + none + } else { + match parse_int(s: fields[5]) { + Absent => none + Present { value: cpu } => Present { + value: FloorCostRow { + identity: fields[0], + module: fields[1], + cpu_ms: millisecond(count: cpu) + } + } + } + } + } +} + +fn parse_claim_cost_tsv(text: String) -> List { + split(s: text, delimiter: "\n") + |> flat_map(line => match parse_claim_cost_line(line: line) { + Absent => [] + Present { value: row } => [row] + }) +} + +// THE BAND HISTOGRAM (question 1: is the population near the ceiling dense). +type CostBand { + lower_ms: Millisecond + upper_ms: Millisecond + rows: Int +} + +// COMPARISON GOES THROUGH `millisecond_count` RATHER THAN OVER THE CARRIER, because the carrier is +// what stops a figure read from one clock being compared against a figure read from another; the +// count is the projection a comparison is entitled to. +fn rows_in_band(rows: List, lower_ms: Millisecond, upper_ms: Millisecond) -> Int { + count(rows |> filter(r => + millisecond_count(m: r.cpu_ms) >= millisecond_count(m: lower_ms) + && millisecond_count(m: r.cpu_ms) < millisecond_count(m: upper_ms))) +} + +fn rows_at_or_above(rows: List, ms: Millisecond) -> Int { + count(rows |> filter(r => millisecond_count(m: r.cpu_ms) >= millisecond_count(m: ms))) +} + +fn cost_bands(rows: List, edges: List) -> List { + enumerate(edges) + |> flat_map(e => + if e.first + 1 < count(edges) { + [CostBand { + lower_ms: e.second, + upper_ms: edges[e.first + 1], + rows: rows_in_band(rows: rows, lower_ms: e.second, upper_ms: edges[e.first + 1]) + }] + } else { [] }) +} + +// CROSSINGS AND THE COUNTERFACTUAL REPLAY. +// +// `excluded_modules` is what makes the replay possible: it answers "what would this run have done +// if these modules had been repaired", which is the only way to separate a repair's effect from +// the ambient contention of the run it landed on. It is NOT a claim that excluded rows cost zero; +// it is a claim about which rows could have crossed. +fn row_is_excluded(row: FloorCostRow, excluded_modules: List) -> Bool { + any(excluded_modules, m => m == row.module) +} + +fn run_crossings(run: RunCost, ceiling_ms: Millisecond, excluded_modules: List) -> List { + run.rows + |> filter(r => millisecond_count(m: r.cpu_ms) >= millisecond_count(m: ceiling_ms) + && !row_is_excluded(row: r, excluded_modules: excluded_modules)) + |> map(r => r.identity) +} + +fn run_worst_cpu(run: RunCost, excluded_modules: List) -> Millisecond { + fold( + run.rows |> filter(r => !row_is_excluded(row: r, excluded_modules: excluded_modules)), + init: millisecond(count: 0), + f: (worst, r) => if millisecond_count(m: r.cpu_ms) > millisecond_count(m: worst) { r.cpu_ms } else { worst } + ) +} + +// A FLOOR RUN REFUSES IF ANY ONE ROW CROSSES, so the run-level verdict is existential over rows +// and the fleet-level figure is a COUNT OF RUNS, never a count of rows: fifteen crossings in one +// run and one crossing in another are the same number of refused merges. +fn runs_refused(runs: List, ceiling_ms: Millisecond, excluded_modules: List) -> Int { + count(runs |> filter(run => count(run_crossings(run: run, ceiling_ms: ceiling_ms, excluded_modules: excluded_modules)) > 0)) +} + +fn worst_over_runs(runs: List, excluded_modules: List) -> Millisecond { + fold(runs, init: millisecond(count: 0), f: (worst, run) => { + let w = run_worst_cpu(run: run, excluded_modules: excluded_modules) + if millisecond_count(m: w) > millisecond_count(m: worst) { w } else { worst } + }) +} + +// MAX-OVER-RUNS, the contention-independent ranking. Membership of any single run's crossing set is +// a function of that run's load as much as of the row, so a candidate population must be ranked on +// the worst each row has been observed to cost, never on one run's sample. +fn identity_max_cpu(runs: List, identity: String) -> Millisecond { + fold(runs, init: millisecond(count: 0), f: (worst, run) => + fold(run.rows |> filter(r => r.identity == identity), init: worst, f: (w, r) => + if millisecond_count(m: r.cpu_ms) > millisecond_count(m: w) { r.cpu_ms } else { w })) +} + +fn module_max_cpu(runs: List, module: String) -> Millisecond { + fold(runs, init: millisecond(count: 0), f: (worst, run) => + fold(run.rows |> filter(r => r.module == module), init: worst, f: (w, r) => + if millisecond_count(m: r.cpu_ms) > millisecond_count(m: w) { r.cpu_ms } else { w })) +} + +// THE REPAIR LADDER: what removing one more module actually buys, which is the quantity that +// decides whether per-producer repair is the right unit of work. +type LadderStep { + removed: String + worst_ms: Millisecond + runs_refused: Int +} + +fn repair_ladder( + runs: List, + ceiling_ms: Millisecond, + already_excluded: List, + order: List +) -> List { + enumerate(order) + |> map(e => { + let excluded = concat(already_excluded, order |> take(n: e.first + 1)) + LadderStep { + removed: e.second, + worst_ms: worst_over_runs(runs: runs, excluded_modules: excluded), + runs_refused: runs_refused(runs: runs, ceiling_ms: ceiling_ms, excluded_modules: excluded) + } + }) +} + +// RUN-TO-RUN INFLATION, PAIRED ON IDENTITY. +// +// CARRIED IN PERMILLE AS AN Int RATHER THAN AS A RATIO, deliberately. The quantity is a ratio, but +// an exact integer permille is comparable and orderable without a float, and the percentile below +// is an ORDER STATISTIC -- it selects an observed value and never interpolates between two, so no +// figure it returns is one this repository did not measure. +// +// `min_base_ms` exists because the artifact records whole milliseconds: at a 2ms baseline a +// one-millisecond scheduling difference is a 50% "inflation" that says nothing about contention. +// Rows below the floor are dropped rather than smoothed. +fn identity_inflation_permille(base: RunCost, other: RunCost, min_base_ms: Millisecond) -> List { + base.rows + |> filter(b => millisecond_count(m: b.cpu_ms) >= millisecond_count(m: min_base_ms)) + |> flat_map(b => match other.rows |> filter(o => o.identity == b.identity) |> first { + Absent => [] + Present { value: o } => [(millisecond_count(m: o.cpu_ms) * 1000) / millisecond_count(m: b.cpu_ms)] + }) +} + +// An order statistic over the observed values: `percent` is 0-100 and the result is always a +// member of the input, so it is a measurement and not an interpolation. +fn permille_percentile(values: List, percent: Int) -> Int? { + let sorted = values |> sort_by(v => v) + let n = count(sorted) + if n == 0 { + none + } else { + let idx = (percent * n) / 100 + if idx >= n { Present { value: sorted[n - 1] } } else { Present { value: sorted[idx] } } + } +} + +// THE MARGIN THE MEASURED INFLATION IMPLIES: the largest clean-run cost that still lands under the +// ceiling after inflation. Integer division floors, which rounds the budget DOWN -- the safe +// direction for a limit. +fn implied_clean_run_budget(ceiling_ms: Millisecond, inflation_permille: Int) -> Millisecond { + if inflation_permille <= 0 { + ceiling_ms + } else { + millisecond(count: (millisecond_count(m: ceiling_ms) * 1000) / inflation_permille) + } +} diff --git a/dag/gunbc/instruments/floor_cost_distribution_instrument.dag b/dag/gunbc/instruments/floor_cost_distribution_instrument.dag new file mode 100644 index 00000000000..6b49b1750f1 --- /dev/null +++ b/dag/gunbc/instruments/floor_cost_distribution_instrument.dag @@ -0,0 +1,277 @@ +module tools.floor_cost_distribution_instrument + +import extdeps.filesystem.filesystem_io +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.measure { Millisecond, millisecond, millisecond_count } +import std.types { Bool, Int, String, NonEmptyStr } +import gunbc.floor_cost_distribution { + FloorCostRow, + RunCost, + CostBand, + LadderStep, + parse_claim_cost_tsv, + cost_bands, + rows_at_or_above, + runs_refused, + worst_over_runs, + module_max_cpu, + repair_ladder, + identity_inflation_permille, + permille_percentile, + implied_clean_run_budget +} + +// THE ENTRY POINT FOR docs/plans/floor-cost-distribution-near-the-ceiling.md. +// +// Recipe, and it needs no floor run -- the inputs are artifacts the required floor already +// uploads on every run: +// +// gh run download -n required-floor-claim-cost -D / +// gunbc run --source-root dag --source-root src/v2 \ +// --entry dag/gunbc/instruments/floor_cost_distribution_instrument.dag \ +// --function floor_cost_distribution_report +// +// WHY THIS IS A SEPARATE MODULE FROM `gunbc.floor_cost_distribution`: reading bytes off disk is a +// realization and the analysis is an interface (DESIGN section 3). The split is what lets the +// witness exercise every derivation on a hand-built fixture with no artifacts present, so the +// evidence does not depend on which runs happen to be downloaded. +// +// AN ABSENT OR UNREADABLE FILE REFUSES AND IS NAMED. It does not contribute an empty population: +// an empty artifact silently emptied a ten-way identity intersection during the original analysis, +// and a run that reads as zero rows is indistinguishable from a run where nothing crossed. That is +// the absorbing fallback DESIGN section 5 forbids, so the arm below is typed and counted. + +// THE CEILING CARRIES ITS UNIT. `v2.workflow.required_floor` `required_floor_claim_cpu_safety_limit_ms` +// is the authority for the value; this row is the analysis-side reading of it, and it is a +// Millisecond rather than a bare Int so it cannot be compared against a figure read from another +// clock -- the exact confusion `std.measure` exists to make unwritable. +data floor_cost_ceiling: Millisecond = millisecond(count: 500) + +// The default clean-run/contended pair the memo's inflation figures rest on, and the default +// download root. Both are arguments to the functions below; these rows only name the defaults so +// the recipe above is runnable as written. +data floor_cost_artifact_root: String = "/tmp/floor-cost" +data floor_cost_artifact_file: String = "required_floor_claim_cost.tsv" + +type RunLoad + = RunLoaded { run: RunCost } + | RunUnreadable { run: NonEmptyStr, path: String } + | RunEmpty { run: NonEmptyStr, path: String } + +fn run_cost_path(root: String, run: NonEmptyStr) -> String { + concat(concat(root, concat("/", run as String)), concat("/", floor_cost_artifact_file)) +} + +fn load_run_cost(root: String, run: NonEmptyStr) -> RunLoad { + let path = run_cost_path(root: root, run: run) + let read = Filesystem.Read(path: path) + if !read.success { + RunUnreadable { run: run, path: path } + } else { + let rows = parse_claim_cost_tsv(text: read.content) + if count(rows) == 0 { + RunEmpty { run: run, path: path } + } else { + RunLoaded { run: RunCost { run: run, rows: rows } } + } + } +} + +fn loaded_runs(loads: List) -> List { + loads |> flat_map(l => match l { + RunLoaded { run: r } => [r] + RunUnreadable { run: _, path: _ } => [] + RunEmpty { run: _, path: _ } => [] + }) +} + +fn refused_load_lines(loads: List) -> List { + loads |> flat_map(l => match l { + RunLoaded { run: _ } => [] + RunUnreadable { run: r, path: p } => [concat(concat("REFUSED unreadable run=", r as String), concat(" path=", p))] + RunEmpty { run: r, path: p } => [concat(concat("REFUSED empty run=", r as String), concat(" path=", p))] + }) +} + +fn band_line(b: CostBand) -> String { + concat( + concat(to_string(millisecond_count(m: b.lower_ms)), concat("-", to_string(millisecond_count(m: b.upper_ms)))), + concat("\t", to_string(b.rows)) + ) +} + +fn ladder_line(s: LadderStep) -> String { + concat( + s.removed, + concat(concat("\tworst=", to_string(millisecond_count(m: s.worst_ms))), concat("\trefused=", to_string(s.runs_refused))) + ) +} + +fn percentile_line(values: List, percent: Int) -> String { + concat( + concat("p", to_string(percent)), + match permille_percentile(values: values, percent: percent) { + Absent => "\tno-observation" + Present { value: v } => concat( + concat("\t", concat(to_string(v), " permille")), + concat("\tclean-run budget ", concat(to_string(millisecond_count(m: implied_clean_run_budget(ceiling_ms: floor_cost_ceiling, inflation_permille: v))), "ms")) + ) + } + ) +} + +// The whole analysis over one loaded set of runs. `baseline` and `contended` name the pair the +// inflation percentiles are paired over; `repaired` is the counterfactual exclusion set; `order` +// is the ladder's removal order. +fn floor_cost_distribution_lines( + loads: List, + baseline: NonEmptyStr, + contended: NonEmptyStr, + repaired: List, + order: List, + band_edges: List, + min_base_ms: Millisecond +) -> List { + let runs = loaded_runs(loads: loads) + let refusals = refused_load_lines(loads: loads) + let latest = runs |> filter(r => (r.run as String) == (contended as String)) |> first + concat( + concat( + concat( + [concat("runs loaded\t", to_string(count(runs)))], + refusals + ), + match latest { + Absent => [concat("REFUSED no population for contended run ", contended as String)] + Present { value: c } => concat( + concat( + [concat("band histogram for run ", c.run as String)], + cost_bands(rows: c.rows, edges: band_edges) |> map(b => band_line(b: b)) + ), + [ + concat("rows at or above 300ms\t", to_string(rows_at_or_above(rows: c.rows, ms: millisecond(count: 300)))), + concat("rows at or above 400ms\t", to_string(rows_at_or_above(rows: c.rows, ms: millisecond(count: 400)))) + ] + ) + } + ), + concat( + [ + concat("runs refused as observed\t", to_string(runs_refused(runs: runs, ceiling_ms: floor_cost_ceiling, excluded_modules: []))), + concat("runs refused with the repaired modules excluded\t", to_string(runs_refused(runs: runs, ceiling_ms: floor_cost_ceiling, excluded_modules: repaired))), + concat("worst row over all runs, repaired excluded\t", to_string(millisecond_count(m: worst_over_runs(runs: runs, excluded_modules: repaired)))) + ], + concat( + repair_ladder(runs: runs, ceiling_ms: floor_cost_ceiling, already_excluded: repaired, order: order) + |> map(s => ladder_line(s: s)), + match runs |> filter(r => (r.run as String) == (baseline as String)) |> first { + Absent => [concat("REFUSED no population for baseline run ", baseline as String)] + Present { value: b } => match latest { + Absent => [] + Present { value: c } => { + let infl = identity_inflation_permille(base: b, other: c, min_base_ms: min_base_ms) + concat( + [concat("paired inflation observations\t", to_string(count(infl)))], + [50, 90, 95, 99] |> map(p => percentile_line(values: infl, percent: p)) + ) + } + } + } + ) + ) + ) +} + +fn floor_cost_distribution_report_for( + root: String, + run_ids: List, + baseline: NonEmptyStr, + contended: NonEmptyStr, + repaired: List, + order: List +) -> String { + join( + floor_cost_distribution_lines( + loads: run_ids |> map(r => load_run_cost(root: root, run: r)), + baseline: baseline, + contended: contended, + repaired: repaired, + order: order, + band_edges: [0, 10, 50, 100, 200, 300, 400, 500, 1000000] |> map(e => millisecond(count: e)), + min_base_ms: millisecond(count: 50) + ), + "\n" + ) +} + +// THE ZERO-ARGUMENT ENTRY POINT, so the recipe at the top of this module is runnable as written and +// the memo can cite a producer rather than carry figures. +// +// THE RUN IDS ARE DATA HERE BECAUSE THE FINDING IS ABOUT THESE RUNS. A measurement over "whatever +// happened to be downloaded" is not re-derivable -- a second reader would get different numbers and +// have no way to tell a real change from a different sample. Naming them is the citation. +data floor_cost_sampled_runs: List = [ + "33668368846" as NonEmptyStr, + "33670164259" as NonEmptyStr, + "33673806092" as NonEmptyStr, + "33680659583" as NonEmptyStr, + "33681059548" as NonEmptyStr, + "33682522296" as NonEmptyStr, + "33683651328" as NonEmptyStr, + "33688140761" as NonEmptyStr, + "33690512169" as NonEmptyStr, + "33691893294" as NonEmptyStr +] + +// The clean/contended pair the inflation percentiles are paired over. `33690512169` is the last +// FloorClean run before gunbc#10133 landed; `33691893294` is the merge commit of #10133 and the +// most contended run in the sample, which is why it is the one the band histogram is read from. +data floor_cost_baseline_run: NonEmptyStr = "33690512169" as NonEmptyStr +data floor_cost_contended_run: NonEmptyStr = "33691893294" as NonEmptyStr + +// The modules gunbc#10133 repaired. Excluding them replays every run as if the repair had already +// landed, which is what separates the repair's effect from the contention of the run it landed on. +data floor_cost_repaired_modules: List = [ + "test.claim.self_host_compile_phase_live_gate_witness", + "test.claim.self_host_compile_phase_frontier_witness", + "test.claim.compiler_frontend_program_status_witness" +] + +// Ladder order: remaining modules by max-over-runs cost, worst first. +data floor_cost_ladder_order: List = [ + "v2.test.emit.rust_produced_decl_emit", + "v2.test.execution.emit_host_field_access_equals_eval", + "v2.test.execution.emit_host_fold_closure_equals_eval", + "v2.test.emit.semantic_decl_serialize_parity", + "v2.test.execution.emit_host_meet_join_equals_eval", + "v2.test.execution.emit_host_complement_equals_eval", + "v2.test.emit.produced_decl_two_target", + "v2.test.execution.emit_host_add_equals_eval", + "v2.test.emit.semantic_decl_routing_generic", + "v2.test.claim.enforcement.lens_module_gate_witness" +] + +fn floor_cost_distribution_report() -> String { + floor_cost_distribution_report_for( + root: floor_cost_artifact_root, + run_ids: floor_cost_sampled_runs, + baseline: floor_cost_baseline_run, + contended: floor_cost_contended_run, + repaired: floor_cost_repaired_modules, + order: floor_cost_ladder_order + ) +} + +// THE ACTUATOR. It REFUSES when any named run failed to load rather than reporting over whatever +// was readable: a report derived from six of ten runs is a different measurement wearing the same +// name, and the percentiles below it would be a statement about which downloads happened to +// succeed. The refusal names the runs, so the operator's next move is a download and not a guess. +fn floor_cost_distribution_check() -> ProcessExit { + let loads = floor_cost_sampled_runs |> map(r => load_run_cost(root: floor_cost_artifact_root, run: r)) + let refusals = refused_load_lines(loads: loads) + if count(refusals) > 0 { + exit_failure(reason: join(concat(["floor cost distribution: refusing an incomplete sample"], refusals), "\n")) + } else { + ExitSuccess + } +} diff --git a/dag/test/claim/floor_cost_distribution_witness_test.dag b/dag/test/claim/floor_cost_distribution_witness_test.dag new file mode 100644 index 00000000000..aab4ffafeed --- /dev/null +++ b/dag/test/claim/floor_cost_distribution_witness_test.dag @@ -0,0 +1,178 @@ +module test.claim.floor_cost_distribution_witness + +import std.types { Bool, Int, String, NonEmptyStr } +import std.measure { Millisecond, millisecond, millisecond_count } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import tools.floor_cost_distribution_instrument { + RunLoad, + RunLoaded, + RunEmpty, + RunUnreadable, + loaded_runs, + refused_load_lines +} +import gunbc.floor_cost_distribution { + FloorCostRow, + RunCost, + CostBand, + LadderStep, + parse_claim_cost_line, + parse_claim_cost_tsv, + rows_in_band, + rows_at_or_above, + cost_bands, + run_crossings, + run_worst_cpu, + runs_refused, + worst_over_runs, + identity_max_cpu, + module_max_cpu, + repair_ladder, + identity_inflation_permille, + permille_percentile, + implied_clean_run_budget +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// THE FIXTURE IS HAND-BUILT AND TINY, DELIBERATELY. These derivations are consumed against ten +// downloaded run artifacts, and a witness that read those artifacts would be measuring whichever +// runs happened to be on disk -- green or red for reasons outside the diff. The subject here is +// the ARITHMETIC, so the fixture is small enough that every expected value is decidable by hand. +// +// Shape: two runs over the same three identities in two modules, one of which ("hot") crosses the +// 500ms ceiling on run B only. That is the exact shape the real analysis turns on -- a row whose +// membership in the crossing set depends on the run rather than on the row. +data row_a1: FloorCostRow = FloorCostRow { identity: "m.hot.x", module: "m.hot", cpu_ms: millisecond(count: 400) } +data row_a2: FloorCostRow = FloorCostRow { identity: "m.mid.y", module: "m.mid", cpu_ms: millisecond(count: 200) } +data row_a3: FloorCostRow = FloorCostRow { identity: "m.mid.z", module: "m.mid", cpu_ms: millisecond(count: 50) } + +data row_b1: FloorCostRow = FloorCostRow { identity: "m.hot.x", module: "m.hot", cpu_ms: millisecond(count: 600) } +data row_b2: FloorCostRow = FloorCostRow { identity: "m.mid.y", module: "m.mid", cpu_ms: millisecond(count: 300) } +data row_b3: FloorCostRow = FloorCostRow { identity: "m.mid.z", module: "m.mid", cpu_ms: millisecond(count: 100) } + +data run_a: RunCost = RunCost { run: "runA" as NonEmptyStr, rows: [row_a1, row_a2, row_a3] } +data run_b: RunCost = RunCost { run: "runB" as NonEmptyStr, rows: [row_b1, row_b2, row_b3] } +data both_runs: List = [run_a, run_b] + +// PARSING: the artifact's own preamble must not become rows. +data sample_tsv: String = "# summary\texecuted=3\tcost_basis=cpu\nidentity\tmodule\toutcome\tverdict_reached\twall_ms\tcpu_ms\teval_steps\tcost_line_ms\nm.hot.x\tm.hot\tpass\ttrue\t402\t400\t1000\t100\nm.mid.y\tm.mid\tpass\ttrue\t201\t200\t900\t100" + +test fn the_summary_and_header_lines_are_not_rows() -> Bool { + count(parse_claim_cost_tsv(text: sample_tsv)) == 2 +} + +test fn a_parsed_row_takes_cpu_from_the_cpu_column_not_the_wall_column() -> Bool { + match parse_claim_cost_tsv(text: sample_tsv) |> first { + Absent => false + Present { value: r } => millisecond_count(m: r.cpu_ms) == 400 && r.identity == "m.hot.x" && r.module == "m.hot" + } +} + +// THE DISCRIMINATING RED FOR THE PARSER, and it is the one that matters: a short or unparseable +// line must produce NO row rather than a zero-cost one. A fabricated zero would sit silently in +// the bottom band and make the population look safer than it is. +test fn an_unparseable_line_yields_no_row_rather_than_a_zero_cost_row() -> Bool { + match parse_claim_cost_line(line: "m.broken\tm\tpass") { + Present { value: _ } => false + Absent => match parse_claim_cost_line(line: "m.x\tm\tpass\ttrue\t10\tNOT-A-NUMBER\t1\t100") { + Present { value: _ } => false + Absent => true + } + } +} + +test fn bands_partition_the_population_and_the_cumulative_counts_agree() -> Bool { + let bands = cost_bands(rows: run_b.rows, edges: [0, 100, 300, 1000] |> map(e => millisecond(count: e))) + count(bands) == 3 + && rows_in_band(rows: run_b.rows, lower_ms: millisecond(count: 0), upper_ms: millisecond(count: 100)) == 0 + && rows_in_band(rows: run_b.rows, lower_ms: millisecond(count: 100), upper_ms: millisecond(count: 300)) == 1 + && rows_in_band(rows: run_b.rows, lower_ms: millisecond(count: 300), upper_ms: millisecond(count: 1000)) == 2 + && rows_at_or_above(rows: run_b.rows, ms: millisecond(count: 300)) == 2 + && rows_at_or_above(rows: run_b.rows, ms: millisecond(count: 500)) == 1 +} + +// A RUN REFUSES EXISTENTIALLY OVER ROWS: runA has no crossing, runB has one. +test fn crossings_are_per_run_and_a_refusal_needs_only_one_row() -> Bool { + count(run_crossings(run: run_a, ceiling_ms: millisecond(count: 500), excluded_modules: [])) == 0 + && count(run_crossings(run: run_b, ceiling_ms: millisecond(count: 500), excluded_modules: [])) == 1 + && runs_refused(runs: both_runs, ceiling_ms: millisecond(count: 500), excluded_modules: []) == 1 +} + +// THE COUNTERFACTUAL REPLAY, which is the derivation the memo's central correction rests on: +// excluding the crossing module must take the refused-run count to zero. +test fn excluding_the_crossing_module_replays_the_runs_clean() -> Bool { + runs_refused(runs: both_runs, ceiling_ms: millisecond(count: 500), excluded_modules: ["m.hot"]) == 0 + && runs_refused(runs: both_runs, ceiling_ms: millisecond(count: 500), excluded_modules: ["m.mid"]) == 1 +} + +// AND ITS DISCRIMINATING RED: excluding an UNRELATED module must not change the verdict. Without +// this, an exclusion that silently dropped every row would also read as "clean" and the +// counterfactual would be vacuous. +test fn excluding_an_unrelated_module_leaves_the_refusal_standing() -> Bool { + runs_refused(runs: both_runs, ceiling_ms: millisecond(count: 500), excluded_modules: ["m.absent"]) == 1 +} + +test fn the_worst_row_is_taken_over_every_run_not_the_latest() -> Bool { + millisecond_count(m: run_worst_cpu(run: run_a, excluded_modules: [])) == 400 + && millisecond_count(m: worst_over_runs(runs: both_runs, excluded_modules: [])) == 600 + && millisecond_count(m: worst_over_runs(runs: both_runs, excluded_modules: ["m.hot"])) == 300 + && millisecond_count(m: identity_max_cpu(runs: both_runs, identity: "m.mid.y")) == 300 + && millisecond_count(m: module_max_cpu(runs: both_runs, module: "m.mid")) == 300 +} + +// THE LADDER: each step must be CUMULATIVE with the ones before it, not independent. +test fn the_repair_ladder_accumulates_its_removals() -> Bool { + let steps = repair_ladder(runs: both_runs, ceiling_ms: millisecond(count: 500), already_excluded: [], order: ["m.hot", "m.mid"]) + count(steps) == 2 + && millisecond_count(m: steps[0].worst_ms) == 300 && steps[0].runs_refused == 0 + && millisecond_count(m: steps[1].worst_ms) == 0 && steps[1].runs_refused == 0 +} + +// INFLATION IS PAIRED ON IDENTITY, and the min_base filter drops rows too small to measure. +// m.hot.x 400->600 is 1500 permille; m.mid.y 200->300 is 1500; m.mid.z 50->100 is 2000. +test fn inflation_is_paired_on_identity_and_carried_in_permille() -> Bool { + let all = identity_inflation_permille(base: run_a, other: run_b, min_base_ms: millisecond(count: 50)) + let big = identity_inflation_permille(base: run_a, other: run_b, min_base_ms: millisecond(count: 100)) + count(all) == 3 && count(big) == 2 + && any(all, v => v == 2000) + && !any(big, v => v == 2000) +} + +// THE PERCENTILE IS AN ORDER STATISTIC: every value it returns was observed. +test fn the_percentile_returns_an_observed_value_and_refuses_an_empty_population() -> Bool { + match permille_percentile(values: [1000, 1200, 1500], percent: 50) { + Absent => false + Present { value: v } => v == 1200 && match permille_percentile(values: [], percent: 50) { + Present { value: _ } => false + Absent => true + } + } +} + +test fn the_implied_budget_is_the_ceiling_deflated_by_the_measured_inflation() -> Bool { + millisecond_count(m: implied_clean_run_budget(ceiling_ms: millisecond(count: 500), inflation_permille: 1600)) == 312 + && millisecond_count(m: implied_clean_run_budget(ceiling_ms: millisecond(count: 500), inflation_permille: 1000)) == 500 +} + +// THE LOAD REFUSAL ARMS. `RunEmpty` exists because an artifact DID download empty during the +// original analysis and silently emptied a ten-way identity intersection; a run that reads as zero +// rows is indistinguishable from a run in which nothing crossed, so it must refuse rather than +// contribute an empty population. These two rows keep both arms produced rather than merely +// declared. +test fn an_unreadable_or_empty_run_is_named_and_never_counted_as_loaded() -> Bool { + let loads = [ + RunLoaded { run: run_a }, + RunEmpty { run: "runEmpty" as NonEmptyStr, path: "/x/runEmpty/required_floor_claim_cost.tsv" }, + RunUnreadable { run: "runGone" as NonEmptyStr, path: "/x/runGone/required_floor_claim_cost.tsv" } + ] + count(loaded_runs(loads: loads)) == 1 + && count(refused_load_lines(loads: loads)) == 2 + && any(refused_load_lines(loads: loads), l => l == "REFUSED empty run=runEmpty path=/x/runEmpty/required_floor_claim_cost.tsv") +} + +// AND ITS DISCRIMINATING RED: an all-loaded set must produce NO refusal lines, so the arm above +// cannot be satisfied by a function that always reports something. +test fn a_complete_sample_produces_no_refusal_lines() -> Bool { + count(refused_load_lines(loads: [RunLoaded { run: run_a }, RunLoaded { run: run_b }])) == 0 +} diff --git a/docs/plans/floor-cost-distribution-near-the-ceiling.md b/docs/plans/floor-cost-distribution-near-the-ceiling.md new file mode 100644 index 00000000000..ba82d9da522 --- /dev/null +++ b/docs/plans/floor-cost-distribution-near-the-ceiling.md @@ -0,0 +1,165 @@ +# The floor cost distribution near the 500ms ceiling + +*Measurement, not a repair. Commissioned by tidy-lynx-804 after gunbc#10133 to decide whether +further per-producer work is a treadmill. The question is whether the population near the ceiling +is dense, whether the rows now crossing share a root, and what an honest margin policy would be.* + +## The instrument + +**Every figure below is produced by `tools.floor_cost_distribution_instrument` +`floor_cost_distribution_report`, and the derivations it composes are +`gunbc.floor_cost_distribution`.** Re-derive rather than trust this page: + +``` +# one download per run id in `floor_cost_sampled_runs`, into `floor_cost_artifact_root`: +gh run download -n required-floor-claim-cost -D /tmp/floor-cost/ + +gunbc run --source-root dag --source-root src/v2 \ + --entry dag/gunbc/instruments/floor_cost_distribution_instrument.dag \ + --function floor_cost_distribution_report +``` + +The first version of this memo named `gunbc.witness_floor_workflow` as its instrument. That +workflow produces the raw rows and performs none of the analysis — the ten-run intersection, the +band histogram, the counterfactual replay and the inflation percentiles existed only as prose plus +numbers, so a reader could check the arithmetic of nothing. Review 58983 refused it on exactly that +ground. The figures that remain below are illustrative of what the instrument returned on the runs +it names; the instrument is the authority, and where the two disagree the instrument is right. + +**The input needs no floor run.** `required-floor-claim-cost` is uploaded by every required-floor +run and carries the COMPLETE executed population — the `[over-cost]` lines in the job log are a +25-row preview of the same data and say so. The runs are named as data in +`floor_cost_sampled_runs`, because a measurement over "whatever happened to be downloaded" is not +re-derivable: a second reader would get different numbers with no way to tell a real change from a +different sample. + +**An incomplete sample refuses.** `floor_cost_distribution_check` exits non-zero naming every run +that failed to load. One artifact in an eleventh run (`33667468330`) downloaded empty during the +original analysis and silently emptied a ten-way identity intersection before it was caught — a run +that reads as zero rows is indistinguishable from a run in which nothing crossed, which is the +absorbing fallback DESIGN §5 forbids. Both refusal arms are witnessed in +`test.claim.floor_cost_distribution_witness`, along with a discriminating red for each derivation +below. + +## 1. The band is dense, and the tail is a plateau + +Post-repair run `33691893294`, CPU basis, 3,498 rows: median 2ms, mean 34ms, p90 86ms, p99 397ms. + +| CPU band | rows | cumulative from top | +| --- | --- | --- | +| ≥500 | 5 | 5 | +| 400–500 | 27 | 32 | +| 300–400 | 55 | 87 | +| 200–300 | 89 | 176 | +| 100–200 | 148 | 324 | +| <100 | 3,174 | 3,498 | + +**There is no gap below the ceiling.** The decision-relevant form of that is what happens if modules +are repaired top-down. Taking the worst row remaining across all ten runs, removing modules in +max-cost order after the three gunbc#10133 already fixed: + +| modules repaired | worst row remaining | runs refused /10 | +| --- | --- | --- | +| (the 3 from #10133) | 533 | 1 | +| +`rust_produced_decl_emit` | 525 | 1 | +| +`emit_host_field_access_equals_eval` | 511 | 1 | +| +`emit_host_fold_closure_equals_eval` | 490 | 0 | +| +`semantic_decl_serialize_parity` | 487 | 0 | +| +`emit_host_meet_join_equals_eval` | 486 | 0 | +| +5 more modules | 429 | 0 | + +The first three modules buy 43ms. The next seven buy 61ms — roughly **10ms per module**. That is the +treadmill, quantified: below the top of the band each repair returns about one fiftieth of the +ceiling. + +## 2. The new crossers do NOT share a root the way the #10133 three did + +`eval_steps` is a cheap structural fingerprint of shared work, and it separates the two cases +cleanly. + +The three modules gunbc#10133 repaired reported **612,662–658,336** eval steps — a 1.07x spread +across seventeen rows in three modules with unrelated subjects. That uniformity *is* the signature of +one shared dominant producer, and it was: `compile_phase_frontier_standing`. + +The families now near the ceiling report **63,196–197,227** eval steps — a 3.1x spread. They are +three distinct shapes rather than one: + +| family | rows ≥200ms | eval_steps | wall/cpu | +| --- | --- | --- | --- | +| `v2.test.execution.emit_host_*_equals_eval` | 33 | 63k–174k | **2.05** | +| `v2.test.emit.semantic_decl_*` | 27 | 78k–141k | 1.01 | +| `v2.test.emit.rust_*` / `produced_decl_*` | 48 | 77k–197k | 1.00 | + +The `emit_host_*` family spends half its wall clock off-CPU, which is the region +`v2.workflow.required_floor` already carries a §4b rung drop for (`OpaqueHostCallUnbounded`). + +**There is one shared-producer lead, and it is not a plan.** `required-floor-cross-claim-demand` +ranks `bind_outcome` (`v2.std.diagnostic`, unkeyed, 679 claims / 108,085 evals / 183 modules) at the +top by a factor of 13, and its module samples reach both the emit and the execution families. But +that artifact's own header states `cost_columns=inclusive_of_callees_do_not_sum`, and its column +totals 1,736,654ms against the run's actual `claim_cpu_total_ms=120,177` — a factor of 14. It ranks +candidates; it cannot quantify a saving, and it must not be cited as one. + +## 3. The margin, measured + +Run-to-run variation on the **same identity** across the sample, restricted to rows with enough +signal that integer milliseconds are not noise: + +- median **1.16x**, p90 **1.35x**, p95 ~1.43x, max ~**2.0x**. + +Two mechanism hypotheses were tested and both are refuted, which is what makes a single global margin +defensible: + +- **Expensive rows do not inflate more.** By baseline-cost bucket the median inflation is 1.18, 1.13, + 1.19, 1.17, 1.07, 0.92 for the 20–50 … 400–500ms buckets. +- **Host-call-heavy rows do not inflate more.** By wall/cpu class the median inflation is 1.16 (pure + CPU), 1.13, 1.16 (host-heavy ≥1.8). + +Run-level contention alone spans only 0.92x–1.15x across the ten runs, so most of the spread is +per-row rather than per-run. And run `33688140761` **refused with a below-median run factor of +0.96**, which is why a refusal cannot be read as evidence that the fleet was busy. + +**PR runs are harsher than `main` runs.** All four modules reported crossing on PR runs +(#10109/#10122/#10123) have `main`-run medians of 314–391ms and `main`-run maxima of 417–487ms, so +the PR path reaches roughly 1.6x on a median row. + +At 1.6x a row must sit under **312ms** on a clean run to be safe; at the observed 2.0x, under +**250ms**. **87 rows exceed 300ms today.** + +## 4. What this corrects + +The framing that prompted this measurement was that gunbc#10133 promoted the next rows down the +ranking into the margin. **The promotion mechanism is refuted, and the repair was disproportionate +rather than one step of a treadmill.** + +*Refuted:* rank does not cause a crossing, absolute cost does, and removing rows above a row does not +make it slower. Measured directly, the five rows that crossed on the post-repair run went **above +their own pre-repair maxima** — pre-repair median 343ms and max 472ms over 45 observations, against +504–533ms post-repair. They did not inherit a margin; they got slower, on the most contended run in +the sample (run factor 1.15, the highest of the ten). + +*Disproportionate:* replaying all ten runs with the three repaired modules excluded — + +| | runs refused | +| --- | --- | +| as observed | **5 / 10** | +| with the #10133 modules excluded | **1 / 10** | + +Across the nine pre-repair runs, **no row outside those three modules ever crossed 500ms.** Those +three modules were 17 of the 22 distinct identities that crossed anywhere in the sample. They were an +outlier — one shared 612k-step producer — not a sample of the plateau. + +## 5. What follows + +The plateau is real and per-producer repair is the wrong unit for 87 rows returning ~10ms each. Two +remedies are sanctioned by `v2.workflow.required_floor`: reduce what the witness reaches for, or +enroll it in a lane declaring its own dated ceiling **and** naming the row as an executing consumer. + +**Raising the ceiling is not a third option and should not be re-proposed.** That module records the +value being raised 500→1552→5000 and both raises being repudiated: 1552 was measured-max plus a +scheduler quantum on the same current tree, which is the oracle DESIGN §5 forbids by name, and 5000 +rode in as a passenger on a ruling about something else. The population that accumulated under that +drift is frozen in `v2.workflow.floor_cost_debt` precisely so it is not granted a higher ceiling. + +Which of the two remedies fits the emit and execution families — and whether a `bind_outcome` repair +buys real margin or only a rank change — is a decision this measurement informs and does not make.