From 473dd2997857cf804da8a9ab7eea094b2b4d8e23 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 11 Jul 2026 01:28:01 +0000 Subject: [PATCH 1/3] WIP: ci floor measurement --- dag/gunbc/ci_workflow.dag | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/dag/gunbc/ci_workflow.dag b/dag/gunbc/ci_workflow.dag index 3702c1d1c44..54fb372913a 100644 --- a/dag/gunbc/ci_workflow.dag +++ b/dag/gunbc/ci_workflow.dag @@ -399,7 +399,7 @@ data gunbc_ci_release_build_step_timeout_minutes: Int = 45 data gunbc_ci_deploy_job_backstop_timeout_minutes: Int = 10 fn gunbc_ci_job_backstop_timeout_minutes() -> Int { - gunbc_ci_release_build_step_timeout_minutes + gunbc_ci_floor_step_timeout_minutes + gunbc_ci_selection_control_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_prelude_allowance_minutes + gunbc_ci_release_build_step_timeout_minutes + gunbc_ci_floor_step_timeout_minutes + gunbc_ci_floor_step_timeout_minutes + gunbc_ci_selection_control_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_prelude_allowance_minutes } data gunbc_ci_job_timeout_policy_disposition: Disposition = Terminal { @@ -432,7 +432,7 @@ fn ci_job() -> Job { id: "ci", name: none, runner: gunbc_ci_runner_spec(), - steps: concat(ci_prelude_steps(), [ci_release_build_step(), ci_floor_peak_pre_step(), ci_floor_step(), ci_floor_peak_post_step(), ci_floor_resolve_receipt_gate_step(), ci_floor_materialization_receipt_gate_step(), ci_selection_control_step(), ci_merge_admission_gate_step()]), + steps: concat(ci_prelude_steps(), [ci_release_build_step(), ci_floor_peak_pre_step(), ci_floor_step(), ci_floor_peak_post_step(), ci_floor_resolve_receipt_gate_step(), ci_floor_materialization_receipt_gate_step(), ci_selection_control_step(), ci_merge_admission_gate_step(), ci_regen_floor_step()]), needs: [], env: none, outputs: none, @@ -446,7 +446,7 @@ fn ci_job() -> Job { fn ci_regen_floor_step() -> Step { RunStep { - name: Present { value: "gunbc ci regen (self-host fixed-point — non-required)" }, + name: Present { value: "gunbc ci regen (self-host fixed-point — required; folded into ci job 2026-07-11)" }, id: none, run: gunbc_ci_regen_floor_run(spec: gunbc_ci_regen_spec), shell: none, @@ -458,23 +458,6 @@ fn ci_regen_floor_step() -> Step { } } -fn ci_regen_job() -> Job { - Job { - id: "ci_regen", - name: none, - runner: gunbc_ci_runner_spec(), - steps: concat(ci_prelude_steps(), [ci_release_build_step(), ci_regen_floor_step()]), - needs: [], - env: none, - outputs: none, - if_condition: none, - strategy: none, - timeout_minutes: Present { value: gunbc_ci_job_backstop_timeout_minutes() }, - continue_on_error: none, - concurrency: none - } -} - data ci_workflow: Workflow = { name: "ci", on: [ @@ -508,5 +491,5 @@ data ci_workflow: Workflow = { id_token: none } }, - jobs: concat([ci_job(), ci_regen_job()], ci_deploy_jobs()) + jobs: concat([ci_job()], ci_deploy_jobs()) } From 0a28be19358954e8e8151a46066011bdfa9a93ad Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 11 Jul 2026 01:38:29 +0000 Subject: [PATCH 2/3] WIP: ci floor measurement --- .github/workflows/ci.yml | 71 +------------------ dag/gunbc/ci_materialization_gate.dag | 9 ++- dag/gunbc/ci_spec.dag | 2 +- dag/gunbc/commit_workflow.dag | 19 ++++- .../claim/ci_materialization_witness_test.dag | 22 +++--- .../placement_grain_witness_test.dag | 8 +-- 6 files changed, 44 insertions(+), 87 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d87dabcf00..6f01aed2a60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ env: jobs: ci: runs-on: [self-hosted, linux, arm64] - timeout-minutes: 150 + timeout-minutes: 210 steps: - name: Checkout uses: actions/checkout@v5 @@ -169,74 +169,7 @@ jobs: git fetch --no-tags origin main "$ROOT/target/release/gunbc" run --source-root dag --source-root src/v2 --entry dag/tools/merge_admission_gate.dag --function main timeout-minutes: 5 - ci_regen: - runs-on: [self-hosted, linux, arm64] - timeout-minutes: 150 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - name: Isolate toolchain dirs - run: | - rm -rf "$RUNNER_TEMP/rustup" "$RUNNER_TEMP/cargo" - echo "HOME=$RUNNER_TEMP" >> "$GITHUB_ENV" - echo "CARGO_HOME=$RUNNER_TEMP/cargo" >> "$GITHUB_ENV" - echo "RUSTUP_HOME=$RUNNER_TEMP/rustup" >> "$GITHUB_ENV" - echo "MAKEFLAGS=" >> "$GITHUB_ENV" - echo "CARGO_BUILD_JOBS=6" >> "$GITHUB_ENV" - if sccache --show-stats >/dev/null 2>&1; then - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - fi - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1.16.0 - with: - components: rustfmt - cache: false - rustflags: '' - env: - HOME: ${{ runner.temp }} - - name: Pin rustup default (isolated RUSTUP_HOME has no default toolchain) - run: rustup default "$(rustup show active-toolchain | awk '{print $1; exit}')" - - name: Cache Cargo - uses: actions/cache@v5 - with: - path: | - ${{ runner.temp }}/cargo/registry/index/ - ${{ runner.temp }}/cargo/registry/cache/ - ${{ runner.temp }}/cargo/git/db/ - target/ - key: cargo-ci-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} - restore-keys: | - cargo-ci-${{ runner.os }}-${{ runner.arch }}- - - name: Build release bins (claim_executor + gunbc) - run: | - ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) - set -o pipefail - BUILD_LOG=$(mktemp) - if ! ("$CARGO_HOME/bin/cargo" build -p v1-compiler --release --features text_lookup_work_counter --bin claim_executor --bin gunbc --bin regen_stage0 --bin discover_source_root_ingest --bin floor_skip_discovery_witness) 2>&1 | tee "$BUILD_LOG"; then - if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error|(exit status: 254)|sccache: error: failed to execute compile' "$BUILD_LOG"; then - echo "::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)" - RETRY_LOG=$(mktemp) - if ! ( CARGO_BUILD_JOBS=1 "$CARGO_HOME/bin/cargo" build -p v1-compiler --release --features text_lookup_work_counter --bin claim_executor --bin gunbc --bin regen_stage0 --bin discover_source_root_ingest --bin floor_skip_discovery_witness ) 2>&1 | tee "$RETRY_LOG"; then - if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error|(exit status: 254)|sccache: error: failed to execute compile' "$RETRY_LOG"; then - echo "::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER" - env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 "$CARGO_HOME/bin/cargo" build -p v1-compiler --release --features text_lookup_work_counter --bin claim_executor --bin gunbc --bin regen_stage0 --bin discover_source_root_ingest --bin floor_skip_discovery_witness || exit 1 - else - exit 1 - fi - fi - rm -f "$RETRY_LOG" - else - exit 1 - fi - fi - rm -f "$BUILD_LOG" - "$ROOT/target/release/claim_executor" --verify-build-artifacts "$ROOT/target/release/claim_executor" "$ROOT/target/release/gunbc" "$ROOT/target/release/floor_skip_discovery_witness" - sccache --show-stats 2>/dev/null || true - timeout-minutes: 45 - - name: gunbc ci regen (self-host fixed-point — non-required) + - name: gunbc ci regen (self-host fixed-point — required; folded into ci job 2026-07-11) run: | ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) git fetch --no-tags origin main 2>/dev/null || true diff --git a/dag/gunbc/ci_materialization_gate.dag b/dag/gunbc/ci_materialization_gate.dag index fbb2a76d6b2..f1200a32c9e 100644 --- a/dag/gunbc/ci_materialization_gate.dag +++ b/dag/gunbc/ci_materialization_gate.dag @@ -9,6 +9,7 @@ import std.materialization_ladder { RefusedNatureConflict, materialization_ladder, materialization_ladder_holds, verdict_is_discharged_by, verdict_is_refused_no_provider, + verdict_is_accepted_single_recompute, no_cache_providers } import gunbc.ci_materialization { @@ -23,7 +24,7 @@ import gunbc.ci_spec { DeployStage, gunbc_ci_deploy_stages, gunbc_ci_deploy_run_script } -data ci_materialization_gate_note: String = "The live binding: the ladder run over the ACTUAL emitted workflow (gunbc.ci_workflow data row), never a fixture copy of it. Demands, exemption identities, and fresh-effect scripts are all resolved against the same authority the ci.yml is emitted from, so a workflow edit that introduces cross-job duplication without a covering provider reds the enrolled witness in the same CI run that would have paid the duplication. Fresh effects: the floor cgroup-peak measurement scripts (a memory.peak read is a new fact each run, never memoizable — the rust_tests cgroup-peak measurement was the prior subject; it retired with the rust_tests job 2026-07-11, and the ci-job floor peak pre/post reads are the same measurement category) and every deploy stage script (a deploy is the effect being the use); all resolved by projection, not copied strings." +data ci_materialization_gate_note: String = "The live binding: the ladder run over the ACTUAL emitted workflow (gunbc.ci_workflow data row), never a fixture copy of it. Demands, exemption identities, and fresh-effect scripts are all resolved against the same authority the ci.yml is emitted from, so a workflow edit that introduces cross-job duplication without a covering provider reds the enrolled witness in the same CI run that would have paid the duplication. Post-fold reality (ci_regen job folded into the ci job 2026-07-11): the workflow now has a SINGLE building job, so the release build is one demand -> AcceptedSingleRecompute (state 5: a single undeclared demand needs no provider, a cache there is dead weight), and the ladder holds WITH AND WITHOUT the sccache provider. The cross-job duplication the ladder refutes is now carried as the discriminating RED control by the synthetic placement_grain witness (two independently-building jobs -> AuthoredDuplication/RefusedNoProvider), which the live witness's demand==1 / discharged==0 pins guard against reintroducing. Fresh effects: the floor cgroup-peak measurement scripts (a memory.peak read is a new fact each run, never memoizable — the rust_tests cgroup-peak measurement was the prior subject; it retired with the rust_tests job 2026-07-11, and the ci-job floor peak pre/post reads are the same measurement category) and every deploy stage script (a deploy is the effect being the use); all resolved by projection, not copied strings." fn ci_fresh_effect_scripts() -> List { concat( @@ -166,3 +167,9 @@ fn ci_release_build_refused_without_provider_count() -> Int { |> filter(v => verdict_is_refused_no_provider(v: v, identity: ci_release_build_script())) |> count() } + +fn ci_release_build_single_recompute_count() -> Int { + ci_materialization_verdicts() + |> filter(v => verdict_is_accepted_single_recompute(v: v, identity: ci_release_build_script())) + |> count() +} diff --git a/dag/gunbc/ci_spec.dag b/dag/gunbc/ci_spec.dag index 590a72bbed7..607a2477c77 100644 --- a/dag/gunbc/ci_spec.dag +++ b/dag/gunbc/ci_spec.dag @@ -106,7 +106,7 @@ data gunbc_ci_regen_spec: CiSpec = { head: "HEAD", mode: DiffMergeBase }, - notice_title: "self-host fixed-point (regen + staleness) — non-required", + notice_title: "self-host fixed-point (regen + staleness) — required; folded into ci job", deploy_stages: [], } diff --git a/dag/gunbc/commit_workflow.dag b/dag/gunbc/commit_workflow.dag index 0d764a49380..87ea3be9532 100644 --- a/dag/gunbc/commit_workflow.dag +++ b/dag/gunbc/commit_workflow.dag @@ -142,8 +142,8 @@ data commit_gate_roster: List = [ entry: "dag/test/claim/ci_materialization_witness_test.dag", check_fns: [ "ci_ladder_holds_on_live_workflow", - "ci_release_build_is_plural_and_discharged_by_sccache", - "ci_ladder_reds_without_sccache_provider", + "ci_single_release_build_is_accepted_single_recompute", + "ci_ladder_holds_without_sccache_single_build", "ci_prelude_scripts_are_typed_cost_floor_acceptances", "ci_measurement_is_exempt_fresh_effect", "ci_catalog_projection_has_no_refusals", @@ -155,6 +155,21 @@ data commit_gate_roster: List = [ }, surfaces: [GithubActionsCiJob] }, + CommitCheckEnrollment { + check: CommitWitnessClaim { + entry: "src/v2/test/claim/duplicate_computation/placement_grain_witness_test.dag", + check_fns: [ + "ci_build_across_two_jobs_is_one_violation", + "shared_artifact_rewire_is_clean", + "single_producer_is_clean", + "declared_per_placement_is_admissible", + "per_placement_observed_exceeds_declared_is_violation" + ], + kind: CorpusWitnessKind, + span: SpanUndeclared + }, + surfaces: [GithubActionsCiJob] + }, CommitCheckEnrollment { check: CommitWitnessClaim { entry: "dag/test/claim/materialization_ladder_witness_test.dag", diff --git a/dag/test/claim/ci_materialization_witness_test.dag b/dag/test/claim/ci_materialization_witness_test.dag index 7f1c504adfd..287405a37bf 100644 --- a/dag/test/claim/ci_materialization_witness_test.dag +++ b/dag/test/claim/ci_materialization_witness_test.dag @@ -9,6 +9,7 @@ import gunbc.ci_materialization_gate { ci_release_build_demand_count, ci_discharged_verdict_count, ci_release_build_discharged_by_sccache_count, + ci_release_build_single_recompute_count, ci_below_cost_floor_verdict_count, ci_exempt_fresh_verdict_count, ci_release_build_refused_without_provider_count, @@ -16,21 +17,22 @@ import gunbc.ci_materialization_gate { ci_run_step_natures_claimed_not_derived_count } -data ci_materialization_witness_note: String = "The ladder run against the LIVE emitted workflow, enrolled on the CI surface — the see-it-working-up-front requirement (operator, 2026-07-08): shell -> binary with appropriate caching installed or a typed refusal, in the same pipeline the redundancy would otherwise be paid in. The RED control removes the sccache provider row and demands the refusal, so the gate discriminates: it is green because the obligation is DISCHARGED, not because nothing looked." +data ci_materialization_witness_note: String = "The ladder run against the LIVE emitted workflow, enrolled on the CI surface — the see-it-working-up-front requirement (operator, 2026-07-08): shell -> binary in the same pipeline the redundancy would otherwise be paid in. Post-fold (ci_regen folded into ci 2026-07-11): the workflow is a SINGLE building job, so the release build is one demand -> AcceptedSingleRecompute and the ladder holds with OR without the sccache provider (state 5: a lone undeclared demand needs no provider). The demand==1 / discharged==0 / refused-without-provider==0 pins are the reintroduction guard: re-adding a second independently-building job flips demand to 2 and the refusal arm fires. The discriminating RED control for the duplication->refusal law now lives in the synthetic placement_grain witness (two building jobs -> AuthoredDuplication/RefusedNoProvider), enrolled alongside this one." test fn ci_ladder_holds_on_live_workflow() -> Bool { ci_materialization_ladder_holds_live() } -test fn ci_release_build_is_plural_and_discharged_by_sccache() -> Bool { - ci_release_build_demand_count() >= 2 - && ci_release_build_discharged_by_sccache_count() == 1 - && ci_discharged_verdict_count() == 1 +test fn ci_single_release_build_is_accepted_single_recompute() -> Bool { + ci_release_build_demand_count() == 1 + && ci_release_build_single_recompute_count() == 1 + && ci_release_build_discharged_by_sccache_count() == 0 + && ci_discharged_verdict_count() == 0 } -test fn ci_ladder_reds_without_sccache_provider() -> Bool { - !ci_materialization_holds_without_sccache() - && ci_release_build_refused_without_provider_count() == 1 +test fn ci_ladder_holds_without_sccache_single_build() -> Bool { + ci_materialization_holds_without_sccache() + && ci_release_build_refused_without_provider_count() == 0 } test fn ci_prelude_scripts_are_typed_cost_floor_acceptances() -> Bool { @@ -47,9 +49,9 @@ test fn ci_catalog_projection_has_no_refusals() -> Bool { } test fn ci_uses_steps_are_counted_not_silent() -> Bool { - ci_uses_steps_skipped_count() == 6 + ci_uses_steps_skipped_count() == 3 } test fn ci_run_step_natures_are_claims_counted_not_silent() -> Bool { - ci_run_step_natures_claimed_not_derived_count() == 14 + ci_run_step_natures_claimed_not_derived_count() == 11 } diff --git a/src/v2/test/claim/duplicate_computation/placement_grain_witness_test.dag b/src/v2/test/claim/duplicate_computation/placement_grain_witness_test.dag index dee43475228..406782e6244 100644 --- a/src/v2/test/claim/duplicate_computation/placement_grain_witness_test.dag +++ b/src/v2/test/claim/duplicate_computation/placement_grain_witness_test.dag @@ -10,7 +10,7 @@ import v2.lens.duplicate_computation { import v2.std.logic { Bool } import v2.std.text { String } -data placement_grain_witness_note: String = "Placement-grain discriminating witness for the codebase-wide duplicate-work lens (docs/plans/duplicate-work-graph-lens-design.md). Flagship real case: CI builds the release binary on TWO independent jobs (ci_job / ci_regen_job; both needs:[], no artifact Share edge; rust_tests warm step retired with the rust_tests job 2026-07-11). This proves the SAME violation logic #6372 ships already catches placement duplication once each independently-building job is projected as one demand for the shared build identity — the general lens SUBSUMES #6372 (argv grain and job/placement grain are one law); the only piece #6372 lacked was the cross-job projection. Multiplicity source = placement (one identity, N runners, no Share edge), not fork. A job that consumes the shared artifact emits no build demand and never flags — exactly the fix the lens forces. Per the materialization reframe this is the content-hash generalization of std.realization reconcile keyed on the identity lattice; the ComputationDemandFact rows here stand in for materialization records pending that unification." +data placement_grain_witness_note: String = "Placement-grain discriminating witness for the codebase-wide duplicate-work lens (docs/plans/duplicate-work-graph-lens-design.md). The real cross-job build duplication (ci_job + a second building job) was DISSOLVED by the ci_regen->ci fold 2026-07-11: the live workflow now builds the release binary on a SINGLE job (the live materialization witness attests demand==1 -> AcceptedSingleRecompute). These SYNTHETIC facts are therefore the reintroduction RED control — they model two independently-building jobs (both needs:[], no artifact Share edge) and prove the lens flags it, so re-adding a second building job reds here in the same run. This is the SAME violation logic #6372 ships once each independently-building job is projected as one demand for the shared build identity — the general lens SUBSUMES #6372 (argv grain and job/placement grain are one law); the only piece #6372 lacked was the cross-job projection. Multiplicity source = placement (one identity, N runners, no Share edge), not fork. A job that consumes the shared artifact emits no build demand and never flags — exactly the fix the lens forces. Per the materialization reframe this is the content-hash generalization of std.realization reconcile keyed on the identity lattice; the ComputationDemandFact rows here stand in for materialization records pending that unification." data release_build_key: String = "cargo build -p v1-compiler --release --bins" @@ -21,8 +21,8 @@ data ci_job_build_demand: ComputationDemandFact = { replication: SingleRun } -data ci_regen_job_build_demand: ComputationDemandFact = { - site: "ci_workflow.ci_regen_job", +data second_building_job_build_demand: ComputationDemandFact = { + site: "ci_workflow.second_building_job", computation_key: release_build_key, guard: UnguardedDemand, replication: SingleRun @@ -59,7 +59,7 @@ data per_placement_b: ComputationDemandFact = { test fn ci_build_across_two_jobs_is_one_violation() -> Bool { duplicate_computation_violation_count(facts: [ ci_job_build_demand, - ci_regen_job_build_demand + second_building_job_build_demand ]) == 1 } From fb65e1073058e80c425fc1d30d546f201803a208 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 11 Jul 2026 01:48:59 +0000 Subject: [PATCH 3/3] WIP: ci floor measurement --- .github/workflows/ci.yml | 2 +- dag/test/claim/ci_yaml_serializer_witness_test.dag | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f01aed2a60..c43444fd095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,5 +173,5 @@ jobs: run: | ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) git fetch --no-tags origin main 2>/dev/null || true - "$ROOT/target/release/claim_executor" --source-root dag --source-root src/v2 --plan-entry src/v2/workflow/ci_floor_plan.dag --plan-function gunbc_ci_regen_floor_batches --notice-title "self-host fixed-point (regen + staleness) — non-required" + "$ROOT/target/release/claim_executor" --source-root dag --source-root src/v2 --plan-entry src/v2/workflow/ci_floor_plan.dag --plan-function gunbc_ci_regen_floor_batches --notice-title "self-host fixed-point (regen + staleness) — required; folded into ci job" timeout-minutes: 60 diff --git a/dag/test/claim/ci_yaml_serializer_witness_test.dag b/dag/test/claim/ci_yaml_serializer_witness_test.dag index 225738bebce..733330ec538 100644 --- a/dag/test/claim/ci_yaml_serializer_witness_test.dag +++ b/dag/test/claim/ci_yaml_serializer_witness_test.dag @@ -34,7 +34,7 @@ fn witness_rustflags_empty_string_quoted() -> Bool { fn witness_steps_sequence_elements_are_yaml_values() -> Bool { let job = ci_workflow.jobs.first() match steps_yaml(steps: job.steps) { - YamlSequence { elements: els } => els.length() == 8 + YamlSequence { elements: els } => els.length() == 14 _ => false } }