diff --git a/dag/gunbc/ci_spec.dag b/dag/gunbc/ci_spec.dag index ba82daec2e0..2b90241496e 100644 --- a/dag/gunbc/ci_spec.dag +++ b/dag/gunbc/ci_spec.dag @@ -318,6 +318,7 @@ fn git_fetch_script(policy: DiffPolicy) -> String { data floor_plan_entry: String = "src/v2/workflow/ci_floor_plan.dag" data floor_plan_function: String = "gunbc_ci_floor_batches" +data plan_artifact_plan_function: String = "gunbc_ci_plan_artifact_batches" data regen_floor_plan_function: String = "gunbc_ci_regen_floor_batches" diff --git a/dag/gunbc/falsifier_workflow.dag b/dag/gunbc/falsifier_workflow.dag index 41efbebe724..667bc09f3be 100644 --- a/dag/gunbc/falsifier_workflow.dag +++ b/dag/gunbc/falsifier_workflow.dag @@ -15,7 +15,7 @@ import gunbc.ci_workflow { gunbc_ci_release_build_step_timeout_minutes, gunbc_ci_aux_step_timeout_minutes, gunbc_ci_prelude_allowance_minutes } import gunbc.ci_runner_target { gunbc_ci_selected_runner_spec } -import gunbc.ci_spec { ci_repo_root_shell } +import gunbc.ci_spec { ci_repo_root_shell, plan_artifact_plan_function } import gunbc.ci_layer_roots { witness_layer_roots } import gunbc.cli_invoke { claim_executor_run_plan_shell @@ -71,7 +71,7 @@ fn compile_clean_cold_control_invoke() -> String { claim_executor_run_plan_shell( source_roots: witness_layer_roots, plan_entry: "src/v2/workflow/ci_floor_plan.dag", - plan_function: "gunbc_ci_plan_artifact_batches", + plan_function: plan_artifact_plan_function, notice_title: Absent, rooted: false ) diff --git a/dag/gunbc/host_runner_memory_cap_verify.dag b/dag/gunbc/host_runner_memory_cap_verify.dag index 4caf8a4d86e..04ae57cb806 100644 --- a/dag/gunbc/host_runner_memory_cap_verify.dag +++ b/dag/gunbc/host_runner_memory_cap_verify.dag @@ -20,9 +20,11 @@ import gunbc.host_runner_memory_provision { runner_memory_cap_readback_verdict, } import gunbc.fleet_show_effective_read { - fleet_runner_unit_memory_props_read_script, runner_unit_memory_live_read_from_stdout, } +import gunbc.fleet_show_effective_read_script { + fleet_runner_unit_memory_props_read_script, +} import gunbc.runner_unit_live_read { RunnerUnitMemoryLiveRead } data runner_memory_cap_verify_probe_unit_assumption: String = "Path (b) verify read-back samples one representative unit (actions-runner@-01.service; precedent: host_converge_slice1_probe_unit) while apply knobs target runner_instance_glob actions-runner@-*.service (5 slots/host per gunbc.runner_slot_allocation). ASSUMPTION glob-uniform: systemd drop-ins apply the same MemoryMax/MemoryHigh/MemorySwapMax to every slot instance, so the knob tally receipt from -01 is the host-level verdict. Drift confined to -02..-05 alone is invisible until multi-unit fold lands (dissolve-on trigger for path (b) verify close)." diff --git a/dag/gunbc/runner_slot_allocation.dag b/dag/gunbc/runner_slot_allocation.dag index 10d93686f1b..01b350a7538 100644 --- a/dag/gunbc/runner_slot_allocation.dag +++ b/dag/gunbc/runner_slot_allocation.dag @@ -6,6 +6,7 @@ import std.disposition { Disposition, Terminal, SingleAuthority, Scaffold } import std.decl_ref { DeclarationRef, WholeDeclaration } import product.placement_supply { HostIdentity } import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3 } +import gunbc.ci_failure_class { FloorOutcome, NoFailure, FloorFailed, Structural, floor_outcome_blocks_merge } type RunnerSlotLabels { github_labels: List @@ -115,3 +116,122 @@ data gunbc_runner_slot_allocation_live_cap_dissolve_on: Disposition = Scaffold { field: WholeDeclaration } } + +data gunbc_floor_armed_budget_doomed_exit_137_outcome: FloorOutcome = FloorFailed { + class: Structural { reason: "Process completed with exit code 137." } +} + +type FloorArmedBudgetDoomedWitness + = FloorArmedBudgetDoomedMeasured { + run_id: String, + runner_unit: String, + armed_budget: ByteSize, + budget_source: String, + floor_outcome: FloorOutcome, + } + | FloorArmedBudgetDoomedUnmeasured + +type FloorArmedBudgetSuccessClassWitness + = FloorArmedBudgetSuccessClassMeasured { + run_id: String, + runner_unit: String, + armed_budget: ByteSize, + floor_peak_post: ByteSize, + floor_outcome: FloorOutcome, + } + | FloorArmedBudgetSuccessClassUnmeasured + +data gunbc_floor_armed_budget_doomed_witness_receipts: List = [ + FloorArmedBudgetDoomedMeasured { + run_id: "29834380839", + runner_unit: "srv3-05", + armed_budget: byte_size(6987137024), + budget_source: "/proc/meminfo MemAvailable", + floor_outcome: gunbc_floor_armed_budget_doomed_exit_137_outcome, + }, + FloorArmedBudgetDoomedMeasured { + run_id: "29845210061", + runner_unit: "srv3-02", + armed_budget: byte_size(5971566592), + budget_source: "/proc/meminfo MemAvailable", + floor_outcome: gunbc_floor_armed_budget_doomed_exit_137_outcome, + }, +] + +data gunbc_floor_armed_budget_success_class_receipt: FloorArmedBudgetSuccessClassWitness = FloorArmedBudgetSuccessClassMeasured { + run_id: "29835983525", + runner_unit: "srv1-05", + armed_budget: byte_size(16106127360), + floor_peak_post: byte_size(16108544000), + floor_outcome: NoFailure, +} + +data gunbc_floor_minimum_viable_armed_budget: ByteSize = byte_size(12884901888) + +data gunbc_floor_minimum_viable_armed_budget_disposition: Disposition = Terminal { + reason: "MEASURED minimum viable armed budget for the CI floor (operator ruling 2026-07-21, post-#6995): 12GiB (12884901888 bytes) is the conservative lower bound of the healthy capped-box success class (~12.5-16GiB floor_peak_post), NOT the typical peak. Derivation: doomed witnesses gunbc_floor_armed_budget_doomed_witness_receipts prove armed budgets 5.97-6.99GiB at arm time (MemAvailable on crowded uncapped srv3) cannot complete the floor walk - runs 29834380839 and 29845210061 died in batch witness phase after ~30min; success-class receipt gunbc_floor_armed_budget_success_class_receipt (run 29835983525 post-#6995, srv1-05, armed_budget=15GiB memory.high, floor_peak_post=16108544000) proves the healthy class sits above this bound. bright-seal #6999 fill-deferral is perf-only (entry-closure memo); coordinate via gunbc_floor_minimum_viable_armed_budget_dissolve_on if a future change cuts mature index residency. Consumed by claim_executor arm-time FloorBudgetBelowMinimumFootprint refusal (fail-fast, requeue-able)." +} + +data gunbc_floor_minimum_viable_armed_budget_dissolve_on: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.runner_slot_allocation", + decl_name: "gunbc_floor_minimum_viable_armed_budget", + field: WholeDeclaration + } +} + +fn gunbc_floor_minimum_viable_armed_budget_bytes() -> ByteSize { + gunbc_floor_minimum_viable_armed_budget +} + +fn witness_floor_minimum_viable_above_doomed_receipts() -> Bool { + all( + gunbc_floor_armed_budget_doomed_witness_receipts, + r => + match r { + FloorArmedBudgetDoomedMeasured { run_id: _, runner_unit: _, armed_budget: budget, budget_source: _, floor_outcome: _ } => + byte_size_count(b: budget) < byte_size_count(b: gunbc_floor_minimum_viable_armed_budget) + FloorArmedBudgetDoomedUnmeasured => false + } + ) +} + +fn witness_floor_minimum_viable_below_success_class() -> Bool { + match gunbc_floor_armed_budget_success_class_receipt { + FloorArmedBudgetSuccessClassMeasured { run_id: _, runner_unit: _, armed_budget: budget, floor_peak_post: _, floor_outcome: _ } => + byte_size_count(b: budget) >= byte_size_count(b: gunbc_floor_minimum_viable_armed_budget) + FloorArmedBudgetSuccessClassUnmeasured => false + } +} + +fn witness_floor_armed_budget_doomed_outcomes_are_failures() -> Bool { + all( + gunbc_floor_armed_budget_doomed_witness_receipts, + r => + match r { + FloorArmedBudgetDoomedMeasured { run_id: _, runner_unit: _, armed_budget: _, budget_source: _, floor_outcome: o } => + floor_outcome_blocks_merge(o: o) + FloorArmedBudgetDoomedUnmeasured => false + } + ) +} + +fn witness_floor_armed_budget_success_outcome_is_no_failure() -> Bool { + match gunbc_floor_armed_budget_success_class_receipt { + FloorArmedBudgetSuccessClassMeasured { run_id: _, runner_unit: _, armed_budget: _, floor_peak_post: _, floor_outcome: o } => + match o { + NoFailure => true + FloorFailed { class: _ } => false + } + FloorArmedBudgetSuccessClassUnmeasured => false + } +} + +fn gunbc_floor_minimum_viable_armed_budget_wall_holds() -> Bool { + witness_floor_minimum_viable_above_doomed_receipts() + && witness_floor_minimum_viable_below_success_class() + && witness_floor_armed_budget_doomed_outcomes_are_failures() + && witness_floor_armed_budget_success_outcome_is_no_failure() + && byte_size_count(b: gunbc_floor_minimum_viable_armed_budget) == byte_size_count(b: byte_size(12884901888)) +} diff --git a/dag/test/claim/runner_slot_allocation_witness_test.dag b/dag/test/claim/runner_slot_allocation_witness_test.dag index e273c10f144..d875e553a5f 100644 --- a/dag/test/claim/runner_slot_allocation_witness_test.dag +++ b/dag/test/claim/runner_slot_allocation_witness_test.dag @@ -8,6 +8,8 @@ import gunbc.runner_slot_allocation { gunbc_runner_slice_cap_declared, gunbc_committed_runner_slot_allocations, gunbc_runner_slot_enforcement_expected_cap, + gunbc_floor_minimum_viable_armed_budget_wall_holds, + gunbc_floor_minimum_viable_armed_budget_bytes, } import std.measure { byte_size, byte_size_count } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } @@ -43,3 +45,11 @@ test fn runner_slot_allocation_RED_oversubscription_exceeds_slice() -> Bool { test fn runner_slot_allocation_enforcement_expected_cap_is_desired_memory_max() -> Bool { byte_size_count(b: gunbc_runner_slot_enforcement_expected_cap()) == byte_size_count(b: gunbc_runner_slot_desired().memory_max) } + +test fn runner_slot_allocation_minimum_viable_armed_budget_wall_holds() -> Bool { + gunbc_floor_minimum_viable_armed_budget_wall_holds() +} + +test fn runner_slot_allocation_minimum_viable_armed_budget_is_12_gib() -> Bool { + byte_size_count(b: gunbc_floor_minimum_viable_armed_budget_bytes()) == byte_size_count(b: byte_size(12884901888)) +} diff --git a/src/v1/stage0/src/bin/claim_executor.rs b/src/v1/stage0/src/bin/claim_executor.rs index aedc298af2a..e3578ed101f 100644 --- a/src/v1/stage0/src/bin/claim_executor.rs +++ b/src/v1/stage0/src/bin/claim_executor.rs @@ -19,9 +19,9 @@ use v1_compiler::cli_run::{ TimingPercentiles, WitnessTimingRow, DEFAULT_SLOWEST_WITNESS_ATTRIBUTION_N, }; use v1_compiler::memory_governor::{ - binding_cap_cgroup_dir, binding_high_cgroup_dir, leaf_cgroup_dir, mem_total_bytes, - memory_events_field, memory_pressure_some_avg10, read_cgroup_raw, read_cgroup_u64, - AdmittedSlot, MemoryGovernor, + binding_cap_cgroup_dir, binding_high_cgroup_dir, floor_budget_below_minimum_footprint, + leaf_cgroup_dir, mem_total_bytes, memory_events_field, memory_pressure_some_avg10, + read_cgroup_raw, read_cgroup_u64, AdmittedSlot, MemoryGovernor, }; use v1_compiler::v1_interpreter::{ color_enabled, paint, run_in_context_with_args, sgr, ExecutionMode, InterpContext, Value, @@ -1786,6 +1786,22 @@ fn perturb_function_to_false(path: &Path, function: &str) -> Result<(), String> fs::write(path, out).map_err(|e| format!("write {}: {e}", path.display())) } +/// SCAFFOLD (§7 seed-retained HAND-RUST — authority: +/// `v2.workflow.ci_floor_plan.gunbc_floor_arm_time_budget_refusal_plan_functions_claim_executor_seed`, +/// symbols `gunbc.ci_spec.floor_plan_function` / `plan_artifact_plan_function` / +/// `gunbc.falsifier_workflow.falsifier_plan_function`): +/// plan functions whose schedule is a floor walk and therefore subject to arm-time +/// `FloorBudgetBelowMinimumFootprint` refusal in claim_executor. +const FLOOR_ARM_TIME_BUDGET_REFUSAL_PLAN_FUNCTIONS: &[&str] = &[ + "gunbc_ci_floor_batches", + "gunbc_ci_plan_artifact_batches", + "gunbc_falsifier_batches", +]; + +fn plan_requires_floor_arm_time_budget_refusal(plan_function: &str) -> bool { + FLOOR_ARM_TIME_BUDGET_REFUSAL_PLAN_FUNCTIONS.contains(&plan_function) +} + fn run_perturb_check( source_roots: &[String], plan_entry: &str, @@ -2148,6 +2164,12 @@ fn run() -> Result { .map(|n| n.get()) .unwrap_or(1), )); + if plan_requires_floor_arm_time_budget_refusal(&plan_function) { + if let Some(msg) = floor_budget_below_minimum_footprint(governor.budget_bytes()) { + eprintln!("claim_executor: {msg}"); + return Err(ExitCode::from(1)); + } + } phase_mark("memory governor armed; starting batch walk"); spawn_floor_memory_heartbeat(); @@ -2220,6 +2242,34 @@ fn main() -> ExitCode { mod tests { use super::*; + #[test] + fn floor_arm_time_budget_refusal_plan_functions_match_dag_seed_roster() { + let root = workspace_root(); + let roots = vec![ + root.join("src/v2").to_string_lossy().into_owned(), + root.join("dag").to_string_lossy().into_owned(), + ]; + let entry = root + .join("src/v2/workflow/ci_floor_plan.dag") + .to_string_lossy() + .into_owned(); + let (graph, indices) = resolve_entry_graph(&roots, &entry).expect("resolve ci_floor_plan"); + let ctx = make_eval_context(&graph, indices, ExecutionMode::Hermetic); + let value = run_value( + &ctx, + "gunbc_floor_arm_time_budget_refusal_plan_function_roster", + ) + .expect("evaluate materialized plan roster"); + let dag_roster = + str_list_from_value(&value, &ctx).expect("plan roster must be List"); + assert_eq!( + dag_roster.as_slice(), + FLOOR_ARM_TIME_BUDGET_REFUSAL_PLAN_FUNCTIONS, + "claim_executor seed const must match ci_floor_plan materialized roster \ + (dissolve-on: v2 emit of stage0 host constants)" + ); + } + // The materialization-receipt chain by execution: a real entry resolves, a // claim evaluates on its InterpContext, and the ctx Drop absorbs ledger // totals into the process accumulator. The env latch is process-global and diff --git a/src/v1/stage0/src/memory_governor.rs b/src/v1/stage0/src/memory_governor.rs index 2a06956f9be..372f4f80b45 100644 --- a/src/v1/stage0/src/memory_governor.rs +++ b/src/v1/stage0/src/memory_governor.rs @@ -651,6 +651,10 @@ impl MemoryGovernor { /// `memory.max` > `/proc/meminfo` MemTotal. Announces itself on stderr — the /// governor replaces the plan-evaluated spawn width, so its one log line is the /// width story for the run. + pub fn budget_bytes(&self) -> Option { + self.limits.budget_bytes + } + pub fn from_environment(max_width: usize) -> MemoryGovernor { let (budget, source_label) = read_host_budget_bytes(); let sensor_dir = binding_high_cgroup_dir() @@ -899,6 +903,35 @@ impl Drop for AdmittedSlot { /// (self-host frontier row for `memory_governor` cgroup-budget readers). pub const DECLARED_RUNNER_SLOT_MEMORY_HIGH_BYTES: u64 = 16_106_127_360; +/// SCAFFOLD (§7 seed-retained HAND-RUST — authority: `dag/gunbc/runner_slot_allocation.dag` +/// `gunbc_floor_minimum_viable_armed_budget` = `byte_size(12884901888)`; doomed/success witness +/// receipts in the same module): +/// arm-time floor refusal when the governor budget is below the measured minimum viable +/// footprint — crowded uncapped hosts with low MemAvailable would otherwise start a doomed +/// ~30min walk (runs 29834380839, 29845210061). +/// dissolve-on: v2 emit of stage0 host-budget constants from `gunbc.runner_slot_allocation` +/// (self-host frontier row for `memory_governor` cgroup-budget readers); re-measure when +/// bright-seal #6999 fill-deferral cuts mature index residency. +pub const DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES: u64 = 12_884_901_888; + +/// Fail-fast refusal when a floor walk's armed budget is provably below the measured +/// minimum viable footprint. +/// +/// - `Some(diagnostic)` — `budget` is known and strictly below the declaration. +/// - `None` — `budget` is `None` (unreadable; refusal does not fire — the governor +/// already logs its budget source on stderr) **or** budget is at/above the threshold. +pub fn floor_budget_below_minimum_footprint(budget: Option) -> Option { + let budget = budget?; + if budget < DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES { + Some(format!( + "FloorBudgetBelowMinimumFootprint: armed budget={budget} bytes < minimum viable {} bytes (gunbc.runner_slot_allocation.gunbc_floor_minimum_viable_armed_budget; doomed witnesses 29834380839, 29845210061) — requeue on a less-crowded runner (fail-fast, not a doomed walk)", + DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES + )) + } else { + None + } +} + /// Cap an uncapped-host MemAvailable sample at the declared runner-slot throttle /// line. Returns `(budget, capped)` where `capped` is true when `avail` exceeded /// the declaration. @@ -1082,6 +1115,29 @@ mod tests { } } + #[test] + fn floor_budget_below_minimum_footprint_refuses_doomed_class() { + let doomed = floor_budget_below_minimum_footprint(Some(6_987_137_024)); + assert!(doomed.is_some()); + assert!( + doomed.unwrap().contains("FloorBudgetBelowMinimumFootprint"), + "typed refusal variant" + ); + assert!(floor_budget_below_minimum_footprint(Some( + DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES + )) + .is_none()); + assert!(floor_budget_below_minimum_footprint(Some( + DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES - 1 + )) + .is_some()); + assert!(floor_budget_below_minimum_footprint(None).is_none()); + assert!( + floor_budget_below_minimum_footprint(Some(DECLARED_RUNNER_SLOT_MEMORY_HIGH_BYTES)) + .is_none() + ); + } + #[test] fn uncapped_host_budget_caps_mem_available_at_declared_runner_slot_high() { let (budget, capped) = uncapped_host_budget_from_mem_available(35_161_423_872); diff --git a/src/v2/test/claim/ci_floor_plan_witness_test.dag b/src/v2/test/claim/ci_floor_plan_witness_test.dag index f450de6e3ba..40e1b817c84 100644 --- a/src/v2/test/claim/ci_floor_plan_witness_test.dag +++ b/src/v2/test/claim/ci_floor_plan_witness_test.dag @@ -22,6 +22,9 @@ import v2.workflow.ci_floor_plan { witness_floor_discovery_flip_serializes_gates_after_corpus, witness_floor_discovery_flip_no_corpus_edge_to_compile_anchor, witness_regen_spec_carries_no_witness_corpus, + gunbc_floor_arm_time_budget_refusal_applies, + witness_floor_arm_time_budget_refusal_plan_roster_authority, + witness_floor_arm_time_budget_refusal_materialized_roster_matches_seed, gunbc_falsifier_self_host_wet_receipt_wall_budget_ms, gunbc_falsifier_self_host_wet_interp_eval_budget_ms, gunbc_ci_fast_lane_eval_budget_ms, @@ -444,6 +447,8 @@ test fn ci_floor_plan_witnesses() -> Bool { && falsifier_plan_structure_holds() && falsifier_self_host_wet_budget_declared_holds() && witness_hermetic_floor_posture_declared() + && witness_floor_arm_time_budget_refusal_plan_roster_authority() + && witness_floor_arm_time_budget_refusal_materialized_roster_matches_seed() } test fn ci_floor_optin_roster_witnesses() -> Bool { diff --git a/src/v2/workflow/ci_floor_plan.dag b/src/v2/workflow/ci_floor_plan.dag index bcdefea6fe5..b51d02f5cbc 100644 --- a/src/v2/workflow/ci_floor_plan.dag +++ b/src/v2/workflow/ci_floor_plan.dag @@ -5,8 +5,11 @@ import gunbc.ci_layer_roots { witness_layer_roots, witness_exclusion_substrings, import v2.compiler.self_host.wet_receipt_enrollment { falsifier_self_host_wet_entries, falsifier_native_cache_cold_roster } import gunbc.ci_spec { gunbc_ci_spec, gunbc_ci_regen_spec, CiSpec, - ci_spec_with_discovery_scan_dirs + ci_spec_with_discovery_scan_dirs, + floor_plan_function, + plan_artifact_plan_function, } +import gunbc.falsifier_workflow { falsifier_plan_function } import gunbc.ci_gate { Gate, EmitHostGate, LayeringImportsGate, ExtdepsExternalAuthorityGate, DagCompileCleanGate, GeneratedArtifactDriftGate, @@ -846,3 +849,38 @@ fn witness_floor_no_corpus_resource_edge_to_compile_anchor(spec: CiSpec) -> Bool }) } +data gunbc_floor_arm_time_budget_refusal_plan_functions_claim_executor_seed: List = [ + floor_plan_function, + plan_artifact_plan_function, + falsifier_plan_function, +] + +data gunbc_floor_arm_time_budget_refusal_plan_functions: List = gunbc_floor_arm_time_budget_refusal_plan_functions_claim_executor_seed + +fn gunbc_floor_arm_time_budget_refusal_plan_function_roster() -> List { + gunbc_floor_arm_time_budget_refusal_plan_functions_claim_executor_seed +} + +fn gunbc_floor_arm_time_budget_refusal_applies(plan_function: String) -> Bool { + any(gunbc_floor_arm_time_budget_refusal_plan_functions, f => f == plan_function) +} + +fn witness_string_lists_have_same_members(a: List, b: List) -> Bool { + (length(xs: a) == length(xs: b)) + && all(a, item => any(b, x => x == item)) + && all(b, item => any(a, x => x == item)) +} + +fn witness_floor_arm_time_budget_refusal_plan_roster_authority() -> Bool { + (length(xs: gunbc_floor_arm_time_budget_refusal_plan_functions) == 3) + && gunbc_floor_arm_time_budget_refusal_applies(plan_function: floor_plan_function) + && gunbc_floor_arm_time_budget_refusal_applies(plan_function: plan_artifact_plan_function) + && gunbc_floor_arm_time_budget_refusal_applies(plan_function: falsifier_plan_function) +} + +fn witness_floor_arm_time_budget_refusal_materialized_roster_matches_seed() -> Bool { + witness_string_lists_have_same_members( + a: gunbc_floor_arm_time_budget_refusal_plan_functions_claim_executor_seed, + b: gunbc_floor_arm_time_budget_refusal_plan_function_roster() + ) +}