Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dag/gunbc/ci_spec.dag
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ fn git_fetch_script(policy: DiffPolicy) -> String {

data floor_plan_entry: String = "src/v2/workflow/ci_floor_plan.dag"
data floor_plan_function: String = "gunbc_ci_floor_batches"
data plan_artifact_plan_function: String = "gunbc_ci_plan_artifact_batches"
data regen_floor_plan_function: String = "gunbc_ci_regen_floor_batches"


Expand Down
4 changes: 2 additions & 2 deletions dag/gunbc/falsifier_workflow.dag
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import gunbc.ci_workflow {
gunbc_ci_release_build_step_timeout_minutes, gunbc_ci_aux_step_timeout_minutes, gunbc_ci_prelude_allowance_minutes
}
import gunbc.ci_runner_target { gunbc_ci_selected_runner_spec }
import gunbc.ci_spec { ci_repo_root_shell }
import gunbc.ci_spec { ci_repo_root_shell, plan_artifact_plan_function }
import gunbc.ci_layer_roots { witness_layer_roots }
import gunbc.cli_invoke {
claim_executor_run_plan_shell
Expand Down Expand Up @@ -71,7 +71,7 @@ fn compile_clean_cold_control_invoke() -> String {
claim_executor_run_plan_shell(
source_roots: witness_layer_roots,
plan_entry: "src/v2/workflow/ci_floor_plan.dag",
plan_function: "gunbc_ci_plan_artifact_batches",
plan_function: plan_artifact_plan_function,
notice_title: Absent,
rooted: false
)
Expand Down
4 changes: 3 additions & 1 deletion dag/gunbc/host_runner_memory_cap_verify.dag
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@ import gunbc.host_runner_memory_provision {
runner_memory_cap_readback_verdict,
}
import gunbc.fleet_show_effective_read {
fleet_runner_unit_memory_props_read_script,
runner_unit_memory_live_read_from_stdout,
}
import gunbc.fleet_show_effective_read_script {
fleet_runner_unit_memory_props_read_script,
}
import gunbc.runner_unit_live_read { RunnerUnitMemoryLiveRead }

data runner_memory_cap_verify_probe_unit_assumption: String = "Path (b) verify read-back samples one representative unit (actions-runner@<host>-01.service; precedent: host_converge_slice1_probe_unit) while apply knobs target runner_instance_glob actions-runner@<host>-*.service (5 slots/host per gunbc.runner_slot_allocation). ASSUMPTION glob-uniform: systemd drop-ins apply the same MemoryMax/MemoryHigh/MemorySwapMax to every slot instance, so the knob tally receipt from -01 is the host-level verdict. Drift confined to -02..-05 alone is invisible until multi-unit fold lands (dissolve-on trigger for path (b) verify close)."
Expand Down
120 changes: 120 additions & 0 deletions dag/gunbc/runner_slot_allocation.dag
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import std.disposition { Disposition, Terminal, SingleAuthority, Scaffold }
import std.decl_ref { DeclarationRef, WholeDeclaration }
import product.placement_supply { HostIdentity }
import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3 }
import gunbc.ci_failure_class { FloorOutcome, NoFailure, FloorFailed, Structural, floor_outcome_blocks_merge }

type RunnerSlotLabels {
github_labels: List<String>
Expand Down Expand Up @@ -115,3 +116,122 @@ data gunbc_runner_slot_allocation_live_cap_dissolve_on: Disposition = Scaffold {
field: WholeDeclaration
}
}

data gunbc_floor_armed_budget_doomed_exit_137_outcome: FloorOutcome = FloorFailed {
class: Structural { reason: "Process completed with exit code 137." }
}

type FloorArmedBudgetDoomedWitness
= FloorArmedBudgetDoomedMeasured {
run_id: String,
runner_unit: String,
armed_budget: ByteSize,
budget_source: String,
floor_outcome: FloorOutcome,
}
| FloorArmedBudgetDoomedUnmeasured

type FloorArmedBudgetSuccessClassWitness
= FloorArmedBudgetSuccessClassMeasured {
run_id: String,
runner_unit: String,
armed_budget: ByteSize,
floor_peak_post: ByteSize,
floor_outcome: FloorOutcome,
}
| FloorArmedBudgetSuccessClassUnmeasured

data gunbc_floor_armed_budget_doomed_witness_receipts: List<FloorArmedBudgetDoomedWitness> = [
FloorArmedBudgetDoomedMeasured {
run_id: "29834380839",
runner_unit: "srv3-05",
armed_budget: byte_size(6987137024),
budget_source: "/proc/meminfo MemAvailable",
floor_outcome: gunbc_floor_armed_budget_doomed_exit_137_outcome,
},
FloorArmedBudgetDoomedMeasured {
run_id: "29845210061",
runner_unit: "srv3-02",
armed_budget: byte_size(5971566592),
budget_source: "/proc/meminfo MemAvailable",
floor_outcome: gunbc_floor_armed_budget_doomed_exit_137_outcome,
},
]

data gunbc_floor_armed_budget_success_class_receipt: FloorArmedBudgetSuccessClassWitness = FloorArmedBudgetSuccessClassMeasured {
run_id: "29835983525",
runner_unit: "srv1-05",
armed_budget: byte_size(16106127360),
floor_peak_post: byte_size(16108544000),
floor_outcome: NoFailure,
}

data gunbc_floor_minimum_viable_armed_budget: ByteSize = byte_size(12884901888)

data gunbc_floor_minimum_viable_armed_budget_disposition: Disposition = Terminal {
reason: "MEASURED minimum viable armed budget for the CI floor (operator ruling 2026-07-21, post-#6995): 12GiB (12884901888 bytes) is the conservative lower bound of the healthy capped-box success class (~12.5-16GiB floor_peak_post), NOT the typical peak. Derivation: doomed witnesses gunbc_floor_armed_budget_doomed_witness_receipts prove armed budgets 5.97-6.99GiB at arm time (MemAvailable on crowded uncapped srv3) cannot complete the floor walk - runs 29834380839 and 29845210061 died in batch witness phase after ~30min; success-class receipt gunbc_floor_armed_budget_success_class_receipt (run 29835983525 post-#6995, srv1-05, armed_budget=15GiB memory.high, floor_peak_post=16108544000) proves the healthy class sits above this bound. bright-seal #6999 fill-deferral is perf-only (entry-closure memo); coordinate via gunbc_floor_minimum_viable_armed_budget_dissolve_on if a future change cuts mature index residency. Consumed by claim_executor arm-time FloorBudgetBelowMinimumFootprint refusal (fail-fast, requeue-able)."
}

data gunbc_floor_minimum_viable_armed_budget_dissolve_on: Disposition = Scaffold {
dissolves_to: SingleAuthority,
bind: DeclarationRef {
module_path: "gunbc.runner_slot_allocation",
decl_name: "gunbc_floor_minimum_viable_armed_budget",
field: WholeDeclaration
}
}

fn gunbc_floor_minimum_viable_armed_budget_bytes() -> ByteSize {
gunbc_floor_minimum_viable_armed_budget
}

fn witness_floor_minimum_viable_above_doomed_receipts() -> Bool {
all(
gunbc_floor_armed_budget_doomed_witness_receipts,
r =>
match r {
FloorArmedBudgetDoomedMeasured { run_id: _, runner_unit: _, armed_budget: budget, budget_source: _, floor_outcome: _ } =>
byte_size_count(b: budget) < byte_size_count(b: gunbc_floor_minimum_viable_armed_budget)
FloorArmedBudgetDoomedUnmeasured => false
}
)
}

fn witness_floor_minimum_viable_below_success_class() -> Bool {
match gunbc_floor_armed_budget_success_class_receipt {
FloorArmedBudgetSuccessClassMeasured { run_id: _, runner_unit: _, armed_budget: budget, floor_peak_post: _, floor_outcome: _ } =>
byte_size_count(b: budget) >= byte_size_count(b: gunbc_floor_minimum_viable_armed_budget)
FloorArmedBudgetSuccessClassUnmeasured => false
}
}

fn witness_floor_armed_budget_doomed_outcomes_are_failures() -> Bool {
all(
gunbc_floor_armed_budget_doomed_witness_receipts,
r =>
match r {
FloorArmedBudgetDoomedMeasured { run_id: _, runner_unit: _, armed_budget: _, budget_source: _, floor_outcome: o } =>
floor_outcome_blocks_merge(o: o)
FloorArmedBudgetDoomedUnmeasured => false
}
)
}

fn witness_floor_armed_budget_success_outcome_is_no_failure() -> Bool {
match gunbc_floor_armed_budget_success_class_receipt {
FloorArmedBudgetSuccessClassMeasured { run_id: _, runner_unit: _, armed_budget: _, floor_peak_post: _, floor_outcome: o } =>
match o {
NoFailure => true
FloorFailed { class: _ } => false
}
FloorArmedBudgetSuccessClassUnmeasured => false
}
}

fn gunbc_floor_minimum_viable_armed_budget_wall_holds() -> Bool {
witness_floor_minimum_viable_above_doomed_receipts()
&& witness_floor_minimum_viable_below_success_class()
&& witness_floor_armed_budget_doomed_outcomes_are_failures()
&& witness_floor_armed_budget_success_outcome_is_no_failure()
&& byte_size_count(b: gunbc_floor_minimum_viable_armed_budget) == byte_size_count(b: byte_size(12884901888))
}
10 changes: 10 additions & 0 deletions dag/test/claim/runner_slot_allocation_witness_test.dag
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import gunbc.runner_slot_allocation {
gunbc_runner_slice_cap_declared,
gunbc_committed_runner_slot_allocations,
gunbc_runner_slot_enforcement_expected_cap,
gunbc_floor_minimum_viable_armed_budget_wall_holds,
gunbc_floor_minimum_viable_armed_budget_bytes,
}
import std.measure { byte_size, byte_size_count }
import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly }
Expand Down Expand Up @@ -43,3 +45,11 @@ test fn runner_slot_allocation_RED_oversubscription_exceeds_slice() -> Bool {
test fn runner_slot_allocation_enforcement_expected_cap_is_desired_memory_max() -> Bool {
byte_size_count(b: gunbc_runner_slot_enforcement_expected_cap()) == byte_size_count(b: gunbc_runner_slot_desired().memory_max)
}

test fn runner_slot_allocation_minimum_viable_armed_budget_wall_holds() -> Bool {
gunbc_floor_minimum_viable_armed_budget_wall_holds()
}

test fn runner_slot_allocation_minimum_viable_armed_budget_is_12_gib() -> Bool {
byte_size_count(b: gunbc_floor_minimum_viable_armed_budget_bytes()) == byte_size_count(b: byte_size(12884901888))
}
56 changes: 53 additions & 3 deletions src/v1/stage0/src/bin/claim_executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ use v1_compiler::cli_run::{
TimingPercentiles, WitnessTimingRow, DEFAULT_SLOWEST_WITNESS_ATTRIBUTION_N,
};
use v1_compiler::memory_governor::{
binding_cap_cgroup_dir, binding_high_cgroup_dir, leaf_cgroup_dir, mem_total_bytes,
memory_events_field, memory_pressure_some_avg10, read_cgroup_raw, read_cgroup_u64,
AdmittedSlot, MemoryGovernor,
binding_cap_cgroup_dir, binding_high_cgroup_dir, floor_budget_below_minimum_footprint,
leaf_cgroup_dir, mem_total_bytes, memory_events_field, memory_pressure_some_avg10,
read_cgroup_raw, read_cgroup_u64, AdmittedSlot, MemoryGovernor,
};
use v1_compiler::v1_interpreter::{
color_enabled, paint, run_in_context_with_args, sgr, ExecutionMode, InterpContext, Value,
Expand Down Expand Up @@ -1786,6 +1786,22 @@ fn perturb_function_to_false(path: &Path, function: &str) -> Result<(), String>
fs::write(path, out).map_err(|e| format!("write {}: {e}", path.display()))
}

/// SCAFFOLD (§7 seed-retained HAND-RUST — authority:
/// `v2.workflow.ci_floor_plan.gunbc_floor_arm_time_budget_refusal_plan_functions_claim_executor_seed`,
/// symbols `gunbc.ci_spec.floor_plan_function` / `plan_artifact_plan_function` /
/// `gunbc.falsifier_workflow.falsifier_plan_function`):
/// plan functions whose schedule is a floor walk and therefore subject to arm-time
/// `FloorBudgetBelowMinimumFootprint` refusal in claim_executor.
const FLOOR_ARM_TIME_BUDGET_REFUSAL_PLAN_FUNCTIONS: &[&str] = &[
"gunbc_ci_floor_batches",
"gunbc_ci_plan_artifact_batches",
"gunbc_falsifier_batches",
];

fn plan_requires_floor_arm_time_budget_refusal(plan_function: &str) -> bool {
FLOOR_ARM_TIME_BUDGET_REFUSAL_PLAN_FUNCTIONS.contains(&plan_function)
}

fn run_perturb_check(
source_roots: &[String],
plan_entry: &str,
Expand Down Expand Up @@ -2148,6 +2164,12 @@ fn run() -> Result<ExitCode, ExitCode> {
.map(|n| n.get())
.unwrap_or(1),
));
if plan_requires_floor_arm_time_budget_refusal(&plan_function) {
if let Some(msg) = floor_budget_below_minimum_footprint(governor.budget_bytes()) {
eprintln!("claim_executor: {msg}");
return Err(ExitCode::from(1));
}
}
phase_mark("memory governor armed; starting batch walk");
spawn_floor_memory_heartbeat();

Expand Down Expand Up @@ -2220,6 +2242,34 @@ fn main() -> ExitCode {
mod tests {
use super::*;

#[test]
fn floor_arm_time_budget_refusal_plan_functions_match_dag_seed_roster() {
let root = workspace_root();
let roots = vec![
root.join("src/v2").to_string_lossy().into_owned(),
root.join("dag").to_string_lossy().into_owned(),
];
let entry = root
.join("src/v2/workflow/ci_floor_plan.dag")
.to_string_lossy()
.into_owned();
let (graph, indices) = resolve_entry_graph(&roots, &entry).expect("resolve ci_floor_plan");
let ctx = make_eval_context(&graph, indices, ExecutionMode::Hermetic);
let value = run_value(
&ctx,
"gunbc_floor_arm_time_budget_refusal_plan_function_roster",
)
.expect("evaluate materialized plan roster");
let dag_roster =
str_list_from_value(&value, &ctx).expect("plan roster must be List<String>");
assert_eq!(
dag_roster.as_slice(),
FLOOR_ARM_TIME_BUDGET_REFUSAL_PLAN_FUNCTIONS,
"claim_executor seed const must match ci_floor_plan materialized roster \
(dissolve-on: v2 emit of stage0 host constants)"
);
}

// The materialization-receipt chain by execution: a real entry resolves, a
// claim evaluates on its InterpContext, and the ctx Drop absorbs ledger
// totals into the process accumulator. The env latch is process-global and
Expand Down
56 changes: 56 additions & 0 deletions src/v1/stage0/src/memory_governor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,10 @@ impl MemoryGovernor {
/// `memory.max` > `/proc/meminfo` MemTotal. Announces itself on stderr — the
/// governor replaces the plan-evaluated spawn width, so its one log line is the
/// width story for the run.
pub fn budget_bytes(&self) -> Option<u64> {
self.limits.budget_bytes
}

pub fn from_environment(max_width: usize) -> MemoryGovernor {
let (budget, source_label) = read_host_budget_bytes();
let sensor_dir = binding_high_cgroup_dir()
Expand Down Expand Up @@ -899,6 +903,35 @@ impl Drop for AdmittedSlot {
/// (self-host frontier row for `memory_governor` cgroup-budget readers).
pub const DECLARED_RUNNER_SLOT_MEMORY_HIGH_BYTES: u64 = 16_106_127_360;

/// SCAFFOLD (§7 seed-retained HAND-RUST — authority: `dag/gunbc/runner_slot_allocation.dag`
/// `gunbc_floor_minimum_viable_armed_budget` = `byte_size(12884901888)`; doomed/success witness
/// receipts in the same module):
/// arm-time floor refusal when the governor budget is below the measured minimum viable
/// footprint — crowded uncapped hosts with low MemAvailable would otherwise start a doomed
/// ~30min walk (runs 29834380839, 29845210061).
/// dissolve-on: v2 emit of stage0 host-budget constants from `gunbc.runner_slot_allocation`
/// (self-host frontier row for `memory_governor` cgroup-budget readers); re-measure when
/// bright-seal #6999 fill-deferral cuts mature index residency.
pub const DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES: u64 = 12_884_901_888;

/// Fail-fast refusal when a floor walk's armed budget is provably below the measured
/// minimum viable footprint.
///
/// - `Some(diagnostic)` — `budget` is known and strictly below the declaration.
/// - `None` — `budget` is `None` (unreadable; refusal does not fire — the governor
/// already logs its budget source on stderr) **or** budget is at/above the threshold.
pub fn floor_budget_below_minimum_footprint(budget: Option<u64>) -> Option<String> {
let budget = budget?;
if budget < DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES {
Some(format!(
"FloorBudgetBelowMinimumFootprint: armed budget={budget} bytes < minimum viable {} bytes (gunbc.runner_slot_allocation.gunbc_floor_minimum_viable_armed_budget; doomed witnesses 29834380839, 29845210061) — requeue on a less-crowded runner (fail-fast, not a doomed walk)",
DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES
))
} else {
None
}
}

/// Cap an uncapped-host MemAvailable sample at the declared runner-slot throttle
/// line. Returns `(budget, capped)` where `capped` is true when `avail` exceeded
/// the declaration.
Expand Down Expand Up @@ -1082,6 +1115,29 @@ mod tests {
}
}

#[test]
fn floor_budget_below_minimum_footprint_refuses_doomed_class() {
let doomed = floor_budget_below_minimum_footprint(Some(6_987_137_024));
assert!(doomed.is_some());
assert!(
doomed.unwrap().contains("FloorBudgetBelowMinimumFootprint"),
"typed refusal variant"
);
assert!(floor_budget_below_minimum_footprint(Some(
DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES
))
.is_none());
assert!(floor_budget_below_minimum_footprint(Some(
DECLARED_FLOOR_MINIMUM_VIABLE_ARMED_BUDGET_BYTES - 1
))
.is_some());
assert!(floor_budget_below_minimum_footprint(None).is_none());
assert!(
floor_budget_below_minimum_footprint(Some(DECLARED_RUNNER_SLOT_MEMORY_HIGH_BYTES))
.is_none()
);
}

#[test]
fn uncapped_host_budget_caps_mem_available_at_declared_runner_slot_high() {
let (budget, capped) = uncapped_host_budget_from_mem_available(35_161_423_872);
Expand Down
5 changes: 5 additions & 0 deletions src/v2/test/claim/ci_floor_plan_witness_test.dag
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ import v2.workflow.ci_floor_plan {
witness_floor_discovery_flip_serializes_gates_after_corpus,
witness_floor_discovery_flip_no_corpus_edge_to_compile_anchor,
witness_regen_spec_carries_no_witness_corpus,
gunbc_floor_arm_time_budget_refusal_applies,
witness_floor_arm_time_budget_refusal_plan_roster_authority,
witness_floor_arm_time_budget_refusal_materialized_roster_matches_seed,
gunbc_falsifier_self_host_wet_receipt_wall_budget_ms,
gunbc_falsifier_self_host_wet_interp_eval_budget_ms,
gunbc_ci_fast_lane_eval_budget_ms,
Expand Down Expand Up @@ -444,6 +447,8 @@ test fn ci_floor_plan_witnesses() -> Bool {
&& falsifier_plan_structure_holds()
&& falsifier_self_host_wet_budget_declared_holds()
&& witness_hermetic_floor_posture_declared()
&& witness_floor_arm_time_budget_refusal_plan_roster_authority()
&& witness_floor_arm_time_budget_refusal_materialized_roster_matches_seed()
}

test fn ci_floor_optin_roster_witnesses() -> Bool {
Expand Down
Loading
Loading