diff --git a/.githooks/pre-push b/.githooks/pre-push index 8f3d3dc9169..b45cce93ddb 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -40,6 +40,7 @@ fresh_artifact_path() { DESIGN.md*) return 0 ;; .githooks/pre-push*) return 0 ;; .github/fleet-converge.sh*) return 0 ;; + .github/fleet-converge-timer-upsert.sh*) return 0 ;; docs/plans/idea-machine.md*) return 0 ;; docs/plans/algebraic-rewrite-optimization.md*) return 0 ;; docs/plans/dsl-v2-defork-audit.md*) return 0 ;; diff --git a/.github/fleet-converge-timer-upsert.sh b/.github/fleet-converge-timer-upsert.sh new file mode 100644 index 00000000000..ef956ccfbf4 --- /dev/null +++ b/.github/fleet-converge-timer-upsert.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +# GENERATED by dsl/gunbc/fleet_converge_timer_emit.dag (gunbc.fleet_converge_timer_emit). DO NOT HAND-EDIT. +# Idempotent per-host upsert: write service+timer units, daemon-reload, enable timer. +# The converge script self-selects via $(hostname -s); no SSH star required for steady-state. + +svc_path="/etc/systemd/system/gunbc-fleet-converge.service" +timer_path="/etc/systemd/system/gunbc-fleet-converge.timer" +mkdir -p "$(dirname "$svc_path")" + +cat > "$svc_path" <<'GUNBC_FLEET_CONVERGE_SERVICE_EOF' +[Unit] +Description=gunbc fleet converge (host-self-selecting reconcile) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=briansrls +ExecStart=/usr/bin/bash /home/briansrls/ctrl/third_party/gunbc/.github/fleet-converge.sh +RemainAfterExit=no +GUNBC_FLEET_CONVERGE_SERVICE_EOF + +cat > "$timer_path" <<'GUNBC_FLEET_CONVERGE_TIMER_EOF' +[Unit] +Description=gunbc fleet converge periodic reconcile + +[Timer] +OnBootSec=120s +OnUnitActiveSec=600s +Unit=gunbc-fleet-converge.service + +[Install] +WantedBy=timers.target +GUNBC_FLEET_CONVERGE_TIMER_EOF + +systemctl daemon-reload +systemctl enable --now gunbc-fleet-converge.timer +printf 'fleet-converge-timer-receipt host=%s verdict=converged\n' "$(hostname -s 2>/dev/null || echo unknown)" diff --git a/.github/fleet-converge.sh b/.github/fleet-converge.sh index 774f2e03b42..f197651fd69 100644 --- a/.github/fleet-converge.sh +++ b/.github/fleet-converge.sh @@ -273,10 +273,12 @@ emit_sessions_membership() { } host_begin "srv1" -converge_gunbc_pinned_tree "srv1" "gunbc" "pinned_tree_sha" "existing_host_quiescent" "skip_process_pin_coherent_and_marker" "GUNBC_ROOT" "$HOME/.local/share/gunbc/pinned" "defer_submodule" "third_party/gunbc" "$HOME/ctrl" "ghcr.io/gunb-ai/gunbc/bootstrap" "sha256:BOOTSTRAP_DIGEST_PLACEHOLDER" "gunbc" "scripts/session-dashboard/container/read-gunbc-pin.sh" ".dashboard/converge-green-place.marker" "pin=" "first_green_at=" "scripts/session-dashboard/bin/gunbc-pin-quiescent-reload.mjs" -converge_slice_property "srv1" "runner" "runner_slice_cap_bytes" "system-actions-runner.slice" "MemoryMax" "85899345920" "85899345920" "85899345920" -converge_per_slot_cap "srv1" "runner" "per_slot_memory_max_bytes" "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" "actions-runner@srv1-*.service" "MemoryMax" "8589934592" -converge_per_slot_cap "srv1" "runner" "per_slot_memory_swap_max_bytes" "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" "actions-runner@srv1-*.service" "MemorySwapMax" "0" +converge_gunbc_pinned_tree "srv1" "gunbc" "pinned_tree_sha" "existing_host_quiescent" "skip_process_pin_coherent_and_marker" "GUNBC_ROOT" "/home/briansrls/.local/share/gunbc/pinned" "defer_submodule" "third_party/gunbc" "/home/briansrls/ctrl" "ghcr.io/gunb-ai/gunbc/bootstrap" "sha256:BOOTSTRAP_DIGEST_PLACEHOLDER" "gunbc" "scripts/session-dashboard/container/read-gunbc-pin.sh" ".dashboard/converge-green-place.marker" "pin=" "first_green_at=" "scripts/session-dashboard/bin/gunbc-pin-quiescent-reload.mjs" +converge_slice_property "srv1" "runner" "runner_slice_cap_bytes" "system-actions-runner.slice" "MemoryMax" "111669149696" "111669149696" "111669149696" +converge_per_slot_cap "srv1" "runner" "per_slot_memory_max_bytes" "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" "actions-runner@srv1-*.service" "MemoryMax" "25769803776" +converge_per_slot_cap "srv1" "runner" "per_slot_memory_swap_max_bytes" "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" "actions-runner@srv1-*.service" "MemorySwapMax" "8589934592" +converge_per_slot_cap "srv1" "runner" "per_slot_memory_high_bytes" "/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf" "actions-runner@srv1-*.service" "MemoryHigh" "15032385536" +converge_slice_property "srv1" "runner" "oom_pressure_auto" "system-actions-runner.slice" "ManagedOOMMemoryPressure" "auto" "auto" "true" converge_slice_property "srv1" "runner" "cpu_weight" "system-actions-runner.slice" "CPUWeight" "100" "100" "100" converge_jobserver_tokens "srv1" "runner" "build_tokens" "/etc/default/ctrl-jobserver" "CTRL_JOBSERVER_TOKENS" "ctrl-jobserver.service" "128" converge_runner_width "srv1" "runner" "runner_count" "actions-runner@srv1-*.service" "10" @@ -288,10 +290,12 @@ converge_slice_property "srv1" "sessions" "cpu_weight" "sessions.slice" "CPUWeig emit_sessions_membership "srv1" "/sys/fs/cgroup/sessions.slice" "/sys/fs/cgroup/system.slice" host_summary "srv1" host_begin "srv2" -converge_gunbc_pinned_tree "srv2" "gunbc" "pinned_tree_sha" "existing_host_quiescent" "skip_process_pin_coherent_and_marker" "GUNBC_ROOT" "$HOME/.local/share/gunbc/pinned" "defer_submodule" "third_party/gunbc" "$HOME/ctrl" "ghcr.io/gunb-ai/gunbc/bootstrap" "sha256:BOOTSTRAP_DIGEST_PLACEHOLDER" "gunbc" "scripts/session-dashboard/container/read-gunbc-pin.sh" ".dashboard/converge-green-place.marker" "pin=" "first_green_at=" "scripts/session-dashboard/bin/gunbc-pin-quiescent-reload.mjs" -converge_slice_property "srv2" "runner" "runner_slice_cap_bytes" "system-actions-runner.slice" "MemoryMax" "85899345920" "85899345920" "85899345920" -converge_per_slot_cap "srv2" "runner" "per_slot_memory_max_bytes" "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" "actions-runner@srv2-*.service" "MemoryMax" "8589934592" -converge_per_slot_cap "srv2" "runner" "per_slot_memory_swap_max_bytes" "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" "actions-runner@srv2-*.service" "MemorySwapMax" "0" +converge_gunbc_pinned_tree "srv2" "gunbc" "pinned_tree_sha" "existing_host_quiescent" "skip_process_pin_coherent_and_marker" "GUNBC_ROOT" "/home/briansrls/.local/share/gunbc/pinned" "defer_submodule" "third_party/gunbc" "/home/briansrls/ctrl" "ghcr.io/gunb-ai/gunbc/bootstrap" "sha256:BOOTSTRAP_DIGEST_PLACEHOLDER" "gunbc" "scripts/session-dashboard/container/read-gunbc-pin.sh" ".dashboard/converge-green-place.marker" "pin=" "first_green_at=" "scripts/session-dashboard/bin/gunbc-pin-quiescent-reload.mjs" +converge_slice_property "srv2" "runner" "runner_slice_cap_bytes" "system-actions-runner.slice" "MemoryMax" "111669149696" "111669149696" "111669149696" +converge_per_slot_cap "srv2" "runner" "per_slot_memory_max_bytes" "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" "actions-runner@srv2-*.service" "MemoryMax" "25769803776" +converge_per_slot_cap "srv2" "runner" "per_slot_memory_swap_max_bytes" "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" "actions-runner@srv2-*.service" "MemorySwapMax" "8589934592" +converge_per_slot_cap "srv2" "runner" "per_slot_memory_high_bytes" "/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf" "actions-runner@srv2-*.service" "MemoryHigh" "15032385536" +converge_slice_property "srv2" "runner" "oom_pressure_auto" "system-actions-runner.slice" "ManagedOOMMemoryPressure" "auto" "auto" "true" converge_slice_property "srv2" "runner" "cpu_weight" "system-actions-runner.slice" "CPUWeight" "100" "100" "100" converge_jobserver_tokens "srv2" "runner" "build_tokens" "/etc/default/ctrl-jobserver" "CTRL_JOBSERVER_TOKENS" "ctrl-jobserver.service" "128" converge_runner_width "srv2" "runner" "runner_count" "actions-runner@srv2-*.service" "10" @@ -303,10 +307,12 @@ converge_slice_property "srv2" "sessions" "cpu_weight" "sessions.slice" "CPUWeig emit_sessions_membership "srv2" "/sys/fs/cgroup/sessions.slice" "/sys/fs/cgroup/system.slice" host_summary "srv2" host_begin "srv3" -converge_gunbc_pinned_tree "srv3" "gunbc" "pinned_tree_sha" "fresh_standup" "skip_process_pin_coherent_and_marker" "GUNBC_ROOT" "$HOME/.local/share/gunbc/pinned" "defer_submodule" "third_party/gunbc" "$HOME/ctrl" "ghcr.io/gunb-ai/gunbc/bootstrap" "sha256:BOOTSTRAP_DIGEST_PLACEHOLDER" "gunbc" "scripts/session-dashboard/container/read-gunbc-pin.sh" ".dashboard/converge-green-place.marker" "pin=" "first_green_at=" "scripts/session-dashboard/bin/gunbc-pin-quiescent-reload.mjs" -converge_slice_property "srv3" "runner" "runner_slice_cap_bytes" "system-actions-runner.slice" "MemoryMax" "85899345920" "85899345920" "85899345920" -converge_per_slot_cap "srv3" "runner" "per_slot_memory_max_bytes" "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" "actions-runner@srv3-*.service" "MemoryMax" "8589934592" -converge_per_slot_cap "srv3" "runner" "per_slot_memory_swap_max_bytes" "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" "actions-runner@srv3-*.service" "MemorySwapMax" "0" +converge_gunbc_pinned_tree "srv3" "gunbc" "pinned_tree_sha" "fresh_standup" "skip_process_pin_coherent_and_marker" "GUNBC_ROOT" "/home/briansrls/.local/share/gunbc/pinned" "defer_submodule" "third_party/gunbc" "/home/briansrls/ctrl" "ghcr.io/gunb-ai/gunbc/bootstrap" "sha256:BOOTSTRAP_DIGEST_PLACEHOLDER" "gunbc" "scripts/session-dashboard/container/read-gunbc-pin.sh" ".dashboard/converge-green-place.marker" "pin=" "first_green_at=" "scripts/session-dashboard/bin/gunbc-pin-quiescent-reload.mjs" +converge_slice_property "srv3" "runner" "runner_slice_cap_bytes" "system-actions-runner.slice" "MemoryMax" "111669149696" "111669149696" "111669149696" +converge_per_slot_cap "srv3" "runner" "per_slot_memory_max_bytes" "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" "actions-runner@srv3-*.service" "MemoryMax" "25769803776" +converge_per_slot_cap "srv3" "runner" "per_slot_memory_swap_max_bytes" "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" "actions-runner@srv3-*.service" "MemorySwapMax" "8589934592" +converge_per_slot_cap "srv3" "runner" "per_slot_memory_high_bytes" "/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf" "actions-runner@srv3-*.service" "MemoryHigh" "15032385536" +converge_slice_property "srv3" "runner" "oom_pressure_auto" "system-actions-runner.slice" "ManagedOOMMemoryPressure" "auto" "auto" "true" converge_slice_property "srv3" "runner" "cpu_weight" "system-actions-runner.slice" "CPUWeight" "100" "100" "100" converge_jobserver_tokens "srv3" "runner" "build_tokens" "/etc/default/ctrl-jobserver" "CTRL_JOBSERVER_TOKENS" "ctrl-jobserver.service" "128" converge_runner_width "srv3" "runner" "runner_count" "actions-runner@srv3-*.service" "10" diff --git a/dsl/extdeps/os/systemd.dag b/dsl/extdeps/os/systemd.dag index 9dc3f10e3b2..4a807c5ceb9 100644 --- a/dsl/extdeps/os/systemd.dag +++ b/dsl/extdeps/os/systemd.dag @@ -30,6 +30,8 @@ data systemd_memory_max_property: NonEmptyStr = "MemoryMax" data systemd_memory_swap_max_property: NonEmptyStr = "MemorySwapMax" +data systemd_memory_high_property: NonEmptyStr = "MemoryHigh" + data systemd_cpu_weight_property: NonEmptyStr = "CPUWeight" fn parse_systemd_unit_active_state(raw: String) -> SystemdUnitActiveState? { diff --git a/dsl/gunbc/bmc_onboarding.dag b/dsl/gunbc/bmc_onboarding.dag index af409066a3a..cd5767bb058 100644 --- a/dsl/gunbc/bmc_onboarding.dag +++ b/dsl/gunbc/bmc_onboarding.dag @@ -3,8 +3,16 @@ module gunbc.bmc_onboarding import std.bmc { BmcEndpoint, Redfish } import std.credentials { FactoryLogin, CredentialFlow, Stored } import std.os.types { OperatingSystemProduct, UbuntuOs, NobleNumbat2404Lts } -import std.types { NonEmptyStr, Int, List } +import std.types { NonEmptyStr, Int, List, Map } import std.logic { Bool } +import std.disposition { Disposition, Scaffold, SingleAuthority, Terminal } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import product.placement_supply { HostIdentity } +import gunbc.fleet_intent_network { + operator_host_srv1, + operator_host_srv2, + operator_host_srv3, +} import extdeps.bmc.types { RedfishBootSourceOverrideTarget, BootTargetPxe, RedfishAccountRole, AccountRoleAdministrator } import extdeps.bmc.openbmc { openbmc_factory_login } import extdeps.boards.asrock_rack { asrock_altrad8ud_redfish_system_id } @@ -232,3 +240,58 @@ fn bmc_credential_probe_to_phase(probe: BmcCredentialProbeResult) -> BmcOnboardi CredentialStateUnknown { reason: _ } => Absent } } + +type HostOnboardingLifecycleRow { + identity: HostIdentity + phase: BmcOnboardingPhase +} + +data fleet_host_onboarding_lifecycle_disposition: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.bmc_onboarding", + decl_name: "fleet_host_onboarding_lifecycle", + field: WholeDeclaration + } +} + +data srv1_bmc_lifecycle_phase: BmcOnboardingPhase = OsInstalled + +data srv2_bmc_lifecycle_phase: BmcOnboardingPhase = OsInstalled + +data srv3_bmc_lifecycle_phase: BmcOnboardingPhase = CredentialsRotated + +data fleet_host_onboarding_lifecycle: List = [ + HostOnboardingLifecycleRow { identity: operator_host_srv1, phase: srv1_bmc_lifecycle_phase }, + HostOnboardingLifecycleRow { identity: operator_host_srv2, phase: srv2_bmc_lifecycle_phase }, + HostOnboardingLifecycleRow { identity: operator_host_srv3, phase: srv3_bmc_lifecycle_phase }, +] + +fn fleet_host_lifecycle_phase_map_derived(rows: List) -> Map { + rows |> fold(init: empty_map(), f: (acc, row) => map_insert(acc, row.identity, row.phase)) +} + +fn host_onboarding_lifecycle_row(host: HostIdentity) -> HostOnboardingLifecycleRow? { + match map_get(fleet_host_lifecycle_phase_map_derived(rows: fleet_host_onboarding_lifecycle), host) { + Present { value: phase } => + Present { value: HostOnboardingLifecycleRow { identity: host, phase: phase } } + Absent => Absent + } +} + +fn host_onboarding_lifecycle_phase(host: HostIdentity) -> BmcOnboardingPhase? { + map_get(fleet_host_lifecycle_phase_map_derived(rows: fleet_host_onboarding_lifecycle), host) +} + +data bmc_phase_is_pre_os_installed_disposition: Disposition = Terminal { + reason: "Canonical exhaustive partition on closed BmcOnboardingPhase ladder: FactoryDefault|CredentialsRotated are pre-OsInstalled; OsInstalled|FabricJoined are installed+. FreshStandup apply boundary in host_converge.gunbc_apply_mode_for_host mirrors this match — not a nickname predicate." +} + +fn bmc_phase_is_pre_os_installed(phase: BmcOnboardingPhase) -> Bool { + match phase { + FactoryDefault => true + CredentialsRotated => true + OsInstalled => false + FabricJoined => false + } +} diff --git a/dsl/gunbc/ci_deploy_access.dag b/dsl/gunbc/ci_deploy_access.dag index c2bbe633ed4..ad6d11508fd 100644 --- a/dsl/gunbc/ci_deploy_access.dag +++ b/dsl/gunbc/ci_deploy_access.dag @@ -18,7 +18,13 @@ type DeployAccess { data gunbc_deploy_sudoers_dropin: NonEmptyStr = "/etc/sudoers.d/gunbc-deploy" -data ci_deploy_runner_principal: NonEmptyStr = "runner" +data gunbc_deploy_runner_principal: NonEmptyStr = "gunbc-deploy" + +data ci_deploy_runner_principal: NonEmptyStr = gunbc_deploy_runner_principal + +fn deploy_access_command_paths(access: DeployAccess) -> List { + map(access.privileged_commands, cmd => cmd.path) +} fn srv1_live_deploy_privileged_commands() -> List { [ diff --git a/dsl/gunbc/ci_runner_placement.dag b/dsl/gunbc/ci_runner_placement.dag index ff2899a9819..f757e714dd9 100644 --- a/dsl/gunbc/ci_runner_placement.dag +++ b/dsl/gunbc/ci_runner_placement.dag @@ -12,10 +12,16 @@ import std.decl_ref { DeclarationRef, WholeDeclaration } import gunbc.runner_spec_from_offer { runner_spec_from_offer } import gunbc.fleet_host_budget { gunbc_fleet_oomd_evidence, + gunbc_runner_slice_cap, gunbc_runner_pool_budget, + gunbc_runner_slot_memory_contract, + RunnerSlotMemoryContract, host_allocation_conserves, declared_runner_count, oomd_actuation_floor, + gunbc_host_swap_available, + fabric_memory_accounting_admissible, + host_fixed_overhead_bytes, } import extdeps.cpu { cpu_facts_threads } import gunbc.ci_floor_measurement { @@ -67,7 +73,7 @@ type RunnerHostDeployment { identity: HostIdentity runner_count: Int runner_slice_cap: ByteSize - per_runner_memory_cap: ByteSize + slot_memory: RunnerSlotMemoryContract build_tokens: HardwareThreadCount runner: RunnerSpec workload_class: WorkloadClass @@ -143,21 +149,14 @@ fn resolve_session_slice(mode: SessionReservationMode, oomd: OomdEnforcementEvid } } -data gunbc_ci_runner_count_from_budget_disposition: Disposition = Terminal { reason: "DESIGN conservation model: the runner count divides the DECLARED runner_pool_budget by the DECLARED per_runner_memory_cap (gunbc.fleet_host_budget). Soundness is host_allocation_conserves (node_conserves over runner_pool + session_pool + fixed_overhead + headroom <= host RAM); it replaces the old chicken-and-egg RunnerSlotUnenforced gate (which refused to derive a count premised on a cap not yet enforced live). 'Is the cap live on the host' is no longer a plan-soundness question - it is a converge-receipt question. Session pool is the uniform 1 GiB demand class shared with the ctrl session dashboard." } - -fn host_fixed_overhead_bytes(o: HostFixedOverhead) -> ByteSize { - match o { - HostFixedOverheadMeasured { bytes: b } => b - HostFixedOverheadUnmeasured => byte_size(0) - } -} +data gunbc_ci_runner_count_from_budget_disposition: Disposition = Terminal { reason: "DESIGN standalone runner model: runner_count is operator-declared width gunbc_declared_runner_slot_count (10) per host — NOT budget-derived from runner_pool/slot_memory.memory_max. Soundness is host_allocation_conserves (runner_pool + fixed_overhead + headroom <= host RAM) plus fabric_memory_accounting_admissible at plan gate — runners only; sessions are an independent standalone contract in cool-swift P4 session-dispatch, NOT conserved here. Replaces the old chicken-and-egg RunnerSlotUnenforced gate. 'Is the cap live on the host' is a converge-receipt question, not plan-soundness." } fn conservation_host_deployment(offer: ComputeOffer, nproc: HardwareThreadCount) -> RunnerHostDeployment { RunnerHostDeployment { identity: offer.supply.physical.identity, runner_count: declared_runner_count(), - runner_slice_cap: gunbc_runner_pool_budget, - per_runner_memory_cap: gunbc_ci_runner_cgroup_memory_cap, + runner_slice_cap: gunbc_runner_slice_cap, + slot_memory: gunbc_runner_slot_memory_contract, build_tokens: nproc, runner: runner_spec_from_offer(offer: offer), workload_class: BatchLatencyTolerant, @@ -181,8 +180,7 @@ fn accumulate_conservation_deployment(acc: DeploymentAccumulator, offer: Compute host: host_ram, overhead: overhead, headroom: headroom, - runner_pool: gunbc_runner_pool_budget, - session_pool: byte_size(0) + runner_pool: gunbc_runner_slice_cap ) { let dep = conservation_host_deployment(offer: offer, nproc: cpu_facts_threads(cpu: cpu)) DeploymentAccumulator { @@ -194,7 +192,7 @@ fn accumulate_conservation_deployment(acc: DeploymentAccumulator, offer: Compute } else { DeploymentAccumulator { deployments: acc.deployments, - unsound_reason: if acc.unsound_reason == "" { concat("host allocation over-committed (runner_pool + session_pool + overhead + headroom exceeds host RAM): ", offer.supply.physical.identity) } else { acc.unsound_reason }, + unsound_reason: if acc.unsound_reason == "" { concat("host allocation over-committed (runner_pool + overhead + headroom exceeds host RAM): ", offer.supply.physical.identity) } else { acc.unsound_reason }, any_unfit: acc.any_unfit, total: acc.total, } @@ -204,17 +202,28 @@ fn accumulate_conservation_deployment(acc: DeploymentAccumulator, offer: Compute } fn runner_deployment_plan() -> RunnerDeploymentPlan { - let acc = fold( - fleet_intent_offers, - init: DeploymentAccumulator { deployments: [], unsound_reason: "", any_unfit: false, total: 0 }, - f: (a, o) => accumulate_conservation_deployment(acc: a, offer: o) - ) - if acc.unsound_reason != "" { - PlanUnsound { reason: acc.unsound_reason } - } else if acc.any_unfit { - PlanUnsound { reason: "declared runner_pool_budget admits zero runners at the declared per_runner_memory_cap: raise runner_pool_budget or lower per_runner_memory_cap" } + let overhead = host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead) + if !fabric_memory_accounting_admissible( + offers: fleet_intent_offers, + overhead: overhead, + slot_count: declared_runner_count(), + contract: gunbc_runner_slot_memory_contract, + host_swap: gunbc_host_swap_available, + ) { + PlanUnsound { reason: "fabric runner memory accounting inadmissible: per-host slot_count×memory_max exceeds slice_cap and burst escape fails (per-slot memory_swap_max > host swap pool and/or single-slot memory_max > slice_cap + host_swap)" } } else { - PlanDerived { hosts: acc.deployments, fleet_concurrent_runs: acc.total } + let acc = fold( + fleet_intent_offers, + init: DeploymentAccumulator { deployments: [], unsound_reason: "", any_unfit: false, total: 0 }, + f: (a, o) => accumulate_conservation_deployment(acc: a, offer: o) + ) + if acc.unsound_reason != "" { + PlanUnsound { reason: acc.unsound_reason } + } else if acc.any_unfit { + PlanUnsound { reason: "declared runner_pool_budget admits zero runners at the declared slot_memory.memory_max: raise runner_pool_budget or lower memory_max" } + } else { + PlanDerived { hosts: acc.deployments, fleet_concurrent_runs: acc.total } + } } } diff --git a/dsl/gunbc/ci_workflow.dag b/dsl/gunbc/ci_workflow.dag index 61ad318aa2c..10e0348e40c 100644 --- a/dsl/gunbc/ci_workflow.dag +++ b/dsl/gunbc/ci_workflow.dag @@ -2,7 +2,8 @@ module gunbc.ci_workflow import extdeps.formats.yaml { yaml_string, yaml_int, yaml_bool, kv } import gunbc.ci_workflow_expressions { - ci_concurrency_group, ci_setup_rust_home, ci_cache_path, ci_cache_key, ci_cache_restore_keys + ci_concurrency_group, ci_setup_rust_home, ci_cache_path, ci_cache_key, ci_cache_restore_keys, + ci_deploy_push_to_main_if, ci_deploy_srv1_job_if, } import gunbc.ci_spec { CiSpec, gunbc_ci_spec, gunbc_ci_workflow_run, gunbc_ci_rust_job_run, @@ -11,9 +12,9 @@ import gunbc.ci_spec { import gunbc.merge_admission_produce { ci_merge_admission_gate_script, } -import gunbc.ci_workflow_expressions { ci_deploy_push_to_main_if } import gunbc.commit_workflow { commit_gate_roster, project_ci_floor_gates } import gunbc.runner_spec_from_offer { gunbc_ci_runner_spec } +import gunbc.host_runner_provision { gunbc_deploy_runner_spec } import std.disposition { Disposition, Scaffold, SingleAuthority, Terminal } import std.decl_ref { DeclarationRef, WholeDeclaration } import extdeps.github.actions { @@ -272,12 +273,12 @@ fn ci_deploy_job(stage: DeployStage) -> Job { Job { id: stage.id, name: none, - runner: gunbc_ci_runner_spec(), + runner: gunbc_deploy_runner_spec(), steps: concat(ci_prelude_steps(), [ci_deploy_step(stage: stage)]), needs: stage.needs, env: none, outputs: none, - if_condition: Present { value: ci_deploy_push_to_main_if }, + if_condition: Present { value: ci_deploy_srv1_job_if }, strategy: none, timeout_minutes: Present { value: default_job_timeout_minutes }, continue_on_error: none, diff --git a/dsl/gunbc/ci_workflow_expressions.dag b/dsl/gunbc/ci_workflow_expressions.dag index be406586bbd..4ed9e9f3fa7 100644 --- a/dsl/gunbc/ci_workflow_expressions.dag +++ b/dsl/gunbc/ci_workflow_expressions.dag @@ -80,6 +80,17 @@ data ci_workflow_expression_templates: List = [ data ci_deploy_push_to_main_if: String = "github.ref == 'refs/heads/main' && github.event_name == 'push'" +data gunbc_deploy_runner_ready_repo_var: String = "GUNBC_DEPLOY_RUNNER_READY" + +data gunbc_deploy_runner_ready_basis: String = "Observed readiness gate for deploy_dashboard_srv1: vars.GUNBC_DEPLOY_RUNNER_READY is the single authority for 'deploy runner live' (runtime inventory proxy, not a declared .dag Bool). Unset/false -> job skips on main; operator sets true after srv1 gunbc-deploy + label deploy is confirmed. Flip needs no ci.yml re-emit." + +data ci_deploy_runner_ready_repo_var_if: String = concat("vars.", concat(gunbc_deploy_runner_ready_repo_var, " == 'true'")) + +data ci_deploy_srv1_job_if: String = concat( + concat("(", ci_deploy_push_to_main_if), + concat(") && (", concat(ci_deploy_runner_ready_repo_var_if, ")")) +) + fn ci_concurrency_group() -> String { serialize_template(template: ci_concurrency_group_template) } fn ci_setup_rust_home() -> String { serialize_template(template: ci_setup_rust_home_template) } fn ci_cache_path() -> String { serialize_template(template: ci_cache_path_template) } diff --git a/dsl/gunbc/fleet_converge_emit.dag b/dsl/gunbc/fleet_converge_emit.dag index fa0fdf0ce9d..6a8d281399c 100644 --- a/dsl/gunbc/fleet_converge_emit.dag +++ b/dsl/gunbc/fleet_converge_emit.dag @@ -38,6 +38,8 @@ import gunbc.host_converge { data fleet_converge_protocol: LayoutProtocol = LayoutProtocol { indent_unit: "" } +data fleet_converge_shell_emit_dissolution_trigger: String = "dissolve-on: emit(intent,Bash) orchestration registry seam (#6092) — hand-shell concat in fleet_converge_emit.dag + fleet_converge_timer_emit.dag dissolves when Slice 2 fleet_converge (+ timer host-effect rows) lower through std.orchestration intent via 05_emit_orchestration; retire fleet_converge_sh_for + project_timer_upsert_to_doc and paired medium_structure_exception_roster entries together (gunbc.plans.shell_emission_model §5 Slice 2)" + fn sh_arg(s: String) -> String { concat(concat("\"", s), "\"") diff --git a/dsl/gunbc/fleet_converge_timer_emit.dag b/dsl/gunbc/fleet_converge_timer_emit.dag new file mode 100644 index 00000000000..3074431bb1f --- /dev/null +++ b/dsl/gunbc/fleet_converge_timer_emit.dag @@ -0,0 +1,167 @@ +module gunbc.fleet_converge_timer_emit + +import std.types { String, List, Int, Bool } +import std.measure { second_count } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import std.layout { + Doc, DocConcat, + LayoutProtocol, + doc_concat, doc_line_of, + render +} +import gunbc.fleet_converge_emit { fleet_converge_shell_emit_dissolution_trigger } +import gunbc.host_converge { + ConvergePolicy, + ConvergePolicyDerived, + ConvergePolicyUnsound, + fleet_converge_policy, + gunbc_host_ctrl_checkout_path, + gunbc_dsl_root_submodule_path, + gunbc_spawn_retry_backstop_cadence, + gunbc_fleet_converge_timer_on_boot_delay, + gunbc_fleet_operator_unix_user, +} + +data fleet_converge_timer_protocol: LayoutProtocol = LayoutProtocol { indent_unit: "" } + +data fleet_converge_timer_emit_disposition: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.fleet_converge_emit", + decl_name: "fleet_converge_shell_emit_dissolution_trigger", + field: WholeDeclaration + } +} + +data fleet_converge_timer_emit_shell_authority: String = fleet_converge_shell_emit_dissolution_trigger + +data gunbc_fleet_converge_service_unit_name: String = "gunbc-fleet-converge.service" +data gunbc_fleet_converge_timer_unit_name: String = "gunbc-fleet-converge.timer" +data gunbc_fleet_converge_systemd_unit_dir: String = "/etc/systemd/system" + +fn fleet_converge_script_path() -> String { + concat( + concat(concat(gunbc_host_ctrl_checkout_path, "/"), gunbc_dsl_root_submodule_path), + "/.github/fleet-converge.sh" + ) +} + +fn fleet_converge_service_unit_path() -> String { + concat(concat(gunbc_fleet_converge_systemd_unit_dir, "/"), gunbc_fleet_converge_service_unit_name) +} + +fn fleet_converge_timer_unit_path() -> String { + concat(concat(gunbc_fleet_converge_systemd_unit_dir, "/"), gunbc_fleet_converge_timer_unit_name) +} + +fn timer_on_unit_active_sec() -> String { + concat(to_string(second_count(s: gunbc_spawn_retry_backstop_cadence)), "s") +} + +fn timer_on_boot_sec() -> String { + concat(to_string(second_count(s: gunbc_fleet_converge_timer_on_boot_delay)), "s") +} + +fn emit_service_unit_doc() -> Doc { + let script = fleet_converge_script_path() + doc_concat(parts: [ + doc_line_of(text: "[Unit]"), + doc_line_of(text: "Description=gunbc fleet converge (host-self-selecting reconcile)"), + doc_line_of(text: "After=network-online.target"), + doc_line_of(text: "Wants=network-online.target"), + doc_line_of(text: ""), + doc_line_of(text: "[Service]"), + doc_line_of(text: "Type=oneshot"), + doc_line_of(text: concat("User=", gunbc_fleet_operator_unix_user)), + doc_line_of(text: concat("ExecStart=/usr/bin/bash ", script)), + doc_line_of(text: "RemainAfterExit=no"), + ]) +} + +fn emit_timer_unit_doc() -> Doc { + let interval = timer_on_unit_active_sec() + let boot = timer_on_boot_sec() + doc_concat(parts: [ + doc_line_of(text: "[Unit]"), + doc_line_of(text: "Description=gunbc fleet converge periodic reconcile"), + doc_line_of(text: ""), + doc_line_of(text: "[Timer]"), + doc_line_of(text: concat("OnBootSec=", boot)), + doc_line_of(text: concat("OnUnitActiveSec=", interval)), + doc_line_of(text: "Unit=gunbc-fleet-converge.service"), + doc_line_of(text: ""), + doc_line_of(text: "[Install]"), + doc_line_of(text: "WantedBy=timers.target"), + ]) +} + +fn emit_timer_upsert_header() -> Doc { + doc_concat(parts: map([ + "#!/usr/bin/env bash", + "set -euo pipefail", + "# GENERATED by dsl/gunbc/fleet_converge_timer_emit.dag (gunbc.fleet_converge_timer_emit). DO NOT HAND-EDIT.", + "# Idempotent per-host upsert: write service+timer units, daemon-reload, enable timer.", + "# The converge script self-selects via $(hostname -s); no SSH star required for steady-state.", + "", + concat("svc_path=\"", concat(fleet_converge_service_unit_path(), "\"")), + concat("timer_path=\"", concat(fleet_converge_timer_unit_path(), "\"")), + "mkdir -p \"$(dirname \"$svc_path\")\"", + "", + "cat > \"$svc_path\" <<'GUNBC_FLEET_CONVERGE_SERVICE_EOF'", + ], line => doc_line_of(text: line))) +} + +fn emit_timer_upsert_footer() -> Doc { + doc_concat(parts: [ + doc_line_of(text: "GUNBC_FLEET_CONVERGE_SERVICE_EOF"), + doc_line_of(text: ""), + doc_line_of(text: "cat > \"$timer_path\" <<'GUNBC_FLEET_CONVERGE_TIMER_EOF'"), + ]) +} + +fn emit_timer_upsert_trailer() -> Doc { + doc_concat(parts: map([ + "GUNBC_FLEET_CONVERGE_TIMER_EOF", + "", + "systemctl daemon-reload", + "systemctl enable --now gunbc-fleet-converge.timer", + "printf 'fleet-converge-timer-receipt host=%s verdict=converged\\n' \"$(hostname -s 2>/dev/null || echo unknown)\"", + ], line => doc_line_of(text: line))) +} + +fn timer_upsert_unsound_doc(reason: String) -> Doc { + doc_concat(parts: map([ + "#!/usr/bin/env bash", + "set -euo pipefail", + "# GENERATED by dsl/gunbc/fleet_converge_timer_emit.dag (gunbc.fleet_converge_timer_emit). DO NOT HAND-EDIT.", + concat("echo \"fleet-converge-timer UNSOUND: ", concat(reason, "\" >&2")), + "exit 1" + ], line => doc_line_of(text: line))) +} + +fn project_timer_upsert_to_doc(policy: ConvergePolicy) -> Doc { + match policy { + ConvergePolicyUnsound { reason: reason } => timer_upsert_unsound_doc(reason: reason) + ConvergePolicyDerived { hosts: _ } => + doc_concat(parts: [ + emit_timer_upsert_header(), + emit_service_unit_doc(), + emit_timer_upsert_footer(), + emit_timer_unit_doc(), + emit_timer_upsert_trailer(), + ]) + } +} + +fn fleet_converge_timer_upsert_sh_for(policy: ConvergePolicy) -> String { + render(doc: project_timer_upsert_to_doc(policy: policy), proto: fleet_converge_timer_protocol) +} + +fn expected_fleet_converge_timer_upsert_sh() -> String { + fleet_converge_timer_upsert_sh_for(policy: fleet_converge_policy()) +} + +fn fleet_converge_timer_upsert_drifted(committed: String) -> Bool { + !(committed == expected_fleet_converge_timer_upsert_sh()) +} diff --git a/dsl/gunbc/fleet_host_budget.dag b/dsl/gunbc/fleet_host_budget.dag index 815301a961b..f927b994f3d 100644 --- a/dsl/gunbc/fleet_host_budget.dag +++ b/dsl/gunbc/fleet_host_budget.dag @@ -1,6 +1,8 @@ module gunbc.fleet_host_budget import std.types { Int, List, String, Bool } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } import std.nat { Nat } import std.measure { ByteSize, byte_size, byte_size_count, @@ -8,6 +10,8 @@ import std.measure { time_measure, watt, measure_scale_fraction_floor, measure_fit_count_floor, + measure_le, + measure_add, HardwareThreadCount, hardware_thread_count, } import std.realization_width { byte_size_saturating_sub, cpu_cores_available } @@ -16,7 +20,13 @@ import std.effects { EffectShape, CreateEffect, CreateCause, CreateIfAbsent, Key import std.realization_schedule { CostAccount, CostBasis, Predicted } import extdeps.accounting.budget { Appropriation, LineItem } import product.budget_tree { BudgetNode, BudgetClaim, Guaranteed, node_conserves } -import gunbc.ci_fleet { ComputeOffer } +import gunbc.ci_fleet { fleet_intent_offers, ComputeOffer } +import gunbc.ci_floor_measurement { + gunbc_ci_host_fixed_overhead, + HostFixedOverhead, + HostFixedOverheadMeasured, + HostFixedOverheadUnmeasured, +} import gunbc.fleet_intent { compute_host_ram_bytes_total, compute_host_primary_cpu } import extdeps.cpu { cpu_facts_threads } import extdeps.ctrl.jobserver { @@ -145,22 +155,184 @@ fn child_node(name: String, b: ByteSize) -> BudgetNode { } -data gunbc_runner_pool_budget: ByteSize = byte_size(85899345920) +fn host_fixed_overhead_bytes(o: HostFixedOverhead) -> ByteSize { + match o { + HostFixedOverheadMeasured { bytes: b } => b + HostFixedOverheadUnmeasured => byte_size(0) + } +} -data gunbc_per_runner_memory_cap: ByteSize = gunbc_ci_runner_cgroup_memory_cap +fn runner_slice_cap(host: ByteSize, overhead: ByteSize, headroom: ByteSize) -> ByteSize { + let after_overhead = byte_size_saturating_sub(total: host, reserved: overhead) + byte_size_saturating_sub(total: after_overhead, reserved: headroom) +} -data gunbc_session_container_memory: ByteSize = byte_size(1073741824) +type RunnerSlotMemoryContract { + memory_max: ByteSize + memory_high: ByteSize + memory_swap_max: ByteSize +} -fn session_pool_bytes(active_sessions: Nat) -> ByteSize { - measure_scale_fraction_floor(m: gunbc_session_container_memory, num: active_sessions, den: 1) +data gunbc_runner_slot_memory_contract_disposition: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "product.compute_fabric", + decl_name: "connect", + field: WholeDeclaration + } } +data gunbc_runner_slot_memory_contract: RunnerSlotMemoryContract = RunnerSlotMemoryContract { + memory_max: byte_size(25769803776), + memory_high: byte_size(15032385536), + memory_swap_max: byte_size(8589934592), +} + +data gunbc_per_runner_memory_cap: ByteSize = gunbc_runner_slot_memory_contract.memory_max + +data gunbc_per_runner_swap_cap: ByteSize = gunbc_runner_slot_memory_contract.memory_swap_max + +data gunbc_per_runner_memory_high: ByteSize = gunbc_runner_slot_memory_contract.memory_high + +data gunbc_declared_runner_slot_count: Int = 10 + +data gunbc_host_swap_available: ByteSize = byte_size(8589934592) + +data gunbc_ci_heavy_job_working_set: ByteSize = byte_size(11811160064) + +fn runner_ram_overcommit_per_host( + slot_count: Int, + contract: RunnerSlotMemoryContract, + slice_cap: ByteSize +) -> ByteSize { + byte_size_saturating_sub( + total: fabric_contract_memory_demand_per_host(slot_count: slot_count, contract: contract), + reserved: slice_cap + ) +} + +fn runner_swap_backs_ram_overcommit( + slot_count: Int, + contract: RunnerSlotMemoryContract, + slice_cap: ByteSize, + host_swap: ByteSize +) -> Bool { + measure_le( + a: runner_ram_overcommit_per_host(slot_count: slot_count, contract: contract, slice_cap: slice_cap), + b: host_swap + ) +} + +fn effective_heavy_job_concurrency(slice_cap: ByteSize, working_set: ByteSize) -> Int { + measure_fit_count_floor(capacity: slice_cap, each: working_set) +} + +fn fabric_host_allocatable_runner_memory(offer: ComputeOffer, overhead: ByteSize) -> ByteSize { + let host_ram = compute_host_ram_bytes_total(host: offer.supply.physical) + let headroom = oomd_actuation_floor(host: host_ram) + runner_slice_cap(host: host_ram, overhead: overhead, headroom: headroom) +} + +fn fabric_total_allocatable_runner_memory(offers: List, overhead: ByteSize) -> ByteSize { + fold( + offers, + init: byte_size(0), + f: (acc, o) => measure_add(a: acc, b: fabric_host_allocatable_runner_memory(offer: o, overhead: overhead)) + ) +} + +fn fabric_host_count(offers: List) -> Int { + offers.length() +} + +fn fabric_runner_slice_cap_per_host(offers: List, overhead: ByteSize) -> ByteSize { + let count = fabric_host_count(offers: offers) + let divisor = if count <= 0 { 1 } else { count } + measure_scale_fraction_floor( + m: fabric_total_allocatable_runner_memory(offers: offers, overhead: overhead), + num: 1, + den: divisor + ) +} + +fn fabric_contract_memory_demand_per_host(slot_count: Int, contract: RunnerSlotMemoryContract) -> ByteSize { + measure_scale_fraction_floor(m: contract.memory_max, num: slot_count, den: 1) +} + +fn fabric_total_contract_memory_demand( + offers: List, + slot_count: Int, + contract: RunnerSlotMemoryContract +) -> ByteSize { + measure_scale_fraction_floor( + m: fabric_contract_memory_demand_per_host(slot_count: slot_count, contract: contract), + num: fabric_host_count(offers: offers), + den: 1 + ) +} + +fn host_memory_burst_admissible( + contract: RunnerSlotMemoryContract, + slice_cap: ByteSize, + host_swap: ByteSize +) -> Bool { + measure_le(a: contract.memory_swap_max, b: host_swap) + && measure_le(a: contract.memory_max, b: measure_add(a: slice_cap, b: host_swap)) +} + +fn host_memory_accounting_admissible( + offer: ComputeOffer, + overhead: ByteSize, + slot_count: Int, + contract: RunnerSlotMemoryContract, + host_swap: ByteSize +) -> Bool { + let slice_cap = fabric_host_allocatable_runner_memory(offer: offer, overhead: overhead) + let demand = fabric_contract_memory_demand_per_host(slot_count: slot_count, contract: contract) + measure_le(a: demand, b: slice_cap) + || host_memory_burst_admissible(contract: contract, slice_cap: slice_cap, host_swap: host_swap) +} + +fn fabric_memory_accounting_admissible( + offers: List, + overhead: ByteSize, + slot_count: Int, + contract: RunnerSlotMemoryContract, + host_swap: ByteSize +) -> Bool { + all( + offers, + o => host_memory_accounting_admissible( + offer: o, + overhead: overhead, + slot_count: slot_count, + contract: contract, + host_swap: host_swap + ) + ) +} + +fn fabric_effective_heavy_job_concurrency(offers: List, overhead: ByteSize, working_set: ByteSize) -> Int { + effective_heavy_job_concurrency( + slice_cap: fabric_total_allocatable_runner_memory(offers: offers, overhead: overhead), + working_set: working_set + ) +} + +data gunbc_runner_slice_cap: ByteSize = fabric_runner_slice_cap_per_host( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead) +) + +data gunbc_runner_pool_budget: ByteSize = gunbc_runner_slice_cap + +data gunbc_runner_standalone_memory_contract_basis: String = "Fabric-level runner budget (operator 2026-07-01 reframe): authority is Σ(contracts) ≤ Σ(capacity) across fleet_intent_offers — hosts are fungible opportunities, not per-host literals. RunnerSlotMemoryContract (burst: high=14GiB, max=24GiB, swap=8GiB/slot) is the workload contract; gunbc_runner_slice_cap is a REALIZATION projection (systemd slice MemoryMax per host) derived via fabric_runner_slice_cap_per_host = Σallocatable / host_count, NOT a hardcoded 80G input. fabric_effective_heavy_job_concurrency = Σallocatable / working_set (~11GiB), emitted on runner-deploy manifest. Per-host burst overcommit (slot_count×memory_max > slice_cap) is ADMISSIBLE (not aggregate-sound) only when host_memory_burst_admissible: per-slot memory_swap_max ≤ host swap pool AND single-slot memory_max ≤ slice_cap + host_swap (one slot may page against the shared pool; runner_swap_backs_ram_overcommit documents aggregate simultaneous demand is NOT swap-backed). fabric_memory_accounting_admissible = all hosts pass host_memory_accounting_admissible and gates runner_deployment_plan. srv3 is mixed (runners+sessions), not dedicated. session_pool dropped from runner budget (cool-swift P4). dissolve-on: product.compute_fabric connect() memory axis + clever-raven orchestration emit." + fn host_allocation_tree( host: ByteSize, overhead: ByteSize, headroom: ByteSize, - runner_pool: ByteSize, - session_pool: ByteSize + runner_pool: ByteSize ) -> BudgetNode { BudgetNode { name: "host_allocation", @@ -168,7 +340,6 @@ fn host_allocation_tree( claims: [overhead_claim(overhead: overhead)], children: [ child_node(name: "runner_pool", b: runner_pool), - child_node(name: "session_pool", b: session_pool), child_node(name: "headroom", b: headroom), ], } @@ -178,31 +349,18 @@ fn host_allocation_conserves( host: ByteSize, overhead: ByteSize, headroom: ByteSize, - runner_pool: ByteSize, - session_pool: ByteSize + runner_pool: ByteSize ) -> Bool { node_conserves(node: host_allocation_tree( host: host, overhead: overhead, headroom: headroom, - runner_pool: runner_pool, - session_pool: session_pool + runner_pool: runner_pool )) } fn declared_runner_count() -> Int { - measure_fit_count_floor(capacity: gunbc_runner_pool_budget, each: gunbc_ci_runner_cgroup_memory_cap) -} - -fn max_concurrent_sessions(session_pool: ByteSize) -> Int { - measure_fit_count_floor(capacity: session_pool, each: gunbc_session_container_memory) -} - - -fn runner_slice_cap(host: ByteSize, overhead: ByteSize, session_slice: ByteSize, headroom: ByteSize) -> ByteSize { - let after_overhead = byte_size_saturating_sub(total: host, reserved: overhead) - let after_session = byte_size_saturating_sub(total: after_overhead, reserved: session_slice) - byte_size_saturating_sub(total: after_session, reserved: headroom) + gunbc_declared_runner_slot_count } fn runner_slice_node(runner_slice: ByteSize, runner_baseline: ByteSize) -> BudgetNode { @@ -219,7 +377,6 @@ fn fleet_host_budget_tree( host: ByteSize, overhead: ByteSize, runner_baseline: ByteSize, - session_slice: ByteSize, headroom: ByteSize, runner_slice: ByteSize ) -> BudgetNode { @@ -228,7 +385,6 @@ fn fleet_host_budget_tree( appropriation: appropriation_of(b: host), claims: [overhead_claim(overhead: overhead)], children: [ - child_node(name: "session_worstcase_reservation", b: session_slice), child_node(name: "headroom", b: headroom), runner_slice_node(runner_slice: runner_slice, runner_baseline: runner_baseline), ], @@ -260,7 +416,6 @@ fn fleet_host_plan_for( overhead: ByteSize, nproc: HardwareThreadCount, runner: FleetContainerType, - session_slice: ByteSize, citation: CtrlJobserverCitation, policy: HeadroomPolicy, oomd: OomdEnforcementEvidence @@ -280,8 +435,8 @@ fn fleet_host_plan_for( match resolve_headroom(policy: policy, floor: floor) { HeadroomUnsound { reason: r } => FleetHostPlanUnsound { reason: r } HeadroomResolved { bytes: hr } => { - let slice = runner_slice_cap(host: host, overhead: overhead, session_slice: session_slice, headroom: hr) - let tree = fleet_host_budget_tree(name: name, host: host, overhead: overhead, runner_baseline: rb, session_slice: session_slice, headroom: hr, runner_slice: slice) + let slice = runner_slice_cap(host: host, overhead: overhead, headroom: hr) + let tree = fleet_host_budget_tree(name: name, host: host, overhead: overhead, runner_baseline: rb, headroom: hr, runner_slice: slice) FleetHostPlanDerived { name: name, tree: tree, @@ -305,7 +460,6 @@ fn fleet_host_plan() -> FleetHostPlan { overhead: byte_size(17179869184), nproc: hardware_thread_count(count: 128), runner: gha_runner_container, - session_slice: byte_size(0), citation: ctrl_jobserver_config, policy: gunbc_fleet_headroom_policy, oomd: gunbc_fleet_oomd_evidence @@ -316,7 +470,6 @@ fn fleet_host_plan() -> FleetHostPlan { fn fleet_host_plan_for_offer( offer: ComputeOffer, overhead: ByteSize, - session_slice: ByteSize, citation: CtrlJobserverCitation, policy: HeadroomPolicy, oomd: OomdEnforcementEvidence @@ -331,7 +484,6 @@ fn fleet_host_plan_for_offer( overhead: overhead, nproc: cpu_facts_threads(cpu: cpu), runner: gha_runner_container, - session_slice: session_slice, citation: citation, policy: policy, oomd: oomd diff --git a/dsl/gunbc/fleet_intent_network.dag b/dsl/gunbc/fleet_intent_network.dag index fabaa60ab4e..a1a6ade399f 100644 --- a/dsl/gunbc/fleet_intent_network.dag +++ b/dsl/gunbc/fleet_intent_network.dag @@ -15,8 +15,24 @@ import product.network_topology { DockerBridge, TailscaleOverlay, } +import std.types { String, List } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } import product.placement_supply { HostIdentity } +data operator_fleet_unix_user_disposition: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.fleet_intent_network", + decl_name: "operator_fleet_unix_user", + field: WholeDeclaration + } +} + +data operator_fleet_unix_user: String = "briansrls" + +data operator_fleet_unix_home: String = concat("/home/", operator_fleet_unix_user) + data operator_host_srv1: HostIdentity = "srv1" data operator_host_srv2: HostIdentity = "srv2" data operator_host_srv3: HostIdentity = "srv3" diff --git a/dsl/gunbc/generated_artifact.dag b/dsl/gunbc/generated_artifact.dag index fe9b4944490..c9996a0caff 100644 --- a/dsl/gunbc/generated_artifact.dag +++ b/dsl/gunbc/generated_artifact.dag @@ -13,6 +13,7 @@ type GeneratedArtifact | GithooksPrePushArtifact | RunnerDeployArtifact | FleetConvergeArtifact + | FleetConvergeTimerUpsertArtifact | AutoinstallUserDataArtifact { spec: HostAutoinstall } | ProxyDhcpDnsmasqArtifact { spec: InstallServerSpec } | PlanArtifact { plan: Plan } @@ -20,7 +21,7 @@ type GeneratedArtifact data generated_artifact_registry: List = concat( concat( concat( - [CiYamlArtifact, GitignoreArtifact, RoadmapArtifact, DesignArtifact, GithooksPrePushArtifact, RunnerDeployArtifact, FleetConvergeArtifact], + [CiYamlArtifact, GitignoreArtifact, RoadmapArtifact, DesignArtifact, GithooksPrePushArtifact, RunnerDeployArtifact, FleetConvergeArtifact, FleetConvergeTimerUpsertArtifact], map(fleet_autoinstall_specs, s => AutoinstallUserDataArtifact { spec: s }) ), map(fleet_install_server_specs, s => ProxyDhcpDnsmasqArtifact { spec: s }) @@ -47,6 +48,7 @@ fn artifact_path(a: GeneratedArtifact) -> String { GithooksPrePushArtifact => ".githooks/pre-push" RunnerDeployArtifact => ".github/fleet-runner-deploy.manifest" FleetConvergeArtifact => ".github/fleet-converge.sh" + FleetConvergeTimerUpsertArtifact => ".github/fleet-converge-timer-upsert.sh" AutoinstallUserDataArtifact { spec } => concat(concat("provisioning/", spec.host), "/user-data") ProxyDhcpDnsmasqArtifact { spec } => concat(concat("provisioning/", spec.host), "/dnsmasq.conf") PlanArtifact { plan } => plan_path(p: plan) @@ -63,6 +65,7 @@ fn artifact_commit_policy(a: GeneratedArtifact) -> CommitPolicy { PlanArtifact { plan: _ } => CommitRequired { consumer: ProjectDocumentation } RunnerDeployArtifact => NotCommitted FleetConvergeArtifact => CommitRequired { consumer: HostReconciler } + FleetConvergeTimerUpsertArtifact => CommitRequired { consumer: HostReconciler } AutoinstallUserDataArtifact { spec: _ } => NotCommitted ProxyDhcpDnsmasqArtifact { spec: _ } => NotCommitted } @@ -88,6 +91,7 @@ fn artifact_eq(a: GeneratedArtifact, b: GeneratedArtifact) -> Bool { GithooksPrePushArtifact => match b { GithooksPrePushArtifact => true _ => false } RunnerDeployArtifact => match b { RunnerDeployArtifact => true _ => false } FleetConvergeArtifact => match b { FleetConvergeArtifact => true _ => false } + FleetConvergeTimerUpsertArtifact => match b { FleetConvergeTimerUpsertArtifact => true _ => false } AutoinstallUserDataArtifact { spec } => match b { AutoinstallUserDataArtifact { spec: spec2 } => spec.host == spec2.host _ => false } ProxyDhcpDnsmasqArtifact { spec } => match b { ProxyDhcpDnsmasqArtifact { spec: spec2 } => spec.host == spec2.host _ => false } PlanArtifact { plan } => match b { PlanArtifact { plan: plan2 } => plan.slug == plan2.slug _ => false } diff --git a/dsl/gunbc/generated_artifact_emit.dag b/dsl/gunbc/generated_artifact_emit.dag index bed03ff6d52..eb800963fa1 100644 --- a/dsl/gunbc/generated_artifact_emit.dag +++ b/dsl/gunbc/generated_artifact_emit.dag @@ -4,7 +4,7 @@ module gunbc.generated_artifact_emit import gunbc.generated_artifact { GeneratedArtifact, CiYamlArtifact, GitignoreArtifact, RoadmapArtifact, DesignArtifact, - RunnerDeployArtifact, FleetConvergeArtifact, GithooksPrePushArtifact, + RunnerDeployArtifact, FleetConvergeArtifact, FleetConvergeTimerUpsertArtifact, GithooksPrePushArtifact, AutoinstallUserDataArtifact, ProxyDhcpDnsmasqArtifact, PlanArtifact, artifact_path } @@ -15,6 +15,7 @@ import gunbc.roadmap_authority { expected_roadmap_md } import gunbc.design_document { expected_design_md } import gunbc.runner_deploy_emit { expected_runner_deploy_manifest } import gunbc.fleet_converge_emit { expected_fleet_converge_sh } +import gunbc.fleet_converge_timer_emit { expected_fleet_converge_timer_upsert_sh } import gunbc.githooks_pre_push_emit { expected_githooks_pre_push_sh } import gunbc.os_install_emit { autoinstall_user_data } import gunbc.install_server_emit { install_server_dnsmasq_conf } @@ -29,6 +30,7 @@ fn artifact_generate(a: GeneratedArtifact) -> String { GithooksPrePushArtifact => expected_githooks_pre_push_sh() RunnerDeployArtifact => expected_runner_deploy_manifest() FleetConvergeArtifact => expected_fleet_converge_sh() + FleetConvergeTimerUpsertArtifact => expected_fleet_converge_timer_upsert_sh() AutoinstallUserDataArtifact { spec } => autoinstall_user_data(payload: spec.payload) ProxyDhcpDnsmasqArtifact { spec } => install_server_dnsmasq_conf(spec: spec) PlanArtifact { plan } => expected_plan_md(p: plan) @@ -44,6 +46,7 @@ fn artifact_extra_valid(a: GeneratedArtifact) -> Bool { GithooksPrePushArtifact => true RunnerDeployArtifact => true FleetConvergeArtifact => true + FleetConvergeTimerUpsertArtifact => true AutoinstallUserDataArtifact { spec: _ } => true ProxyDhcpDnsmasqArtifact { spec: _ } => true PlanArtifact { plan: _ } => true diff --git a/dsl/gunbc/host_converge.dag b/dsl/gunbc/host_converge.dag index 88e4d07b8a0..a53822d32c8 100644 --- a/dsl/gunbc/host_converge.dag +++ b/dsl/gunbc/host_converge.dag @@ -25,14 +25,25 @@ import gunbc.ci_floor_measurement { SessionAggregateUnmeasured, gunbc_ci_session_reservation_mode, } -import gunbc.ci_fleet { fleet_intent_offers } -import gunbc.fleet_intent_network { operator_host_srv3 } +import gunbc.fleet_intent_network { + operator_fleet_unix_home, + operator_fleet_unix_user, +} +import gunbc.bmc_onboarding { + host_onboarding_lifecycle_phase, + BmcOnboardingPhase, + FactoryDefault, + CredentialsRotated, + OsInstalled, + FabricJoined, +} import extdeps.ctrl.gunbc_pin { gunbc_pin_match, gunbc_pin_skewed } import gunbc.oomd_install { gunbc_oomd_sessions_policy, gunbc_oomd_sessions_slice } import gunbc.runner_slot_enforcement { gunbc_runner_slot_unit_placeholder } import extdeps.os.systemd { systemd_memory_max_property, systemd_memory_swap_max_property, + systemd_memory_high_property, systemd_cpu_weight_property, } import extdeps.os.oomd { @@ -40,6 +51,7 @@ import extdeps.os.oomd { oomd_managed_oom_memory_pressure_limit_property, managed_oom_action_value, managed_oom_pressure_limit_show_value, + ManagedOomAuto, } @@ -49,6 +61,7 @@ data gunbc_sessions_legacy_parent_slice: String = "system.slice" data gunbc_runner_unit_template: String = "actions-runner@.service" data gunbc_runner_per_slot_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" data gunbc_runner_per_slot_swap_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" +data gunbc_runner_per_slot_high_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf" data gunbc_runner_jobserver_env_file: String = "/etc/default/ctrl-jobserver" data gunbc_runner_jobserver_env_key: String = "CTRL_JOBSERVER_TOKENS" data gunbc_runner_jobserver_service: String = "ctrl-jobserver.service" @@ -57,9 +70,13 @@ data gunbc_root_env_key: String = "GUNBC_ROOT" data gunbc_dsl_root_submodule_path: String = "third_party/gunbc" -data gunbc_host_ctrl_checkout_path: String = "$HOME/ctrl" +data gunbc_fleet_operator_unix_home: String = operator_fleet_unix_home + +data gunbc_fleet_operator_unix_user: String = operator_fleet_unix_user + +data gunbc_host_ctrl_checkout_path: String = concat(gunbc_fleet_operator_unix_home, "/ctrl") -data gunbc_host_gunbc_root_path: String = "$HOME/.local/share/gunbc/pinned" +data gunbc_host_gunbc_root_path: String = concat(gunbc_fleet_operator_unix_home, "/.local/share/gunbc/pinned") data gunbc_pinned_tree_desired_label: String = "ctrl:third_party/gunbc" @@ -92,6 +109,8 @@ data gunbc_ctrl_quiescent_reload_hook: String = "scripts/session-dashboard/bin/g data gunbc_spawn_retry_backstop_cadence: Second = second(count: 600) +data gunbc_fleet_converge_timer_on_boot_delay: Second = second(count: 120) + type GreenPlaceReadinessMarker { path: String pin_prefix: String @@ -367,7 +386,16 @@ fn gunbc_pinned_binary_provenance_policy() -> GunbcBinaryProvenancePolicy { } fn gunbc_apply_mode_for_host(host: HostIdentity) -> ConvergeApplyMode { - if host == operator_host_srv3 { FreshStandup } else { ExistingHostQuiescentReload } + match host_onboarding_lifecycle_phase(host: host) { + Present { value: phase } => + match phase { + FactoryDefault => FreshStandup + CredentialsRotated => FreshStandup + OsInstalled => ExistingHostQuiescentReload + FabricJoined => ExistingHostQuiescentReload + } + Absent => ExistingHostQuiescentReload + } } fn gunbc_pinned_noop_policy() -> ConvergeNoOpPolicy { @@ -435,23 +463,45 @@ fn runner_host_knobs(h: RunnerHostDeployment) -> List { legacy_converge_knob( slice: RunnerSlice, name: "per_slot_memory_max_bytes", - desired: to_string(byte_size_count(b: h.per_runner_memory_cap)), + desired: to_string(byte_size_count(b: h.slot_memory.memory_max)), target: PerSlotMemoryCap { dropin_path: gunbc_runner_per_slot_dropin_path, instance_glob: glob, property: systemd_memory_max_property, - bytes_value: to_string(byte_size_count(b: h.per_runner_memory_cap)), + bytes_value: to_string(byte_size_count(b: h.slot_memory.memory_max)), } ), legacy_converge_knob( slice: RunnerSlice, name: "per_slot_memory_swap_max_bytes", - desired: "0", + desired: to_string(byte_size_count(b: h.slot_memory.memory_swap_max)), target: PerSlotMemoryCap { dropin_path: gunbc_runner_per_slot_swap_dropin_path, instance_glob: glob, property: systemd_memory_swap_max_property, - bytes_value: "0", + bytes_value: to_string(byte_size_count(b: h.slot_memory.memory_swap_max)), + } + ), + legacy_converge_knob( + slice: RunnerSlice, + name: "per_slot_memory_high_bytes", + desired: to_string(byte_size_count(b: h.slot_memory.memory_high)), + target: PerSlotMemoryCap { + dropin_path: gunbc_runner_per_slot_high_dropin_path, + instance_glob: glob, + property: systemd_memory_high_property, + bytes_value: to_string(byte_size_count(b: h.slot_memory.memory_high)), + } + ), + legacy_converge_knob( + slice: RunnerSlice, + name: "oom_pressure_auto", + desired: "true", + target: SliceProperty { + unit: gunbc_runner_slot_unit_placeholder, + property: oomd_managed_oom_memory_pressure_property, + apply_value: managed_oom_action_value(a: ManagedOomAuto), + expected_effective: managed_oom_action_value(a: ManagedOomAuto), } ), slice_numeric_knob( diff --git a/dsl/gunbc/host_runner_provision.dag b/dsl/gunbc/host_runner_provision.dag new file mode 100644 index 00000000000..84ae596c34d --- /dev/null +++ b/dsl/gunbc/host_runner_provision.dag @@ -0,0 +1,103 @@ +module gunbc.host_runner_provision + +import std.types { String, List, NonEmptyStr } +import extdeps.github.actions { RunnerSpec, SelfHosted, HostedRunner, RunsOnExpression } +import gunbc.fleet_intent { ComputeHost, srv1_host } +import gunbc.ci_fleet { ComputeOffer, srv1_offer, srv2_offer, srv3_offer } +import gunbc.runner_spec_from_offer { runner_spec_from_offer } +import gunbc.ci_deploy_access { + DeployAccess, + ci_deploy_srv1_access, + gunbc_deploy_runner_principal, + deploy_access_command_paths, +} + +type RunnerProvisionRole = Ci | Deploy + +type HostRunnerProvision { + role: RunnerProvisionRole + principal: NonEmptyStr + host: ComputeHost + privileged_commands: List + labels: List +} + +data gunbc_ci_runner_principal: NonEmptyStr = "gunbc-runner" + +data gunbc_runner_principal: NonEmptyStr = gunbc_ci_runner_principal + +data gunbc_deploy_runner_label: NonEmptyStr = "deploy" + +data gunbc_runner_provision_privileged_commands_basis: String = "Dual-role HostRunnerProvision (operator L8 / cool-swift): ci role = gunbc-runner, quarantined (privileged_commands empty); deploy role = gunbc-deploy, trusted, privileged_commands aggregated from DeployAccess consumer-declares/provisioning-grants (bash, apt-get, systemctl, tailscale, install) realized as /etc/sudoers.d/gunbc-deploy NOPASSWD. Deploy GHA label = deploy so runs-on [self-hosted, linux, arm64, deploy] lands only on the trusted identity. srv1 carries both roles; other fleet hosts CI-only until deploy is declared. Deploy job readiness gate is observed via vars.GUNBC_DEPLOY_RUNNER_READY (ci_workflow_expressions), not a declared Bool here. dissolve-on: aggregate_authority fold over all HostEffect consumers (#6093)." + +fn compute_offer_for_host(host: ComputeHost) -> ComputeOffer { + match host.identity { + "srv1" => srv1_offer + "srv2" => srv2_offer + "srv3" => srv3_offer + _ => srv1_offer + } +} + +fn runner_spec_label_list(spec: RunnerSpec) -> List { + match spec { + SelfHosted { labels: labels } => labels + HostedRunner { label: l } => [l] + RunsOnExpression { expression: e } => [e] + } +} + +fn runner_spec_with_extra_label(spec: RunnerSpec, extra: String) -> RunnerSpec { + match spec { + SelfHosted { labels: labels } => SelfHosted { labels: concat(labels, [extra]) } + other => other + } +} + +fn host_has_deploy_runner(host: ComputeHost) -> Bool { + host.identity == srv1_host.identity +} + +fn gunbc_ci_runner_provision_for(host: ComputeHost) -> HostRunnerProvision { + HostRunnerProvision { + role: Ci, + principal: gunbc_ci_runner_principal, + host: host, + privileged_commands: [], + labels: runner_spec_label_list(spec: runner_spec_from_offer(offer: compute_offer_for_host(host: host))), + } +} + +fn gunbc_deploy_runner_provision_for(host: ComputeHost) -> HostRunnerProvision { + let base = runner_spec_from_offer(offer: compute_offer_for_host(host: host)) + HostRunnerProvision { + role: Deploy, + principal: gunbc_deploy_runner_principal, + host: host, + privileged_commands: deploy_access_command_paths(access: ci_deploy_srv1_access), + labels: runner_spec_label_list(spec: runner_spec_with_extra_label(spec: base, extra: gunbc_deploy_runner_label)), + } +} + +fn host_runner_provisions_for(host: ComputeHost) -> List { + if host_has_deploy_runner(host: host) { + [gunbc_ci_runner_provision_for(host: host), gunbc_deploy_runner_provision_for(host: host)] + } else { + [gunbc_ci_runner_provision_for(host: host)] + } +} + +fn gunbc_runner_provision_for(host: ComputeHost) -> HostRunnerProvision { + gunbc_ci_runner_provision_for(host: host) +} + +fn host_runner_provision_principal(p: HostRunnerProvision) -> String { + p.principal +} + +fn gunbc_deploy_runner_spec() -> RunnerSpec { + runner_spec_with_extra_label( + spec: runner_spec_from_offer(offer: srv1_offer), + extra: gunbc_deploy_runner_label + ) +} diff --git a/dsl/gunbc/host_standup.dag b/dsl/gunbc/host_standup.dag index f59cb64c01e..319d6f33ae8 100644 --- a/dsl/gunbc/host_standup.dag +++ b/dsl/gunbc/host_standup.dag @@ -17,6 +17,7 @@ import gunbc.os_install_mechanism { srv3_install_mechanism, } import gunbc.host_converge { fleet_converge_policy, ConvergePolicy } +import gunbc.host_runner_provision { HostRunnerProvision, gunbc_runner_provision_for } import gunbc.fleet_intent_network { operator_host_srv3 } import product.placement_supply { HostIdentity } @@ -212,6 +213,8 @@ data host_standup_spine: List = [ label: "P1:runner-deploy-slot", disposition: ModeledCompose { authorities: [ + decl_ref(module_path: "gunbc.host_runner_provision", decl_name: "HostRunnerProvision"), + decl_ref(module_path: "gunbc.host_runner_provision", decl_name: "gunbc_runner_provision_for"), decl_ref(module_path: "gunbc.runner_spec_from_offer", decl_name: "runner_spec_from_offer"), decl_ref(module_path: "gunbc.runner_deploy_emit", decl_name: "expected_runner_deploy_manifest"), decl_ref(module_path: "gunbc.runner_slot_enforcement", decl_name: "reconcile_runner_slot"), @@ -291,7 +294,7 @@ data host_standup_gap_ledger: List = [ GapLedgerRow { phase_label: "P1:runner-deploy-slot", modeling: "modeled", - authority_or_interim: "gunbc.runner_spec_from_offer + gunbc.runner_deploy_emit + gunbc.runner_slot_enforcement + gunbc.ci_runner_placement" + authority_or_interim: "gunbc.host_runner_provision.HostRunnerProvision + gunbc.runner_spec_from_offer + gunbc.runner_deploy_emit + gunbc.runner_slot_enforcement + gunbc.ci_runner_placement" }, GapLedgerRow { phase_label: "P2:compute-fabric-enroll", diff --git a/dsl/gunbc/runner_deploy_emit.dag b/dsl/gunbc/runner_deploy_emit.dag index 3baf96dafcb..210d373867e 100644 --- a/dsl/gunbc/runner_deploy_emit.dag +++ b/dsl/gunbc/runner_deploy_emit.dag @@ -28,6 +28,12 @@ import gunbc.ci_floor_measurement { gunbc_ci_session_reservation_mode, } import gunbc.ci_fleet { fleet_intent_offers } +import gunbc.fleet_host_budget { + host_fixed_overhead_bytes, + gunbc_ci_heavy_job_working_set, + fabric_effective_heavy_job_concurrency, +} +import gunbc.ci_floor_measurement { gunbc_ci_host_fixed_overhead } import gunbc.oomd_install { gunbc_oomd_sessions_policy } import extdeps.github.actions { RunnerSpec, SelfHosted, HostedRunner, RunsOnExpression } import std.measure { byte_size_count, hardware_thread_count_value, percent_count } @@ -66,7 +72,9 @@ fn manifest_host_fields(h: RunnerHostDeployment) -> List { serial_field(key: "host", value: serial_scalar(text: h.identity)), serial_field(key: "runner_count", value: serial_scalar(text: to_string(h.runner_count))), serial_field(key: "runner_slice_cap_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.runner_slice_cap)))), - serial_field(key: "per_slot_memory_max_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.per_runner_memory_cap)))), + serial_field(key: "per_slot_memory_max_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.slot_memory.memory_max)))), + serial_field(key: "per_slot_memory_high_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.slot_memory.memory_high)))), + serial_field(key: "per_slot_memory_swap_max_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.slot_memory.memory_swap_max)))), serial_field(key: "build_tokens", value: serial_scalar(text: to_string(hardware_thread_count_value(t: h.build_tokens)))), serial_field(key: "labels", value: serial_scalar(text: runner_labels_text(spec: h.runner))), serial_field(key: "cpu_weight", value: serial_scalar(text: to_string(workload_class_cpu_weight(wc: h.workload_class)))) @@ -84,7 +92,17 @@ fn runner_plan_doc() -> Doc { PlanDerived { hosts: hosts, fleet_concurrent_runs: total } => doc_concat(parts: append( map(hosts, h => doc_concat(parts: [doc_text(text: "runner-deploy "), manifest_host_doc(h: h), DocLine])), - items: [doc_line_of(text: concat("runner-deploy fleet_concurrent_runs=", to_string(total)))] + items: [ + doc_line_of(text: concat("runner-deploy fleet_concurrent_runs=", to_string(total))), + doc_line_of(text: concat( + "runner-deploy fabric_effective_heavy_job_concurrency=", + to_string(fabric_effective_heavy_job_concurrency( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead), + working_set: gunbc_ci_heavy_job_working_set, + )) + )), + ] )) } } diff --git a/dsl/test/claim/bmc_onboarding_lifecycle_witness_test.dag b/dsl/test/claim/bmc_onboarding_lifecycle_witness_test.dag index a4413598777..5a55adf5c6c 100644 --- a/dsl/test/claim/bmc_onboarding_lifecycle_witness_test.dag +++ b/dsl/test/claim/bmc_onboarding_lifecycle_witness_test.dag @@ -36,6 +36,14 @@ import gunbc.bmc_onboarding { RotatedCredentialActive, CredentialStateUnknown, bmc_credential_probe_to_phase, + host_onboarding_lifecycle_phase, + bmc_phase_is_pre_os_installed, + srv3_bmc_lifecycle_phase, +} +import gunbc.fleet_intent_network { + operator_host_srv1, + operator_host_srv2, + operator_host_srv3, } import extdeps.bmc.types { BootTargetPxe, AccountRoleAdministrator, redfish_boot_target_wire, redfish_account_role_wire } import gunbc.assimilate.bmc_token_federation { emit_bmc_token_smoke_workflow_yaml } @@ -216,3 +224,26 @@ test fn credential_probe_maps_to_correct_phase() -> Bool { Present { value: _ } => false } } + +test fn fleet_host_lifecycle_phase_is_single_authority_for_apply_mode() -> Bool { + bmc_phase_is_pre_os_installed(phase: FactoryDefault) + && bmc_phase_is_pre_os_installed(phase: CredentialsRotated) + && !bmc_phase_is_pre_os_installed(phase: OsInstalled) + && !bmc_phase_is_pre_os_installed(phase: FabricJoined) + && match host_onboarding_lifecycle_phase(host: operator_host_srv1) { + Present { value: OsInstalled } => true + Present { value: _ } => false + Absent => false + } + && match host_onboarding_lifecycle_phase(host: operator_host_srv2) { + Present { value: OsInstalled } => true + Present { value: _ } => false + Absent => false + } + && match host_onboarding_lifecycle_phase(host: operator_host_srv3) { + Present { value: CredentialsRotated } => true + Present { value: _ } => false + Absent => false + } + && srv3_bmc_lifecycle_phase == CredentialsRotated +} diff --git a/dsl/test/claim/ci_budget_tree_witness_test.dag b/dsl/test/claim/ci_budget_tree_witness_test.dag index 2c3d8852b21..4bf14b33513 100644 --- a/dsl/test/claim/ci_budget_tree_witness_test.dag +++ b/dsl/test/claim/ci_budget_tree_witness_test.dag @@ -20,7 +20,7 @@ import gunbc.ci_budget_tree { budget_tree_dashboard_session_pool_node, witness_floor_budget_equals_runner_cgroup_cap, } -import gunbc.fleet_host_budget { gunbc_per_runner_memory_cap } +import gunbc.fleet_host_budget { gunbc_runner_slot_memory_contract } import gunbc.ci_floor_measurement { gunbc_ci_runner_cgroup_memory_cap, gunbc_ci_dashboard_session_container_cap, @@ -105,11 +105,11 @@ fn witness_unknown_child_is_absent() -> Bool { } } -fn witness_spawn_width_budget_is_per_runner_cap() -> Bool { +fn witness_spawn_width_budget_is_floor_cgroup_cap() -> Bool { byte_size_count(b: srv1_floor_memory_budget()) > 0 && byte_size_count(b: srv1_floor_memory_budget()) == byte_size_count(b: gunbc_ci_runner_cgroup_memory_cap) - && byte_size_count(b: srv1_floor_memory_budget()) == byte_size_count(b: gunbc_per_runner_memory_cap) && byte_size_count(b: srv1_floor_memory_budget()) < byte_size_count(b: srv1_host_ram()) + && byte_size_count(b: gunbc_runner_slot_memory_contract.memory_max) > byte_size_count(b: gunbc_ci_runner_cgroup_memory_cap) } fn witness_hard_cap_is_runner_cgroup_cap() -> Bool { @@ -149,7 +149,7 @@ test fn ci_budget_tree_holds() -> Bool { && witness_srv3_symmetric_to_srv1() && witness_two_pools_present() && witness_unknown_child_is_absent() - && witness_spawn_width_budget_is_per_runner_cap() + && witness_spawn_width_budget_is_floor_cgroup_cap() && witness_hard_cap_is_runner_cgroup_cap() && witness_cap_split_runner_lt_session() && witness_floor_budget_equals_runner_cgroup_cap_holds() diff --git a/dsl/test/claim/ci_deploy_witness_test.dag b/dsl/test/claim/ci_deploy_witness_test.dag index e262abc0947..0027ad09348 100644 --- a/dsl/test/claim/ci_deploy_witness_test.dag +++ b/dsl/test/claim/ci_deploy_witness_test.dag @@ -11,6 +11,7 @@ import gunbc.ci_spec { import gunbc.ci_deploy_access { ci_deploy_srv1_access, ci_deploy_runner_principal, + gunbc_deploy_runner_principal, deploy_access_is_local_apply, deploy_access_preflight_script, gunbc_deploy_sudoers_dropin, @@ -18,7 +19,12 @@ import gunbc.ci_deploy_access { srv1_live_deploy_privileged_commands, } import gunbc.ci_workflow { ci_workflow } -import gunbc.ci_workflow_expressions { ci_deploy_push_to_main_if } +import gunbc.ci_workflow_expressions { + ci_deploy_push_to_main_if, + ci_deploy_srv1_job_if, + ci_deploy_runner_ready_repo_var_if, + gunbc_deploy_runner_ready_repo_var, +} import gunbc.ci_yaml_emit { expected_ci_yml } import gunbc.fleet_intent { srv1_host } import gunbc.host_effect { LocalShell } @@ -32,7 +38,9 @@ fn witness_deploy_stage_targets_live_deploy_wet() -> Bool { fn witness_deploy_access_uses_local_apply_on_srv1() -> Bool { let access = ci_deploy_srv1_access access.host == srv1_host + && access.principal == gunbc_deploy_runner_principal && access.principal == ci_deploy_runner_principal + && access.principal == "gunbc-deploy" && deploy_access_is_local_apply(access: access) && match access.transport { LocalShell => true @@ -70,18 +78,25 @@ fn witness_workflow_has_deploy_job_gated_on_ci() -> Bool { length(deploy_jobs) == 1 && deploy_jobs.first().needs == ["ci"] && match deploy_jobs.first().if_condition { - Present { value: cond } => cond == ci_deploy_push_to_main_if + Present { value: cond } => cond == ci_deploy_srv1_job_if Absent => false } } +fn witness_deploy_job_gated_on_observed_readiness_var() -> Bool { + string_contains(s: ci_deploy_srv1_job_if, pattern: ci_deploy_push_to_main_if) + && string_contains(s: ci_deploy_srv1_job_if, pattern: ci_deploy_runner_ready_repo_var_if) + && string_contains(s: ci_deploy_srv1_job_if, pattern: gunbc_deploy_runner_ready_repo_var) +} + fn witness_ci_yml_emits_deploy_job() -> Bool { let yaml = expected_ci_yml() string_contains(s: yaml, pattern: " deploy_dashboard_srv1:") && string_contains(s: yaml, pattern: " needs: [ci]") - && string_contains(s: yaml, pattern: concat(" if: ", ci_deploy_push_to_main_if)) + && string_contains(s: yaml, pattern: "vars.GUNBC_DEPLOY_RUNNER_READY == 'true'") && string_contains(s: yaml, pattern: "Deploy dashboard to srv1 (live_deploy_apply_srv1_wet)") && string_contains(s: yaml, pattern: live_deploy_apply_srv1_wet_function) + && string_contains(s: yaml, pattern: "runs-on: [self-hosted, linux, arm64, deploy]") } test fn ci_deploy_witnesses() -> Bool { @@ -91,5 +106,6 @@ test fn ci_deploy_witnesses() -> Bool { && witness_ci_spec_carries_deploy_stages() && witness_deploy_run_script_invokes_gunbc_wet() && witness_workflow_has_deploy_job_gated_on_ci() + && witness_deploy_job_gated_on_observed_readiness_var() && witness_ci_yml_emits_deploy_job() } diff --git a/dsl/test/claim/fleet_converge_emit_test.dag b/dsl/test/claim/fleet_converge_emit_test.dag index 190042aa481..9e6725d4eea 100644 --- a/dsl/test/claim/fleet_converge_emit_test.dag +++ b/dsl/test/claim/fleet_converge_emit_test.dag @@ -1,8 +1,13 @@ module test.claim.fleet_converge_emit import std.types { Bool, String, List, Int } -import std.measure { byte_size, hardware_thread_count, second_count } +import std.measure { byte_size, byte_size_count, hardware_thread_count, second_count } import extdeps.github.actions { RunnerSpec, SelfHosted } +import gunbc.fleet_host_budget { + gunbc_runner_slice_cap, + gunbc_runner_slot_memory_contract, + RunnerSlotMemoryContract, +} import gunbc.ci_runner_placement { RunnerDeploymentPlan, PlanDerived, @@ -59,19 +64,27 @@ import gunbc.host_converge { fleet_converge_policy, } import extdeps.ctrl.gunbc_pin { gunbc_pin_match } -import gunbc.fleet_intent_network { operator_host_srv3 } +import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3 } import gunbc.fleet_converge_emit { expected_fleet_converge_sh, fleet_converge_sh_for, fleet_converge_drifted, } +fn fixture_slot_memory(max_bytes: Int) -> RunnerSlotMemoryContract { + RunnerSlotMemoryContract { + memory_max: byte_size(max_bytes), + memory_high: gunbc_runner_slot_memory_contract.memory_high, + memory_swap_max: gunbc_runner_slot_memory_contract.memory_swap_max, + } +} + fn fixture_runner_host() -> RunnerHostDeployment { RunnerHostDeployment { identity: "srv1", runner_count: 7, runner_slice_cap: byte_size(70391285350), - per_runner_memory_cap: byte_size(30000000000), + slot_memory: fixture_slot_memory(max_bytes: 30000000000), build_tokens: hardware_thread_count(count: 12), runner: SelfHosted { labels: ["self-hosted", "linux", "arm64"] }, workload_class: BatchLatencyTolerant, @@ -103,15 +116,17 @@ fn has(s: String, p: String) -> Bool { } fn witness_live_converge_is_conservation_sound() -> Bool { + let slice = to_string(byte_size_count(b: gunbc_runner_slice_cap)) let sh = expected_fleet_converge_sh() !has(s: sh, p: "fleet-converge UNSOUND:") && has(s: sh, p: "converge-receipt") - && has(s: sh, p: "converge_slice_property \"srv1\" \"runner\" \"runner_slice_cap_bytes\" \"system-actions-runner.slice\" \"MemoryMax\" \"85899345920\" \"85899345920\" \"85899345920\"") - && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf\" \"actions-runner@srv1-*.service\" \"MemoryMax\" \"8589934592\"") - && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_swap_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf\" \"actions-runner@srv1-*.service\" \"MemorySwapMax\" \"0\"") + && has(s: sh, p: concat("converge_slice_property \"srv1\" \"runner\" \"runner_slice_cap_bytes\" \"system-actions-runner.slice\" \"MemoryMax\" \"", concat(slice, concat("\" \"", concat(slice, concat("\" \"", concat(slice, "\""))))))) + && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf\" \"actions-runner@srv1-*.service\" \"MemoryMax\" \"25769803776\"") + && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_swap_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf\" \"actions-runner@srv1-*.service\" \"MemorySwapMax\" \"8589934592\"") + && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_high_bytes\" \"/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf\" \"actions-runner@srv1-*.service\" \"MemoryHigh\" \"15032385536\"") && has(s: sh, p: "converge_runner_width \"srv1\" \"runner\" \"runner_count\" \"actions-runner@srv1-*.service\" \"10\"") && has(s: sh, p: "host_begin \"srv3\"") - && has(s: sh, p: "converge_gunbc_pinned_tree \"srv3\" \"gunbc\" \"pinned_tree_sha\" \"fresh_standup\"") + && has(s: sh, p: concat("converge_gunbc_pinned_tree \"srv3\" \"gunbc\" \"pinned_tree_sha\" \"fresh_standup\"")) && has(s: sh, p: "converge_runner_width \"srv3\" \"runner\" \"runner_count\" \"actions-runner@srv3-*.service\" \"10\"") } @@ -135,7 +150,7 @@ fn witness_sound_emits_runner_knobs() -> Bool { let sh = fixture_sound_sh() has(s: sh, p: "converge_slice_property \"srv1\" \"runner\" \"runner_slice_cap_bytes\" \"system-actions-runner.slice\" \"MemoryMax\" \"70391285350\" \"70391285350\" \"70391285350\"") && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf\" \"actions-runner@srv1-*.service\" \"MemoryMax\" \"30000000000\"") - && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_swap_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf\" \"actions-runner@srv1-*.service\" \"MemorySwapMax\" \"0\"") + && has(s: sh, p: "converge_per_slot_cap \"srv1\" \"runner\" \"per_slot_memory_swap_max_bytes\" \"/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf\" \"actions-runner@srv1-*.service\" \"MemorySwapMax\" \"8589934592\"") && has(s: sh, p: "converge_runner_width \"srv1\" \"runner\" \"runner_count\" \"actions-runner@srv1-*.service\" \"7\"") } @@ -200,7 +215,7 @@ fn witness_drift_perturbed_desired_flips_emit() -> Bool { identity: "srv1", runner_count: 99, runner_slice_cap: byte_size(70391285350), - per_runner_memory_cap: byte_size(30000000000), + slot_memory: fixture_slot_memory(max_bytes: 30000000000), build_tokens: hardware_thread_count(count: 12), runner: SelfHosted { labels: ["self-hosted"] }, workload_class: BatchLatencyTolerant, @@ -248,9 +263,10 @@ fn witness_gunbc_pinned_tree_emitted() -> Bool { && has(s: sh, p: "first_green_at=") } -fn witness_gunbc_apply_mode_srv3_fresh_standup() -> Bool { - converge_apply_mode_label(m: gunbc_apply_mode_for_host(host: operator_host_srv3)) == "fresh_standup" - && converge_apply_mode_label(m: gunbc_apply_mode_for_host(host: "srv1")) == "existing_host_quiescent" +fn witness_gunbc_apply_mode_derives_from_lifecycle() -> Bool { + converge_apply_mode_label(m: gunbc_apply_mode_for_host(host: operator_host_srv1)) == "existing_host_quiescent" + && converge_apply_mode_label(m: gunbc_apply_mode_for_host(host: operator_host_srv2)) == "existing_host_quiescent" + && converge_apply_mode_label(m: gunbc_apply_mode_for_host(host: operator_host_srv3)) == "fresh_standup" } fn witness_gunbc_noop_predicate_process_and_green_place() -> Bool { @@ -314,7 +330,7 @@ test fn fleet_converge_emit_holds() -> Bool { && witness_drift_perturbed_desired_flips_emit() && witness_gunbc_pinned_tree_emitted() && witness_gunbc_noop_predicate_process_and_green_place() - && witness_gunbc_apply_mode_srv3_fresh_standup() + && witness_gunbc_apply_mode_derives_from_lifecycle() && witness_gunbc_pinned_tree_runs_first() && witness_sessions_membership_line_emitted() } diff --git a/dsl/test/claim/fleet_converge_timer_emit_test.dag b/dsl/test/claim/fleet_converge_timer_emit_test.dag new file mode 100644 index 00000000000..5319b1cebb1 --- /dev/null +++ b/dsl/test/claim/fleet_converge_timer_emit_test.dag @@ -0,0 +1,61 @@ +module test.claim.fleet_converge_timer_emit + +import std.types { Bool, String } +import gunbc.fleet_converge_timer_emit { + expected_fleet_converge_timer_upsert_sh, + fleet_converge_timer_upsert_drifted, + gunbc_fleet_converge_service_unit_name, + gunbc_fleet_converge_timer_unit_name, +} +import gunbc.host_converge { + gunbc_host_ctrl_checkout_path, + gunbc_dsl_root_submodule_path, + gunbc_spawn_retry_backstop_cadence, + gunbc_fleet_converge_timer_on_boot_delay, + gunbc_fleet_operator_unix_user, +} +import std.measure { second_count } + +fn has(s: String, p: String) -> Bool { + string_contains(s: s, pattern: p) +} + +fn witness_timer_upsert_emits_units_and_enables() -> Bool { + let sh = expected_fleet_converge_timer_upsert_sh() + !has(s: sh, p: "fleet-converge-timer UNSOUND:") + && has(s: sh, p: "#!/usr/bin/env bash") + && has(s: sh, p: gunbc_fleet_converge_service_unit_name) + && has(s: sh, p: gunbc_fleet_converge_timer_unit_name) + && has(s: sh, p: "systemctl daemon-reload") + && has(s: sh, p: "systemctl enable --now gunbc-fleet-converge.timer") + && has(s: sh, p: "fleet-converge-timer-receipt") +} + +fn witness_service_execs_converge_script() -> Bool { + let sh = expected_fleet_converge_timer_upsert_sh() + let script = concat( + concat(concat(gunbc_host_ctrl_checkout_path, "/"), gunbc_dsl_root_submodule_path), + "/.github/fleet-converge.sh" + ) + has(s: sh, p: concat("ExecStart=/usr/bin/bash ", script)) + && has(s: sh, p: concat("User=", gunbc_fleet_operator_unix_user)) + && !has(s: sh, p: "$HOME") +} + +fn witness_timer_cadence_matches_spawn_backstop() -> Bool { + let sh = expected_fleet_converge_timer_upsert_sh() + has(s: sh, p: concat("OnUnitActiveSec=", concat(to_string(second_count(s: gunbc_spawn_retry_backstop_cadence)), "s"))) + && has(s: sh, p: concat("OnBootSec=", concat(to_string(second_count(s: gunbc_fleet_converge_timer_on_boot_delay)), "s"))) +} + +fn witness_drift_red_receipt() -> Bool { + !fleet_converge_timer_upsert_drifted(committed: expected_fleet_converge_timer_upsert_sh()) + && fleet_converge_timer_upsert_drifted(committed: concat(expected_fleet_converge_timer_upsert_sh(), "x")) +} + +test fn fleet_converge_timer_emit_holds() -> Bool { + witness_timer_upsert_emits_units_and_enables() + && witness_service_execs_converge_script() + && witness_timer_cadence_matches_spawn_backstop() + && witness_drift_red_receipt() +} diff --git a/dsl/test/claim/fleet_host_budget_test.dag b/dsl/test/claim/fleet_host_budget_test.dag index 07dd045a3bb..3c0307d4fda 100644 --- a/dsl/test/claim/fleet_host_budget_test.dag +++ b/dsl/test/claim/fleet_host_budget_test.dag @@ -84,10 +84,6 @@ fn overhead_16gib() -> ByteSize { byte_size(17179869184) } -fn session_slice_fixture() -> ByteSize { - byte_size(40000000000) -} - fn fitting_headroom() -> HeadroomPolicy { HeadroomSet { bytes: byte_size(16106127360) } } @@ -106,7 +102,6 @@ fn is_unsound(plan: FleetHostPlan) -> Bool { fn full_plan( citation: CtrlJobserverCitation, runner: FleetContainerType, - session_slice: ByteSize, policy: HeadroomPolicy, oomd: OomdEnforcementEvidence ) -> FleetHostPlan { @@ -116,7 +111,6 @@ fn full_plan( overhead: overhead_16gib(), nproc: hardware_thread_count(count: 128), runner: runner, - session_slice: session_slice, citation: citation, policy: policy, oomd: oomd @@ -124,7 +118,7 @@ fn full_plan( } fn sound_plan() -> FleetHostPlan { - full_plan(citation: fixture_citation(), runner: fixture_runner_measured(), session_slice: session_slice_fixture(), policy: fitting_headroom(), oomd: OomdEnforced) + full_plan(citation: fixture_citation(), runner: fixture_runner_measured(), policy: fitting_headroom(), oomd: OomdEnforced) } fn witness_live_plan_fail_closed() -> Bool { @@ -132,19 +126,19 @@ fn witness_live_plan_fail_closed() -> Bool { } fn witness_jobserver_absent_unsound() -> Bool { - is_unsound(plan: full_plan(citation: CtrlJobserverAbsent, runner: fixture_runner_measured(), session_slice: session_slice_fixture(), policy: fitting_headroom(), oomd: OomdEnforced)) + is_unsound(plan: full_plan(citation: CtrlJobserverAbsent, runner: fixture_runner_measured(), policy: fitting_headroom(), oomd: OomdEnforced)) } fn witness_runner_baseline_unmeasured_unsound() -> Bool { - is_unsound(plan: full_plan(citation: fixture_citation(), runner: fixture_runner_unmeasured(), session_slice: session_slice_fixture(), policy: fitting_headroom(), oomd: OomdEnforced)) + is_unsound(plan: full_plan(citation: fixture_citation(), runner: fixture_runner_unmeasured(), policy: fitting_headroom(), oomd: OomdEnforced)) } fn witness_headroom_unset_unsound() -> Bool { - is_unsound(plan: full_plan(citation: fixture_citation(), runner: fixture_runner_measured(), session_slice: session_slice_fixture(), policy: HeadroomUnset, oomd: OomdEnforced)) + is_unsound(plan: full_plan(citation: fixture_citation(), runner: fixture_runner_measured(), policy: HeadroomUnset, oomd: OomdEnforced)) } fn witness_oomd_unverified_unsound() -> Bool { - is_unsound(plan: full_plan(citation: fixture_citation(), runner: fixture_runner_measured(), session_slice: session_slice_fixture(), policy: fitting_headroom(), oomd: OomdUnverified)) + is_unsound(plan: full_plan(citation: fixture_citation(), runner: fixture_runner_measured(), policy: fitting_headroom(), oomd: OomdUnverified)) } fn witness_headroom_below_floor_unsound() -> Bool { @@ -201,36 +195,36 @@ fn witness_plan_derives_and_conserves() -> Bool { } } -fn witness_runner_slice_is_session_worstcase_only() -> Bool { - let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), session_slice: session_slice_fixture(), headroom: fitting_headroom_bytes()) - let expected = byte_size_count(b: host_125gib()) - byte_size_count(b: overhead_16gib()) - byte_size_count(b: session_slice_fixture()) - byte_size_count(b: fitting_headroom_bytes()) +fn witness_runner_slice_excludes_session_pool() -> Bool { + let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), headroom: fitting_headroom_bytes()) + let expected = byte_size_count(b: host_125gib()) - byte_size_count(b: overhead_16gib()) - byte_size_count(b: fitting_headroom_bytes()) byte_size_count(b: slice) == expected } fn witness_runner_slice_excludes_build_pool() -> Bool { let cfg = fixture_cfg() - let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), session_slice: session_slice_fixture(), headroom: fitting_headroom_bytes()) + let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), headroom: fitting_headroom_bytes()) let build_pool_bytes = byte_size_count(b: cfg.mib_per_job) * hardware_thread_count_value(t: derive_jobserver_tokens(nproc: hardware_thread_count(count: 128), mem_total: host_125gib(), cfg: cfg)) - let after_session = byte_size_count(b: host_125gib()) - byte_size_count(b: overhead_16gib()) - byte_size_count(b: session_slice_fixture()) - let double_subtract_would_starve = after_session < (build_pool_bytes + byte_size_count(b: fitting_headroom_bytes())) + let after_overhead = byte_size_count(b: host_125gib()) - byte_size_count(b: overhead_16gib()) + let double_subtract_would_starve = after_overhead < (build_pool_bytes + byte_size_count(b: fitting_headroom_bytes())) byte_size_count(b: slice) > 0 && double_subtract_would_starve } fn witness_host_conserves_exactly() -> Bool { - let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), session_slice: session_slice_fixture(), headroom: fitting_headroom_bytes()) - let whole = byte_size_count(b: overhead_16gib()) + byte_size_count(b: session_slice_fixture()) + byte_size_count(b: fitting_headroom_bytes()) + byte_size_count(b: slice) + let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), headroom: fitting_headroom_bytes()) + let whole = byte_size_count(b: overhead_16gib()) + byte_size_count(b: fitting_headroom_bytes()) + byte_size_count(b: slice) whole == byte_size_count(b: host_125gib()) } fn witness_tree_conserves() -> Bool { - let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), session_slice: session_slice_fixture(), headroom: fitting_headroom_bytes()) - let tree = fleet_host_budget_tree(name: "fixture", host: host_125gib(), overhead: overhead_16gib(), runner_baseline: runner_idle_bytes(), session_slice: session_slice_fixture(), headroom: fitting_headroom_bytes(), runner_slice: slice) + let slice = runner_slice_cap(host: host_125gib(), overhead: overhead_16gib(), headroom: fitting_headroom_bytes()) + let tree = fleet_host_budget_tree(name: "fixture", host: host_125gib(), overhead: overhead_16gib(), runner_baseline: runner_idle_bytes(), headroom: fitting_headroom_bytes(), runner_slice: slice) node_conserves(node: tree) } fn witness_overcommit_tree_refused() -> Bool { let bloated_slice = byte_size(byte_size_count(b: host_125gib())) - let tree = fleet_host_budget_tree(name: "overcommit", host: host_125gib(), overhead: overhead_16gib(), runner_baseline: runner_idle_bytes(), session_slice: session_slice_fixture(), headroom: fitting_headroom_bytes(), runner_slice: bloated_slice) + let tree = fleet_host_budget_tree(name: "overcommit", host: host_125gib(), overhead: overhead_16gib(), runner_baseline: runner_idle_bytes(), headroom: fitting_headroom_bytes(), runner_slice: bloated_slice) !node_conserves(node: tree) } @@ -253,7 +247,7 @@ test fn fleet_model_holds() -> Bool { && witness_margin_exceeds_nproc_saturates() && witness_runner_cpu_cap_is_cores_minus_margin() && witness_plan_derives_and_conserves() - && witness_runner_slice_is_session_worstcase_only() + && witness_runner_slice_excludes_session_pool() && witness_runner_slice_excludes_build_pool() && witness_host_conserves_exactly() && witness_tree_conserves() diff --git a/dsl/test/claim/generated_artifact_drift_test.dag b/dsl/test/claim/generated_artifact_drift_test.dag index 7f80db78b70..e67afca5ba6 100644 --- a/dsl/test/claim/generated_artifact_drift_test.dag +++ b/dsl/test/claim/generated_artifact_drift_test.dag @@ -2,7 +2,7 @@ module test.claim.generated_artifact_drift import gunbc.generated_artifact { - GeneratedArtifact, CiYamlArtifact, GitignoreArtifact, RoadmapArtifact, DesignArtifact, GithooksPrePushArtifact, RunnerDeployArtifact, FleetConvergeArtifact, AutoinstallUserDataArtifact, ProxyDhcpDnsmasqArtifact, PlanArtifact, + GeneratedArtifact, CiYamlArtifact, GitignoreArtifact, RoadmapArtifact, DesignArtifact, GithooksPrePushArtifact, RunnerDeployArtifact, FleetConvergeArtifact, FleetConvergeTimerUpsertArtifact, AutoinstallUserDataArtifact, ProxyDhcpDnsmasqArtifact, PlanArtifact, RepoConsumer, GitProtocol, GithubActionsWorkflow, ProjectDocumentation, HostReconciler, CommitPolicy, CommitRequired, NotCommitted, generated_artifact_registry, @@ -35,6 +35,7 @@ fn witness_all_known_committed() -> Bool { && artifact_is_committed(a: GithooksPrePushArtifact) && artifact_is_committed(a: PlanArtifact { plan: idea_machine_plan }) && artifact_is_committed(a: FleetConvergeArtifact) + && artifact_is_committed(a: FleetConvergeTimerUpsertArtifact) && !artifact_is_committed(a: RunnerDeployArtifact) && !artifact_is_committed(a: AutoinstallUserDataArtifact { spec: srv3_autoinstall_spec }) && !artifact_is_committed(a: ProxyDhcpDnsmasqArtifact { spec: srv3_install_server_spec }) @@ -44,6 +45,7 @@ fn witness_all_known_committed() -> Bool { && !path_is_ignored(path: "DESIGN.md") && !path_is_ignored(path: ".githooks/pre-push") && !path_is_ignored(path: ".github/fleet-converge.sh") + && !path_is_ignored(path: ".github/fleet-converge-timer-upsert.sh") && !path_is_ignored(path: "docs/plans/idea-machine.md") } @@ -58,7 +60,7 @@ fn witness_ignored_paths_excluded() -> Bool { } fn witness_registry_complete() -> Bool { - count(generated_artifact_registry) == (7 + count(fleet_autoinstall_specs) + count(fleet_install_server_specs) + count(all_plans)) + count(generated_artifact_registry) == (8 + count(fleet_autoinstall_specs) + count(fleet_install_server_specs) + count(all_plans)) && registry_contains(a: CiYamlArtifact) && registry_contains(a: GitignoreArtifact) && registry_contains(a: RoadmapArtifact) @@ -66,6 +68,7 @@ fn witness_registry_complete() -> Bool { && registry_contains(a: GithooksPrePushArtifact) && registry_contains(a: RunnerDeployArtifact) && registry_contains(a: FleetConvergeArtifact) + && registry_contains(a: FleetConvergeTimerUpsertArtifact) && registry_contains(a: AutoinstallUserDataArtifact { spec: srv3_autoinstall_spec }) && registry_contains(a: ProxyDhcpDnsmasqArtifact { spec: srv3_install_server_spec }) && registry_contains(a: PlanArtifact { plan: idea_machine_plan }) @@ -76,6 +79,7 @@ fn witness_registry_complete() -> Bool { && artifact_is_committed(a: GithooksPrePushArtifact) && artifact_is_committed(a: PlanArtifact { plan: idea_machine_plan }) && artifact_is_committed(a: FleetConvergeArtifact) + && artifact_is_committed(a: FleetConvergeTimerUpsertArtifact) && !artifact_is_committed(a: RunnerDeployArtifact) && !artifact_is_committed(a: AutoinstallUserDataArtifact { spec: srv3_autoinstall_spec }) && !artifact_is_committed(a: ProxyDhcpDnsmasqArtifact { spec: srv3_install_server_spec }) @@ -101,6 +105,7 @@ fn witness_commit_policy_reasons() -> Bool { && consumer_is(a: DesignArtifact, expected: ProjectDocumentation) && consumer_is(a: PlanArtifact { plan: idea_machine_plan }, expected: ProjectDocumentation) && consumer_is(a: FleetConvergeArtifact, expected: HostReconciler) + && consumer_is(a: FleetConvergeTimerUpsertArtifact, expected: HostReconciler) && match artifact_commit_policy(a: RunnerDeployArtifact) { NotCommitted => true _ => false } && match artifact_commit_policy(a: AutoinstallUserDataArtifact { spec: srv3_autoinstall_spec }) { NotCommitted => true _ => false } && match artifact_commit_policy(a: ProxyDhcpDnsmasqArtifact { spec: srv3_install_server_spec }) { NotCommitted => true _ => false } diff --git a/dsl/test/claim/host_allocation_conservation_test.dag b/dsl/test/claim/host_allocation_conservation_test.dag index b52b45444ef..edc0fd5e520 100644 --- a/dsl/test/claim/host_allocation_conservation_test.dag +++ b/dsl/test/claim/host_allocation_conservation_test.dag @@ -5,22 +5,19 @@ import std.measure { byte_size } import gunbc.fleet_host_budget { host_allocation_conserves, declared_runner_count, - max_concurrent_sessions, - session_pool_bytes, - gunbc_runner_pool_budget, + gunbc_runner_slice_cap, } test fn declared_runner_count_is_ten() -> Bool { declared_runner_count() == 10 } -test fn allocation_conserves_when_pools_fit() -> Bool { +test fn allocation_conserves_when_runner_pool_fits() -> Bool { host_allocation_conserves( host: byte_size(134217728000), overhead: byte_size(17179869184), headroom: byte_size(8589934592), - runner_pool: gunbc_runner_pool_budget, - session_pool: byte_size(0) + runner_pool: gunbc_runner_slice_cap ) } @@ -29,21 +26,16 @@ test fn allocation_unsound_when_runner_pool_overcommits() -> Bool { host: byte_size(134217728000), overhead: byte_size(17179869184), headroom: byte_size(8589934592), - runner_pool: byte_size(128849018880), - session_pool: byte_size(0) + runner_pool: byte_size(128849018880) ) } -test fn sessions_subtract_from_the_same_budget() -> Bool { - !host_allocation_conserves( +test fn allocation_is_runner_only_no_session_term() -> Bool { + host_allocation_conserves( host: byte_size(134217728000), overhead: byte_size(17179869184), headroom: byte_size(8589934592), - runner_pool: gunbc_runner_pool_budget, - session_pool: session_pool_bytes(active_sessions: 40) + runner_pool: gunbc_runner_slice_cap ) -} - -test fn max_concurrent_sessions_falls_out_of_pool() -> Bool { - max_concurrent_sessions(session_pool: session_pool_bytes(active_sessions: 12)) == 12 + && declared_runner_count() > 0 } diff --git a/dsl/test/claim/host_runner_provision_witness_test.dag b/dsl/test/claim/host_runner_provision_witness_test.dag new file mode 100644 index 00000000000..24179816cc1 --- /dev/null +++ b/dsl/test/claim/host_runner_provision_witness_test.dag @@ -0,0 +1,85 @@ +module test.claim.host_runner_provision + +import std.types { Bool, String } +import gunbc.host_runner_provision { + HostRunnerProvision, + RunnerProvisionRole, + gunbc_ci_runner_principal, + gunbc_deploy_runner_principal, + gunbc_deploy_runner_label, + gunbc_ci_runner_provision_for, + gunbc_deploy_runner_provision_for, + gunbc_runner_provision_for, + host_runner_provisions_for, + host_runner_provision_principal, + gunbc_deploy_runner_spec, +} +import gunbc.fleet_intent { srv1_host, srv2_host, srv3_host } +import gunbc.ci_deploy_access { ci_deploy_srv1_access, deploy_access_command_paths } +import extdeps.github.actions { SelfHosted } + +fn witness_ci_principal_is_gunbc_runner() -> Bool { + gunbc_ci_runner_principal == "gunbc-runner" + && host_runner_provision_principal(p: gunbc_ci_runner_provision_for(host: srv1_host)) == "gunbc-runner" +} + +fn witness_ci_role_has_no_privileged_commands() -> Bool { + let p = gunbc_ci_runner_provision_for(host: srv1_host) + p.role == Ci && p.privileged_commands.length() == 0 +} + +fn witness_deploy_principal_and_sudoers_contract() -> Bool { + let p = gunbc_deploy_runner_provision_for(host: srv1_host) + p.role == Deploy + && p.principal == gunbc_deploy_runner_principal + && p.principal == "gunbc-deploy" + && p.privileged_commands == deploy_access_command_paths(access: ci_deploy_srv1_access) + && p.privileged_commands.length() == 5 +} + +fn witness_deploy_labels_include_deploy() -> Bool { + let p = gunbc_deploy_runner_provision_for(host: srv1_host) + any(p.labels, l => l == gunbc_deploy_runner_label) + && any(p.labels, l => l == "self-hosted") + && any(p.labels, l => l == "linux") + && any(p.labels, l => l == "arm64") +} + +fn witness_deploy_runner_spec_matches_provision() -> Bool { + match gunbc_deploy_runner_spec() { + SelfHosted { labels: labels } => + any(labels, l => l == gunbc_deploy_runner_label) + && any(labels, l => l == "arm64") + _ => false + } +} + +fn witness_srv1_has_ci_and_deploy_roles() -> Bool { + let provisions = host_runner_provisions_for(host: srv1_host) + provisions.length() == 2 + && provisions.first().role == Ci + && provisions.last().role == Deploy +} + +fn witness_srv2_is_ci_only() -> Bool { + let provisions = host_runner_provisions_for(host: srv2_host) + provisions.length() == 1 && provisions.first().role == Ci +} + +fn witness_srv3_greenfield_ci_only() -> Bool { + let p = gunbc_runner_provision_for(host: srv3_host) + p.host.identity == "srv3" + && p.principal == gunbc_ci_runner_principal + && p.role == Ci +} + +test fn host_runner_provision_holds() -> Bool { + witness_ci_principal_is_gunbc_runner() + && witness_ci_role_has_no_privileged_commands() + && witness_deploy_principal_and_sudoers_contract() + && witness_deploy_labels_include_deploy() + && witness_deploy_runner_spec_matches_provision() + && witness_srv1_has_ci_and_deploy_roles() + && witness_srv2_is_ci_only() + && witness_srv3_greenfield_ci_only() +} diff --git a/dsl/test/claim/runner_memory_contract_witness_test.dag b/dsl/test/claim/runner_memory_contract_witness_test.dag new file mode 100644 index 00000000000..9b34d3afe4e --- /dev/null +++ b/dsl/test/claim/runner_memory_contract_witness_test.dag @@ -0,0 +1,189 @@ +module test.claim.runner_memory_contract + +import std.types { Bool, String } +import std.measure { byte_size_count, byte_size } +import gunbc.fleet_host_budget { + gunbc_runner_slot_memory_contract, + RunnerSlotMemoryContract, + gunbc_runner_slice_cap, + gunbc_declared_runner_slot_count, + gunbc_host_swap_available, + gunbc_ci_heavy_job_working_set, + gunbc_per_runner_memory_cap, + gunbc_per_runner_swap_cap, + gunbc_per_runner_memory_high, + runner_swap_backs_ram_overcommit, + runner_ram_overcommit_per_host, + fabric_effective_heavy_job_concurrency, + fabric_memory_accounting_admissible, + fabric_runner_slice_cap_per_host, + host_fixed_overhead_bytes, + declared_runner_count, +} +import gunbc.ci_fleet { fleet_intent_offers } +import gunbc.ci_floor_measurement { gunbc_ci_host_fixed_overhead } +import gunbc.fleet_converge_emit { expected_fleet_converge_sh } +import gunbc.host_converge { fleet_converge_policy, ConvergePolicyDerived, RunnerSlice } +import gunbc.ci_runner_placement { runner_deployment_plan, PlanDerived } + +fn has(s: String, p: String) -> Bool { + string_contains(s: s, pattern: p) +} + +fn witness_contract_is_parameterized_record() -> Bool { + let c = gunbc_runner_slot_memory_contract + byte_size_count(b: c.memory_max) == 25769803776 + && byte_size_count(b: c.memory_high) == 15032385536 + && byte_size_count(b: c.memory_swap_max) == 8589934592 + && gunbc_per_runner_memory_cap == c.memory_max + && gunbc_per_runner_swap_cap == c.memory_swap_max + && gunbc_per_runner_memory_high == c.memory_high +} + +fn witness_slice_cap_derives_from_fabric() -> Bool { + byte_size_count(b: gunbc_runner_slice_cap) == byte_size_count(b: fabric_runner_slice_cap_per_host( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead) + )) + && declared_runner_count() == gunbc_declared_runner_slot_count + && declared_runner_count() == 10 +} + +fn witness_fabric_concurrency_is_derived() -> Bool { + fabric_effective_heavy_job_concurrency( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead), + working_set: gunbc_ci_heavy_job_working_set + ) > 0 +} + +fn witness_ram_overcommit_exceeds_host_swap_backing() -> Bool { + let slice = fabric_runner_slice_cap_per_host( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead) + ) + let overcommit = runner_ram_overcommit_per_host( + slot_count: declared_runner_count(), + contract: gunbc_runner_slot_memory_contract, + slice_cap: slice + ) + byte_size_count(b: overcommit) > byte_size_count(b: gunbc_host_swap_available) + && !runner_swap_backs_ram_overcommit( + slot_count: declared_runner_count(), + contract: gunbc_runner_slot_memory_contract, + slice_cap: slice, + host_swap: gunbc_host_swap_available + ) +} + +fn per_slot_line(host: String, knob: String, property: String, bytes: String) -> String { + concat( + concat( + concat( + concat( + concat( + concat( + concat("converge_per_slot_cap \"", host), + "\" \"runner\" \"" + ), + knob + ), + if knob == "per_slot_memory_swap_max_bytes" { + "\" \"/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf\" \"actions-runner@" + } else if knob == "per_slot_memory_max_bytes" { + "\" \"/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf\" \"actions-runner@" + } else { + "\" \"/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf\" \"actions-runner@" + } + ), + host + ), + "-*.service\" \"" + ), + concat(concat(property, "\" \""), concat(bytes, "\"")) + ) +} + +fn witness_emit_projects_burst_contract() -> Bool { + let c = gunbc_runner_slot_memory_contract + let max = to_string(byte_size_count(b: c.memory_max)) + let swap = to_string(byte_size_count(b: c.memory_swap_max)) + let high = to_string(byte_size_count(b: c.memory_high)) + let slice = to_string(byte_size_count(b: gunbc_runner_slice_cap)) + let sh = expected_fleet_converge_sh() + has(s: sh, p: concat("converge_slice_property \"srv1\" \"runner\" \"runner_slice_cap_bytes\" \"system-actions-runner.slice\" \"MemoryMax\" \"", concat(slice, "\""))) + && has(s: sh, p: per_slot_line(host: "srv1", knob: "per_slot_memory_max_bytes", property: "MemoryMax", bytes: max)) + && has(s: sh, p: per_slot_line(host: "srv1", knob: "per_slot_memory_swap_max_bytes", property: "MemorySwapMax", bytes: swap)) + && has(s: sh, p: per_slot_line(host: "srv1", knob: "per_slot_memory_high_bytes", property: "MemoryHigh", bytes: high)) + && has(s: sh, p: "converge_slice_property \"srv1\" \"runner\" \"oom_pressure_auto\" \"system-actions-runner.slice\" \"ManagedOOMMemoryPressure\" \"auto\" \"auto\" \"true\"") + && has(s: sh, p: per_slot_line(host: "srv3", knob: "per_slot_memory_max_bytes", property: "MemoryMax", bytes: max)) + && has(s: sh, p: per_slot_line(host: "srv3", knob: "per_slot_memory_high_bytes", property: "MemoryHigh", bytes: high)) + && !has(s: sh, p: "\"MemorySwapMax\" \"0\"") + && !has(s: sh, p: "\"MemoryMax\" \"8589934592\"") +} + +fn witness_deployment_carries_slot_memory_and_slice() -> Bool { + match runner_deployment_plan() { + PlanDerived { hosts: hosts } => + hosts.length() > 0 + && all(hosts, h => + byte_size_count(b: h.slot_memory.memory_max) == byte_size_count(b: gunbc_runner_slot_memory_contract.memory_max) + && byte_size_count(b: h.slot_memory.memory_high) == byte_size_count(b: gunbc_runner_slot_memory_contract.memory_high) + && byte_size_count(b: h.slot_memory.memory_swap_max) == byte_size_count(b: gunbc_runner_slot_memory_contract.memory_swap_max) + && byte_size_count(b: h.runner_slice_cap) == byte_size_count(b: gunbc_runner_slice_cap) + ) + _ => false + } +} + +fn witness_policy_has_per_slot_memory_knobs() -> Bool { + match fleet_converge_policy() { + ConvergePolicyDerived { hosts: hosts } => + hosts.length() > 0 + && all(hosts, h => + any(h.knobs, k => k.name == "per_slot_memory_high_bytes" && k.slice == RunnerSlice) + && any(h.knobs, k => k.name == "per_slot_memory_max_bytes" && k.slice == RunnerSlice) + && any(h.knobs, k => k.name == "per_slot_memory_swap_max_bytes" && k.slice == RunnerSlice) + && any(h.knobs, k => k.name == "runner_slice_cap_bytes" && k.slice == RunnerSlice) + && any(h.knobs, k => k.name == "oom_pressure_auto" && k.slice == RunnerSlice) + ) + _ => false + } +} + +fn witness_burst_gate_rejects_excess_per_slot_swap() -> Bool { + let inflated_swap = RunnerSlotMemoryContract { + memory_max: gunbc_runner_slot_memory_contract.memory_max, + memory_high: gunbc_runner_slot_memory_contract.memory_high, + memory_swap_max: byte_size(17179869184), + } + !fabric_memory_accounting_admissible( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead), + slot_count: gunbc_declared_runner_slot_count, + contract: inflated_swap, + host_swap: gunbc_host_swap_available + ) +} + +fn witness_fabric_memory_accounting_admissible_under_burst_contract() -> Bool { + fabric_memory_accounting_admissible( + offers: fleet_intent_offers, + overhead: host_fixed_overhead_bytes(o: gunbc_ci_host_fixed_overhead), + slot_count: gunbc_declared_runner_slot_count, + contract: gunbc_runner_slot_memory_contract, + host_swap: gunbc_host_swap_available + ) +} + +test fn runner_memory_contract_holds() -> Bool { + witness_contract_is_parameterized_record() + && witness_slice_cap_derives_from_fabric() + && witness_fabric_concurrency_is_derived() + && witness_ram_overcommit_exceeds_host_swap_backing() + && witness_emit_projects_burst_contract() + && witness_deployment_carries_slot_memory_and_slice() + && witness_policy_has_per_slot_memory_knobs() + && witness_fabric_memory_accounting_admissible_under_burst_contract() + && witness_burst_gate_rejects_excess_per_slot_swap() +} diff --git a/dsl/test/claim/runner_placement_witness_test.dag b/dsl/test/claim/runner_placement_witness_test.dag index 0308b14bdd2..4a38dbb201d 100644 --- a/dsl/test/claim/runner_placement_witness_test.dag +++ b/dsl/test/claim/runner_placement_witness_test.dag @@ -30,7 +30,10 @@ import gunbc.ci_runner_placement { } import gunbc.fleet_host_budget { declared_runner_count, + gunbc_runner_slice_cap, gunbc_runner_pool_budget, + gunbc_runner_slot_memory_contract, + gunbc_host_swap_available, } import gunbc.ci_floor_measurement { gunbc_ci_runner_cgroup_memory_cap, @@ -76,8 +79,8 @@ fn witness_live_plan_is_conservation_sound() -> Bool { && total == declared_runner_count() * fleet_intent_offers.length() && all(hosts, h => h.runner_count == declared_runner_count() - && byte_size_count(b: h.per_runner_memory_cap) == byte_size_count(b: gunbc_ci_runner_cgroup_memory_cap) - && byte_size_count(b: h.runner_slice_cap) == byte_size_count(b: gunbc_runner_pool_budget) + && byte_size_count(b: h.slot_memory.memory_max) == byte_size_count(b: gunbc_runner_slot_memory_contract.memory_max) + && byte_size_count(b: h.runner_slice_cap) == byte_size_count(b: gunbc_runner_slice_cap) && hardware_thread_count_value(t: h.build_tokens) > 0 ) PlanUnsound { reason: _ } => false @@ -89,7 +92,8 @@ fn witness_per_runner_cap_within_runner_pool() -> Bool { PlanDerived { hosts: hosts, fleet_concurrent_runs: _ } => hosts.length() > 0 && all(hosts, h => - h.runner_count * byte_size_count(b: h.per_runner_memory_cap) <= byte_size_count(b: h.runner_slice_cap) + byte_size_count(b: h.slot_memory.memory_max) <= byte_size_count(b: h.runner_slice_cap) + && byte_size_count(b: h.slot_memory.memory_swap_max) <= byte_size_count(b: gunbc_host_swap_available) ) PlanUnsound { reason: _ } => false } @@ -187,6 +191,13 @@ fn witness_curve_has_full_coverage_option() -> Bool { any(operating_curve(), r => r.coverage_count == r.sample_count) } +fn witness_runner_manifest_emits_fabric_concurrency() -> Bool { + string_contains( + s: expected_runner_deploy_manifest(), + pattern: "runner-deploy fabric_effective_heavy_job_concurrency=" + ) +} + fn witness_manifest_drift_red_receipt() -> Bool { !runner_deploy_drifted(committed: expected_runner_deploy_manifest()) && runner_deploy_drifted(committed: concat(expected_runner_deploy_manifest(), "x")) @@ -267,6 +278,7 @@ test fn runner_placement_holds() -> Bool { && witness_curve_coverage_monotone_in_cap() && witness_curve_has_full_coverage_option() && witness_manifest_drift_red_receipt() + && witness_runner_manifest_emits_fabric_concurrency() && witness_session_cpu_weight_exceeds_runner_cpu_weight() && witness_runner_manifest_line_contains_cpu_weight() && witness_runner_and_session_cpu_weights_are_distinct() diff --git a/src/v2/lens/medium_structure_containment.dag b/src/v2/lens/medium_structure_containment.dag index ad4426349fb..a488ea5b3fd 100644 --- a/src/v2/lens/medium_structure_containment.dag +++ b/src/v2/lens/medium_structure_containment.dag @@ -74,6 +74,8 @@ data emitted_medium_emit_fns: List = [ "witness_layer_source_flags", "fleet_converge_sh_for", "expected_fleet_converge_sh", + "fleet_converge_timer_upsert_sh_for", + "expected_fleet_converge_timer_upsert_sh", "expected_githooks_pre_push_sh", "toolchain_provision_shell_exec", "autoinstall_user_data", @@ -145,6 +147,7 @@ data medium_structure_exception_roster: List = [ "src/v2/test/claim/ci_spec_witness_test.dag", "dsl/gunbc/fleet_converge_emit.dag", + "dsl/gunbc/fleet_converge_timer_emit.dag", "dsl/gunbc/tailscale_acl_emit.dag", "dsl/gunbc/live_deploy/emit.dag", "dsl/gunbc/live_deploy/spec.dag", @@ -159,6 +162,7 @@ data medium_structure_exception_roster: List = [ "dsl/gunbc/assimilate/bmc_token_federation.dag", "dsl/gunbc/tools/review.dag", "dsl/test/claim/fleet_converge_emit_test.dag", + "dsl/test/claim/fleet_converge_timer_emit_test.dag", "dsl/test/claim/live_deploy/emit_test.dag", "dsl/test/claim/githooks_pre_push_emit_test.dag",