From b50b812623ef8eeb63b2b360f8d14d98b8bb2b8f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 23 Aug 2026 23:48:02 +0000 Subject: [PATCH 1/9] Give the compile-pool slice a converger: wire compile_pool_ensure into fleet convergence Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ptiy92YHu7ig5W7ktiBzpZ --- dag/extdeps/systemd/systemctl.dag | 33 ++ dag/gunbc/compile_pool_observe.dag | 179 ++++++++++ dag/gunbc/host_effect.dag | 1 + dag/gunbc/host_effect_realize.dag | 327 +++++++++++++++++- dag/gunbc/host_toolchain_ensure.dag | 16 +- dag/gunbc/live_deploy/operations.dag | 8 + dag/gunbc/systemctl_show_read.dag | 75 ++++ ...ompile_pool_ensure_wiring_witness_test.dag | 123 +++++++ 8 files changed, 757 insertions(+), 5 deletions(-) create mode 100644 dag/gunbc/compile_pool_observe.dag create mode 100644 dag/test/claim/compile_pool_ensure_wiring_witness_test.dag diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index b24e5c4ac97..1fe14e24e2e 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -53,6 +53,15 @@ fn systemctl_enable_argv(unit: NonEmptyStr) -> List { ["systemctl", "enable", unit as String] } +// START IS NOT RESTART, AND FOR A SLICE THE DIFFERENCE IS DESTRUCTIVE RATHER THAN STYLISTIC. Stopping a +// slice kills every process in it (systemd.slice(5)), so `restart` applied to a compile pool would terminate +// the compilers the pool exists to hold. `start` on an already-active slice is a no-op that re-applies the +// unit's resource directives, which is what a converger wants. The modeled `Start` operation already exists +// on the service surface; this is its argv authority for the shell-body realization edge. +fn systemctl_start_argv(unit: NonEmptyStr) -> List { + ["systemctl", "start", unit as String] +} + fn systemctl_restart_argv(unit: NonEmptyStr) -> List { ["systemctl", "restart", unit as String] } @@ -302,6 +311,30 @@ service systemd.Systemctl { } } + // LoadState is the only observation that distinguishes an ABSENT unit from a present one, and it + // needs its own operation rather than a SystemdUnitProperty arm: that coproduct is numeric cgroup + // evidence sharing one numeric mock, and a wire value of `not-found` is not a magnitude. Cited to + // systemd.unit(5) LoadState -- `loaded`, `not-found`, `bad-setting`, `error`, `masked` -- of which + // `not-found` is the one this corpus reads positively. `systemctl show` exits 0 for a unit the + // manager has never heard of and prints not-found, so a nonzero exit here means the MANAGER was + // unreadable, which is a different refusal from an absent unit and must not be folded into it. + operation ShowLoadState { + input { unit: NonEmptyStr } + output { + value: String from "stdout" + success: Bool from "exit_success" + } + readonly + transport shell { argv: ["systemctl", "show", "{unit}", "--property=LoadState", "--value"] } + exit { + 0 => Unit + nonzero => String "systemctl show LoadState failed" + } + mock_response { + 0 => { value: "not-found", success: true } "hermetic systemd.Systemctl.ShowLoadState" + } + } + operation ListUnits { input { pattern: NonEmptyStr, diff --git a/dag/gunbc/compile_pool_observe.dag b/dag/gunbc/compile_pool_observe.dag new file mode 100644 index 00000000000..9fceca05407 --- /dev/null +++ b/dag/gunbc/compile_pool_observe.dag @@ -0,0 +1,179 @@ +module gunbc.compile_pool_observe + +import std.types { String, NonEmptyStr, Bool } +import std.measure { ByteSize } +import std.disposition { Disposition, SingleAuthority } +import std.algebra { trim } +import extdeps.systemd { + SystemdUnitProperty, + MemoryMax, + MemoryHigh, + MemoryCurrent, + SystemdCgroupMemoryLimit, + MemoryLimitBytes, + MemoryLimitUnbounded, + MemoryLimitAbsent, + parse_systemd_memory_limit_show, + systemd_unit_load_state_not_found_wire, +} +import gunbc.host_effect { HostEffectTransport } +import gunbc.systemctl_show_read { + SystemdPropertyCapture, + SystemdPropertyCaptured, + SystemdPropertyCaptureRefused, + systemctl_show_property_read, + systemctl_show_load_state_read, +} +import gunbc.build_cache_instance { + CgroupMemoryLimits, + EndpointObservationUnavailableReason, + ReachRefused, +} +import gunbc.host_compile_pool { + CompilePoolObservation, + CompilePoolSliceObserved, + CompilePoolSliceAbsent, + CompilePoolObservationUnavailable, +} + +data compile_pool_observe_scope_disposition: Disposition = SingleAuthority + +// THE MISSING PRODUCER. gunbc.host_compile_pool declares CompilePoolObservation and gunbc.compile_pool_ensure +// consumes it, and until this module existed every value of that type in the tree was hand-authored inside a +// test -- so the admission that decides whether a host may have its compile-pool slice written was proven over +// fixtures and computed over nothing. That is the same shape gunbc.build_cache_endpoint_observe closed for the +// endpoint observation, and this is the slice half of it. +// +// WHAT IT READS AND IN WHICH ORDER. LoadState first, because it is the only reading that separates an absent +// slice from a present one (extdeps.systemd.systemctl ShowLoadState carries the citation: `systemctl show` +// answers `infinity` for MemoryMax on a unit the manager never heard of, identically to a loaded unit with no +// ceiling). Only under a loaded unit are the three cgroup magnitudes read. A slice that is loaded but reports +// no numeric ceiling is NOT folded into absent and NOT folded into a zero: `infinity` and an unparseable value +// are both refusals carrying what was read, because a pool with no ceiling is precisely the unbounded charge +// the managed slice exists to replace and reporting it as a size would fabricate the thing being checked. +// +// WHAT IT DOES NOT DO: it decides nothing. Whether the observed slice is large enough, correctly named, or +// permitted to be written is gunbc.compile_pool_ensure's and gunbc.host_compile_pool's judgement over this +// value; this module only reports what the host said. + +fn compile_pool_reach_refusal(slice_unit: NonEmptyStr, what: String, why: String) -> CompilePoolObservation { + CompilePoolObservationUnavailable { + reason: ReachRefused { + detail: join( + ["compile-pool slice ", slice_unit as String, ": ", what, " -- ", why], + "", + ) as NonEmptyStr, + }, + } +} + +fn compile_pool_limit_bytes(lim: SystemdCgroupMemoryLimit) -> ByteSize? { + match lim { + MemoryLimitBytes { bytes: b } => Present { value: b } + MemoryLimitUnbounded => none + MemoryLimitAbsent => none + } +} + +fn compile_pool_limit_label(lim: SystemdCgroupMemoryLimit) -> String { + match lim { + MemoryLimitBytes { bytes: _ } => "a byte count" + MemoryLimitUnbounded => "infinity (no ceiling -- an unbounded slice is the charge the managed pool replaces, not a size)" + MemoryLimitAbsent => "empty or unparseable" + } +} + +// MemoryCurrent IS READ EVEN THOUGH NO JUDGEMENT USES IT, AND AN UNREADABLE ONE REFUSES. CgroupMemoryLimits +// carries three magnitudes and the two judgements over this observation read only two -- the ensure admission +// compares memory.max and memory.high against the derived size, the readiness check compares memory.max. The +// third is required to CONSTRUCT the record, so this module cannot report an observation without it, and it +// does not invent one: a zero would be a fabricated live charge that reads as an idle pool, which is exactly +// the state an operator would act on. So a slice that is loaded but never started -- systemd answers +// `[not set]` for MemoryCurrent until the cgroup exists -- is reported unavailable with that reading quoted, +// and the next pass observes it after the start. RUNG: mitigatable, and the deficit is a carrier shape rather +// than a read -- a limit PAIR and a live magnitude fused into one record. NEXT-RUNG TRIGGER: CgroupMemoryLimits +// split so observed ceilings can be reported without a current charge, at which point this arm is +// unrepresentable rather than refused. +type CompilePoolLimitRead + = CompilePoolLimitBytes { bytes: ByteSize } + | CompilePoolLimitRefused { observation: CompilePoolObservation } + +fn compile_pool_limit_of_capture( + slice_unit: NonEmptyStr, + property_name: String, + capture: SystemdPropertyCapture, +) -> CompilePoolLimitRead { + match capture { + SystemdPropertyCaptureRefused { reason: why } => + CompilePoolLimitRefused { + observation: compile_pool_reach_refusal( + slice_unit: slice_unit, + what: join(["could not read ", property_name], ""), + why: why, + ), + } + SystemdPropertyCaptured { value: raw } => { + let lim = parse_systemd_memory_limit_show(raw: trim(raw)) + match compile_pool_limit_bytes(lim: lim) { + Present { value: b } => CompilePoolLimitBytes { bytes: b } + Absent => + CompilePoolLimitRefused { + observation: compile_pool_reach_refusal( + slice_unit: slice_unit, + what: join([property_name, " is not a ceiling"], ""), + why: compile_pool_limit_label(lim: lim), + ), + } + } + } + } +} + +fn compile_pool_slice_limits_from_reads( + slice_unit: NonEmptyStr, + max_read: SystemdPropertyCapture, + high_read: SystemdPropertyCapture, + current_read: SystemdPropertyCapture, +) -> CompilePoolObservation { + match compile_pool_limit_of_capture(slice_unit: slice_unit, property_name: "MemoryMax", capture: max_read) { + CompilePoolLimitRefused { observation: o } => o + CompilePoolLimitBytes { bytes: max } => + match compile_pool_limit_of_capture(slice_unit: slice_unit, property_name: "MemoryHigh", capture: high_read) { + CompilePoolLimitRefused { observation: o } => o + CompilePoolLimitBytes { bytes: high } => + match compile_pool_limit_of_capture(slice_unit: slice_unit, property_name: "MemoryCurrent", capture: current_read) { + CompilePoolLimitRefused { observation: o } => o + CompilePoolLimitBytes { bytes: current } => + CompilePoolSliceObserved { + slice_unit: slice_unit, + limits: CgroupMemoryLimits { + memory_max: max, + memory_high: high, + memory_current: current, + }, + } + } + } + } +} + +fn observe_compile_pool_slice( + transport: HostEffectTransport, + slice_unit: NonEmptyStr, +) -> CompilePoolObservation { + match systemctl_show_load_state_read(transport: transport, unit: slice_unit) { + SystemdPropertyCaptureRefused { reason: why } => + compile_pool_reach_refusal(slice_unit: slice_unit, what: "could not read LoadState", why: why) + SystemdPropertyCaptured { value: raw } => + if trim(raw) == (systemd_unit_load_state_not_found_wire as String) { + CompilePoolSliceAbsent { slice_unit: slice_unit } + } else { + compile_pool_slice_limits_from_reads( + slice_unit: slice_unit, + max_read: systemctl_show_property_read(transport: transport, unit: slice_unit, property: MemoryMax), + high_read: systemctl_show_property_read(transport: transport, unit: slice_unit, property: MemoryHigh), + current_read: systemctl_show_property_read(transport: transport, unit: slice_unit, property: MemoryCurrent), + ) + } + } +} diff --git a/dag/gunbc/host_effect.dag b/dag/gunbc/host_effect.dag index bc54467409d..a2cc0cb1ea4 100644 --- a/dag/gunbc/host_effect.dag +++ b/dag/gunbc/host_effect.dag @@ -42,6 +42,7 @@ type HostEffect = | OsInstallActuatorToolchainEnsure {} | EnsureBuildCacheInstance { desired: BuildCacheInstance } | ProvisionCodexRuntime {} + | EnsureCompilePoolSlice {} | SetHostnameCas { hostnamectl_path: NonEmptyStr, base: DnsLabel, desired: DnsLabel } | ReadEffectivePosixPrincipal { read: EffectivePosixPrincipalRead } | SudoNopasswdExecuteProbe { probe: SudoNopasswdExecuteProbeShape } diff --git a/dag/gunbc/host_effect_realize.dag b/dag/gunbc/host_effect_realize.dag index 7d0df8c5cc4..772b5adb145 100644 --- a/dag/gunbc/host_effect_realize.dag +++ b/dag/gunbc/host_effect_realize.dag @@ -8,6 +8,7 @@ import std.decl_ref { DeclarationRef, WholeDeclaration, decl_ref, declaration_re import gunbc.host_effect { HostEffect, ShellCommand, RedfishAction, ConvergePlan, OsInstallActuatorToolchainEnsure, EnsureBuildCacheInstance, ProvisionCodexRuntime, + EnsureCompilePoolSlice, SetHostnameCas, ReadEffectivePosixPrincipal, SudoNopasswdExecuteProbe, @@ -95,7 +96,7 @@ import gunbc.typed_argv_exec { } import extdeps.bmc.types { RedfishSystemAction } import gunbc.output_policy { OutcomeIsData } -import gunbc.fleet_intent { ComputeHost, srv1_host, compute_host_primary_cpu } +import gunbc.fleet_intent { ComputeHost, srv1_host, compute_host_primary_cpu, cpu_facts_threads, compute_host_ram_bytes_total } import product.placement_supply { HostIdentity } import extdeps.toolchain.types { Architecture, architecture_diagnostic_name } import gunbc.host_converge { HostConverge, FreshStandup, ExistingHostQuiescentReload, gunbc_apply_mode_for_host } @@ -260,13 +261,42 @@ import gunbc.live_deploy.operations { systemctl_daemon_reload_command, systemctl_enable_command, systemctl_restart_command, -} + systemctl_start_command, +} +import extdeps.jobserver { jobserver_config } +import gunbc.fleet_host_budget { + CompilePoolPlacement, + CompilePoolSliceDeclared, + CompilePoolInRunnerSlots, + CompilePoolResolution, + CompilePoolUnresolved, + compile_pool_bytes_for, + gunbc_compile_pool_placement, +} +import gunbc.compile_pool_ensure { + EnsureCompilePoolAdmission, + EnsureCompilePoolAlreadyConverged, + EnsureCompilePoolAdmitted, + EnsureCompilePoolRefused, + CompilePoolEnsureRefusal, + EnsurePoolTopologyUndeclared, + EnsurePoolSizingUnresolved, + EnsureCachePlacementUnknown, + EnsurePoolSliceUnobservable, + EnsurePoolSliceNamedDoesNotMatch, + admit_ensure_compile_pool_slice, + compile_pool_ensure_is_converged, +} +import gunbc.compile_pool_observe { observe_compile_pool_slice } +import gunbc.host_layout { gunbc_managed_compile_pool_slice } +import extdeps.systemd.unit_file { SystemdUnitFile, serialize_systemd_unit_file } + import gunbc.build_cache_instance { ci_cache_instance } import gunbc.srv3_boot_once_cd { srv3_boot_once_apply_read_back_sleep, } import extdeps.filesystem.filesystem_io -import std.measure { second_count } +import std.measure { second_count, ByteSize, byte_size_count } import gunbc.srv3_ubuntu_install_media_artifact { srv3_ubuntu_install_media_artifact } import gunbc.srv3_seeded_install_media_artifact { srv3_seeded_install_media_artifact } import extdeps.provisioning.ubuntu_install_media_fetch { @@ -333,6 +363,7 @@ type ResolvedHostEffectCell = | OsInstallActuatorToolchainOnHost { node: ComputeHost } | ProvisionBuildCacheOnHost { node: ComputeHost, desired: BuildCacheInstance } | ProvisionCodexRuntimeOnHost { node: ComputeHost } + | EnsureCompilePoolSliceOnHost { node: ComputeHost } | HostnameOnHost { node: ComputeHost, hostnamectl_path: NonEmptyStr, base: DnsLabel, desired: DnsLabel } | HostnameReadOnHost { node: ComputeHost } | NbdProxySessionOnHost { @@ -400,6 +431,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res ProvisionBuildCacheOnHost { node: n, desired: d } ProvisionCodexRuntime {} => ProvisionCodexRuntimeOnHost { node: n } + EnsureCompilePoolSlice {} => + EnsureCompilePoolSliceOnHost { node: n } Srv3ReceiptEmit { lines: lines } => ReceiptEmitOnHost { node: n, lines: lines } Srv3BootOnceReadBackSleep => @@ -439,6 +472,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res ProvisionBuildCacheOnHost { node: n, desired: d } ProvisionCodexRuntime {} => ProvisionCodexRuntimeOnHost { node: n } + EnsureCompilePoolSlice {} => + EnsureCompilePoolSliceOnHost { node: n } Srv3InstallMediaFetch => TypedReconcileOnHost { node: n, kind: TypedReconcileFetch } Srv3SeededInstallMediaRemaster => TypedReconcileOnHost { node: n, kind: TypedReconcileRemaster } Srv3SeededInstallMediaToolchainEnsure => @@ -505,6 +540,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res IncompatibleCell { reason: "host_effect: EnsureBuildCacheInstance is in-band (per-host sccache daemon) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } ProvisionCodexRuntime {} => IncompatibleCell { reason: "host_effect: ProvisionCodexRuntime is in-band (per-host provider runtime bundle) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } + EnsureCompilePoolSlice {} => + IncompatibleCell { reason: "host_effect: EnsureCompilePoolSlice is in-band (per-host systemd slice) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } Srv3NbdProxyServe { intent: _, lease_key: _ } => IncompatibleCell { reason: "host_effect: Srv3NbdProxyServe is in-band (actuator host long-running session) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } CodexSupervisedWorkerTurn { intent: _, lease_key: _ } => @@ -522,6 +559,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res IncompatibleCell { reason: "host_effect: EnsureBuildCacheInstance is in-band (per-host sccache daemon) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } ProvisionCodexRuntime {} => IncompatibleCell { reason: "host_effect: ProvisionCodexRuntime is in-band (per-host provider runtime bundle) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } + EnsureCompilePoolSlice {} => + IncompatibleCell { reason: "host_effect: EnsureCompilePoolSlice is in-band (per-host systemd slice) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } Srv3InstallMediaFetch => IncompatibleCell { reason: "host_effect: Srv3InstallMediaFetch is in-band (actuator host typed reconcile) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } Srv3SeededInstallMediaRemaster => @@ -3675,6 +3714,8 @@ fn host_effect_apply(target: NodeControlPlane, effect: HostEffect, evidence: Ide realize_provision_build_cache(effect_intent: intent) ProvisionCodexRuntimeOnHost { node: _ } => realize_provision_codex_runtime(effect_intent: intent) + EnsureCompilePoolSliceOnHost { node: _ } => + realize_ensure_compile_pool_slice(transport: transport, effect_intent: intent) HostnameOnHost { node: _, hostnamectl_path: p, base: b, desired: d } => realize_set_hostname_cas_on_host(hostnamectl_path: p, base: b, desired: d, transport: transport, intent: intent) HostnameReadOnHost { node: _ } => @@ -3853,6 +3894,8 @@ fn host_effect_apply_gated( realize_provision_build_cache(effect_intent: intent) ProvisionCodexRuntimeOnHost { node: _ } => realize_provision_codex_runtime(effect_intent: intent) + EnsureCompilePoolSliceOnHost { node: _ } => + realize_ensure_compile_pool_slice(transport: transport, effect_intent: intent) HostnameOnHost { node: _, hostnamectl_path: p, base: b, desired: d } => realize_set_hostname_cas_on_host(hostnamectl_path: p, base: b, desired: d, transport: transport, intent: intent) HostnameReadOnHost { node: _ } => @@ -3946,6 +3989,7 @@ fn host_effect_drive_directive(r: Reconciliation not_converged_directive(drive: host_effect_drive(intent: intent)) ProvisionBuildCacheOnHost { node: _, desired: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) ProvisionCodexRuntimeOnHost { node: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) + EnsureCompilePoolSliceOnHost { node: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) HostnameOnHost { node: _, hostnamectl_path: _, base: _, desired: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) HostnameReadOnHost { node: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) NbdProxySessionOnHost { node: _, intent: _, lease_key: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) @@ -4350,6 +4394,283 @@ fn build_cache_ensure_step( data already_converged_evidence_is_the_observation_note: String = "THE ALREADY-CONVERGED ARM RETURNS ShellEffectApplied WITH exit_code 0 AND NO SHELL HAVING RUN, and that is worth stating because it looks like a fabricated receipt and is not. HostEffectEvidence has two arms and neither names an observation, so the evidence carrier cannot yet distinguish 'a command ran and succeeded' from 'a read established that nothing needed to run'. What the receipt carries is the observation itself -- the unit found owning the endpoint -- rather than invented command output, so a reader sees WHY it converged. DISSOLVE-ON: a HostEffectEvidence arm for an observation-grounded convergence, at which point this arm names it and the exit_code 0 goes away. Until then the honest thing is to say so here rather than let the zero read as a process exit." +// THE COMPILE-POOL SLICE GETS ITS OWN EFFECT RATHER THAN RIDING THE CACHE ENSURE, AND THE SEAM IS THE POINT. +// gunbc.compile_pool_ensure admit_ensure_compile_pool_slice refuses on TOPOLOGY before it observes anything, +// and gunbc_compile_pool_placement is CompilePoolInRunnerSlots today, so folding this admission into +// realize_provision_build_cache_body as a precondition would make EnsureBuildCacheInstance refuse on every +// host in the fleet. A lane that halts another lane to express a dependency has chosen the wrong seam: the +// pool is a separate desired state with a separate unit file, so it is a separate effect whose refusal costs +// nothing that was already converging. (Parent ruling, warm-tern-755, 2026-08-23.) +// +// WHAT IT ANSWERS TODAY, so this is not read as a description of a working fleet: EnsurePoolTopologyUndeclared, +// on every host, naming gunbc_compile_pool_placement as the cause. That is the correct answer and the landing +// state -- no host has a managed slice, and the flip of that row is the change this converger exists to make +// possible. gunbc_compile_pool_placement's own dissolution trigger says to flip it in the same change that +// converges the managed unit AND ITS SLICE; the unit had a converger and the slice had none. +// +// THE EFFECT CARRIES NO PAYLOAD, AND THAT IS DELIBERATE AFTER THE A5 LESSON. EnsureBuildCacheInstance carries +// its BuildCacheInstance because the desired daemon is a row a realizer would otherwise have to guess. The +// pool's desired state is not a row: the slice NAME comes from the topology and the SIZE is derived by +// gunbc.fleet_host_budget compile_pool_bytes_for from the cited jobserver knobs. Putting either on the effect +// would mint a second answer to a question that already has one authority, so the effect names the target host +// and the derivation happens here from that host's own facts. +// WHICH SLICE IS OBSERVED WHEN THE TOPOLOGY DECLARES NONE. Under CompilePoolInRunnerSlots there is no declared +// slice to read, and the observation is a parameter the admission takes whether or not it consults it -- it +// refuses on topology first. The subject is therefore the slice gunbc.host_layout names, which makes the read +// a true statement about the host (that unit really is what a managed pool would be called there) rather than +// a fabricated subject, and keeps the read unconditional so it cannot become a function of the answer it +// feeds. It also produces a real finding for free: on a host where the topology says no pool is managed and +// the slice nevertheless exists, the reading is there in the observation rather than being unasked. Note the +// layout row and the topology row are two authorities on ONE question -- whether a managed pool is declared -- +// and they disagree today; that disagreement is a standing hazard recorded on this PR, not something this +// function resolves. +fn compile_pool_observation_subject(placement: CompilePoolPlacement) -> NonEmptyStr { + match placement { + CompilePoolSliceDeclared { slice_unit: declared } => declared as NonEmptyStr + CompilePoolInRunnerSlots => gunbc_managed_compile_pool_slice + } +} + +fn compile_pool_ensure_refusal_detail(reason: CompilePoolEnsureRefusal) -> String { + match reason { + EnsurePoolTopologyUndeclared { detail: d } => d as String + EnsurePoolSizingUnresolved { detail: d } => d as String + EnsureCachePlacementUnknown { detail: d } => d as String + EnsurePoolSliceUnobservable { detail: d } => d as String + EnsurePoolSliceNamedDoesNotMatch { declared: dec, observed: obs } => + join( + ["the declared compile-pool slice is ", dec as String, " but the host answers for ", obs as String], + "", + ) + } +} + +fn compile_pool_slice_system_path(slice_unit: NonEmptyStr) -> String { + concat("/etc/systemd/system/", slice_unit as String) +} + +// THE BYTES INSTALLED ARE THE BYTES THE ADMISSION ADMITTED. EnsureCompilePoolAdmitted already carries the +// SystemdUnitFile gunbc.build_cache_unit rendered from the derived size, so this serializes THAT value rather +// than re-rendering from the same inputs. Re-rendering would be a second construction of one artifact, and +// the failure it admits is not hypothetical: the admission could be granted against one size and the file +// written from another with nothing between them to notice. +// +// Install then daemon-reload then START -- never restart. Stopping a slice kills every process in it +// (systemd.slice(5)), so restarting the compile pool would terminate the compilers it exists to hold, which +// is the opposite of converging it. `start` on an already-active slice re-applies the unit's resource +// directives and kills nothing. +fn compile_pool_slice_install_body(slice_unit: NonEmptyStr, slice_unit_file: SystemdUnitFile) -> String { + join([ + "set -euo pipefail", + "_gunbc_stage=\"$(mktemp -d)\"", + "trap 'rm -rf \"$_gunbc_stage\"' EXIT", + join(["cat > \"$_gunbc_stage/", slice_unit as String, "\" <<'__GUNBC_COMPILE_POOL_SLICE__'"], ""), + concat(serialize_systemd_unit_file(file: slice_unit_file), "__GUNBC_COMPILE_POOL_SLICE__"), + join(["sudo -n install -m 0644 \"$_gunbc_stage/", slice_unit as String, "\" ", compile_pool_slice_system_path(slice_unit: slice_unit)], ""), + systemctl_daemon_reload_command(privileged: true), + systemctl_start_command(unit: slice_unit, privileged: true), + "", + ], "\n") +} + +data compile_pool_slice_install_shell_dissolution_trigger: DissolutionCondition = unbound_dissolution(description: "dissolve-on: gunbc.host_effect_realize.compile_pool_slice_install_body -- concat-built systemd slice install shell run over shell_exec_via_bash transport, the same shape and the same debt as build_cache_provision_shell_script beside it. DISSOLVES WHEN bash-emit (#5828) realizes a unit-file install as typed host-effect argv, at which point both bodies retire together rather than this one being converted alone.") + +fn compile_pool_ensure_admission_for( + node: ComputeHost, + transport: HostEffectTransport, +) -> EnsureCompilePoolAdmission { + let placement = gunbc_compile_pool_placement + let instance = ci_cache_instance(host: node.identity) + let survey = survey_build_cache_endpoint( + instance: instance, + reads: build_cache_endpoint_reads(instance: instance, transport: transport), + ) + admit_ensure_compile_pool_slice( + topology: placement, + sizing: compile_pool_sizing_for_host(node: node, placement: placement), + cache_placement: survey.placement, + observed: observe_compile_pool_slice( + transport: transport, + slice_unit: compile_pool_observation_subject(placement: placement), + ), + ) +} + +// The size is DERIVED here from the target host's own facts rather than travelling on the effect, through the +// same compile_pool_bytes_for gunbc.ci_runner_placement uses to shrink the runner pool. The two must agree or +// the host stops conserving, and they agree because there is one derivation, not because two call sites were +// kept in step. +fn compile_pool_sizing_for_host(node: ComputeHost, placement: CompilePoolPlacement) -> CompilePoolResolution { + match compute_host_primary_cpu(host: node) { + Absent => + CompilePoolUnresolved { + reason: "compile pool unresolved: the target ComputeHost declares no primary CPU, so the jobserver token count that sizes the pool cannot be derived; fail-closed, no default size", + } + Present { value: cpu } => + compile_pool_bytes_for( + placement: placement, + nproc: cpu_facts_threads(cpu: cpu), + host: compute_host_ram_bytes_total(host: node), + citation: jobserver_config, + ) + } +} + +// The admission runs TWICE and the second run is not a formality: it is the postcondition, re-derived from a +// FRESH observation of the host rather than from the state we just wrote. A read-back that reused the +// preflight observation would green on evidence it produced -- the exact defect +// provision_observes_before_it_acts_note records from srv3 -- and the whole point is that the pre-state and +// the post-state can differ. Convergence is claimed only when the second admission answers +// EnsureCompilePoolAlreadyConverged, which is the same predicate the preflight uses to skip the write. +fn compile_pool_slice_install_step( + node: ComputeHost, + slice_unit: NonEmptyStr, + sized: ByteSize, + slice_unit_file: SystemdUnitFile, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + let observed = run_shell_transport( + script: retained_srvn( + body: shell_exec_via_bash(body: compile_pool_slice_install_body(slice_unit: slice_unit, slice_unit_file: slice_unit_file)), + reason: "srvN compile-pool slice install (managed cgroup parent for cache-miss compilers)", + ), + transport: transport, + ) + if observed.success == false { + NotConverged { + reason: join( + [ + "host_effect_apply: EnsureCompilePoolSlice ran its install and the shell exited ", + to_string(observed.exit_code), + ": ", + if observed.stderr == "" { observed.stdout } else { observed.stderr }, + ], + "", + ), + applied: effect_intent, + } + } else { + let after = compile_pool_ensure_admission_for(node: node, transport: transport) + if compile_pool_ensure_is_converged(admission: after) { + Converged { + evidence: ShellEffectApplied { + exit_code: observed.exit_code, + stdout: join( + [ + "CompilePoolSliceReceipt: converged unit=", + slice_unit as String, + " memory.max=memory.high=", + to_string(byte_size_count(b: sized)), + " (re-observed after install; the admission answered already-converged)", + ], + "", + ), + stderr: observed.stderr, + }, + applied: effect_intent, + } + } else { + NotConverged { + reason: join( + [ + "host_effect_apply: EnsureCompilePoolSlice installed ", + slice_unit as String, + " and the read-back did not converge -- ", + compile_pool_admission_diagnostic(admission: after), + ], + "", + ), + applied: effect_intent, + } + } + } +} + +fn compile_pool_admission_diagnostic(admission: EnsureCompilePoolAdmission) -> String { + match admission { + EnsureCompilePoolAlreadyConverged { slice_unit: u, sized_bytes: _ } => + join(["the slice ", u as String, " is converged"], "") + EnsureCompilePoolAdmitted { slice_unit: u, sized_bytes: _, slice_unit_file: _ } => + join( + ["the slice ", u as String, " is still admitted for writing, so the installed unit did not take effect on the host"], + "", + ) + EnsureCompilePoolRefused { reason: r } => compile_pool_ensure_refusal_detail(reason: r) + } +} + +fn realize_ensure_compile_pool_slice( + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + match effect_intent.target { + BmcController { node: _ } => + NotConverged { + reason: "host_effect_apply: EnsureCompilePoolSlice requires a HostOs target; fail-closed.", + applied: effect_intent, + } + HostOs { node: node } => + match transport { + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => + NotConverged { + reason: "host_effect_apply: EnsureCompilePoolSlice cannot use EmitArtifactThenThinRun transport; typed mismatch.", + applied: effect_intent, + } + LocalShell => realize_ensure_compile_pool_slice_body(node: node, transport: transport, effect_intent: effect_intent) + SshShell { ssh_host: _ } => realize_ensure_compile_pool_slice_body(node: node, transport: transport, effect_intent: effect_intent) + FleetSsh { target: _, context: _ } => realize_ensure_compile_pool_slice_body(node: node, transport: transport, effect_intent: effect_intent) + } + } +} + +fn realize_ensure_compile_pool_slice_body( + node: ComputeHost, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + match compile_pool_ensure_admission_for(node: node, transport: transport) { + EnsureCompilePoolRefused { reason: r } => + NotConverged { + reason: join( + [ + "host_effect_apply: EnsureCompilePoolSlice refused before writing anything -- ", + compile_pool_ensure_refusal_detail(reason: r), + ], + "", + ), + applied: effect_intent, + } + EnsureCompilePoolAlreadyConverged { slice_unit: u, sized_bytes: b } => + Converged { + evidence: ShellEffectApplied { + exit_code: 0, + stdout: join( + [ + "CompilePoolSliceReceipt: already_converged unit=", + u as String, + " memory.max=memory.high=", + to_string(byte_size_count(b: b)), + " (observed on the host; no unit written, no start issued)", + ], + "", + ), + stderr: "", + }, + applied: effect_intent, + } + EnsureCompilePoolAdmitted { slice_unit: u, sized_bytes: b, slice_unit_file: f } => + compile_pool_slice_install_step( + node: node, + slice_unit: u, + sized: b, + slice_unit_file: f, + transport: transport, + effect_intent: effect_intent, + ) + } +} + fn build_cache_already_converged( unit: NonEmptyStr, effect_intent: HostEffectIntent, diff --git a/dag/gunbc/host_toolchain_ensure.dag b/dag/gunbc/host_toolchain_ensure.dag index c75002d973a..04c20fc22dd 100644 --- a/dag/gunbc/host_toolchain_ensure.dag +++ b/dag/gunbc/host_toolchain_ensure.dag @@ -18,6 +18,7 @@ import gunbc.fleet_intent { ComputeHost } import gunbc.host_effect { HostEffect, EnsureBuildCacheInstance, + EnsureCompilePoolSlice, ProvisionCodexRuntime, OsInstallActuatorToolchainEnsure, HostOs, @@ -29,9 +30,18 @@ import gunbc.os_install_actuator_selection { srv3_os_install_actuator_host } import gunbc.build_cache_instance { ci_cache_instance } import std.realization_reconcile { Converged, NotConverged } -type HostToolchainKind = BuildCache | CodexRuntime | OsInstallActuator +type HostToolchainKind = BuildCache | CodexRuntime | OsInstallActuator | CompilePool -// Compatibility specialization, not the Ensure abstraction: HostToolchainKind is the closed selector for two existing HostEffect adapters in this entrypoint. Generic convergence semantics live in gunbc.ensure and dispatch requirements live in gunbc.roadmap_execution_contract. Cargo, rustc, provider auth, and future task dependencies must not be added to this enum merely because their callers say ensure; they require their own typed observation/plan/readback adapters and may then reuse generic Ensure. +// Compatibility specialization, not the Ensure abstraction: HostToolchainKind is the closed selector for the +// existing HostEffect adapters reachable from this entrypoint. Generic convergence semantics live in +// gunbc.ensure and dispatch requirements live in gunbc.roadmap_execution_contract. Cargo, rustc, provider auth, +// and future task dependencies must not be added to this enum merely because their callers say ensure; they +// require their own typed observation/plan/readback adapters and may then reuse generic Ensure. +// +// CompilePool was admitted against that bar rather than beside it, and the three parts are nameable: the +// observation is gunbc.compile_pool_observe observe_compile_pool_slice, the plan is gunbc.compile_pool_ensure +// admit_ensure_compile_pool_slice, and the read-back is that same admission re-derived from a fresh +// observation after the install. It is listed here because the bar was met, not because its caller says ensure. data host_toolchain_ensure_srv3_fork_scaffold: Disposition = Terminal { reason: "srv3_os_install_actuator_toolchain_ensure thinned to host_toolchain_ensure(host: srv3, kind: OsInstallActuator) — no behavioral fork remains." } @@ -89,6 +99,7 @@ fn host_toolchain_ensure_effect(kind: HostToolchainKind, host: HostIdentity) -> BuildCache => EnsureBuildCacheInstance { desired: ci_cache_instance(host: host) } CodexRuntime => ProvisionCodexRuntime {} OsInstallActuator => OsInstallActuatorToolchainEnsure {} + CompilePool => EnsureCompilePoolSlice {} } } @@ -96,6 +107,7 @@ fn host_toolchain_ensure_target_host(kind: HostToolchainKind, host: HostIdentity match kind { BuildCache => fleet_compute_host_for_identity(identity: host) CodexRuntime => fleet_compute_host_for_identity(identity: host) + CompilePool => fleet_compute_host_for_identity(identity: host) OsInstallActuator => if host == operator_host_srv3 { Present { value: srv3_os_install_actuator_host } diff --git a/dag/gunbc/live_deploy/operations.dag b/dag/gunbc/live_deploy/operations.dag index 5072e671a97..f38cb9cb65f 100644 --- a/dag/gunbc/live_deploy/operations.dag +++ b/dag/gunbc/live_deploy/operations.dag @@ -70,6 +70,14 @@ fn systemctl_enable_command(unit: NonEmptyStr, privileged: Bool) -> String { } } +fn systemctl_start_command(unit: NonEmptyStr, privileged: Bool) -> String { + if privileged { + shell_privileged_command_text_of_argv(argv: systemctl_start_argv(unit: unit)) + } else { + shell_command_text_of_argv(argv: systemctl_start_argv(unit: unit)) + } +} + fn systemctl_restart_command(unit: NonEmptyStr, privileged: Bool) -> String { if privileged { shell_privileged_command_text_of_argv(argv: systemctl_restart_argv(unit: unit)) diff --git a/dag/gunbc/systemctl_show_read.dag b/dag/gunbc/systemctl_show_read.dag index caff1b4bef2..63eb436052f 100644 --- a/dag/gunbc/systemctl_show_read.dag +++ b/dag/gunbc/systemctl_show_read.dag @@ -131,3 +131,78 @@ fn systemctl_show_property_operation_argv_matches_transport( ArgvMaterializationRefused { at: _, cause: _ } => false } } + +// THE LOAD-STATE READ IS A SIBLING OF THE PROPERTY READ, NOT A SPECIAL CASE OF IT. It shares the +// capture type because the shape of the answer is identical -- one `--value` line, or a located +// refusal -- but it cannot share the operation: extdeps.systemd.systemctl's own annotation records +// that SystemdUnitProperty is numeric cgroup evidence with one numeric mock, and `not-found` is not +// a magnitude. Sharing the operation would have meant widening that coproduct with a member whose +// mock lies about every other member's shape. +// +// WHY A CONSUMER NEEDS IT AT ALL: `systemctl show --property=MemoryMax --value` answers +// `infinity` for a unit the manager has never heard of, exactly as it does for a loaded unit with no +// ceiling. Reading only the limits therefore cannot tell an ABSENT slice from an UNBOUNDED one -- +// two states whose remedies are opposite (install it vs refuse and report the drift). LoadState is +// the discriminator, so it is read first and the limits are read only under a loaded unit. +fn systemctl_show_load_state_operation_argv(unit: NonEmptyStr) -> ArgvMaterialization { + shell_materialize_operation_argv( + systemctl_show_property_path, + systemctl_show_property_service, + systemctl_show_load_state_operation, + [operation_argv_bind_text(name: "unit", text: unit as String)] + ) +} + +data systemctl_show_load_state_operation: String = "ShowLoadState" + +fn systemctl_show_load_state_read_local(unit: NonEmptyStr) -> SystemdPropertyCapture { + let result = systemd.Systemctl.ShowLoadState(unit: unit) + systemctl_show_property_value_from_stdout(success: result.success, stdout: result.value) +} + +fn systemctl_show_load_state_read_ssh(host: NonEmptyStr, unit: NonEmptyStr) -> SystemdPropertyCapture { + match systemctl_show_load_state_operation_argv(unit: unit) { + ArgvMaterialized { argv: argv } => + systemctl_show_property_read_ssh_argv(host: host, argv: argv) + ArgvMaterializationRefused { at: _, cause: c } => + SystemdPropertyCaptureRefused { reason: operation_argv_cause_label(cause: c) } + } +} + +fn systemctl_show_load_state_read_fleet_ssh( + target: SshTarget, + context: FleetSshExecutionContext, + unit: NonEmptyStr, +) -> SystemdPropertyCapture { + match systemctl_show_load_state_operation_argv(unit: unit) { + ArgvMaterialized { argv: argv } => + systemd_property_capture_from_outcome( + outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv), + ) + ArgvMaterializationRefused { at: _, cause: c } => + SystemdPropertyCaptureRefused { reason: operation_argv_cause_label(cause: c) } + } +} + +fn systemctl_show_load_state_read( + transport: HostEffectTransport, + unit: NonEmptyStr, +) -> SystemdPropertyCapture { + match transport { + LocalShell => systemctl_show_load_state_read_local(unit: unit) + SshShell { ssh_host: h } => systemctl_show_load_state_read_ssh(host: h, unit: unit) + FleetSsh { target: t, context: c } => systemctl_show_load_state_read_fleet_ssh(target: t, context: c, unit: unit) + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => + SystemdPropertyCaptureRefused { + reason: "systemctl show LoadState refused EmitArtifactThenThinRun transport", + } + } +} + +fn systemctl_show_load_state_operation_argv_matches_transport(unit: NonEmptyStr) -> Bool { + let authority = systemctl_show_load_state_argv(unit: unit) + match systemctl_show_load_state_operation_argv(unit: unit) { + ArgvMaterialized { argv: materialized } => join(materialized, " ") == join(authority, " ") + ArgvMaterializationRefused { at: _, cause: _ } => false + } +} diff --git a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag new file mode 100644 index 00000000000..25c6a6c9b12 --- /dev/null +++ b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag @@ -0,0 +1,123 @@ +module test.claim.compile_pool_ensure_wiring_witness + +import gunbc.host_toolchain_ensure { host_toolchain_ensure_effect, CompilePool } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// WHAT THIS MODULE IS FOR: gunbc.compile_pool_ensure was proven over fixtures and computed by nothing -- +// its only caller was its own test module -- so an admission named as the wall between a host and an +// unbounded compile pool was green without ever being asked a question. These witnesses assert the parts of +// the wiring that make it reachable, and the parts of the observation that decide what it is asked. + +// REACHABILITY, not shape: the entrypoint a person or a converge pass actually calls must produce the effect, +// and the effect must resolve to a cell that dispatches to the realizer. Asserting the arm exists on the +// coproduct would prove nothing -- it existed before this change too, with nothing routing to it. +test fn witness_compile_pool_kind_produces_the_effect() -> Bool { + match host_toolchain_ensure_effect(kind: CompilePool, host: "srv1" as HostIdentity) { + EnsureCompilePoolSlice {} => true + _ => false + } +} + +test fn witness_compile_pool_effect_routes_to_the_host_cell() -> Bool { + match resolve_host_effect_cell(target: HostOs { node: srv1_host }, effect: EnsureCompilePoolSlice {}) { + EnsureCompilePoolSliceOnHost { node: _ } => true + _ => false + } +} + +// The BMC arm is the discriminating negative: a slice is an in-band systemd fact, so a Redfish target must +// refuse rather than resolve. Without it the routing witness above would pass on a fold that resolved +// everything to the same cell. +test fn witness_compile_pool_effect_refuses_a_bmc_target() -> Bool { + match resolve_host_effect_cell(target: BmcController { node: srv1_host }, effect: EnsureCompilePoolSlice {}) { + IncompatibleCell { reason: why } => string_contains(s: why, pattern: "EnsureCompilePoolSlice") + _ => false + } +} + +// THE LIVE TOPOLOGY TRIPWIRE, and it reads the REAL gunbc_compile_pool_placement rather than a fixture, so it +// is a claim about this fleet. It asserts a REFUSAL today: the row is CompilePoolInRunnerSlots, so no host has +// a managed slice and every convergence attempt refuses naming that row. When the row flips to +// CompilePoolSliceDeclared this witness reds -- deliberately, in the same change that shrinks runner_pool -- +// and at that point it becomes an assertion that the pool converges rather than being deleted. +test fn witness_live_topology_refuses_the_slice_ensure() -> Bool { + let a = admit_ensure_compile_pool_slice( + topology: gunbc_compile_pool_placement, + sizing: compile_pool_bytes_for( + placement: gunbc_compile_pool_placement, + nproc: hardware_thread_count(count: 128), + host: byte_size(134317047808), + citation: jobserver_config, + ), + cache_placement: ServerPlacementAbsent, + observed: CompilePoolSliceAbsent { slice_unit: gunbc_managed_compile_pool_slice }, + ) + match a { + EnsureCompilePoolRefused { reason: r } => + match r { + EnsurePoolTopologyUndeclared { detail: d } => + string_contains(s: d as String, pattern: "gunbc_compile_pool_placement") + _ => false + } + _ => false + } +} + +// INFINITY IS NOT A SIZE, AND THIS IS THE RED THAT MATTERS MOST IN THE OBSERVATION. `systemctl show +// --property=MemoryMax --value` answers `infinity` both for a unit the manager never heard of and for a +// loaded slice with no ceiling. Reading either as a magnitude would let a slice with NO LIMIT be reported as +// a sized pool, which is the unbounded charge the managed slice exists to replace -- reported as its own +// remedy. The refusal is what keeps the two apart. +test fn witness_unbounded_memory_max_is_not_read_as_a_size() -> Bool { + match compile_pool_limit_of_capture( + slice_unit: gunbc_managed_compile_pool_slice, + property_name: "MemoryMax", + capture: SystemdPropertyCaptured { value: "infinity" }, + ) { + CompilePoolLimitRefused { observation: o } => + match o { + CompilePoolObservationUnavailable { reason: _ } => true + _ => false + } + CompilePoolLimitBytes { bytes: _ } => false + } +} + +// The paired positive control: a real byte count must decode, or the refusal above would be satisfied by a +// decoder that refuses everything. +test fn witness_numeric_memory_max_decodes_to_bytes() -> Bool { + match compile_pool_limit_of_capture( + slice_unit: gunbc_managed_compile_pool_slice, + property_name: "MemoryMax", + capture: SystemdPropertyCaptured { value: "17179869184" }, + ) { + CompilePoolLimitBytes { bytes: b } => byte_size_count(b: b) == 17179869184 + CompilePoolLimitRefused { observation: _ } => false + } +} + +// A read that could not run is UNAVAILABLE, never ABSENT. The two have opposite remedies -- install the slice +// versus fix the reach -- and folding an unreadable manager into "the slice is not there" would send a +// converger to write a unit on a host it cannot see. +test fn witness_unreadable_property_is_unavailable_not_absent() -> Bool { + match compile_pool_limit_of_capture( + slice_unit: gunbc_managed_compile_pool_slice, + property_name: "MemoryMax", + capture: SystemdPropertyCaptureRefused { reason: "systemctl show exited unsuccessfully" }, + ) { + CompilePoolLimitRefused { observation: o } => + match o { + CompilePoolObservationUnavailable { reason: _ } => true + CompilePoolSliceAbsent { slice_unit: _ } => false + CompilePoolSliceObserved { slice_unit: _, limits: _ } => false + } + CompilePoolLimitBytes { bytes: _ } => false + } +} + +// The load-state argv the transport materializes must be the argv extdeps.systemd.systemctl declares. Without +// this the operation could drift from its own authority and every reading above would be of the wrong unit. +test fn witness_load_state_operation_argv_matches_its_authority() -> Bool { + systemctl_show_load_state_operation_argv_matches_transport(unit: gunbc_managed_compile_pool_slice) +} From d949ccad2bd3b2b2c85d3bf30e72d35a15d536ee Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 24 Aug 2026 00:32:43 +0000 Subject: [PATCH 2/9] Move the ShowLoadState annotation to module-item grain The annotation sat inside the service declaration body, which DESIGN 4c does not model. Found by execution: claim_executor --required-ci parse phase, seven refusals on this file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ptiy92YHu7ig5W7ktiBzpZ --- dag/extdeps/systemd/systemctl.dag | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index 1fe14e24e2e..b93aed211d0 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -137,6 +137,14 @@ fn systemctl_user_start_argv(unit: NonEmptyStr) -> List { ["systemctl", "--user", "start", unit as String] } +// LoadState is the only observation that distinguishes an ABSENT unit from a present one, which is why the +// `ShowLoadState` operation on the service surface above is its own operation rather than a +// SystemdUnitProperty arm: that coproduct is numeric cgroup evidence sharing one numeric mock, and a wire +// value of `not-found` is not a magnitude. Cited to systemd.unit(5) LoadState -- `loaded`, `not-found`, +// `bad-setting`, `error`, `masked` -- of which `not-found` is the one this corpus reads positively. +// `systemctl show` exits 0 for a unit the manager has never heard of and prints not-found, so a nonzero exit +// means the MANAGER was unreadable, which is a different refusal from an absent unit and must not be folded +// into it. fn systemctl_show_load_state_argv(unit: NonEmptyStr) -> List { ["systemctl", "show", unit as String, "--property=LoadState", "--value"] } @@ -311,13 +319,6 @@ service systemd.Systemctl { } } - // LoadState is the only observation that distinguishes an ABSENT unit from a present one, and it - // needs its own operation rather than a SystemdUnitProperty arm: that coproduct is numeric cgroup - // evidence sharing one numeric mock, and a wire value of `not-found` is not a magnitude. Cited to - // systemd.unit(5) LoadState -- `loaded`, `not-found`, `bad-setting`, `error`, `masked` -- of which - // `not-found` is the one this corpus reads positively. `systemctl show` exits 0 for a unit the - // manager has never heard of and prints not-found, so a nonzero exit here means the MANAGER was - // unreadable, which is a different refusal from an absent unit and must not be folded into it. operation ShowLoadState { input { unit: NonEmptyStr } output { From dc3392973540b364e34144665f34f5078204e97b Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 24 Aug 2026 00:53:20 +0000 Subject: [PATCH 3/9] One authority for whether this fleet has a managed compile pool Two rows answered the same question and disagreed. Every BuildCacheInstance carried gunbc_managed_compile_pool_slice as intended_compile_pool unconditionally, while gunbc_compile_pool_placement said CompilePoolInRunnerSlots -- so the rendered cache unit named a Slice= that nothing provisions. Applying it would have had systemd create that cgroup implicitly WITH NO LIMITS: the same unbounded compile charge gunbc_compile_pool_doc measured, wearing the managed topology's name, with the budget model believing a bounded pool existed. Worse than the placement it was meant to improve on, because it looks converged. THE ROW MOVES TO gunbc.host_layout, AND THE HOME WAS FORCED BY MEASUREMENT RATHER THAN CHOSEN. Deriving intended_compile_pool from the placement row where it stood is impossible: gunbc.fleet_host_budget transitively REACHES gunbc.build_cache_instance, so the instance importing it closes a cycle, and acyclicity is the one structural law the import graph has. host_layout reaches neither consumer and both already import it, so the single authority costs no new import edge. The semantic argument agrees with the measurement -- where compile RSS is charged is a fact about how a host is laid out, which is that module's whole subject, and it already owns the slice NAME. The name and whether the name is managed were always one question in two homes. THE FIX IS CONSTRUCTION, NOT A CHECK. intended_compile_pool stops being a bare NonEmptyStr and carries CompilePoolPlacement itself, sourced from the single row. build_cache_unit then renders Slice= through a match, so the CompilePoolInRunnerSlots arm has NO SLICE NAME TO RENDER and the hazardous unit is not constructible -- rather than being constructible and avoided by remembering. runner_activation's pool-receipt binding answers false on that arm instead of comparing a receipt against a pool this fleet does not declare. THE ROW IS NOT FLIPPED. The fleet still declares CompilePoolInRunnerSlots, so both live tripwires keyed on that value stay green: test.claim.compile_pool_ensure_wiring_witness witness_live_topology_refuses_the_slice_ensure and test.claim.host_compile_pool live_topology_doc. The flip belongs with the wet convergence that installs the slice, which is what the dissolution trigger already demands. NOT VERIFIED LOCALLY: a whole-tree compile is OOM-killed in a session container (REAL_EXIT=137, swap disabled, shared slice), so this relies on CI to compile it. Reviewers should treat the required run as the first real check, not a formality. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/build_cache_instance.dag | 12 ++++++++---- dag/gunbc/build_cache_unit.dag | 18 +++++++++++++++++- dag/gunbc/compile_pool_ensure.dag | 7 +++---- dag/gunbc/fleet_host_budget.dag | 11 ++++------- dag/gunbc/host_compile_pool.dag | 5 ++++- dag/gunbc/host_effect_realize.dag | 6 ++++-- dag/gunbc/host_layout.dag | 10 ++++++++++ dag/gunbc/runner_activation.dag | 12 +++++++++++- 8 files changed, 61 insertions(+), 20 deletions(-) diff --git a/dag/gunbc/build_cache_instance.dag b/dag/gunbc/build_cache_instance.dag index 3a4b7dca17b..addfb8d895e 100644 --- a/dag/gunbc/build_cache_instance.dag +++ b/dag/gunbc/build_cache_instance.dag @@ -10,7 +10,11 @@ import extdeps.cache.sccache { SccacheServerEndpoint, SccacheServerUnixSocket, s import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3, operator_host_srv4, } -import gunbc.host_layout { gunbc_managed_compile_pool_slice } +import gunbc.host_layout { + gunbc_managed_compile_pool_slice, + CompilePoolPlacement, CompilePoolSliceDeclared, CompilePoolInRunnerSlots, + gunbc_compile_pool_placement, +} data build_cache_instance_scope_disposition: Disposition = SingleAuthority @@ -34,7 +38,7 @@ type BuildCacheInstance { intended_principal: NonEmptyStr intended_storage: NonEmptyStr intended_capacity: ByteSize? - intended_compile_pool: NonEmptyStr + intended_compile_pool: CompilePoolPlacement } data build_cache_configuration_observation_note: String = "STORAGE ROOT AND CAPACITY ARE TWO OBSERVED CONFIGURATION AXES, and each observation is bound to both the exact deployed BuildCacheInstance and the ProcessIdentity whose environment was read. A path or byte count without both subjects could be replayed across hosts, across the CI and session instances that share one sccache implementation, or across two processes after pid reuse. The owner is the process already proven to hold the instance endpoint; a unit file would answer what systemd WOULD start, not how the live server is configured, and every host in the incident population currently runs a server systemd did not start.\n\nUNSET AND READ UNAVAILABLE ARE DIFFERENT OBSERVATIONS WITH DIFFERENT REMEDIES. Unset means the owner environment was read and carries no assignment; unavailable means the read did not answer, and must never narrow to unset. Endpoint or owner absence does not appear as a third arm here: without the prerequisite endpoint-owner observation there is no configuration subject, so these carriers are not constructed at all. That is not-applicability at the join rather than malformed configuration.\n\nCAPACITY ALSO RETAINS CONFIGURED TEXT THAT CANNOT YET BE GROUNDED AS BYTES. gunbc.build_cache_unit emits a plain decimal byte count, which has one exact decoding. sccache also accepts suffixed text such as its documented 10G default, but this repository has no cited rule deciding whether G denotes 10^9 or 2^30. Choosing either would fabricate a magnitude whose seven-percent difference could silently pass or refuse convergence. BuildCacheCapacityConfiguredUngrounded therefore carries the text verbatim; it narrows when extdeps.cache.sccache cites and models the accepted size grammar. The instance itself retains whether capacity intent is declared, so a later comparison can refuse intended_capacity Absent rather than freezing a vendor default as a decision.\n\nThese are carriers only: they mint no verified flag and perform no host read. A producer must read the endpoint owner's environment into one of these arms before runner activation can replace storage_verified and capacity_verified with typed observed-versus-intended comparisons." @@ -284,7 +288,7 @@ fn ci_cache_instance(host: HostIdentity) -> BuildCacheInstance { intended_principal: "ghrunner" as NonEmptyStr, intended_storage: "/var/lib/ctrl/sccache-ci" as NonEmptyStr, intended_capacity: none, - intended_compile_pool: gunbc_managed_compile_pool_slice, + intended_compile_pool: gunbc_compile_pool_placement, } } @@ -301,7 +305,7 @@ fn session_cache_instance(host: HostIdentity) -> BuildCacheInstance { intended_principal: "briansrls" as NonEmptyStr, intended_storage: "/var/lib/ctrl/sccache" as NonEmptyStr, intended_capacity: Present { value: gibibyte_to_byte_size(g: gibibyte(50)) }, - intended_compile_pool: gunbc_managed_compile_pool_slice, + intended_compile_pool: gunbc_compile_pool_placement, } } diff --git a/dag/gunbc/build_cache_unit.dag b/dag/gunbc/build_cache_unit.dag index 219491f9a08..2397badfe58 100644 --- a/dag/gunbc/build_cache_unit.dag +++ b/dag/gunbc/build_cache_unit.dag @@ -72,6 +72,20 @@ fn runner_ordering_dropin_lines(instance: BuildCacheInstance) -> List { systemd_drop_in_lines(drop_in: runner_ordering_drop_in(instance: instance)) } +// SLICE= IS RENDERED ONLY WHERE A MANAGED POOL IS DECLARED, AND THAT IS A CONSTRUCTION WALL RATHER +// THAN A CHECK. This directive used to be spelled unconditionally from a bare-string field that +// always named the managed slice, while gunbc.host_layout's placement row said the fleet has no +// managed pool. Applying that unit would have named a Slice= nothing provisions, and systemd +// creates such a cgroup implicitly WITH NO LIMITS -- the unbounded compile charge wearing the +// managed topology's name. Now the field carries the placement itself, so the CompilePoolInRunnerSlots +// arm has no slice name to render and the bad unit cannot be built. +fn compile_pool_service_directives(placement: CompilePoolPlacement) -> List { + match placement { + CompilePoolSliceDeclared { slice_unit: u } => [ServiceSlice { name: u as NonEmptyStr }] + CompilePoolInRunnerSlots => [] + } +} + fn build_cache_unit_file(instance: BuildCacheInstance) -> SystemdUnitFile { ServiceUnitFile { unit: [ @@ -84,11 +98,12 @@ fn build_cache_unit_file(instance: BuildCacheInstance) -> SystemdUnitFile { After { target: "network.target" as NonEmptyStr }, ], service: concat( + compile_pool_service_directives(placement: instance.intended_compile_pool), + concat( [ ServiceType { mode: "simple" as NonEmptyStr }, ServiceUser { name: instance.intended_principal }, ServiceGroup { name: instance.intended_principal }, - ServiceSlice { name: instance.intended_compile_pool }, Environment { assignment: concat("SCCACHE_DIR=", instance.intended_storage as String) as NonEmptyStr, }, @@ -111,6 +126,7 @@ fn build_cache_unit_file(instance: BuildCacheInstance) -> SystemdUnitFile { ], ), ), + ), install: Installable { directives: [WantedBy { target: "multi-user.target" as NonEmptyStr }] }, } } diff --git a/dag/gunbc/compile_pool_ensure.dag b/dag/gunbc/compile_pool_ensure.dag index c537bfa08bd..deaf0eaa606 100644 --- a/dag/gunbc/compile_pool_ensure.dag +++ b/dag/gunbc/compile_pool_ensure.dag @@ -3,7 +3,9 @@ module gunbc.compile_pool_ensure import std.types { String, NonEmptyStr, Bool, List } import std.measure { ByteSize, byte_size_count } import std.disposition { Disposition, SingleAuthority } -import gunbc.host_layout { gunbc_managed_compile_pool_slice } +import gunbc.host_layout { + CompilePoolPlacement, CompilePoolSliceDeclared, CompilePoolInRunnerSlots, + gunbc_compile_pool_placement, gunbc_managed_compile_pool_slice } import gunbc.build_cache_unit { compile_pool_slice_unit_file } import extdeps.systemd.unit_file { SystemdUnitFile } import gunbc.host_compile_pool { @@ -13,9 +15,6 @@ import gunbc.host_compile_pool { CompilePoolObservationUnavailable, } import gunbc.fleet_host_budget { - CompilePoolPlacement, - CompilePoolSliceDeclared, - CompilePoolInRunnerSlots, CompilePoolResolution, CompilePoolDerived, CompilePoolUnresolved, diff --git a/dag/gunbc/fleet_host_budget.dag b/dag/gunbc/fleet_host_budget.dag index 68e4d23b102..7a2b019fd53 100644 --- a/dag/gunbc/fleet_host_budget.dag +++ b/dag/gunbc/fleet_host_budget.dag @@ -17,6 +17,10 @@ import std.realization_schedule { CostAccount, CostBasis, Predicted } import extdeps.accounting.budget { Appropriation, LineItem } import product.budget_tree { BudgetNode, BudgetClaim, Guaranteed, node_conserves } import product.placement_supply { HostIdentity } +import gunbc.host_layout { + CompilePoolPlacement, CompilePoolSliceDeclared, CompilePoolInRunnerSlots, + gunbc_compile_pool_placement, +} import gunbc.fabric_control_plane_charge { HostFabricControlPlaneClaim, ControlPlaneNotPlaced, @@ -139,13 +143,6 @@ data gunbc_compile_pool_doc: String = "THE COMPILE POOL IS A CLAIM ON HOST RAM T data gunbc_jobserver_token_override_note: String = "THE LIVE JOBSERVER ADMITS ROUGHLY TWICE THE BUILD PARALLELISM ITS OWN FORMULA ALLOWS, observed on srv1 and srv2 on 2026-08-06 and recorded here because it lands on exactly the axis the compile pool governs.\n\n/usr/local/sbin/ctrl-jobserver derives tokens = min(cores - MARGIN, floor((MemTotal_gib - MEM_RESERVE_GIB) * 1024 / MIB_PER_JOB)) — the same derivation extdeps.jobserver.derive_jobserver_tokens carries. With the cited defaults (margin 8, reserve 16GiB, 1792MiB per job) on these hosts (nproc 128, MemTotal 125.09GiB) that yields min(120, 62) = 62 tokens, the MEMORY bound binding well below the core bound.\n\nBoth hosts set CTRL_JOBSERVER_TOKENS=128 in /etc/default/ctrl-jobserver, and the script short-circuits on that variable — `if [[ -z \"$\{CTRL_JOBSERVER_TOKENS:-}\" ]]` guards the whole derivation, so an explicit setting skips the memory bound entirely rather than being clamped by it. 128 tokens at 1792MiB each is 224GiB of admissible concurrent compile memory on a 125GiB host.\n\nWHY THIS IS RECORDED AND NOT FIXED HERE: it is an operator configuration fact about a ctrl-era unit, not a gunbc-derived value, and gunbc has no realized knob for it (JobserverTokens is KnobUnimplemented — its presence is what makes interpret_host_converge refuse the whole host). Changing it is a host action with real consequences for build throughput and belongs to the jobserver re-homing lane, not to a budget module. What matters for THIS module is that the compile pool cannot be sized honestly against an admission ceiling that its own inputs disown — so when gunbc_compile_pool_placement flips to CompilePoolSliceDeclared, the pool must be sized from the DERIVED tokens, and the gap between derived and configured is the over-admission that flip would expose rather than create.\n\nCorroborating observation on the same hosts: srv1 had 85GiB of swap in use against srv2's 1.3GiB, on identical hardware with identical swap provisioning (248GiB across two files) — consistent with srv1's compile server thrashing under the starved cgroup that gunbc_compile_pool_doc describes." -data gunbc_compile_pool_placement_doc: String = "WHERE COMPILE RSS IS CHARGED IS A TOPOLOGY FACT, AND THE BUDGET IS DERIVED FROM IT — not a fudge factor. CompilePoolInRunnerSlots: no managed compile-pool slice exists, so cache-miss compiles are charged to whichever cgroup the sccache server sits in; the budget's compile_pool child is therefore ZERO because the pool is not a separate claim on host RAM — it is (nominally) already inside runner_pool. That is the honest description of today's topology, NOT an absorbing fallback: it is level-triggered on a declared row, it widens no failure arm, and it fabricates no converged verdict. Note it is also the topology in which a mis-placed server silently exports one slot's caps onto every compile on the host — that deficit is refused by gunbc.host_build_cache_provision's placement verdict, not papered over here. CompilePoolSliceDeclared: the managed unit owns the compiles in its own slice, so the pool becomes a first-class sibling child sized by the admitted jobserver tokens and runner_pool must shrink to keep the host conserving. DISSOLUTION TRIGGER: flip this row to CompilePoolSliceDeclared in the same change that converges the managed gunbc-build-cache-ci.service unit and its slice; conservation then refuses by construction if the slots were not shrunk." - -type CompilePoolPlacement - = CompilePoolSliceDeclared { slice_unit: String } - | CompilePoolInRunnerSlots - -data gunbc_compile_pool_placement: CompilePoolPlacement = CompilePoolInRunnerSlots type CompilePoolResolution = CompilePoolDerived { bytes: ByteSize } diff --git a/dag/gunbc/host_compile_pool.dag b/dag/gunbc/host_compile_pool.dag index 4402bb9a9df..a35c2768f64 100644 --- a/dag/gunbc/host_compile_pool.dag +++ b/dag/gunbc/host_compile_pool.dag @@ -4,8 +4,11 @@ import std.types { String, NonEmptyStr, Bool } import std.measure { ByteSize, byte_size_count } import std.disposition { Disposition, SingleAuthority } import gunbc.build_cache_instance { CgroupMemoryLimits, EndpointObservationUnavailableReason, unavailable_reason_detail } -import gunbc.fleet_host_budget { +import gunbc.host_layout { CompilePoolPlacement, CompilePoolSliceDeclared, CompilePoolInRunnerSlots, + gunbc_compile_pool_placement, +} +import gunbc.fleet_host_budget { CompilePoolResolution, CompilePoolDerived, CompilePoolUnresolved, } import gunbc.runner_activation { HostCompilePoolReady } diff --git a/dag/gunbc/host_effect_realize.dag b/dag/gunbc/host_effect_realize.dag index 772b5adb145..aacc779387a 100644 --- a/dag/gunbc/host_effect_realize.dag +++ b/dag/gunbc/host_effect_realize.dag @@ -264,14 +264,16 @@ import gunbc.live_deploy.operations { systemctl_start_command, } import extdeps.jobserver { jobserver_config } -import gunbc.fleet_host_budget { +import gunbc.host_layout { CompilePoolPlacement, CompilePoolSliceDeclared, CompilePoolInRunnerSlots, + gunbc_compile_pool_placement, +} +import gunbc.fleet_host_budget { CompilePoolResolution, CompilePoolUnresolved, compile_pool_bytes_for, - gunbc_compile_pool_placement, } import gunbc.compile_pool_ensure { EnsureCompilePoolAdmission, diff --git a/dag/gunbc/host_layout.dag b/dag/gunbc/host_layout.dag index 50153a7194b..4de4d6252dc 100644 --- a/dag/gunbc/host_layout.dag +++ b/dag/gunbc/host_layout.dag @@ -100,6 +100,16 @@ data managed_compile_pool_slice_note: String = "The ONE authority for the manage data gunbc_managed_compile_pool_slice: NonEmptyStr = "gunbc-compile-pool.slice" as NonEmptyStr +data compile_pool_placement_home_note: String = "THE PLACEMENT ROW LIVES BESIDE THE SLICE NAME BECAUSE THEY ARE ONE QUESTION, AND IT USED TO LIVE IN TWO HOMES. gunbc_managed_compile_pool_slice names the managed slice; gunbc_compile_pool_placement says whether this fleet HAS one. Split across two modules, both could be read independently and they disagreed: every BuildCacheInstance carried the slice name as intended_compile_pool unconditionally, while the placement row said CompilePoolInRunnerSlots, so the rendered cache unit named a Slice= that nothing provisions. Applying it would have had systemd create that cgroup implicitly WITH NO LIMITS -- the same unbounded compile charge gunbc_compile_pool_doc measured, now wearing the managed topology name, with the budget model believing a bounded pool existed.\n\nWHY HERE AND NOT IN gunbc.fleet_host_budget, where the row was declared. Measured, not preferred: fleet_host_budget transitively REACHES build_cache_instance, so the instance importing it would close a cycle, and acyclicity is the one structural law the import graph has. host_layout reaches neither consumer and both already import it, so the single authority costs no new edge. The semantic argument agrees with the measurement -- where compile RSS is charged is a fact about how a HOST is laid out, which is this module's whole subject, and the budget DERIVES from it rather than owning it.\n\nWHAT THIS DOES NOT DO: it does not flip the row. The fleet still declares CompilePoolInRunnerSlots and every live tripwire keyed on that value stays green. What changed is that a disagreement no longer has anywhere to live." + +data gunbc_compile_pool_placement_doc: String = "WHERE COMPILE RSS IS CHARGED IS A TOPOLOGY FACT, AND THE BUDGET IS DERIVED FROM IT — not a fudge factor. CompilePoolInRunnerSlots: no managed compile-pool slice exists, so cache-miss compiles are charged to whichever cgroup the sccache server sits in; the budget's compile_pool child is therefore ZERO because the pool is not a separate claim on host RAM — it is (nominally) already inside runner_pool. That is the honest description of today's topology, NOT an absorbing fallback: it is level-triggered on a declared row, it widens no failure arm, and it fabricates no converged verdict. Note it is also the topology in which a mis-placed server silently exports one slot's caps onto every compile on the host — that deficit is refused by gunbc.host_build_cache_provision's placement verdict, not papered over here. CompilePoolSliceDeclared: the managed unit owns the compiles in its own slice, so the pool becomes a first-class sibling child sized by the admitted jobserver tokens and runner_pool must shrink to keep the host conserving. DISSOLUTION TRIGGER: flip this row to CompilePoolSliceDeclared in the same change that converges the managed gunbc-build-cache-ci.service unit and its slice; conservation then refuses by construction if the slots were not shrunk." + +type CompilePoolPlacement + = CompilePoolSliceDeclared { slice_unit: String } + | CompilePoolInRunnerSlots + +data gunbc_compile_pool_placement: CompilePoolPlacement = CompilePoolInRunnerSlots + data belt_spawn_workdir_env_var_note: String = "The ONE authority for the environment variable that carries a belt tick's spawn root, homed here for the same reason as the compile-pool slice name above: it is a fact about how a HOST is laid out and it has TWO consumers on opposite sides of a boundary, so neither may own it. gunbc.roadmap_belt_actuate READS it in belt_run_once_cli; gunbc.live_deploy.emit WRITES it as the Environment= line of the belt unit it installs. A literal in either place is the same string in two files, and the failure it produces is silent rather than loud: the writer and the reader would simply stop meeting, the reader would refuse for a missing variable, and nothing would say which of the two spellings was wrong.\\n\\nHoming it in either consumer would also drag one into the other's closure. The deploy emitter needs exactly one string from the belt; importing gunbc.roadmap_belt_actuate to get it would put the belt's whole implementation — tmux, git worktrees, the Claude provider preflight — behind every emission of a deploy script, for a name. That is the coupling this module exists to prevent." data belt_spawn_workdir_env_var: NonEmptyStr = "GUNBC_BELT_SPAWN_WORKDIR" as NonEmptyStr diff --git a/dag/gunbc/runner_activation.dag b/dag/gunbc/runner_activation.dag index 92e762e359a..f6e151f3277 100644 --- a/dag/gunbc/runner_activation.dag +++ b/dag/gunbc/runner_activation.dag @@ -9,6 +9,9 @@ import gunbc.runner_host_deploy { RunnerCommandOrRefusal, RunnerCommandReady, RunnerCommandRefused, runner_enable_command_argv, runner_instance_names, } +import gunbc.host_layout { + CompilePoolSliceDeclared, CompilePoolInRunnerSlots, +} import gunbc.build_cache_instance { BuildCacheInstance, BuildCacheServerPlacement, ServerManagedUnitOwned, ServerRunnerOwned, ServerUnsupervised, @@ -76,8 +79,15 @@ fn pool_receipt_is_for_host(p: HostCompilePoolReady, host: HostIdentity) -> Bool (p.host as String) == (host as String) } +// A POOL RECEIPT CANNOT MATCH AN INSTANCE THAT DECLARES NO MANAGED POOL. intended_compile_pool +// carries the placement itself rather than a bare slice name, so the CompilePoolInRunnerSlots arm +// has no slice to compare against and answers false -- activation refuses rather than comparing a +// receipt to a pool this fleet does not declare. fn pool_receipt_names_intended_pool(p: HostCompilePoolReady, r: BuildCacheInstanceReady) -> Bool { - (p.slice_unit as String) == (r.instance.intended_compile_pool as String) + match r.instance.intended_compile_pool { + CompilePoolSliceDeclared { slice_unit: u } => (p.slice_unit as String) == u + CompilePoolInRunnerSlots => false + } } fn admit_runner_activation( From d30176020011f61f700b6e4d2b85157a31e41484 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 24 Aug 2026 01:31:36 +0000 Subject: [PATCH 4/9] The compile-pool directive fold returned a type that does not exist ServiceDirective is a nickname for SystemdServiceDirective, which is what extdeps.systemd.unit_file declares and what every other directive in this render already uses. Not a missing import: the bare name resolves nowhere in the corpus. Found by execution: required-ci floor, strict preparation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ptiy92YHu7ig5W7ktiBzpZ --- dag/gunbc/build_cache_unit.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/gunbc/build_cache_unit.dag b/dag/gunbc/build_cache_unit.dag index 2397badfe58..a296fc1005c 100644 --- a/dag/gunbc/build_cache_unit.dag +++ b/dag/gunbc/build_cache_unit.dag @@ -79,7 +79,7 @@ fn runner_ordering_dropin_lines(instance: BuildCacheInstance) -> List { // creates such a cgroup implicitly WITH NO LIMITS -- the unbounded compile charge wearing the // managed topology's name. Now the field carries the placement itself, so the CompilePoolInRunnerSlots // arm has no slice name to render and the bad unit cannot be built. -fn compile_pool_service_directives(placement: CompilePoolPlacement) -> List { +fn compile_pool_service_directives(placement: CompilePoolPlacement) -> List { match placement { CompilePoolSliceDeclared { slice_unit: u } => [ServiceSlice { name: u as NonEmptyStr }] CompilePoolInRunnerSlots => [] From e683251cc595742a588669c7e4ec3a74d25ba38f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 24 Aug 2026 01:50:23 +0000 Subject: [PATCH 5/9] Give CompilePool a production entry point, and stop forking the deploy staging authority Two review findings (codex/gpt-5.6-sol, review 55280), both correct. 1. Every other HostToolchainKind is reached through a named func; CompilePool had none, so the kind was reachable from the dispatcher and the dispatcher from nobody. gunbc.host_compile_pool_provision provision_compile_pool is that entry point, thin, carrying no policy. 2. The install body hand-assembled a cat-heredoc and a sudo -n install -- a second authority for staging-and-installing a unit file beside the one gunbc.live_deploy.operations already owns. It now routes through deploy_stage_write_command and deploy_stage_install_command, which build from bash_build nodes rather than string assembly. The residue is the two lines live_deploy itself declares unmodelled (the stage-dir Let and the trap), both derived from deploy_stage_dir_var. Three witnesses added, each with an authorable RED: the first draft of the install body contained exactly the heredoc and sudo-install they refuse. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ptiy92YHu7ig5W7ktiBzpZ --- dag/gunbc/host_compile_pool_provision.dag | 55 +++++++++++++++++++ dag/gunbc/host_effect_realize.dag | 45 +++++++++++++-- ...ompile_pool_ensure_wiring_witness_test.dag | 44 +++++++++++++++ 3 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 dag/gunbc/host_compile_pool_provision.dag diff --git a/dag/gunbc/host_compile_pool_provision.dag b/dag/gunbc/host_compile_pool_provision.dag new file mode 100644 index 00000000000..8dae5efcb53 --- /dev/null +++ b/dag/gunbc/host_compile_pool_provision.dag @@ -0,0 +1,55 @@ +module gunbc.host_compile_pool_provision + +import std.disposition { Disposition, SingleAuthority, Scaffold, RealizationDispatch, Terminal } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import std.process { ProcessExit } +import std.resources { Network } +import std.types { Bool, String } +import gunbc.host_effect { HostEffect, EnsureCompilePoolSlice } +import gunbc.host_toolchain_ensure { host_toolchain_ensure, CompilePool } +import product.placement_supply { HostIdentity } + +data host_compile_pool_provision_scope_disposition: Disposition = SingleAuthority + +// THE ENTRY POINT, WITHOUT WHICH THE WIRING IS NOT WIRING. gunbc.host_toolchain_ensure gained a +// CompilePool kind, but every other kind on that selector is reached through a named `func` of its own +// -- provision_build_cache for BuildCache, provision_codex_runtime for CodexRuntime, +// srv3_os_install_actuator_toolchain_ensure for OsInstallActuator -- and the compile pool had none. A +// kind nothing selects is the same unreachability this lane exists to close, moved one layer out: the +// admission would have been reachable from the dispatcher and the dispatcher reachable from nobody. +// Found by review (codex/gpt-5.6-sol, review 55280) rather than by me, and the finding was correct. +// +// IT IS A THIN CALLER AND CARRIES NO POLICY. Unlike provision_build_cache there is no catalog gate to +// consult first: the compile pool has no cited third-party artifact to admit, so every decision this +// effect makes -- topology declared, size derived, cache placement readable, slice observed -- belongs +// to gunbc.compile_pool_ensure's admission and is taken inside the realizer against the live host. +// Putting any of it here would be a second decision site for one question. +data host_compile_pool_provision_realize_scaffold: Disposition = Scaffold { + dissolves_to: RealizationDispatch, + bind: DeclarationRef { + module_path: "gunbc.host_effect_realize", + decl_name: "realize_ensure_compile_pool_slice", + field: WholeDeclaration, + } +} + +fn provision_compile_pool_effect_for_host(host: HostIdentity) -> HostEffect { + EnsureCompilePoolSlice {} +} + +fn provision_compile_pool_routes_through_host_effect_apply() -> Bool { + match host_compile_pool_provision_realize_scaffold { + Scaffold { + dissolves_to: RealizationDispatch, + bind: DeclarationRef { module_path: m, decl_name: n, field: _ }, + } => + m == "gunbc.host_effect_realize" && n == "realize_ensure_compile_pool_slice" + Terminal { reason: _ } => false + } +} + +func provision_compile_pool(host: HostIdentity) -> ProcessExit + uses net: Network +{ + host_toolchain_ensure(host: host, kind: CompilePool) +} diff --git a/dag/gunbc/host_effect_realize.dag b/dag/gunbc/host_effect_realize.dag index aacc779387a..0cecbf6bb56 100644 --- a/dag/gunbc/host_effect_realize.dag +++ b/dag/gunbc/host_effect_realize.dag @@ -258,6 +258,9 @@ import gunbc.build_cache_endpoint_path { stale_socket_unlink_command, } import gunbc.live_deploy.operations { + deploy_stage_dir_var, + deploy_stage_write_command, + deploy_stage_install_command, systemctl_daemon_reload_command, systemctl_enable_command, systemctl_restart_command, @@ -4464,17 +4467,49 @@ fn compile_pool_slice_system_path(slice_unit: NonEmptyStr) -> String { fn compile_pool_slice_install_body(slice_unit: NonEmptyStr, slice_unit_file: SystemdUnitFile) -> String { join([ "set -euo pipefail", - "_gunbc_stage=\"$(mktemp -d)\"", - "trap 'rm -rf \"$_gunbc_stage\"' EXIT", - join(["cat > \"$_gunbc_stage/", slice_unit as String, "\" <<'__GUNBC_COMPILE_POOL_SLICE__'"], ""), - concat(serialize_systemd_unit_file(file: slice_unit_file), "__GUNBC_COMPILE_POOL_SLICE__"), - join(["sudo -n install -m 0644 \"$_gunbc_stage/", slice_unit as String, "\" ", compile_pool_slice_system_path(slice_unit: slice_unit)], ""), + compile_pool_stage_directory_line(), + compile_pool_stage_trap_line(), + deploy_stage_write_command( + stage_name: slice_unit as String, + content: serialize_systemd_unit_file(file: slice_unit_file), + ), + deploy_stage_install_command( + stage_name: slice_unit as String, + mode: compile_pool_slice_file_mode, + dest: compile_pool_slice_system_path(slice_unit: slice_unit), + ), systemctl_daemon_reload_command(privileged: true), systemctl_start_command(unit: slice_unit, privileged: true), "", ], "\n") } +data compile_pool_slice_file_mode: NonEmptyStr = "0644" as NonEmptyStr + +// THE STAGE-DIRECTORY ASSIGNMENT AND THE TRAP ARE THE ONLY AUTHORED LINES LEFT, AND THEY ARE THE TWO +// gunbc.live_deploy.emit ALREADY DECLARES AS UNMODELLED. That module's deploy_stage_directory_let_step +// builds the assignment as a modeled Let over ExprCmdSubst, but it yields a PipelineStep and this +// realizer's transport takes a script body, so there is no serializer between them to reach for; the +// trap has no grammar row at all (deploy_stage_trap_is_unmodelled_note). Both are spelled here from the +// same deploy_stage_dir_var authority the staged-path words read, so the variable name is not a second +// spelling even though the two lines are. +// +// EVERYTHING ELSE ROUTES THROUGH THE EXISTING AUTHORITY. The unit bytes reach the host through +// deploy_stage_write_command and the privileged install through deploy_stage_install_command -- both +// built from v2.extdeps.languages.bash_build nodes rather than assembled here -- so a slice unit name +// or a unit body carrying a quote, a space or a metacharacter keeps its word boundary structurally +// instead of by whatever quoting this module remembered to write. The first draft of this function did +// assemble them by hand: a `cat >` heredoc for the body and a hand-spelled `sudo -n install`, which was +// a SECOND authority for staging-and-installing a unit file beside the one live_deploy already owns. +// Reusing it is the section 3 repair, and it is also what shrinks the residue to the two lines above. +fn compile_pool_stage_directory_line() -> String { + join([deploy_stage_dir_var, "=\"$(mktemp -d)\""], "") +} + +fn compile_pool_stage_trap_line() -> String { + join(["trap 'rm -rf \"$", deploy_stage_dir_var, "\"' EXIT"], "") +} + data compile_pool_slice_install_shell_dissolution_trigger: DissolutionCondition = unbound_dissolution(description: "dissolve-on: gunbc.host_effect_realize.compile_pool_slice_install_body -- concat-built systemd slice install shell run over shell_exec_via_bash transport, the same shape and the same debt as build_cache_provision_shell_script beside it. DISSOLVES WHEN bash-emit (#5828) realizes a unit-file install as typed host-effect argv, at which point both bodies retire together rather than this one being converted alone.") fn compile_pool_ensure_admission_for( diff --git a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag index 25c6a6c9b12..899bd37a13d 100644 --- a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag +++ b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag @@ -19,6 +19,50 @@ test fn witness_compile_pool_kind_produces_the_effect() -> Bool { } } +// THE ENTRY POINT EXISTS AND SELECTS THIS KIND. Without this the CompilePool arm is reachable from the +// dispatcher and the dispatcher reachable from nobody -- which is the same unreachability this module +// exists to close, moved one layer out. It asserts the production func's own effect selection rather +// than re-asserting the dispatcher's, so deleting provision_compile_pool reds here. +test fn witness_production_entry_point_selects_the_compile_pool_effect() -> Bool { + match provision_compile_pool_effect_for_host(host: "srv1" as HostIdentity) { + EnsureCompilePoolSlice {} => provision_compile_pool_routes_through_host_effect_apply() + _ => false + } +} + +// The install body must not re-assemble the staging and install that gunbc.live_deploy.operations already +// owns. These assert the delegation by its observable residue: the unit bytes reach the host through a +// printf redirect built by deploy_stage_write_command, and the privileged copy through +// deploy_stage_install_command -- so a hand-rolled `cat` heredoc or a hand-spelled `sudo -n install` +// reds here. RED is authorable: the first draft of this function contained exactly those two. +test fn witness_install_body_delegates_to_the_deploy_staging_authority() -> Bool { + let body = compile_pool_slice_install_body( + slice_unit: gunbc_managed_compile_pool_slice, + slice_unit_file: compile_pool_slice_unit_file( + slice_unit: gunbc_managed_compile_pool_slice, + sized: byte_size(17179869184), + ), + ) + (string_contains(s: body, pattern: "<<") == false) + && (string_contains(s: body, pattern: "cat > ") == false) + && string_contains(s: body, pattern: "printf") + && string_contains(s: body, pattern: "/etc/systemd/system/") +} + +// START, NEVER RESTART: stopping a slice kills every process in it, so a restart here would terminate the +// compilers the pool exists to hold. +test fn witness_install_body_starts_and_never_restarts_the_slice() -> Bool { + let body = compile_pool_slice_install_body( + slice_unit: gunbc_managed_compile_pool_slice, + slice_unit_file: compile_pool_slice_unit_file( + slice_unit: gunbc_managed_compile_pool_slice, + sized: byte_size(17179869184), + ), + ) + string_contains(s: body, pattern: "systemctl") + && (string_contains(s: body, pattern: "restart") == false) +} + test fn witness_compile_pool_effect_routes_to_the_host_cell() -> Bool { match resolve_host_effect_cell(target: HostOs { node: srv1_host }, effect: EnsureCompilePoolSlice {}) { EnsureCompilePoolSliceOnHost { node: _ } => true From 4825c5315e1b79ba44597a2c2982a993013ca18f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 24 Aug 2026 02:40:25 +0000 Subject: [PATCH 6/9] Load the entry-point module in the witness closure An import list does not bind names here, but it does decide which modules a run LOADS, and gunbc.host_compile_pool_provision is a leaf entry point that nothing imports. Its symbols were absent rather than unbound, so the entry-point witness failed NoSuchFunction under gunbc run while passing under the floor, which loads all 3873 modules regardless. Measured: all eight witnesses now resolve, typecheck and evaluate to true. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ptiy92YHu7ig5W7ktiBzpZ --- .../compile_pool_ensure_wiring_witness_test.dag | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag index 899bd37a13d..99055686a76 100644 --- a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag +++ b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag @@ -1,6 +1,20 @@ module test.claim.compile_pool_ensure_wiring_witness import gunbc.host_toolchain_ensure { host_toolchain_ensure_effect, CompilePool } +// IMPORTED FOR LOADING, NOT FOR BINDING, and the distinction is what this import is here to record. +// Names in this corpus resolve by global uniqueness across the modules a run has LOADED -- an import +// list does not bind them, which is why the other ten witnesses below reference symbols no import +// names and resolve fine. But a module enters the load graph only by being imported by something in +// the entry's closure, and gunbc.host_compile_pool_provision is a LEAF entry point: like its siblings +// gunbc.host_build_cache_provision and gunbc.srv3_os_install_actuator_toolchain_ensure, nothing in the +// corpus imports it. Without this line its symbols are absent rather than unbound, and the entry-point +// witness fails with NoSuchFunction -- measured, not reasoned: that is exactly how it failed under +// `gunbc run` before this import was added, while the same witness passed under the floor, which +// prepares one subject over all 3873 modules and therefore loads it anyway. +import gunbc.host_compile_pool_provision { + provision_compile_pool_effect_for_host, + provision_compile_pool_routes_through_host_effect_apply, +} data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly From 6e6d3c95f9ba9b5544e835ca73891a0bb35e13d6 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 24 Aug 2026 04:33:25 +0000 Subject: [PATCH 7/9] Three witnesses were reading the live topology row through a production instance The placement carrier made intended_compile_pool conditional, and three floor witnesses went red. None of them was testing what its name says. - build_cache_ensure witness_the_generated_unit_reproduces_the_captured_environment asserted Slice=gunbc-compile-pool.slice as part of a CAPTURED environment. That capture was an env var set, and its SCCACHE_STARTUP_NOTIFY entry is the tell that no unit existed on that host at all -- so the Slice= line was never captured, it was our own intent pinned inside a fidelity witness. Moved to a both-arms witness beside the capacity idiom the same file already uses: absent under InRunnerSlots, present under SliceDeclared. - runner_activation_wall and runner_host_deploy built a fully-ready host from the PRODUCTION ci_cache_instance, so once the live row became the carrier they were asserting a fact about this fleet topology rather than about the activation fold. Fixtures now declare the pool; the live answer gets its own witness that asserts the fleet-wide refusal deliberately. That new witness returned false on its first run -- fully_verified_ready derives its instance from the fixture two calls away, so the live witness silently received the pool-declared one and asserted a refusal that could not happen. Split into fully_verified_ready_for so a witness chooses its own subject. Caught by running it. Measured: all five resolve and return true, including the negative control witness_bare_host_admission_cannot_yield_an_enable_command. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ptiy92YHu7ig5W7ktiBzpZ --- dag/test/claim/build_cache_ensure_test.dag | 39 ++++++++++++- .../claim/runner_activation_wall_test.dag | 57 ++++++++++++++++++- .../claim/runner_host_deploy_witness_test.dag | 23 +++++++- 3 files changed, 115 insertions(+), 4 deletions(-) diff --git a/dag/test/claim/build_cache_ensure_test.dag b/dag/test/claim/build_cache_ensure_test.dag index a98d7ee4964..36eeae4fb53 100644 --- a/dag/test/claim/build_cache_ensure_test.dag +++ b/dag/test/claim/build_cache_ensure_test.dag @@ -31,7 +31,6 @@ test fn witness_the_generated_unit_reproduces_the_captured_environment() -> Bool let u = ci_unit() unit_declares_line(lines: u, needle: "User=ghrunner") && unit_declares_line(lines: u, needle: "Group=ghrunner") - && unit_declares_line(lines: u, needle: "Slice=gunbc-compile-pool.slice") && unit_declares_line(lines: u, needle: "Environment=SCCACHE_DIR=/var/lib/ctrl/sccache-ci") && unit_declares_line(lines: u, needle: "Environment=SCCACHE_SERVER_UDS=/var/lib/ctrl/sccache-ci/server.sock") && unit_declares_line(lines: u, needle: "Environment=SCCACHE_IDLE_TIMEOUT=0") @@ -40,6 +39,44 @@ test fn witness_the_generated_unit_reproduces_the_captured_environment() -> Bool && unit_declares_line(lines: u, needle: "Environment=SCCACHE_ERROR_LOG=/var/lib/ctrl/sccache-ci/sccache-server.log") } +// THE Slice= LINE LEFT THIS WITNESS BECAUSE IT WAS NEVER PART OF THE CAPTURE. This witness is named for +// what the 2026-08-01 srv1 capture contained, and that capture was an ENVIRONMENT: SCCACHE_DIR, +// SCCACHE_ERROR_LOG, SCCACHE_IDLE_TIMEOUT, SCCACHE_SERVER_UDS, SCCACHE_START_SERVER and a +// SCCACHE_STARTUP_NOTIFY temp path whose presence is the tell that NO UNIT EXISTED on that host at all. +// A Slice= assertion was therefore never reproducing a captured line -- it was pinning our own declared +// intent inside a fidelity witness, which is why it broke the moment that intent correctly became +// conditional on the topology. +// +// It is not deleted, it is moved to the both-arms form beside it, mirroring the capacity idiom this file +// already uses: an assertion that only ever checks one arm cannot see a render that emits the line +// unconditionally, which is exactly the defect the placement carrier was introduced to make unwritable. +test fn witness_the_slice_line_follows_the_declared_placement_both_ways() -> Bool { + let undeclared = render_build_cache_unit( + instance: instance_with_pool(base: ci(), placement: CompilePoolInRunnerSlots), + ) + let declared = render_build_cache_unit( + instance: instance_with_pool( + base: ci(), + placement: CompilePoolSliceDeclared { slice_unit: gunbc_managed_compile_pool_slice as String }, + ), + ) + (unit_mentions_prefix(lines: undeclared, prefix: "Slice=") == false) + && unit_declares_line(lines: declared, needle: "Slice=gunbc-compile-pool.slice") +} + +fn instance_with_pool(base: BuildCacheInstance, placement: CompilePoolPlacement) -> BuildCacheInstance { + BuildCacheInstance { + identity: base.identity, + implementation: base.implementation, + endpoint: base.endpoint, + intended_unit: base.intended_unit, + intended_principal: base.intended_principal, + intended_storage: base.intended_storage, + intended_capacity: base.intended_capacity, + intended_compile_pool: placement, + } +} + data absent_capacity_doc: String = "THE LOAD-BEARING PAIR. CI's intended_capacity is Absent and the live capture confirms why: srv1's running CI server carries NO SCCACHE_CACHE_SIZE at all, so its 10 GiB is sccache's built-in default rather than a decision. Rendering a default here would fabricate a decision nobody made AND freeze it -- once written into a generated unit it is indistinguishable from an intended value, and the tracked gap closes silently. Both halves are asserted: Absent must omit the line, Present must emit it, because a render that omitted it unconditionally would pass the first assertion while silently dropping the session instance's real 50G cap." test fn witness_an_absent_capacity_omits_the_line_and_a_present_one_emits_it() -> Bool { diff --git a/dag/test/claim/runner_activation_wall_test.dag b/dag/test/claim/runner_activation_wall_test.dag index bc735ca558d..01d13efe2a3 100644 --- a/dag/test/claim/runner_activation_wall_test.dag +++ b/dag/test/claim/runner_activation_wall_test.dag @@ -8,8 +8,51 @@ fn srv1_deploy_fixture() -> RunnerHostDeploy { srv4_runner_host_deploy } +// A FIXTURE MUST CONSTRUCT THE WORLD IT CLAIMS TO TEST. These witnesses assert what the ACTIVATION FOLD +// does for a fully ready host, so every input has to be a fixture -- and the instance was not one: it +// came from the production ci_cache_instance, whose intended_compile_pool is now the live +// gunbc_compile_pool_placement row rather than a bare slice name. The moment that row became the +// carrier, "a ready host activates" silently turned into a claim about THIS FLEET'S TOPOLOGY, and since +// the row is CompilePoolInRunnerSlots the fold correctly refused and the witness reds. +// +// Declaring the pool on the fixture restores the intended subject. It does not weaken the wall: the +// refusal paths are asserted by the negative witnesses beside these, and the live-topology answer has +// its own tripwire (test.claim.host_compile_pool live_topology_doc, and the fleet-wide refusal witness +// below) rather than being asserted here by accident. +fn pool_declared_instance_fixture(host: HostIdentity) -> BuildCacheInstance { + let base = ci_cache_instance(host: host) + BuildCacheInstance { + identity: base.identity, + implementation: base.implementation, + endpoint: base.endpoint, + intended_unit: base.intended_unit, + intended_principal: base.intended_principal, + intended_storage: base.intended_storage, + intended_capacity: base.intended_capacity, + intended_compile_pool: CompilePoolSliceDeclared { + slice_unit: gunbc_managed_compile_pool_slice as String, + }, + } +} + fn ready_instance_fixture(host: HostIdentity) -> BuildCacheInstance { - ci_cache_instance(host: host) + pool_declared_instance_fixture(host: host) +} + +// THE LIVE ANSWER, ASSERTED ON PURPOSE RATHER THAN INHERITED BY ACCIDENT. On this fleet +// gunbc_compile_pool_placement is CompilePoolInRunnerSlots, so no instance declares a managed pool and +// activation refuses for every host. That is the designed state -- gunbc.host_compile_pool's producer +// note and gunbc.runner_activation's pool-readiness note both say so -- and it was previously invisible +// here because the fixture read the production row without anyone noticing it had become the subject. +test fn witness_live_topology_refuses_activation_fleet_wide() -> Bool { + let d = srv1_deploy_fixture() + let live = ci_cache_instance(host: d.host_label as HostIdentity) + yields_command( + outcome: activation_for( + cache_ready: fully_verified_ready_for(instance: live, unit: live.intended_unit), + pool_ready: ready_pool_fixture(), + ), + ) == false } fn owner_fixture() -> gunbc.build_cache_instance.ProcessIdentity { @@ -21,8 +64,18 @@ fn owner_fixture() -> gunbc.build_cache_instance.ProcessIdentity { } fn fully_verified_ready(host: HostIdentity, unit: NonEmptyStr) -> BuildCacheInstanceReady { + fully_verified_ready_for(instance: ready_instance_fixture(host: host), unit: unit) +} + +// TAKES THE INSTANCE RATHER THAN DERIVING IT, so a witness can choose whether its subject is the +// fixture world or the live fleet. Without this split the two are indistinguishable at the call site: +// the live-topology witness below was written against fully_verified_ready and silently received the +// POOL-DECLARED fixture, so it asserted a refusal that could never happen and returned false. Caught by +// running it -- an assertion whose subject is chosen by a helper two calls away is exactly the kind that +// reads correct and measures something else. +fn fully_verified_ready_for(instance: BuildCacheInstance, unit: NonEmptyStr) -> BuildCacheInstanceReady { BuildCacheInstanceReady { - instance: ready_instance_fixture(host: host), + instance: instance, owner: owner_fixture(), managed_unit: unit, endpoint_verified: true, diff --git a/dag/test/claim/runner_host_deploy_witness_test.dag b/dag/test/claim/runner_host_deploy_witness_test.dag index 5f0341de960..b9f87217747 100644 --- a/dag/test/claim/runner_host_deploy_witness_test.dag +++ b/dag/test/claim/runner_host_deploy_witness_test.dag @@ -189,9 +189,30 @@ test fn red_control_admission_blind_builder_rejected_by_gate() -> Bool { data activation_moved_note: String = "runner_enable_command was deleted with the materialization/activation split; these rows now drive runner_activate_command through a readiness receipt. The receipt below is a fixture, and the enrollment assertions it supports are unchanged: an unenrolled host still refuses, and the refusal still carries no command. What changed is that enrollment alone is no longer sufficient, which is the whole point of the split — so these witnesses now assert the WEAKER half of a stronger wall, and the wall's own controls live in test.claim.runner_activation_wall." +// DEFINED LOCALLY RATHER THAN SHARED WITH runner_activation_wall_test, which carries the same fixture +// and the full reasoning for it. A cross-test-module reference resolves under the floor -- it prepares +// one subject over every module, so the other test module is loaded -- and fails NoSuchFunction under a +// targeted `gunbc run` whose entry closure does not reach it. Twelve duplicated lines are cheaper than a +// witness whose resolvability depends on which harness runs it. +fn pool_declared_instance_fixture(host: HostIdentity) -> BuildCacheInstance { + let base = ci_cache_instance(host: host) + BuildCacheInstance { + identity: base.identity, + implementation: base.implementation, + endpoint: base.endpoint, + intended_unit: base.intended_unit, + intended_principal: base.intended_principal, + intended_storage: base.intended_storage, + intended_capacity: base.intended_capacity, + intended_compile_pool: CompilePoolSliceDeclared { + slice_unit: gunbc_managed_compile_pool_slice as String, + }, + } +} + fn witness_ready_receipt() -> BuildCacheInstanceReady { BuildCacheInstanceReady { - instance: ci_cache_instance(host: srv4_runner_host_deploy.host_label as HostIdentity), + instance: pool_declared_instance_fixture(host: srv4_runner_host_deploy.host_label as HostIdentity), owner: ProcessIdentity { boot_id: "fixture-boot" as NonEmptyStr, pid: 4242, From 3673a7ec4d3d2b46cae02503ef272d9284814548 Mon Sep 17 00:00:00 2001 From: "gunbai-bot[bot]" <289086189+gunbai-bot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:08:48 -0400 Subject: [PATCH 8/9] Produce BuildCacheInstanceReady from exact observations (#9104) * Produce BuildCacheInstanceReady from exact observations * Bind unavailable cache observations to endpoint owner * Refuse foreign unavailable observations before classification --------- Co-authored-by: Brian Searls --- dag/extdeps/cache/sccache.dag | 6 + dag/gunbc/build_cache_instance.dag | 2 +- dag/gunbc/build_cache_instance_readiness.dag | 222 ++++++++++++++++++ dag/gunbc/build_cache_unit.dag | 5 +- dag/gunbc/runner_activation.dag | 33 +-- dag/gunbc/systemd_directive_value_grain.dag | 10 +- .../build_cache_instance_readiness_test.dag | 185 +++++++++++++++ .../claim/runner_activation_wall_test.dag | 23 -- .../claim/runner_host_deploy_witness_test.dag | 6 - .../systemd_directive_value_grain_test.dag | 4 +- 10 files changed, 428 insertions(+), 68 deletions(-) create mode 100644 dag/gunbc/build_cache_instance_readiness.dag create mode 100644 dag/test/claim/build_cache_instance_readiness_test.dag diff --git a/dag/extdeps/cache/sccache.dag b/dag/extdeps/cache/sccache.dag index 5cce53bc516..73f1fdfba37 100644 --- a/dag/extdeps/cache/sccache.dag +++ b/dag/extdeps/cache/sccache.dag @@ -171,6 +171,12 @@ data sccache_installed_binary_path_note: String = "Single authority for WHERE th data sccache_installed_binary_path: String = "/usr/local/bin/sccache" +// These names are part of sccache's cited configuration interface, not facts invented by the +// systemd renderer or by the readiness observer. Keeping the names here lets both consumers compose +// assignments and reads from one authority. +data sccache_cache_dir_env_var: NonEmptyStr = "SCCACHE_DIR" as NonEmptyStr +data sccache_cache_size_env_var: NonEmptyStr = "SCCACHE_CACHE_SIZE" as NonEmptyStr + fn sccache_install_script(rel: SccacheBinaryRelease, arch: NonEmptyStr, digest: Digest) -> String { let dir = join(["sccache-", rel.version as String, "-", arch as String, "-unknown-linux-musl"], "") let url = join(["https://github.com/mozilla/sccache/releases/download/", rel.version as String, "/", dir, ".tar.gz"], "") diff --git a/dag/gunbc/build_cache_instance.dag b/dag/gunbc/build_cache_instance.dag index addfb8d895e..effe31ccfc3 100644 --- a/dag/gunbc/build_cache_instance.dag +++ b/dag/gunbc/build_cache_instance.dag @@ -41,7 +41,7 @@ type BuildCacheInstance { intended_compile_pool: CompilePoolPlacement } -data build_cache_configuration_observation_note: String = "STORAGE ROOT AND CAPACITY ARE TWO OBSERVED CONFIGURATION AXES, and each observation is bound to both the exact deployed BuildCacheInstance and the ProcessIdentity whose environment was read. A path or byte count without both subjects could be replayed across hosts, across the CI and session instances that share one sccache implementation, or across two processes after pid reuse. The owner is the process already proven to hold the instance endpoint; a unit file would answer what systemd WOULD start, not how the live server is configured, and every host in the incident population currently runs a server systemd did not start.\n\nUNSET AND READ UNAVAILABLE ARE DIFFERENT OBSERVATIONS WITH DIFFERENT REMEDIES. Unset means the owner environment was read and carries no assignment; unavailable means the read did not answer, and must never narrow to unset. Endpoint or owner absence does not appear as a third arm here: without the prerequisite endpoint-owner observation there is no configuration subject, so these carriers are not constructed at all. That is not-applicability at the join rather than malformed configuration.\n\nCAPACITY ALSO RETAINS CONFIGURED TEXT THAT CANNOT YET BE GROUNDED AS BYTES. gunbc.build_cache_unit emits a plain decimal byte count, which has one exact decoding. sccache also accepts suffixed text such as its documented 10G default, but this repository has no cited rule deciding whether G denotes 10^9 or 2^30. Choosing either would fabricate a magnitude whose seven-percent difference could silently pass or refuse convergence. BuildCacheCapacityConfiguredUngrounded therefore carries the text verbatim; it narrows when extdeps.cache.sccache cites and models the accepted size grammar. The instance itself retains whether capacity intent is declared, so a later comparison can refuse intended_capacity Absent rather than freezing a vendor default as a decision.\n\nThese are carriers only: they mint no verified flag and perform no host read. A producer must read the endpoint owner's environment into one of these arms before runner activation can replace storage_verified and capacity_verified with typed observed-versus-intended comparisons." +data build_cache_configuration_observation_note: String = "STORAGE ROOT AND CAPACITY ARE TWO OBSERVED CONFIGURATION AXES, and each observation is bound to both the exact deployed BuildCacheInstance and the ProcessIdentity whose environment was read. A path or byte count without both subjects could be replayed across hosts, across the CI and session instances that share one sccache implementation, or across two processes after pid reuse. The owner is the process already proven to hold the instance endpoint; a unit file would answer what systemd WOULD start, not how the live server is configured, and every host in the incident population currently runs a server systemd did not start.\n\nUNSET AND READ UNAVAILABLE ARE DIFFERENT OBSERVATIONS WITH DIFFERENT REMEDIES. Unset means the owner environment was read and carries no assignment; unavailable means the read did not answer, and must never narrow to unset. Endpoint or owner absence does not appear as a third arm here: without the prerequisite endpoint-owner observation there is no configuration subject, so these carriers are not constructed at all. That is not-applicability at the join rather than malformed configuration.\n\nCAPACITY ALSO RETAINS CONFIGURED TEXT THAT CANNOT YET BE GROUNDED AS BYTES. gunbc.build_cache_unit emits a plain decimal byte count, which has one exact decoding. sccache also accepts suffixed text such as its documented 10G default, but this repository has no cited rule deciding whether G denotes 10^9 or 2^30. Choosing either would fabricate a magnitude whose seven-percent difference could silently pass or refuse convergence. BuildCacheCapacityConfiguredUngrounded therefore carries the text verbatim; it narrows when extdeps.cache.sccache cites and models the accepted size grammar. The instance itself retains whether capacity intent is declared, so gunbc.build_cache_instance_readiness refuses intended_capacity Absent rather than freezing a vendor default as a decision.\n\nThese carriers are consumed by gunbc.build_cache_instance_readiness, whose producer preserves unavailable separately from observed mismatch and is the only function that mints the proof-shaped BuildCacheInstanceReady receipt." type BuildCacheStorageConfiguration = BuildCacheStorageRootConfigured { root: NonEmptyStr } diff --git a/dag/gunbc/build_cache_instance_readiness.dag b/dag/gunbc/build_cache_instance_readiness.dag new file mode 100644 index 00000000000..fb4a9309a69 --- /dev/null +++ b/dag/gunbc/build_cache_instance_readiness.dag @@ -0,0 +1,222 @@ +module gunbc.build_cache_instance_readiness + +import std.types { String, NonEmptyStr, Bool } +import std.measure { byte_size_count } +import std.disposition { Disposition, SingleAuthority } +import extdeps.cache.sccache { sccache_local_id, sccache_installed_binary_path } +import gunbc.host_layout { CompilePoolSliceDeclared, CompilePoolInRunnerSlots } +import gunbc.build_cache_instance { + BuildCacheInstance, ProcessIdentity, + BuildCacheEndpointObservation, EndpointAbsent, EndpointOwnerObserved, + EndpointOwnershipAmbiguous, EndpointObservationUnavailable, unavailable_reason_detail, + OwnerInSystemdUnit, OwnerInRunnerSlot, OwnerContextUndecided, + BuildCacheStorageObservation, BuildCacheStorageObserved, + BuildCacheStorageObservationUnavailable, BuildCacheStorageRootConfigured, + BuildCacheStorageRootUnset, + BuildCacheCapacityObservation, BuildCacheCapacityObserved, + BuildCacheCapacityObservationUnavailable, BuildCacheCapacityConfigured, + BuildCacheCapacityConfiguredUngrounded, BuildCacheCapacityUnset, +} +import gunbc.runner_activation { BuildCacheInstanceReady, HostCompilePoolReady } + +data build_cache_instance_readiness_scope_disposition: Disposition = SingleAuthority + +data build_cache_instance_readiness_producer_note: String = "THE PRODUCER PRESERVES THREE STATES, NOT TWO. BuildCacheInstanceReady used to carry six Bool flags, but false could mean either an observation positively disagreed with intent OR the observation never answered. Those states have different remedies and different authority: drift is a host fact; unavailable is a failure of the observation path. Collapsing them is DESIGN's empty-observation narrow. The proof receipt therefore carries no flags. This producer returns Ready only after every exact-subject comparison passes, NotReady only from positive observations that disagree, and ObservationUnavailable whenever an exact-subject input read did not answer.\n\nTHE COMPILE-POOL CHECK IS ABOUT THIS SERVER, NOT MERELY THIS HOST. HostCompilePoolReady proves the declared slice exists and is sized on the host. It does not prove the endpoint owner lives beneath that slice. The producer separately checks the OwnerInSystemdUnit cgroup path contains the declared slice as one complete path segment, so a healthy slice beside a server still running in a runner cgroup cannot mint readiness. CompilePoolInRunnerSlots has no slice name and refuses as PoolNotDeclared; the global managed-slice constant is deliberately not consulted.\n\nTHE JOIN BINDS EVERY INPUT TWICE: the full BuildCacheInstance value and the ProcessIdentity. A storage read from the session server cannot verify the CI server on the same host, and a read from a recycled pid cannot verify its predecessor. An unavailable storage or capacity observation dominates mismatches on other axes only after that observation is bound to the endpoint's instance and owner. A foreign unavailable observation is refused as a subject mismatch before its unavailable result is accepted, so it cannot classify this instance's availability." + +type BuildCacheInstanceNotReadyReason + = CacheEndpointAbsent + | CacheEndpointOwnerAmbiguous + | CacheEndpointWrongUnit + | CacheImplementationUnsupported + | CacheExecutableMismatch + | CachePrincipalMismatch + | CacheObservationSubjectMismatch { axis: NonEmptyStr } + | CacheStorageUnset + | CacheStorageMismatch + | CacheCapacityIntentUndeclared + | CacheCapacityUnset + | CacheCapacityUngrounded + | CacheCapacityMismatch + | CachePoolNotDeclared + | CachePoolReceiptWrongHost + | CachePoolNotReady + | CachePoolSliceMismatch + | CacheOwnerOutsidePool + +type BuildCacheInstanceObservationUnavailableReason + = CacheEndpointObservationUnavailable { detail: NonEmptyStr } + | CacheStorageObservationUnavailable { detail: NonEmptyStr } + | CacheCapacityObservationUnavailable { detail: NonEmptyStr } + +type BuildCacheInstanceReadiness + = BuildCacheInstanceObservedReady { receipt: BuildCacheInstanceReady } + | BuildCacheInstanceObservedNotReady { reason: BuildCacheInstanceNotReadyReason } + | BuildCacheInstanceObservationUnavailable { reason: BuildCacheInstanceObservationUnavailableReason } + +fn same_instance(a: BuildCacheInstance, b: BuildCacheInstance) -> Bool { a == b } +fn same_owner(a: ProcessIdentity, b: ProcessIdentity) -> Bool { a == b } + +fn unavailable_input( + endpoint: BuildCacheEndpointObservation, + storage: BuildCacheStorageObservation, + capacity: BuildCacheCapacityObservation, +) -> BuildCacheInstanceReadiness? { + match endpoint { + EndpointObservationUnavailable { instance: _, reason: r } => + Present { value: BuildCacheInstanceObservationUnavailable { + reason: CacheEndpointObservationUnavailable { detail: unavailable_reason_detail(reason: r) }, + } } + EndpointAbsent { instance: _ } => none + EndpointOwnershipAmbiguous { instance: _, candidate_count: _ } => none + EndpointOwnerObserved { instance: instance, owner: owner, executable: _, principal: _, owner_context: _, memory_limits: _, memory_events: _ } => + unavailable_configuration_input(anchor_instance: instance, anchor_owner: owner, storage: storage, capacity: capacity) + } +} + +fn unavailable_configuration_input( + anchor_instance: BuildCacheInstance, + anchor_owner: ProcessIdentity, + storage: BuildCacheStorageObservation, + capacity: BuildCacheCapacityObservation, +) -> BuildCacheInstanceReadiness? { + match storage { + BuildCacheStorageObservationUnavailable { instance: si, owner: so, detail: d } => + if !same_instance(a: anchor_instance, b: si) || !same_owner(a: anchor_owner, b: so) { + Present { value: BuildCacheInstanceObservedNotReady { + reason: CacheObservationSubjectMismatch { axis: "storage" as NonEmptyStr }, + } } + } else { + Present { value: BuildCacheInstanceObservationUnavailable { + reason: CacheStorageObservationUnavailable { detail: d }, + } } + } + BuildCacheStorageObserved { instance: _, owner: _, configuration: _ } => + match capacity { + BuildCacheCapacityObservationUnavailable { instance: ci, owner: co, detail: d } => + if !same_instance(a: anchor_instance, b: ci) || !same_owner(a: anchor_owner, b: co) { + Present { value: BuildCacheInstanceObservedNotReady { + reason: CacheObservationSubjectMismatch { axis: "capacity" as NonEmptyStr }, + } } + } else { + Present { value: BuildCacheInstanceObservationUnavailable { + reason: CacheCapacityObservationUnavailable { detail: d }, + } } + } + BuildCacheCapacityObserved { instance: _, owner: _, configuration: _ } => none + } + } +} + +fn pool_cgroup_segment(slice: NonEmptyStr) -> String { + concat(concat("/", slice as String), "/") +} + +fn owner_is_nested_in_pool(observed_cgroup: NonEmptyStr, slice: NonEmptyStr) -> Bool { + (observed_cgroup as String).contains(pool_cgroup_segment(slice: slice)) +} + +fn observed_readiness( + endpoint: BuildCacheEndpointObservation, + storage: BuildCacheStorageObservation, + capacity: BuildCacheCapacityObservation, + pool: HostCompilePoolReady, +) -> BuildCacheInstanceReadiness { + match endpoint { + EndpointAbsent { instance: _ } => BuildCacheInstanceObservedNotReady { reason: CacheEndpointAbsent } + EndpointOwnershipAmbiguous { instance: _, candidate_count: _ } => BuildCacheInstanceObservedNotReady { reason: CacheEndpointOwnerAmbiguous } + EndpointObservationUnavailable { instance: _, reason: r } => BuildCacheInstanceObservationUnavailable { + reason: CacheEndpointObservationUnavailable { detail: unavailable_reason_detail(reason: r) }, + } + EndpointOwnerObserved { + instance: instance, owner: owner, executable: executable, principal: principal, + owner_context: context, memory_limits: _, memory_events: _, + } => + match storage { + BuildCacheStorageObservationUnavailable { instance: _, owner: _, detail: d } => BuildCacheInstanceObservationUnavailable { + reason: CacheStorageObservationUnavailable { detail: d }, + } + BuildCacheStorageObserved { instance: si, owner: so, configuration: sc } => + match capacity { + BuildCacheCapacityObservationUnavailable { instance: _, owner: _, detail: d } => BuildCacheInstanceObservationUnavailable { + reason: CacheCapacityObservationUnavailable { detail: d }, + } + BuildCacheCapacityObserved { instance: ci, owner: co, configuration: cc } => + if !same_instance(a: instance, b: si) || !same_owner(a: owner, b: so) { + BuildCacheInstanceObservedNotReady { reason: CacheObservationSubjectMismatch { axis: "storage" as NonEmptyStr } } + } else if !same_instance(a: instance, b: ci) || !same_owner(a: owner, b: co) { + BuildCacheInstanceObservedNotReady { reason: CacheObservationSubjectMismatch { axis: "capacity" as NonEmptyStr } } + } else if instance.implementation != sccache_local_id { + BuildCacheInstanceObservedNotReady { reason: CacheImplementationUnsupported } + } else if (executable as String) != sccache_installed_binary_path { + BuildCacheInstanceObservedNotReady { reason: CacheExecutableMismatch } + } else if (principal as String) != (instance.intended_principal as String) { + BuildCacheInstanceObservedNotReady { reason: CachePrincipalMismatch } + } else { + match context { + OwnerInRunnerSlot { slot: _, lifecycle: _, observed_cgroup: _ } => BuildCacheInstanceObservedNotReady { reason: CacheEndpointWrongUnit } + OwnerContextUndecided { detail: _, observed_cgroup: _ } => BuildCacheInstanceObservedNotReady { reason: CacheEndpointWrongUnit } + OwnerInSystemdUnit { unit: unit, observed_cgroup: cgroup } => + if (unit as String) != (instance.intended_unit as String) { + BuildCacheInstanceObservedNotReady { reason: CacheEndpointWrongUnit } + } else { + match sc { + BuildCacheStorageRootUnset => BuildCacheInstanceObservedNotReady { reason: CacheStorageUnset } + BuildCacheStorageRootConfigured { root: root } => + if (root as String) != (instance.intended_storage as String) { + BuildCacheInstanceObservedNotReady { reason: CacheStorageMismatch } + } else { + match instance.intended_capacity { + Absent => BuildCacheInstanceObservedNotReady { reason: CacheCapacityIntentUndeclared } + Present { value: intended_capacity } => + match cc { + BuildCacheCapacityUnset => BuildCacheInstanceObservedNotReady { reason: CacheCapacityUnset } + BuildCacheCapacityConfiguredUngrounded { configured_text: _ } => BuildCacheInstanceObservedNotReady { reason: CacheCapacityUngrounded } + BuildCacheCapacityConfigured { capacity: observed_capacity } => + if byte_size_count(b: intended_capacity) != byte_size_count(b: observed_capacity) { + BuildCacheInstanceObservedNotReady { reason: CacheCapacityMismatch } + } else if (pool.host as String) != (instance.identity.host as String) { + BuildCacheInstanceObservedNotReady { reason: CachePoolReceiptWrongHost } + } else if !pool.pool_declared || !pool.pool_observed { + BuildCacheInstanceObservedNotReady { reason: CachePoolNotReady } + } else { + match instance.intended_compile_pool { + CompilePoolInRunnerSlots => BuildCacheInstanceObservedNotReady { reason: CachePoolNotDeclared } + CompilePoolSliceDeclared { slice_unit: slice } => + if (pool.slice_unit as String) != slice { + BuildCacheInstanceObservedNotReady { reason: CachePoolSliceMismatch } + } else if !owner_is_nested_in_pool(observed_cgroup: cgroup, slice: pool.slice_unit) { + BuildCacheInstanceObservedNotReady { reason: CacheOwnerOutsidePool } + } else { + BuildCacheInstanceObservedReady { + receipt: BuildCacheInstanceReady { + instance: instance, + owner: owner, + managed_unit: unit, + }, + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} + +fn derive_build_cache_instance_readiness( + endpoint: BuildCacheEndpointObservation, + storage: BuildCacheStorageObservation, + capacity: BuildCacheCapacityObservation, + pool: HostCompilePoolReady, +) -> BuildCacheInstanceReadiness { + match unavailable_input(endpoint: endpoint, storage: storage, capacity: capacity) { + Present { value: unavailable } => unavailable + Absent => observed_readiness(endpoint: endpoint, storage: storage, capacity: capacity, pool: pool) + } +} diff --git a/dag/gunbc/build_cache_unit.dag b/dag/gunbc/build_cache_unit.dag index a296fc1005c..79250a18272 100644 --- a/dag/gunbc/build_cache_unit.dag +++ b/dag/gunbc/build_cache_unit.dag @@ -19,6 +19,7 @@ import extdeps.cache.sccache { SccacheServerTcpPort, SccacheServerUnixSocket, sccache_installed_binary_path, sccache_start_server_env_assignment, sccache_no_daemon_env_assignment, sccache_idle_timeout_never_env_assignment, + sccache_cache_dir_env_var, sccache_cache_size_env_var, } data build_cache_unit_scope_disposition: Disposition = SingleAuthority @@ -43,7 +44,7 @@ fn capacity_env_directives(instance: BuildCacheInstance) -> List [] Present { value: b } => [ Environment { - assignment: concat("SCCACHE_CACHE_SIZE=", to_string(byte_size_count(b: b))) as NonEmptyStr, + assignment: concat(concat(sccache_cache_size_env_var as String, "="), to_string(byte_size_count(b: b))) as NonEmptyStr, }, ] } @@ -105,7 +106,7 @@ fn build_cache_unit_file(instance: BuildCacheInstance) -> SystemdUnitFile { ServiceUser { name: instance.intended_principal }, ServiceGroup { name: instance.intended_principal }, Environment { - assignment: concat("SCCACHE_DIR=", instance.intended_storage as String) as NonEmptyStr, + assignment: concat(concat(sccache_cache_dir_env_var as String, "="), instance.intended_storage as String) as NonEmptyStr, }, Environment { assignment: concat( diff --git a/dag/gunbc/runner_activation.dag b/dag/gunbc/runner_activation.dag index f6e151f3277..63ae51fc259 100644 --- a/dag/gunbc/runner_activation.dag +++ b/dag/gunbc/runner_activation.dag @@ -23,23 +23,15 @@ data runner_activation_scope_disposition: Disposition = SingleAuthority data materialization_is_not_activation_note: String = "RunnerDeploySlot CONFLATED TWO LIFECYCLE MOMENTS, and the conflation is why the fleet-wide birth defect was constructible. Materializing a slot — writing its files, service template, credentials and drop-ins — and ACTIVATING it — enabling the unit so it may accept a job — have different prerequisites, and only the second one can create the defect. A materialized-but-disabled slot compiles nothing and therefore cannot spawn a cache server; the moment a slot may accept work, its first cache-miss compile will connect-or-start, and if no managed endpoint owner exists that start happens inside the runner's own cgroup, where it survives as the host's de facto build cache under one slot's memory ceiling.\\n\\nSo the prerequisite belongs on ACTIVATION and nowhere else. This module owns that moment; gunbc.runner_host_deploy keeps materialization and no longer offers an enable path at all. The two live in different modules because the dependency has to run one way: activation needs the deployed-instance model, materialization does not, and a single module owning both would have to import the cache authority into the installer's own layer.\\n\\nWHY MOVING THE ROW IN host_standup WOULD NOT HAVE DONE THIS. The spine declares P1 RunnerDeploySlot before P1b BuildCacheProvision, and swapping those two rows changes the declared narrative while leaving runner_enable_command constructible from a bare RunnerHostAdmission through exactly the path that produced srv1. Ordering in a status list is a projection of a dependency; it is not the dependency. The wall has to be that the enable command has no constructor that does not consume the readiness receipt." -data build_cache_instance_ready_note: String = "THE RECEIPT IS ABOUT AN EXACT INSTANCE AND AN EXACT OWNER, never about a responsive daemon. It carries the desired BuildCacheInstance it was checked against and the ProcessIdentity actually found holding the endpoint, so a reader can tell WHICH instance was proven ready and WHICH process proved it — a token meaning 'sccache seems fine on this host' would readmit the state this whole lane exists to remove.\\n\\nThe receipt carries managed_unit — a bare unit name — and NOT a BuildCacheServerPlacement, which is the load-bearing choice and was arrived at by having a weaker version fail. The first draft declared the field as placement: ServerManagedUnitOwned, reasoning that naming one variant as the field type made the other four unwritable. That is not what this substrate does: a variant used as a type widens to its coproduct, so the field really had type BuildCacheServerPlacement and a receipt holding ServerRunnerOwned was perfectly constructible — the compiler said so, refusing a match on the field as non-exhaustive over all five arms. The note claiming unrepresentability would have shipped as a false statement about a wall, which is rung inflation (DESIGN section 4b) in the carrier that exists to prevent exactly that.\n\nSo the receipt does not hold a placement at all. A managed unit name is the ONLY ownership fact it can express, and there is no way to write 'ready but runner-owned' or 'ready but placement unknown' into it, because those states have no field to occupy. The placement coproduct stays where it belongs — on the observation — and the bridge between them is placement_admits_activation, which refuses four arms out of five and deliberately does not construct a receipt.\\n\\nThe verified flags stay Bool deliberately and this is the honest weak point, so it is named rather than hidden: each one should eventually be a typed comparison carrying the observed value beside the intended one, the way placement already does. They are Bool today because the observation carriers for principal, storage and capacity do not exist yet — the endpoint observer reads executable and principal, but nothing reads back a unit's configured storage root or capacity. A Bool that nothing can currently compute from an observation is a placeholder, and the dissolve-on is that each flag becomes a typed observed-vs-intended pair as its producer lands. Until then the receipt is only as strong as its producer, which is why no producer ships in this increment." - -data build_cache_instance_ready_observation_carrier_correction: String = "CORRECTION TO build_cache_instance_ready_note's final paragraph: gunbc.build_cache_instance now carries storage and capacity observations bound to the same exact instance and endpoint-owning process. The flags remain Bool because no producer reads that process's environment into the carriers yet, principal still has no typed comparison, and replacing flags before those producers land would fabricate verification. The dissolve-on remains one typed observed-versus-intended comparison per producer; this increment supplies only the two missing carrier shapes." +data build_cache_instance_ready_note: String = "THE RECEIPT IS ABOUT AN EXACT INSTANCE AND AN EXACT OWNER, never about a responsive daemon. It carries the desired BuildCacheInstance it was checked against and the ProcessIdentity actually found holding the endpoint, so a reader can tell WHICH instance was proven ready and WHICH process proved it — a token meaning 'sccache seems fine on this host' would readmit the state this whole lane exists to remove.\\n\\nThe receipt carries managed_unit — a bare unit name — and NOT a BuildCacheServerPlacement, which is the load-bearing choice and was arrived at by having a weaker version fail. The first draft declared the field as placement: ServerManagedUnitOwned, reasoning that naming one variant as the field type made the other four unwritable. That is not what this substrate does: a variant used as a type widens to its coproduct, so the field really had type BuildCacheServerPlacement and a receipt holding ServerRunnerOwned was perfectly constructible — the compiler said so, refusing a match on the field as non-exhaustive over all five arms. The note claiming unrepresentability would have shipped as a false statement about a wall, which is rung inflation (DESIGN section 4b) in the carrier that exists to prevent exactly that.\n\nSo the receipt does not hold a placement at all. A managed unit name is the ONLY ownership fact it can express, and there is no way to write 'ready but runner-owned' or 'ready but placement unknown' into it, because those states have no field to occupy. The placement coproduct stays where it belongs — on the observation — and gunbc.build_cache_instance_readiness is the single producer that joins every required positive observation before constructing this proof-shaped receipt." type BuildCacheInstanceReady { instance: BuildCacheInstance owner: ProcessIdentity managed_unit: NonEmptyStr - endpoint_verified: Bool - executable_verified: Bool - principal_verified: Bool - storage_verified: Bool - capacity_verified: Bool - compile_pool_verified: Bool } -data host_compile_pool_ready_note: String = "The compile pool is a SEPARATE prerequisite from the cache instance because moving the server into a managed unit relocates where every cache-miss compiler's memory is charged. A managed server inside no declared pool is not an improvement on a runner-owned one; it is the same unbounded charge under a different parent. The two receipts are therefore both required and neither implies the other. Today no host realizes the slice and the budget derives a zero pool, so this receipt has no producer either — which is the correct state for a wall whose actuation is deliberately unauthorized." +data host_compile_pool_ready_note: String = "The compile pool is a SEPARATE prerequisite from the cache instance because moving the server into a managed unit relocates where every cache-miss compiler's memory is charged. A managed server inside no declared pool is not an improvement on a runner-owned one; it is the same unbounded charge under a different parent. The two receipts are therefore both required and neither implies the other. gunbc.host_compile_pool derives this receipt from the declared topology, sizing and slice observation; under today's CompilePoolInRunnerSlots topology it correctly produces not-ready for every host." type HostCompilePoolReady { host: HostIdentity @@ -62,15 +54,6 @@ fn ready_receipt_names_intended_unit(r: BuildCacheInstanceReady) -> Bool { (r.managed_unit as String) == (r.instance.intended_unit as String) } -fn readiness_flags_all_verified(r: BuildCacheInstanceReady) -> Bool { - r.endpoint_verified - && r.executable_verified - && r.principal_verified - && r.storage_verified - && r.capacity_verified - && r.compile_pool_verified -} - fn compile_pool_is_ready(p: HostCompilePoolReady) -> Bool { p.pool_declared && p.pool_observed } @@ -119,13 +102,6 @@ fn admit_runner_activation( reason: "runner_activation: the endpoint is owned by a managed unit, but not the unit this instance intends. ServerManagedUnitOwned means SOME unit holds it, which is satisfied by the wrong one — an out-of-band supervisor, a legacy ctrl unit, a hand-started service — and each of those charges cache-miss compilers to a parent the fleet does not manage. Refusing rather than accepting unit ownership as a synonym for correct ownership." as NonEmptyStr, } true => - match readiness_flags_all_verified(r: cache_ready) { - false => - RunnerActivationRefused { - host_label: d.host_label, - reason: "runner_activation: the build-cache instance is managed-unit-owned but at least one declared attribute (endpoint, executable, principal, storage, capacity, compile pool) is unverified against intent. A partially verified instance is not a ready one; refusing rather than activating on the verified subset." as NonEmptyStr, - } - true => match pool_receipt_is_for_host(p: pool_ready, host: d.host_label as HostIdentity) && pool_receipt_names_intended_pool(p: pool_ready, r: cache_ready) { false => @@ -152,12 +128,11 @@ fn admit_runner_activation( } } } - } } } } -data activation_command_has_one_constructor_note: String = "THE WALL ITSELF: there is no function in this module that takes a RunnerHostDeploy, or a bare RunnerHostAdmission, and yields an enable command. runner_activate_command consumes RunnerActivationAdmission, and its refusal arm carries no command at all — so 'enable a runner without a grounded managed cache' is unrepresentable rather than discouraged. gunbc.runner_host_deploy's readiness-blind runner_enable_command is deleted rather than deprecated; leaving it would leave the old path constructible, and a wall with a door beside it is a sign.\\n\\nHONEST RESIDUAL, inherited from the enrollment wall this mirrors and stated for the same reason: .dag has no visibility modifiers yet, so a caller can hand-write RunnerActivationAdmitted, or a BuildCacheInstanceReady whose Bool flags are all true, without any observation behind it. That is an explicit lie in the source rather than a forgotten check, which is the strongest wall this substrate currently supports. It closes when node/subtree visibility grants make these constructors private to their producers (DESIGN open thread), at which point the receipt becomes unforgeable." +data activation_command_has_one_constructor_note: String = "THE WALL ITSELF: there is no function in this module that takes a RunnerHostDeploy, or a bare RunnerHostAdmission, and yields an enable command. runner_activate_command consumes RunnerActivationAdmission, and its refusal arm carries no command at all — so 'enable a runner without a grounded managed cache' is unrepresentable rather than discouraged. gunbc.runner_host_deploy's readiness-blind runner_enable_command is deleted rather than deprecated; leaving it would leave the old path constructible, and a wall with a door beside it is a sign.\\n\\nHONEST RESIDUAL, inherited from the enrollment wall this mirrors and stated for the same reason: .dag has no visibility modifiers yet, so a caller can hand-write RunnerActivationAdmitted or BuildCacheInstanceReady without any observation behind it. That is an explicit lie in source rather than a forgotten check, which is the strongest wall this substrate currently supports. It closes when node/subtree visibility grants make these constructors private to their producers (DESIGN open thread), at which point the receipt becomes unforgeable." fn runner_activate_command(admission: RunnerActivationAdmission) -> RunnerCommandOrRefusal { match admission { @@ -198,7 +173,7 @@ fn runner_activation_gate_accepts( } } -data placement_admits_activation_note: String = "The bridge from an observation to a readiness receipt is deliberately PARTIAL and lives here as a refusal, not a constructor. Given any endpoint observation, this answers only whether its placement could support activation at all — and only ServerManagedUnitOwned can. It does not build a BuildCacheInstanceReady, because the remaining attribute verifications have no producers yet; a function that manufactured the receipt from placement alone would fabricate five verified flags from one observed fact, which is the fabricated-plausible-output shape (DESIGN section 5) applied to the wall's own input." +data placement_admits_activation_note: String = "This projection remains deliberately PARTIAL. Given any endpoint observation, it answers only whether its placement could support activation at all — and only ServerManagedUnitOwned can. It does not build a BuildCacheInstanceReady: gunbc.build_cache_instance_readiness owns that construction and also requires executable, principal, storage, capacity and exact compile-pool nesting. Manufacturing the receipt from placement alone would fabricate those independent observations." fn placement_admits_activation(placement: BuildCacheServerPlacement) -> Bool { match placement { diff --git a/dag/gunbc/systemd_directive_value_grain.dag b/dag/gunbc/systemd_directive_value_grain.dag index 5bb10425afd..5fff8e6d7ff 100644 --- a/dag/gunbc/systemd_directive_value_grain.dag +++ b/dag/gunbc/systemd_directive_value_grain.dag @@ -86,9 +86,9 @@ data build_cache_environment_variables: List = [ BuildCacheEnvironmentVariable { name: "SCCACHE_START_SERVER" as NonEmptyStr, authority: ModeledUpstream }, BuildCacheEnvironmentVariable { name: "SCCACHE_NO_DAEMON" as NonEmptyStr, authority: ModeledUpstream }, BuildCacheEnvironmentVariable { name: "SCCACHE_IDLE_TIMEOUT" as NonEmptyStr, authority: ModeledUpstream }, - BuildCacheEnvironmentVariable { name: "SCCACHE_DIR" as NonEmptyStr, authority: SpelledInConsumer }, + BuildCacheEnvironmentVariable { name: "SCCACHE_DIR" as NonEmptyStr, authority: ModeledUpstream }, BuildCacheEnvironmentVariable { name: "SCCACHE_ERROR_LOG" as NonEmptyStr, authority: SpelledInConsumer }, - BuildCacheEnvironmentVariable { name: "SCCACHE_CACHE_SIZE" as NonEmptyStr, authority: SpelledInConsumer }, + BuildCacheEnvironmentVariable { name: "SCCACHE_CACHE_SIZE" as NonEmptyStr, authority: ModeledUpstream }, BuildCacheEnvironmentVariable { name: "SCCACHE_SERVER_UDS" as NonEmptyStr, authority: SpelledInConsumer }, BuildCacheEnvironmentVariable { name: "SCCACHE_SERVER_PORT" as NonEmptyStr, authority: SpelledInConsumer }, ] @@ -108,8 +108,8 @@ data build_cache_environment_variables: List = [ // measure() == measure(). The correct reading of the green is not that the modeled half is verified // but that it needs no verification: composing a value from its authority IS the construction, and // a check restating it is the second representation section 5 says to prefer construction over. The -// three modeled variables are therefore at structurally guaranteed and have nothing to assert; the -// five unmodeled ones are the entire residue, and they cannot be asserted at all without respelling +// five modeled variables are therefore at structurally guaranteed and have nothing to assert; the +// three unmodeled ones are the entire residue, and they cannot be asserted at all without respelling // their literals, which would pass by construction for the same reason. // // So the claim is about the SPLIT rather than about either side of it: both authorities inhabited, @@ -131,5 +131,5 @@ fn environment_name_authority_is_split() -> Bool { // arriving by the back door, so the row states its condition in prose and is honest that no // mechanism watches it. data systemd_directive_value_grain_dissolution: DissolutionCondition = unbound_dissolution( - description: "the systemd directive vocabulary decomposes its values: SliceMemoryMax and SliceMemoryHigh take a modeled magnitude that can express the byte count, the suffixed form and the percentage form systemd.resource-control(5) admits, and Environment takes a variable name and a value as two fields rather than one fused assignment -- at which point the five sccache variable names gunbc spells as literals move to extdeps.cache.sccache beside the three that already have rows, because a two-field directive gives them somewhere to be. This row deletes when all three land together; landing only the name move would leave five rows whose consumer still concatenates them by hand." as NonEmptyStr, + description: "the systemd directive vocabulary decomposes its values: SliceMemoryMax and SliceMemoryHigh take a modeled magnitude that can express the byte count, the suffixed form and the percentage form systemd.resource-control(5) admits, and Environment takes a variable name and a value as two fields rather than one fused assignment -- at which point the remaining three sccache variable names gunbc spells as literals move to extdeps.cache.sccache beside the five modeled rows. SCCACHE_DIR and SCCACHE_CACHE_SIZE moved early because the readiness producer now reads those exact upstream names as well as the renderer writing them; leaving either literal in both consumers would have grown the fork this roster exists to count. This row deletes when the value-grain work and the remaining name move land together." as NonEmptyStr, ) diff --git a/dag/test/claim/build_cache_instance_readiness_test.dag b/dag/test/claim/build_cache_instance_readiness_test.dag new file mode 100644 index 00000000000..7de931ae4b4 --- /dev/null +++ b/dag/test/claim/build_cache_instance_readiness_test.dag @@ -0,0 +1,185 @@ +module test.claim.build_cache_instance_readiness + +import std.types { String, NonEmptyStr, Bool } +import std.measure { gibibyte, gibibyte_to_byte_size } +import product.placement_supply { HostIdentity } +import extdeps.cache.sccache { sccache_installed_binary_path } +import gunbc.host_layout { CompilePoolSliceDeclared, gunbc_managed_compile_pool_slice } +import gunbc.build_cache_instance { + BuildCacheInstance, ProcessIdentity, + EndpointOwnerObserved, OwnerInSystemdUnit, + BuildCacheStorageObserved, BuildCacheStorageObservationUnavailable, + BuildCacheStorageRootConfigured, + BuildCacheCapacityObserved, BuildCacheCapacityConfigured, + ci_cache_instance, session_cache_instance, +} +import gunbc.runner_activation { HostCompilePoolReady } +import gunbc.build_cache_instance_readiness { + derive_build_cache_instance_readiness, + BuildCacheInstanceObservedReady, BuildCacheInstanceObservedNotReady, + BuildCacheInstanceObservationUnavailable, + CacheOwnerOutsidePool, CacheCapacityIntentUndeclared, + CacheStorageObservationUnavailable, CacheObservationSubjectMismatch, +} + +fn owner_fixture() -> ProcessIdentity { + ProcessIdentity { + boot_id: "73c8153e-0e0e-4480-a36c-bc7560c50e8b" as NonEmptyStr, + pid: 3975946, + start_time: "79870020" as NonEmptyStr, + } +} + +fn declared_session_instance() -> BuildCacheInstance { + let base = session_cache_instance(host: "srv1") + BuildCacheInstance { + identity: base.identity, + implementation: base.implementation, + endpoint: base.endpoint, + intended_unit: base.intended_unit, + intended_principal: base.intended_principal, + intended_storage: base.intended_storage, + intended_capacity: base.intended_capacity, + intended_compile_pool: CompilePoolSliceDeclared { slice_unit: gunbc_managed_compile_pool_slice as String }, + } +} + +fn declared_ci_instance() -> BuildCacheInstance { + let base = ci_cache_instance(host: "srv1") + BuildCacheInstance { + identity: base.identity, + implementation: base.implementation, + endpoint: base.endpoint, + intended_unit: base.intended_unit, + intended_principal: base.intended_principal, + intended_storage: base.intended_storage, + intended_capacity: base.intended_capacity, + intended_compile_pool: CompilePoolSliceDeclared { slice_unit: gunbc_managed_compile_pool_slice as String }, + } +} + +fn ready_pool() -> HostCompilePoolReady { + HostCompilePoolReady { + host: "srv1" as HostIdentity, + slice_unit: gunbc_managed_compile_pool_slice, + pool_declared: true, + pool_observed: true, + } +} + +fn endpoint_for(instance: BuildCacheInstance, cgroup: NonEmptyStr) -> BuildCacheEndpointObservation { + EndpointOwnerObserved { + instance: instance, + owner: owner_fixture(), + executable: sccache_installed_binary_path as NonEmptyStr, + principal: instance.intended_principal, + owner_context: OwnerInSystemdUnit { unit: instance.intended_unit, observed_cgroup: cgroup }, + memory_limits: none, + memory_events: none, + } +} + +fn storage_for(instance: BuildCacheInstance) -> BuildCacheStorageObservation { + BuildCacheStorageObserved { + instance: instance, + owner: owner_fixture(), + configuration: BuildCacheStorageRootConfigured { root: instance.intended_storage }, + } +} + +fn capacity_for(instance: BuildCacheInstance) -> BuildCacheCapacityObservation { + BuildCacheCapacityObserved { + instance: instance, + owner: owner_fixture(), + configuration: BuildCacheCapacityConfigured { capacity: gibibyte_to_byte_size(g: gibibyte(50)) }, + } +} + +test fn exact_observations_mint_the_proof_receipt() -> Bool { + let instance = declared_session_instance() + match derive_build_cache_instance_readiness( + endpoint: endpoint_for(instance: instance, cgroup: "/gunbc-compile-pool.slice/gunbc-build-cache-session.service" as NonEmptyStr), + storage: storage_for(instance: instance), + capacity: capacity_for(instance: instance), + pool: ready_pool(), + ) { + BuildCacheInstanceObservedReady { receipt: r } => + r.instance == instance && r.owner == owner_fixture() + BuildCacheInstanceObservedNotReady { reason: _ } => false + BuildCacheInstanceObservationUnavailable { reason: _ } => false + } +} + +test fn a_ready_slice_beside_the_server_does_not_verify_nesting() -> Bool { + let instance = declared_session_instance() + match derive_build_cache_instance_readiness( + endpoint: endpoint_for(instance: instance, cgroup: "/system.slice/gunbc-build-cache-session.service" as NonEmptyStr), + storage: storage_for(instance: instance), + capacity: capacity_for(instance: instance), + pool: ready_pool(), + ) { + BuildCacheInstanceObservedNotReady { reason: CacheOwnerOutsidePool } => true + BuildCacheInstanceObservedReady { receipt: _ } => false + BuildCacheInstanceObservedNotReady { reason: _ } => false + BuildCacheInstanceObservationUnavailable { reason: _ } => false + } +} + +test fn unavailable_dominates_a_positive_mismatch() -> Bool { + let instance = declared_session_instance() + match derive_build_cache_instance_readiness( + endpoint: endpoint_for(instance: instance, cgroup: "/system.slice/wrong.service" as NonEmptyStr), + storage: BuildCacheStorageObservationUnavailable { + instance: instance, + owner: owner_fixture(), + detail: "/proc owner environment read failed" as NonEmptyStr, + }, + capacity: capacity_for(instance: instance), + pool: ready_pool(), + ) { + BuildCacheInstanceObservationUnavailable { reason: CacheStorageObservationUnavailable { detail: d } } => + (d as String).contains("environment") + BuildCacheInstanceObservedReady { receipt: _ } => false + BuildCacheInstanceObservedNotReady { reason: _ } => false + BuildCacheInstanceObservationUnavailable { reason: _ } => false + } +} + +test fn foreign_unavailable_observation_cannot_classify_this_instance() -> Bool { + let instance = declared_session_instance() + match derive_build_cache_instance_readiness( + endpoint: endpoint_for(instance: instance, cgroup: "/gunbc-compile-pool.slice/gunbc-build-cache-session.service" as NonEmptyStr), + storage: BuildCacheStorageObservationUnavailable { + instance: declared_ci_instance(), + owner: owner_fixture(), + detail: "/proc owner environment read failed" as NonEmptyStr, + }, + capacity: capacity_for(instance: instance), + pool: ready_pool(), + ) { + BuildCacheInstanceObservedNotReady { reason: CacheObservationSubjectMismatch { axis: a } } => + (a as String) == "storage" + BuildCacheInstanceObservedReady { receipt: _ } => false + BuildCacheInstanceObservedNotReady { reason: _ } => false + BuildCacheInstanceObservationUnavailable { reason: _ } => false + } +} + +test fn absent_capacity_intent_is_not_vacuously_verified() -> Bool { + let instance = declared_ci_instance() + match derive_build_cache_instance_readiness( + endpoint: endpoint_for(instance: instance, cgroup: "/gunbc-compile-pool.slice/gunbc-build-cache-ci.service" as NonEmptyStr), + storage: storage_for(instance: instance), + capacity: BuildCacheCapacityObserved { + instance: instance, + owner: owner_fixture(), + configuration: BuildCacheCapacityConfigured { capacity: gibibyte_to_byte_size(g: gibibyte(10)) }, + }, + pool: ready_pool(), + ) { + BuildCacheInstanceObservedNotReady { reason: CacheCapacityIntentUndeclared } => true + BuildCacheInstanceObservedReady { receipt: _ } => false + BuildCacheInstanceObservedNotReady { reason: _ } => false + BuildCacheInstanceObservationUnavailable { reason: _ } => false + } +} diff --git a/dag/test/claim/runner_activation_wall_test.dag b/dag/test/claim/runner_activation_wall_test.dag index 01d13efe2a3..846f24e721c 100644 --- a/dag/test/claim/runner_activation_wall_test.dag +++ b/dag/test/claim/runner_activation_wall_test.dag @@ -78,12 +78,6 @@ fn fully_verified_ready_for(instance: BuildCacheInstance, unit: NonEmptyStr) -> instance: instance, owner: owner_fixture(), managed_unit: unit, - endpoint_verified: true, - executable_verified: true, - principal_verified: true, - storage_verified: true, - capacity_verified: true, - compile_pool_verified: true, } } @@ -213,23 +207,6 @@ test fn witness_unready_compile_pool_refuses_even_with_a_managed_owner() -> Bool yields_command(outcome: declared_not_observed) == false } -test fn witness_partially_verified_instance_refuses() -> Bool { - let d = srv1_deploy_fixture() - let intended = ready_instance_fixture(host: d.host_label as HostIdentity).intended_unit - let partial = BuildCacheInstanceReady { - instance: ready_instance_fixture(host: d.host_label as HostIdentity), - owner: owner_fixture(), - managed_unit: intended, - endpoint_verified: true, - executable_verified: true, - principal_verified: true, - storage_verified: false, - capacity_verified: true, - compile_pool_verified: true, - } - yields_command(outcome: activation_for(cache_ready: partial, pool_ready: ready_pool_fixture())) == false -} - data materialization_does_not_activate_doc: String = "The split's own control. Materializing a slot must remain possible on nothing more than host enrollment — that is the whole point of separating it — while activation must not. If this witness ever fails because the installer command started demanding a readiness receipt, the split has collapsed in the other direction and materialization has become impossible to perform first, which would make the ordering unsatisfiable rather than enforced." test fn witness_materializing_a_slot_does_not_activate_it() -> Bool { diff --git a/dag/test/claim/runner_host_deploy_witness_test.dag b/dag/test/claim/runner_host_deploy_witness_test.dag index b9f87217747..46d29c166c9 100644 --- a/dag/test/claim/runner_host_deploy_witness_test.dag +++ b/dag/test/claim/runner_host_deploy_witness_test.dag @@ -219,12 +219,6 @@ fn witness_ready_receipt() -> BuildCacheInstanceReady { start_time: "80000000" as NonEmptyStr, }, managed_unit: ci_cache_instance(host: srv4_runner_host_deploy.host_label as HostIdentity).intended_unit, - endpoint_verified: true, - executable_verified: true, - principal_verified: true, - storage_verified: true, - capacity_verified: true, - compile_pool_verified: true, } } diff --git a/dag/test/claim/systemd_directive_value_grain_test.dag b/dag/test/claim/systemd_directive_value_grain_test.dag index 3bf1e835323..f3fc2494f9f 100644 --- a/dag/test/claim/systemd_directive_value_grain_test.dag +++ b/dag/test/claim/systemd_directive_value_grain_test.dag @@ -16,7 +16,7 @@ data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly // // Two of the carrier's three findings are unenforceable by construction, and this file says so // rather than faking coverage. The byte-form loss is LATENT -- every producer emits a plain count -// today, so there is no wrong state to detect. The five unmodeled sccache variable names cannot be +// today, so there is no wrong state to detect. The three unmodeled sccache variable names cannot be // checked without spelling the same literals a second time, which would pass by construction and // prove nothing. // @@ -41,7 +41,7 @@ test fn the_roster_carries_both_authorities_at_named_members() -> Bool { |> any(v => (v.name as String) == "SCCACHE_START_SERVER" && match v.authority { ModeledUpstream => true SpelledInConsumer => false })) && (build_cache_environment_variables - |> any(v => (v.name as String) == "SCCACHE_DIR" + |> any(v => (v.name as String) == "SCCACHE_ERROR_LOG" && match v.authority { ModeledUpstream => false SpelledInConsumer => true })) } From 146f554417d51eb84cf6b7be1db9d4a5487fc793 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 24 Aug 2026 21:40:15 +0000 Subject: [PATCH 9/9] WIP: Wire compile_pool_ensure into fleet convergence: the managed slice is mo --- dag/gunbc/host_effect_realize.dag | 446 ++++++++++++++---- dag/gunbc/live_deploy/operations.dag | 8 - ...ompile_pool_ensure_wiring_witness_test.dag | 136 ++++-- 3 files changed, 469 insertions(+), 121 deletions(-) diff --git a/dag/gunbc/host_effect_realize.dag b/dag/gunbc/host_effect_realize.dag index 0cecbf6bb56..f1258e7fd2b 100644 --- a/dag/gunbc/host_effect_realize.dag +++ b/dag/gunbc/host_effect_realize.dag @@ -258,13 +258,9 @@ import gunbc.build_cache_endpoint_path { stale_socket_unlink_command, } import gunbc.live_deploy.operations { - deploy_stage_dir_var, - deploy_stage_write_command, - deploy_stage_install_command, systemctl_daemon_reload_command, systemctl_enable_command, systemctl_restart_command, - systemctl_start_command, } import extdeps.jobserver { jobserver_config } import gunbc.host_layout { @@ -295,6 +291,18 @@ import gunbc.compile_pool_ensure { import gunbc.compile_pool_observe { observe_compile_pool_slice } import gunbc.host_layout { gunbc_managed_compile_pool_slice } import extdeps.systemd.unit_file { SystemdUnitFile, serialize_systemd_unit_file } +import extdeps.systemd.systemctl { systemctl_daemon_reload_argv, systemctl_start_argv } +import extdeps.tools.gnu_coreutils { coreutils_install_binary_name, coreutils_install_mode_flag } +import gunbc.file_access { FileTreeScope } +import gunbc.typed_remote_file_write { + RemoteFileMode, + RemoteFileModeOwnerReadWriteWorldRead, + TypedRemoteFileConvergence, + TypedRemoteFileConverged, + TypedRemoteFileConvergenceRefused, + converge_typed_remote_file, + typed_remote_file_write_seal, +} import gunbc.build_cache_instance { ci_cache_instance } import gunbc.srv3_boot_once_cd { @@ -4464,53 +4472,97 @@ fn compile_pool_slice_system_path(slice_unit: NonEmptyStr) -> String { // (systemd.slice(5)), so restarting the compile pool would terminate the compilers it exists to hold, which // is the opposite of converging it. `start` on an already-active slice re-applies the unit's resource // directives and kills nothing. -fn compile_pool_slice_install_body(slice_unit: NonEmptyStr, slice_unit_file: SystemdUnitFile) -> String { - join([ - "set -euo pipefail", - compile_pool_stage_directory_line(), - compile_pool_stage_trap_line(), - deploy_stage_write_command( - stage_name: slice_unit as String, - content: serialize_systemd_unit_file(file: slice_unit_file), - ), - deploy_stage_install_command( - stage_name: slice_unit as String, - mode: compile_pool_slice_file_mode, - dest: compile_pool_slice_system_path(slice_unit: slice_unit), - ), - systemctl_daemon_reload_command(privileged: true), - systemctl_start_command(unit: slice_unit, privileged: true), - "", - ], "\n") -} +// THE UNIT FILE REACHES THE HOST AS BYTES ON STDIN AND NEVER AS SHELL, and the privileged half is +// argv rather than a script. What stood here was a concat-built program run through shell_exec_via_bash +// -- net-new string-bodied shell on a growth surface bash-emit (#5828) exists to shrink, justified by a +// dissolution trigger reading "the same debt as build_cache_provision_shell_script beside it". That +// justification was invalid under the 2026-08-10 ruling: a dissolution condition describes how admitted +// debt ENDS and does not authorize creating it, and "same as the row beside it" is a NEW row on the +// surface, which makes the conversion larger rather than smaller. +// +// WHY THIS IS NOT MERELY THE SAME INSTALL SPELLED TYPED. converge_typed_remote_file stages to a sibling +// path, sets the mode before the bytes are reachable under the real name, renames within one directory +// (POSIX-atomic), and then READS THE DESTINATION BACK and compares it to the declared content. The shell +// body it replaces published bytes under the destination name and trusted a zero exit -- on a unit file +// that is a window where systemd can read something that was never a valid unit. So this is a rung +// climb, not a respelling, and the two hand-authored lines the old note conceded were unmodelled (the +// mktemp stage directory and the cleanup trap) are GONE rather than relocated: the typed writer owns +// staging, so there is no stage directory for this module to name or clean up. +// +// THE PRIVILEGE SPLIT, because the typed writer cannot write a system path and that is not a defect in +// it. typed_remote_file_stage_argv is a fixed [dd, status=none, of=...] with no privilege arm -- which is +// why spark's consumer is user-scope, and why managed_grant_install installs /etc/sudoers.d by connecting +// AS a privileged principal instead. Neither answer is available here: this realizer receives one +// transport for observation and install both, so changing its principal would change what the OBSERVE +// step reads. The split used instead needs no substrate change and no principal decision -- the typed +// writer lands the bytes UNPRIVILEGED at a staging path under its own read-back guarantee, and one +// privileged typed argv publishes that verified file to the system path. Every word is drawn from an +// existing authority (coreutils_install_binary_name, coreutils_install_mode_flag, systemctl_*_argv), so +// a slice unit name carrying a metacharacter is a word in an argv vector, not text in a program. +fn compile_pool_slice_staging_path(slice_unit: NonEmptyStr) -> String { + join([compile_pool_slice_staging_root as String, "/", slice_unit as String], "") +} + +data compile_pool_slice_staging_root: NonEmptyStr = "/tmp" as NonEmptyStr data compile_pool_slice_file_mode: NonEmptyStr = "0644" as NonEmptyStr -// THE STAGE-DIRECTORY ASSIGNMENT AND THE TRAP ARE THE ONLY AUTHORED LINES LEFT, AND THEY ARE THE TWO -// gunbc.live_deploy.emit ALREADY DECLARES AS UNMODELLED. That module's deploy_stage_directory_let_step -// builds the assignment as a modeled Let over ExprCmdSubst, but it yields a PipelineStep and this -// realizer's transport takes a script body, so there is no serializer between them to reach for; the -// trap has no grammar row at all (deploy_stage_trap_is_unmodelled_note). Both are spelled here from the -// same deploy_stage_dir_var authority the staged-path words read, so the variable name is not a second -// spelling even though the two lines are. +fn compile_pool_slice_staging_scope() -> FileTreeScope { + FileTreeScope { root_path: compile_pool_slice_staging_root } +} + +// PUBLICATION IS TWO STEPS BECAUSE install(1) TRUNCATES, and getting this wrong is the difference +// between a verified file and a verified file that systemd never reads. `install` opens its destination +// O_CREAT|O_TRUNC and writes through it, so publishing STRAIGHT to /etc/systemd/system/ would let +// that path pass through truncated and partially-written states -- exactly the window the staged write +// closes at the staging path and nowhere else. A concurrent systemd read would still see a unit that was +// never valid, and the read-back one step earlier would have proven the STAGED copy while the PUBLISHED +// copy is the only one systemd ever opens. (Caught in review, not by me: the first draft of this function +// published in one install(1) call and claimed the window closed.) // -// EVERYTHING ELSE ROUTES THROUGH THE EXISTING AUTHORITY. The unit bytes reach the host through -// deploy_stage_write_command and the privileged install through deploy_stage_install_command -- both -// built from v2.extdeps.languages.bash_build nodes rather than assembled here -- so a slice unit name -// or a unit body carrying a quote, a space or a metacharacter keeps its word boundary structurally -// instead of by whatever quoting this module remembered to write. The first draft of this function did -// assemble them by hand: a `cat >` heredoc for the body and a hand-spelled `sudo -n install`, which was -// a SECOND authority for staging-and-installing a unit file beside the one live_deploy already owns. -// Reusing it is the section 3 repair, and it is also what shrinks the residue to the two lines above. -fn compile_pool_stage_directory_line() -> String { - join([deploy_stage_dir_var, "=\"$(mktemp -d)\""], "") +// So install(1) places the bytes at a sibling INCOMING name -- where a truncating write harms nothing, +// because no reader knows that name -- with the mode set in the same call that writes them, and a second +// step renames within the one directory. POSIX makes that rename atomic, so the destination goes from the +// old unit to the new one with no observable state in between. `mv -f` is spelled here as +// gunbc.typed_remote_file_write typed_remote_file_rename_argv spells it; there is no coreutils mv row to +// cite, and minting one for this call site alone would be a second authority for a word that module +// already writes. +// THE INCOMING NAME IS A SUFFIX ON THE DESTINATION, NOT A PATH IN A TEMP DIRECTORY, and that is what +// makes the rename atomic rather than merely quick. rename(2) is atomic only WITHIN one filesystem; a +// temp directory elsewhere degrades it to copy-and-unlink, which silently reopens the truncation window +// this two-step publish exists to close. Deriving the incoming name by concatenating onto the +// destination guarantees the same directory, so same-filesystem holds by construction and there is no +// mount-point fact to model or check. +fn compile_pool_slice_incoming_path(slice_unit: NonEmptyStr) -> String { + concat(compile_pool_slice_system_path(slice_unit: slice_unit), ".gunbc-incoming") +} + +fn compile_pool_slice_publish_argv(slice_unit: NonEmptyStr) -> List { + [ + "sudo", + "-n", + coreutils_install_binary_name as String, + coreutils_install_mode_flag as String, + compile_pool_slice_file_mode as String, + compile_pool_slice_staging_path(slice_unit: slice_unit), + compile_pool_slice_incoming_path(slice_unit: slice_unit), + ] } -fn compile_pool_stage_trap_line() -> String { - join(["trap 'rm -rf \"$", deploy_stage_dir_var, "\"' EXIT"], "") +fn compile_pool_slice_rename_argv(slice_unit: NonEmptyStr) -> List { + [ + "sudo", + "-n", + "mv", + "-f", + compile_pool_slice_incoming_path(slice_unit: slice_unit), + compile_pool_slice_system_path(slice_unit: slice_unit), + ] } -data compile_pool_slice_install_shell_dissolution_trigger: DissolutionCondition = unbound_dissolution(description: "dissolve-on: gunbc.host_effect_realize.compile_pool_slice_install_body -- concat-built systemd slice install shell run over shell_exec_via_bash transport, the same shape and the same debt as build_cache_provision_shell_script beside it. DISSOLVES WHEN bash-emit (#5828) realizes a unit-file install as typed host-effect argv, at which point both bodies retire together rather than this one being converted alone.") +fn compile_pool_privileged_argv(argv: List) -> List { + concat(["sudo", "-n"], argv) +} fn compile_pool_ensure_admission_for( node: ComputeHost, @@ -4559,6 +4611,9 @@ fn compile_pool_sizing_for_host(node: ComputeHost, placement: CompilePoolPlaceme // provision_observes_before_it_acts_note records from srv3 -- and the whole point is that the pre-state and // the post-state can differ. Convergence is claimed only when the second admission answers // EnsureCompilePoolAlreadyConverged, which is the same predicate the preflight uses to skip the write. +// FOUR TYPED STEPS, HALTING AT THE FIRST FAILURE, each located by name. The sequence is write -> +// publish -> daemon-reload -> start; nothing after a failed step is attempted, and the reason names +// WHICH step stopped rather than collapsing to one exit code. fn compile_pool_slice_install_step( node: ComputeHost, slice_unit: NonEmptyStr, @@ -4567,60 +4622,279 @@ fn compile_pool_slice_install_step( transport: HostEffectTransport, effect_intent: HostEffectIntent, ) -> Reconciliation { - let observed = run_shell_transport( - script: retained_srvn( - body: shell_exec_via_bash(body: compile_pool_slice_install_body(slice_unit: slice_unit, slice_unit_file: slice_unit_file)), - reason: "srvN compile-pool slice install (managed cgroup parent for cache-miss compilers)", + match transport { + FleetSsh { target: t, context: ctx } => + compile_pool_slice_install_typed( + node: node, + slice_unit: slice_unit, + sized: sized, + slice_unit_file: slice_unit_file, + target: t, + context: ctx, + transport: transport, + effect_intent: effect_intent, + ) + LocalShell => + NotConverged { + reason: compile_pool_slice_transport_refusal(detail: "LocalShell carries no remote target"), + applied: effect_intent, + } + SshShell { ssh_host: h } => + NotConverged { + reason: compile_pool_slice_transport_refusal( + detail: join(["SshShell(", h as String, ") carries no sealed trust context"], ""), + ), + applied: effect_intent, + } + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => + NotConverged { + reason: compile_pool_slice_transport_refusal(detail: "EmitArtifactThenThinRun is not a live session"), + applied: effect_intent, + } + } +} + +// ONE EFFECT, TWO HALVES, DIFFERENT TRANSPORT READINESS -- AND THAT IS THE GENERAL RESULT, not a fact +// about compile pools. A "convert this to the typed substrate" change reads as ONE migration, and here it +// is two facts with different owners: +// +// ARGV -- publish, rename, daemon-reload, start. Typed TODAY on SshShell, via run_typed_argv_transport +// -> typed_argv_exec_over_ssh. No blocker, no context needed. +// BYTES -- the unit file itself. FleetSsh-ONLY, because converge_typed_remote_file shapes its +// invocation through shape_fleet_ssh_exec, which requires a sealed FleetSshExecutionContext. +// +// WHY THAT SPLIT IS LOAD-BEARING RATHER THAN TRIVIA. Every production route to this effect arrives on +// SshShell: host_toolchain_ensure passes reach.access.transport, and host_toolchain_direct_access maps +// srv1-srv4 to host_identity_ssh_access rows that construct SshShell. So a conversion that assumed the +// substrate is available per-MODULE rather than per-HALF would have typed both halves, compiled clean, +// passed its witnesses, and REFUSED ON ALL FOUR HOSTS -- a green-looking change that silently does +// nothing. The availability question has to be asked separately of each half, against what callers +// actually pass, not against what the transport enum can express. +// +// THE MISSING PIECE IS ALREADY OWNED. extdeps.ssh.session ssh_exec_portable_words_supersedes_exec_argv_note +// lists this module FIRST among the consumers that must "route through a FleetSshExecutionContext". So the +// bytes half is not unmodelled work -- it is scheduled work, and the refusal below is what makes its +// absence observable instead of leaving it as a line in someone else's dissolve-on text. + +// THE REFUSAL NAMES THE MISSING CAPABILITY, NOT THE WRONG TRANSPORT, and the difference decides whether +// anyone can act on it. "requires FleetSsh" tells a reader to change a call site, which is the wrong +// remedy: EVERY production route reaches this effect over SshShell, because host_toolchain_ensure passes +// reach.access.transport and host_toolchain_direct_access maps srv1-srv4 to host_identity_ssh_access rows +// that construct SshShell. So there is no caller to fix. What is missing is that host_effect_realize does +// not yet route through a FleetSshExecutionContext -- a migration extdeps.ssh.session names in its own +// dissolve-on (ssh_exec_portable_words_supersedes_exec_argv_note lists this module FIRST) -- and the +// refusal says exactly that, so its remedy points at the lane that can supply it. +// +// WHY REFUSING BEATS WRITING A SMALL SHELL BODY HERE. The bytes are the only step that cannot be typed on +// SshShell; publish, rename, daemon-reload and start all can, and are. Spelling the write as a shell +// program would add a net-new string-bodied row to the surface bash-emit (#5828) exists to shrink, and +// would zero the deficit's frequency by construction -- nobody would learn the capability was missing, +// so the migration that supplies it would never rank. Refusing keeps the gap observable. The effect is +// unreachable on main today, so a loud refusal is not a regression against anything that works. +// +// WHAT THIS IS NOT: countable in the DESIGN sense. The reason is a String on NotConverged, so it is typed +// and located but not enumerated anywhere a consumer can total. Saying otherwise would be the rung +// inflation section 4b names as worse than sitting low. Next-rung trigger: a typed refusal variant on the +// reconciliation carrier that a receipt can count, which is worth doing when a second call site needs it +// and not before. +fn compile_pool_slice_transport_refusal(detail: String) -> String { + join( + [ + "host_effect_apply: EnsureCompilePoolSlice cannot deliver the unit file -- the typed write needs a sealed FleetSshExecutionContext and gunbc.host_effect_realize does not yet route through one (", + detail, + "). The argv steps are typed and reachable on this transport; only the bytes are blocked. No unit written, no start issued.", + ], + "", + ) +} + +// The admission runs TWICE and the second run is not a formality: it is the postcondition, re-derived from a +// FRESH observation of the host rather than from the state we just wrote. A read-back that reused the +// preflight observation would green on evidence it produced -- the exact defect +// provision_observes_before_it_acts_note records from srv3 -- and the whole point is that the pre-state and +// the post-state can differ. Convergence is claimed only when the second admission answers +// EnsureCompilePoolAlreadyConverged, which is the same predicate the preflight uses to skip the write. +// +// NOTE THE TWO READ-BACKS ARE DIFFERENT CLAIMS AND BOTH ARE KEPT. converge_typed_remote_file reads the +// STAGED FILE back and proves the declared bytes are on disk; the admission below re-observes the SLICE +// and proves systemd actually took them. Bytes on disk is not a running cgroup, and neither read-back +// substitutes for the other. +fn compile_pool_slice_install_typed( + node: ComputeHost, + slice_unit: NonEmptyStr, + sized: ByteSize, + slice_unit_file: SystemdUnitFile, + target: SshTarget, + context: FleetSshExecutionContext, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + match converge_typed_remote_file( + target: target, + context: context, + admission: typed_remote_file_write_seal( + path: compile_pool_slice_staging_path(slice_unit: slice_unit) as FilePath, + content: serialize_systemd_unit_file(file: slice_unit_file), + allowed_scope: compile_pool_slice_staging_scope(), ), + mode: RemoteFileModeOwnerReadWriteWorldRead, + ) { + TypedRemoteFileConvergenceRefused { step: st, reason: r } => + NotConverged { + reason: compile_pool_slice_step_refusal(step: join(["write(", st, ")"], ""), detail: r), + applied: effect_intent, + } + TypedRemoteFileConverged { path: _, mode: _ } => + compile_pool_slice_publish_step( + node: node, + slice_unit: slice_unit, + sized: sized, + transport: transport, + effect_intent: effect_intent, + ) + } +} + +fn compile_pool_slice_publish_step( + node: ComputeHost, + slice_unit: NonEmptyStr, + sized: ByteSize, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + match compile_pool_slice_argv_step( + argv: compile_pool_slice_publish_argv(slice_unit: slice_unit), + step: "publish", transport: transport, + effect_intent: effect_intent, + ) { + Present { value: refused } => refused + Absent => + match compile_pool_slice_argv_step( + argv: compile_pool_slice_rename_argv(slice_unit: slice_unit), + step: "rename", + transport: transport, + effect_intent: effect_intent, + ) { + Present { value: refused } => refused + Absent => + match compile_pool_slice_argv_step( + argv: compile_pool_privileged_argv(argv: systemctl_daemon_reload_argv()), + step: "daemon-reload", + transport: transport, + effect_intent: effect_intent, + ) { + Present { value: refused } => refused + Absent => + match compile_pool_slice_argv_step( + argv: compile_pool_privileged_argv(argv: systemctl_start_argv(unit: slice_unit)), + step: "start", + transport: transport, + effect_intent: effect_intent, + ) { + Present { value: refused } => refused + Absent => + compile_pool_slice_readback_step( + node: node, + slice_unit: slice_unit, + sized: sized, + transport: transport, + effect_intent: effect_intent, + ) + } + } + } + } +} + +// PRESENT MEANS THE SEQUENCE STOPPED. The Optional carries the refusal rather than a Bool, so a caller +// cannot continue past a failed step by reading the flag the wrong way round -- there is no "false" +// that means "keep going" beside a reason it discarded. +fn compile_pool_slice_argv_step( + argv: List, + step: String, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Optional> { + match run_typed_argv_transport(argv: argv, transport: transport) { + TypedArgvExecRefused { reason: r } => + Present { + value: NotConverged { + reason: compile_pool_slice_step_refusal(step: step, detail: r), + applied: effect_intent, + }, + } + TypedArgvExecConverged { exit_code: code, stdout: out, stderr: err } => + if code == 0 { + Absent + } else { + Present { + value: NotConverged { + reason: compile_pool_slice_step_refusal( + step: step, + detail: join( + ["exit ", to_string(code), ": ", if err == "" { out } else { err }], + "", + ), + ), + applied: effect_intent, + }, + } + } + } +} + +fn compile_pool_slice_step_refusal(step: String, detail: String) -> String { + join( + [ + "host_effect_apply: EnsureCompilePoolSlice stopped at step ", + step, + " -- ", + detail, + ], + "", ) - if observed.success == false { +} + +fn compile_pool_slice_readback_step( + node: ComputeHost, + slice_unit: NonEmptyStr, + sized: ByteSize, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + let after = compile_pool_ensure_admission_for(node: node, transport: transport) + if compile_pool_ensure_is_converged(admission: after) { + Converged { + evidence: ShellEffectApplied { + exit_code: 0, + stdout: join( + [ + "CompilePoolSliceReceipt: converged unit=", + slice_unit as String, + " memory.max=memory.high=", + to_string(byte_size_count(b: sized)), + " (typed write read back at the staging path, published, started; re-observed after install)", + ], + "", + ), + stderr: "", + }, + applied: effect_intent, + } + } else { NotConverged { reason: join( [ - "host_effect_apply: EnsureCompilePoolSlice ran its install and the shell exited ", - to_string(observed.exit_code), - ": ", - if observed.stderr == "" { observed.stdout } else { observed.stderr }, + "host_effect_apply: EnsureCompilePoolSlice installed ", + slice_unit as String, + " and the read-back did not converge -- ", + compile_pool_admission_diagnostic(admission: after), ], "", ), applied: effect_intent, } - } else { - let after = compile_pool_ensure_admission_for(node: node, transport: transport) - if compile_pool_ensure_is_converged(admission: after) { - Converged { - evidence: ShellEffectApplied { - exit_code: observed.exit_code, - stdout: join( - [ - "CompilePoolSliceReceipt: converged unit=", - slice_unit as String, - " memory.max=memory.high=", - to_string(byte_size_count(b: sized)), - " (re-observed after install; the admission answered already-converged)", - ], - "", - ), - stderr: observed.stderr, - }, - applied: effect_intent, - } - } else { - NotConverged { - reason: join( - [ - "host_effect_apply: EnsureCompilePoolSlice installed ", - slice_unit as String, - " and the read-back did not converge -- ", - compile_pool_admission_diagnostic(admission: after), - ], - "", - ), - applied: effect_intent, - } - } } } diff --git a/dag/gunbc/live_deploy/operations.dag b/dag/gunbc/live_deploy/operations.dag index f38cb9cb65f..5072e671a97 100644 --- a/dag/gunbc/live_deploy/operations.dag +++ b/dag/gunbc/live_deploy/operations.dag @@ -70,14 +70,6 @@ fn systemctl_enable_command(unit: NonEmptyStr, privileged: Bool) -> String { } } -fn systemctl_start_command(unit: NonEmptyStr, privileged: Bool) -> String { - if privileged { - shell_privileged_command_text_of_argv(argv: systemctl_start_argv(unit: unit)) - } else { - shell_command_text_of_argv(argv: systemctl_start_argv(unit: unit)) - } -} - fn systemctl_restart_command(unit: NonEmptyStr, privileged: Bool) -> String { if privileged { shell_privileged_command_text_of_argv(argv: systemctl_restart_argv(unit: unit)) diff --git a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag index 99055686a76..88365e2b3ae 100644 --- a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag +++ b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag @@ -44,37 +44,119 @@ test fn witness_production_entry_point_selects_the_compile_pool_effect() -> Bool } } -// The install body must not re-assemble the staging and install that gunbc.live_deploy.operations already -// owns. These assert the delegation by its observable residue: the unit bytes reach the host through a -// printf redirect built by deploy_stage_write_command, and the privileged copy through -// deploy_stage_install_command -- so a hand-rolled `cat` heredoc or a hand-spelled `sudo -n install` -// reds here. RED is authorable: the first draft of this function contained exactly those two. -test fn witness_install_body_delegates_to_the_deploy_staging_authority() -> Bool { - let body = compile_pool_slice_install_body( - slice_unit: gunbc_managed_compile_pool_slice, - slice_unit_file: compile_pool_slice_unit_file( - slice_unit: gunbc_managed_compile_pool_slice, - sized: byte_size(17179869184), - ), +// THE UNIT FILE IS NO LONGER A PROGRAM, so these assert an argv VECTOR rather than substrings in a shell +// body. That is a stronger claim than the pair they replace: a substring assertion over a program cannot +// tell a word from text that happens to contain it, and both of the old witnesses would have passed on a +// body that spelled the right characters inside the wrong quoting. Word identity has no such gap. +// +// RED IS AUTHORABLE for each: re-fusing the publish into one string, dropping sudo, changing the mode away +// from the declared authority, or reaching for restart all change a word this compares by equality. +test fn witness_publish_is_typed_words_and_never_a_program() -> Bool { + let argv = compile_pool_slice_publish_argv(slice_unit: gunbc_managed_compile_pool_slice) + (argv == + [ + "sudo", + "-n", + "install", + "-m", + "0644", + concat("/tmp/", gunbc_managed_compile_pool_slice as String), + concat("/etc/systemd/system/", gunbc_managed_compile_pool_slice as String), + ]) + && (any(argv, w => string_contains(s: w, pattern: " ")) == false) + && (any(argv, w => string_contains(s: w, pattern: ">")) == false) +} + +// THE MODE IS THE DECLARED AUTHORITY'S, NOT A LITERAL SPELLED TWICE. The vector above pins "0644" as the +// value a reader can see; this pins that the value CAME FROM compile_pool_slice_file_mode, so editing the +// authority moves the argv rather than leaving two spellings to drift apart. +test fn witness_publish_mode_is_read_from_the_declared_authority() -> Bool { + any( + compile_pool_slice_publish_argv(slice_unit: gunbc_managed_compile_pool_slice), + w => w == compile_pool_slice_file_mode as String, ) - (string_contains(s: body, pattern: "<<") == false) - && (string_contains(s: body, pattern: "cat > ") == false) - && string_contains(s: body, pattern: "printf") - && string_contains(s: body, pattern: "/etc/systemd/system/") } -// START, NEVER RESTART: stopping a slice kills every process in it, so a restart here would terminate the -// compilers the pool exists to hold. -test fn witness_install_body_starts_and_never_restarts_the_slice() -> Bool { - let body = compile_pool_slice_install_body( - slice_unit: gunbc_managed_compile_pool_slice, - slice_unit_file: compile_pool_slice_unit_file( - slice_unit: gunbc_managed_compile_pool_slice, - sized: byte_size(17179869184), - ), +// START, NEVER RESTART: stopping a slice kills every process in it (systemd.slice(5)), so a restart here +// would terminate the compilers the pool exists to hold. +test fn witness_start_argv_starts_and_never_restarts_the_slice() -> Bool { + let argv = compile_pool_privileged_argv(argv: systemctl_start_argv(unit: gunbc_managed_compile_pool_slice)) + (argv == ["sudo", "-n", "systemctl", "start", gunbc_managed_compile_pool_slice as String]) + && (any(argv, w => w == "restart") == false) +} + +// THE STAGING PATH IS INSIDE THE SCOPE THE SEAL IS GIVEN, and the negative is the point: a path outside it +// must REFUSE rather than write. Without the second half this would pass on a seal that admitted anything. +test fn witness_staging_seal_admits_its_scope_and_refuses_outside_it() -> Bool { + let inside = typed_remote_file_write_seal( + path: compile_pool_slice_staging_path(slice_unit: gunbc_managed_compile_pool_slice) as FilePath, + content: "[Slice]\n", + allowed_scope: compile_pool_slice_staging_scope(), + ) + let outside = typed_remote_file_write_seal( + path: "/etc/systemd/system/gunbc-compile.slice" as FilePath, + content: "[Slice]\n", + allowed_scope: compile_pool_slice_staging_scope(), ) - string_contains(s: body, pattern: "systemctl") - && (string_contains(s: body, pattern: "restart") == false) + match inside { + TypedRemoteFileWriteAdmitted { write: _ } => + match outside { + TypedRemoteFileWriteAdmissionRefused { cause: _ } => true + _ => false + } + _ => false + } +} + +// PUBLICATION IS TWO STEPS AND THE SECOND ONE IS A RENAME. install(1) opens its destination +// O_CREAT|O_TRUNC -- measured on this fleet: same inode before and after -- so publishing straight to the +// system path would let it pass through a truncated state that a concurrent systemd read can see. The +// staged read-back cannot catch that, because it proves the STAGED copy while systemd opens the PUBLISHED +// one. So install lands the bytes at a sibling incoming name and mv -f renames within the directory. +test fn witness_publication_renames_rather_than_truncating_the_destination() -> Bool { + let publish = compile_pool_slice_publish_argv(slice_unit: gunbc_managed_compile_pool_slice) + let rename = compile_pool_slice_rename_argv(slice_unit: gunbc_managed_compile_pool_slice) + let dest = concat("/etc/systemd/system/", gunbc_managed_compile_pool_slice as String) + let incoming = concat(dest, ".gunbc-incoming") + (any(publish, w => w == dest) == false) + && any(publish, w => w == incoming) + && (rename == ["sudo", "-n", "mv", "-f", incoming, dest]) +} + +// THE RENAME MUST BE WITHIN ONE DIRECTORY OR IT IS NOT ATOMIC. rename(2) across filesystems degrades to +// copy-and-unlink, which reopens the window the two-step publish closes. Deriving the incoming name as a +// suffix on the destination makes same-directory structural, and this asserts that derivation rather than +// trusting it: a future edit relocating staging to /tmp or a scratch dir reds here. +test fn witness_incoming_name_is_a_sibling_of_the_destination() -> Bool { + let dest = compile_pool_slice_system_path(slice_unit: gunbc_managed_compile_pool_slice) + let incoming = compile_pool_slice_incoming_path(slice_unit: gunbc_managed_compile_pool_slice) + starts_with(s: incoming, prefix: concat(dest, ".")) + && (string_contains(s: incoming, pattern: "/tmp") == false) +} + +fn compile_pool_witness_intent() -> HostEffectIntent { + HostEffectIntent { + target: HostOs { node: srv1_host }, + effect: EnsureCompilePoolSlice {}, + evidence: host_effect_identity_evidence, + transport: SshShell { ssh_host: "srv1" as NonEmptyStr }, + privilege_denial: none, + } +} + +// THE TRANSPORT NARROWING, ASSERTED RATHER THAN DESCRIBED. The typed write needs the SshTarget and the +// sealed context, so SshShell must refuse where the deleted shell body would have run. This is the one +// behaviour this conversion REMOVES, so it is the one a reader is most entitled to see enforced. +test fn witness_unanchored_transport_refuses_instead_of_installing() -> Bool { + match realize_ensure_compile_pool_slice( + transport: SshShell { ssh_host: "srv1" as NonEmptyStr }, + effect_intent: compile_pool_witness_intent(), + ) { + NotConverged { reason: r, applied: _ } => + string_contains(s: r, pattern: "FleetSshExecutionContext") + && string_contains(s: r, pattern: "No unit written") + _ => false + } } test fn witness_compile_pool_effect_routes_to_the_host_cell() -> Bool {