diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index b24e5c4ac97..b93aed211d0 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -53,6 +53,15 @@ fn systemctl_enable_argv(unit: NonEmptyStr) -> List { ["systemctl", "enable", unit as String] } +// START IS NOT RESTART, AND FOR A SLICE THE DIFFERENCE IS DESTRUCTIVE RATHER THAN STYLISTIC. Stopping a +// slice kills every process in it (systemd.slice(5)), so `restart` applied to a compile pool would terminate +// the compilers the pool exists to hold. `start` on an already-active slice is a no-op that re-applies the +// unit's resource directives, which is what a converger wants. The modeled `Start` operation already exists +// on the service surface; this is its argv authority for the shell-body realization edge. +fn systemctl_start_argv(unit: NonEmptyStr) -> List { + ["systemctl", "start", unit as String] +} + fn systemctl_restart_argv(unit: NonEmptyStr) -> List { ["systemctl", "restart", unit as String] } @@ -128,6 +137,14 @@ fn systemctl_user_start_argv(unit: NonEmptyStr) -> List { ["systemctl", "--user", "start", unit as String] } +// LoadState is the only observation that distinguishes an ABSENT unit from a present one, which is why the +// `ShowLoadState` operation on the service surface above is its own operation rather than a +// SystemdUnitProperty arm: that coproduct is numeric cgroup evidence sharing one numeric mock, and a wire +// value of `not-found` is not a magnitude. Cited to systemd.unit(5) LoadState -- `loaded`, `not-found`, +// `bad-setting`, `error`, `masked` -- of which `not-found` is the one this corpus reads positively. +// `systemctl show` exits 0 for a unit the manager has never heard of and prints not-found, so a nonzero exit +// means the MANAGER was unreadable, which is a different refusal from an absent unit and must not be folded +// into it. fn systemctl_show_load_state_argv(unit: NonEmptyStr) -> List { ["systemctl", "show", unit as String, "--property=LoadState", "--value"] } @@ -302,6 +319,23 @@ service systemd.Systemctl { } } + operation ShowLoadState { + input { unit: NonEmptyStr } + output { + value: String from "stdout" + success: Bool from "exit_success" + } + readonly + transport shell { argv: ["systemctl", "show", "{unit}", "--property=LoadState", "--value"] } + exit { + 0 => Unit + nonzero => String "systemctl show LoadState failed" + } + mock_response { + 0 => { value: "not-found", success: true } "hermetic systemd.Systemctl.ShowLoadState" + } + } + operation ListUnits { input { pattern: NonEmptyStr, diff --git a/dag/gunbc/compile_pool_observe.dag b/dag/gunbc/compile_pool_observe.dag new file mode 100644 index 00000000000..9fceca05407 --- /dev/null +++ b/dag/gunbc/compile_pool_observe.dag @@ -0,0 +1,179 @@ +module gunbc.compile_pool_observe + +import std.types { String, NonEmptyStr, Bool } +import std.measure { ByteSize } +import std.disposition { Disposition, SingleAuthority } +import std.algebra { trim } +import extdeps.systemd { + SystemdUnitProperty, + MemoryMax, + MemoryHigh, + MemoryCurrent, + SystemdCgroupMemoryLimit, + MemoryLimitBytes, + MemoryLimitUnbounded, + MemoryLimitAbsent, + parse_systemd_memory_limit_show, + systemd_unit_load_state_not_found_wire, +} +import gunbc.host_effect { HostEffectTransport } +import gunbc.systemctl_show_read { + SystemdPropertyCapture, + SystemdPropertyCaptured, + SystemdPropertyCaptureRefused, + systemctl_show_property_read, + systemctl_show_load_state_read, +} +import gunbc.build_cache_instance { + CgroupMemoryLimits, + EndpointObservationUnavailableReason, + ReachRefused, +} +import gunbc.host_compile_pool { + CompilePoolObservation, + CompilePoolSliceObserved, + CompilePoolSliceAbsent, + CompilePoolObservationUnavailable, +} + +data compile_pool_observe_scope_disposition: Disposition = SingleAuthority + +// THE MISSING PRODUCER. gunbc.host_compile_pool declares CompilePoolObservation and gunbc.compile_pool_ensure +// consumes it, and until this module existed every value of that type in the tree was hand-authored inside a +// test -- so the admission that decides whether a host may have its compile-pool slice written was proven over +// fixtures and computed over nothing. That is the same shape gunbc.build_cache_endpoint_observe closed for the +// endpoint observation, and this is the slice half of it. +// +// WHAT IT READS AND IN WHICH ORDER. LoadState first, because it is the only reading that separates an absent +// slice from a present one (extdeps.systemd.systemctl ShowLoadState carries the citation: `systemctl show` +// answers `infinity` for MemoryMax on a unit the manager never heard of, identically to a loaded unit with no +// ceiling). Only under a loaded unit are the three cgroup magnitudes read. A slice that is loaded but reports +// no numeric ceiling is NOT folded into absent and NOT folded into a zero: `infinity` and an unparseable value +// are both refusals carrying what was read, because a pool with no ceiling is precisely the unbounded charge +// the managed slice exists to replace and reporting it as a size would fabricate the thing being checked. +// +// WHAT IT DOES NOT DO: it decides nothing. Whether the observed slice is large enough, correctly named, or +// permitted to be written is gunbc.compile_pool_ensure's and gunbc.host_compile_pool's judgement over this +// value; this module only reports what the host said. + +fn compile_pool_reach_refusal(slice_unit: NonEmptyStr, what: String, why: String) -> CompilePoolObservation { + CompilePoolObservationUnavailable { + reason: ReachRefused { + detail: join( + ["compile-pool slice ", slice_unit as String, ": ", what, " -- ", why], + "", + ) as NonEmptyStr, + }, + } +} + +fn compile_pool_limit_bytes(lim: SystemdCgroupMemoryLimit) -> ByteSize? { + match lim { + MemoryLimitBytes { bytes: b } => Present { value: b } + MemoryLimitUnbounded => none + MemoryLimitAbsent => none + } +} + +fn compile_pool_limit_label(lim: SystemdCgroupMemoryLimit) -> String { + match lim { + MemoryLimitBytes { bytes: _ } => "a byte count" + MemoryLimitUnbounded => "infinity (no ceiling -- an unbounded slice is the charge the managed pool replaces, not a size)" + MemoryLimitAbsent => "empty or unparseable" + } +} + +// MemoryCurrent IS READ EVEN THOUGH NO JUDGEMENT USES IT, AND AN UNREADABLE ONE REFUSES. CgroupMemoryLimits +// carries three magnitudes and the two judgements over this observation read only two -- the ensure admission +// compares memory.max and memory.high against the derived size, the readiness check compares memory.max. The +// third is required to CONSTRUCT the record, so this module cannot report an observation without it, and it +// does not invent one: a zero would be a fabricated live charge that reads as an idle pool, which is exactly +// the state an operator would act on. So a slice that is loaded but never started -- systemd answers +// `[not set]` for MemoryCurrent until the cgroup exists -- is reported unavailable with that reading quoted, +// and the next pass observes it after the start. RUNG: mitigatable, and the deficit is a carrier shape rather +// than a read -- a limit PAIR and a live magnitude fused into one record. NEXT-RUNG TRIGGER: CgroupMemoryLimits +// split so observed ceilings can be reported without a current charge, at which point this arm is +// unrepresentable rather than refused. +type CompilePoolLimitRead + = CompilePoolLimitBytes { bytes: ByteSize } + | CompilePoolLimitRefused { observation: CompilePoolObservation } + +fn compile_pool_limit_of_capture( + slice_unit: NonEmptyStr, + property_name: String, + capture: SystemdPropertyCapture, +) -> CompilePoolLimitRead { + match capture { + SystemdPropertyCaptureRefused { reason: why } => + CompilePoolLimitRefused { + observation: compile_pool_reach_refusal( + slice_unit: slice_unit, + what: join(["could not read ", property_name], ""), + why: why, + ), + } + SystemdPropertyCaptured { value: raw } => { + let lim = parse_systemd_memory_limit_show(raw: trim(raw)) + match compile_pool_limit_bytes(lim: lim) { + Present { value: b } => CompilePoolLimitBytes { bytes: b } + Absent => + CompilePoolLimitRefused { + observation: compile_pool_reach_refusal( + slice_unit: slice_unit, + what: join([property_name, " is not a ceiling"], ""), + why: compile_pool_limit_label(lim: lim), + ), + } + } + } + } +} + +fn compile_pool_slice_limits_from_reads( + slice_unit: NonEmptyStr, + max_read: SystemdPropertyCapture, + high_read: SystemdPropertyCapture, + current_read: SystemdPropertyCapture, +) -> CompilePoolObservation { + match compile_pool_limit_of_capture(slice_unit: slice_unit, property_name: "MemoryMax", capture: max_read) { + CompilePoolLimitRefused { observation: o } => o + CompilePoolLimitBytes { bytes: max } => + match compile_pool_limit_of_capture(slice_unit: slice_unit, property_name: "MemoryHigh", capture: high_read) { + CompilePoolLimitRefused { observation: o } => o + CompilePoolLimitBytes { bytes: high } => + match compile_pool_limit_of_capture(slice_unit: slice_unit, property_name: "MemoryCurrent", capture: current_read) { + CompilePoolLimitRefused { observation: o } => o + CompilePoolLimitBytes { bytes: current } => + CompilePoolSliceObserved { + slice_unit: slice_unit, + limits: CgroupMemoryLimits { + memory_max: max, + memory_high: high, + memory_current: current, + }, + } + } + } + } +} + +fn observe_compile_pool_slice( + transport: HostEffectTransport, + slice_unit: NonEmptyStr, +) -> CompilePoolObservation { + match systemctl_show_load_state_read(transport: transport, unit: slice_unit) { + SystemdPropertyCaptureRefused { reason: why } => + compile_pool_reach_refusal(slice_unit: slice_unit, what: "could not read LoadState", why: why) + SystemdPropertyCaptured { value: raw } => + if trim(raw) == (systemd_unit_load_state_not_found_wire as String) { + CompilePoolSliceAbsent { slice_unit: slice_unit } + } else { + compile_pool_slice_limits_from_reads( + slice_unit: slice_unit, + max_read: systemctl_show_property_read(transport: transport, unit: slice_unit, property: MemoryMax), + high_read: systemctl_show_property_read(transport: transport, unit: slice_unit, property: MemoryHigh), + current_read: systemctl_show_property_read(transport: transport, unit: slice_unit, property: MemoryCurrent), + ) + } + } +} diff --git a/dag/gunbc/host_effect.dag b/dag/gunbc/host_effect.dag index bc54467409d..a2cc0cb1ea4 100644 --- a/dag/gunbc/host_effect.dag +++ b/dag/gunbc/host_effect.dag @@ -42,6 +42,7 @@ type HostEffect = | OsInstallActuatorToolchainEnsure {} | EnsureBuildCacheInstance { desired: BuildCacheInstance } | ProvisionCodexRuntime {} + | EnsureCompilePoolSlice {} | SetHostnameCas { hostnamectl_path: NonEmptyStr, base: DnsLabel, desired: DnsLabel } | ReadEffectivePosixPrincipal { read: EffectivePosixPrincipalRead } | SudoNopasswdExecuteProbe { probe: SudoNopasswdExecuteProbeShape } diff --git a/dag/gunbc/host_effect_realize.dag b/dag/gunbc/host_effect_realize.dag index 7d0df8c5cc4..772b5adb145 100644 --- a/dag/gunbc/host_effect_realize.dag +++ b/dag/gunbc/host_effect_realize.dag @@ -8,6 +8,7 @@ import std.decl_ref { DeclarationRef, WholeDeclaration, decl_ref, declaration_re import gunbc.host_effect { HostEffect, ShellCommand, RedfishAction, ConvergePlan, OsInstallActuatorToolchainEnsure, EnsureBuildCacheInstance, ProvisionCodexRuntime, + EnsureCompilePoolSlice, SetHostnameCas, ReadEffectivePosixPrincipal, SudoNopasswdExecuteProbe, @@ -95,7 +96,7 @@ import gunbc.typed_argv_exec { } import extdeps.bmc.types { RedfishSystemAction } import gunbc.output_policy { OutcomeIsData } -import gunbc.fleet_intent { ComputeHost, srv1_host, compute_host_primary_cpu } +import gunbc.fleet_intent { ComputeHost, srv1_host, compute_host_primary_cpu, cpu_facts_threads, compute_host_ram_bytes_total } import product.placement_supply { HostIdentity } import extdeps.toolchain.types { Architecture, architecture_diagnostic_name } import gunbc.host_converge { HostConverge, FreshStandup, ExistingHostQuiescentReload, gunbc_apply_mode_for_host } @@ -260,13 +261,42 @@ import gunbc.live_deploy.operations { systemctl_daemon_reload_command, systemctl_enable_command, systemctl_restart_command, -} + systemctl_start_command, +} +import extdeps.jobserver { jobserver_config } +import gunbc.fleet_host_budget { + CompilePoolPlacement, + CompilePoolSliceDeclared, + CompilePoolInRunnerSlots, + CompilePoolResolution, + CompilePoolUnresolved, + compile_pool_bytes_for, + gunbc_compile_pool_placement, +} +import gunbc.compile_pool_ensure { + EnsureCompilePoolAdmission, + EnsureCompilePoolAlreadyConverged, + EnsureCompilePoolAdmitted, + EnsureCompilePoolRefused, + CompilePoolEnsureRefusal, + EnsurePoolTopologyUndeclared, + EnsurePoolSizingUnresolved, + EnsureCachePlacementUnknown, + EnsurePoolSliceUnobservable, + EnsurePoolSliceNamedDoesNotMatch, + admit_ensure_compile_pool_slice, + compile_pool_ensure_is_converged, +} +import gunbc.compile_pool_observe { observe_compile_pool_slice } +import gunbc.host_layout { gunbc_managed_compile_pool_slice } +import extdeps.systemd.unit_file { SystemdUnitFile, serialize_systemd_unit_file } + import gunbc.build_cache_instance { ci_cache_instance } import gunbc.srv3_boot_once_cd { srv3_boot_once_apply_read_back_sleep, } import extdeps.filesystem.filesystem_io -import std.measure { second_count } +import std.measure { second_count, ByteSize, byte_size_count } import gunbc.srv3_ubuntu_install_media_artifact { srv3_ubuntu_install_media_artifact } import gunbc.srv3_seeded_install_media_artifact { srv3_seeded_install_media_artifact } import extdeps.provisioning.ubuntu_install_media_fetch { @@ -333,6 +363,7 @@ type ResolvedHostEffectCell = | OsInstallActuatorToolchainOnHost { node: ComputeHost } | ProvisionBuildCacheOnHost { node: ComputeHost, desired: BuildCacheInstance } | ProvisionCodexRuntimeOnHost { node: ComputeHost } + | EnsureCompilePoolSliceOnHost { node: ComputeHost } | HostnameOnHost { node: ComputeHost, hostnamectl_path: NonEmptyStr, base: DnsLabel, desired: DnsLabel } | HostnameReadOnHost { node: ComputeHost } | NbdProxySessionOnHost { @@ -400,6 +431,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res ProvisionBuildCacheOnHost { node: n, desired: d } ProvisionCodexRuntime {} => ProvisionCodexRuntimeOnHost { node: n } + EnsureCompilePoolSlice {} => + EnsureCompilePoolSliceOnHost { node: n } Srv3ReceiptEmit { lines: lines } => ReceiptEmitOnHost { node: n, lines: lines } Srv3BootOnceReadBackSleep => @@ -439,6 +472,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res ProvisionBuildCacheOnHost { node: n, desired: d } ProvisionCodexRuntime {} => ProvisionCodexRuntimeOnHost { node: n } + EnsureCompilePoolSlice {} => + EnsureCompilePoolSliceOnHost { node: n } Srv3InstallMediaFetch => TypedReconcileOnHost { node: n, kind: TypedReconcileFetch } Srv3SeededInstallMediaRemaster => TypedReconcileOnHost { node: n, kind: TypedReconcileRemaster } Srv3SeededInstallMediaToolchainEnsure => @@ -505,6 +540,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res IncompatibleCell { reason: "host_effect: EnsureBuildCacheInstance is in-band (per-host sccache daemon) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } ProvisionCodexRuntime {} => IncompatibleCell { reason: "host_effect: ProvisionCodexRuntime is in-band (per-host provider runtime bundle) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } + EnsureCompilePoolSlice {} => + IncompatibleCell { reason: "host_effect: EnsureCompilePoolSlice is in-band (per-host systemd slice) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } Srv3NbdProxyServe { intent: _, lease_key: _ } => IncompatibleCell { reason: "host_effect: Srv3NbdProxyServe is in-band (actuator host long-running session) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } CodexSupervisedWorkerTurn { intent: _, lease_key: _ } => @@ -522,6 +559,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res IncompatibleCell { reason: "host_effect: EnsureBuildCacheInstance is in-band (per-host sccache daemon) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } ProvisionCodexRuntime {} => IncompatibleCell { reason: "host_effect: ProvisionCodexRuntime is in-band (per-host provider runtime bundle) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } + EnsureCompilePoolSlice {} => + IncompatibleCell { reason: "host_effect: EnsureCompilePoolSlice is in-band (per-host systemd slice) and cannot target BmcController; target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } Srv3InstallMediaFetch => IncompatibleCell { reason: "host_effect: Srv3InstallMediaFetch is in-band (actuator host typed reconcile) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } Srv3SeededInstallMediaRemaster => @@ -3675,6 +3714,8 @@ fn host_effect_apply(target: NodeControlPlane, effect: HostEffect, evidence: Ide realize_provision_build_cache(effect_intent: intent) ProvisionCodexRuntimeOnHost { node: _ } => realize_provision_codex_runtime(effect_intent: intent) + EnsureCompilePoolSliceOnHost { node: _ } => + realize_ensure_compile_pool_slice(transport: transport, effect_intent: intent) HostnameOnHost { node: _, hostnamectl_path: p, base: b, desired: d } => realize_set_hostname_cas_on_host(hostnamectl_path: p, base: b, desired: d, transport: transport, intent: intent) HostnameReadOnHost { node: _ } => @@ -3853,6 +3894,8 @@ fn host_effect_apply_gated( realize_provision_build_cache(effect_intent: intent) ProvisionCodexRuntimeOnHost { node: _ } => realize_provision_codex_runtime(effect_intent: intent) + EnsureCompilePoolSliceOnHost { node: _ } => + realize_ensure_compile_pool_slice(transport: transport, effect_intent: intent) HostnameOnHost { node: _, hostnamectl_path: p, base: b, desired: d } => realize_set_hostname_cas_on_host(hostnamectl_path: p, base: b, desired: d, transport: transport, intent: intent) HostnameReadOnHost { node: _ } => @@ -3946,6 +3989,7 @@ fn host_effect_drive_directive(r: Reconciliation not_converged_directive(drive: host_effect_drive(intent: intent)) ProvisionBuildCacheOnHost { node: _, desired: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) ProvisionCodexRuntimeOnHost { node: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) + EnsureCompilePoolSliceOnHost { node: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) HostnameOnHost { node: _, hostnamectl_path: _, base: _, desired: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) HostnameReadOnHost { node: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) NbdProxySessionOnHost { node: _, intent: _, lease_key: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) @@ -4350,6 +4394,283 @@ fn build_cache_ensure_step( data already_converged_evidence_is_the_observation_note: String = "THE ALREADY-CONVERGED ARM RETURNS ShellEffectApplied WITH exit_code 0 AND NO SHELL HAVING RUN, and that is worth stating because it looks like a fabricated receipt and is not. HostEffectEvidence has two arms and neither names an observation, so the evidence carrier cannot yet distinguish 'a command ran and succeeded' from 'a read established that nothing needed to run'. What the receipt carries is the observation itself -- the unit found owning the endpoint -- rather than invented command output, so a reader sees WHY it converged. DISSOLVE-ON: a HostEffectEvidence arm for an observation-grounded convergence, at which point this arm names it and the exit_code 0 goes away. Until then the honest thing is to say so here rather than let the zero read as a process exit." +// THE COMPILE-POOL SLICE GETS ITS OWN EFFECT RATHER THAN RIDING THE CACHE ENSURE, AND THE SEAM IS THE POINT. +// gunbc.compile_pool_ensure admit_ensure_compile_pool_slice refuses on TOPOLOGY before it observes anything, +// and gunbc_compile_pool_placement is CompilePoolInRunnerSlots today, so folding this admission into +// realize_provision_build_cache_body as a precondition would make EnsureBuildCacheInstance refuse on every +// host in the fleet. A lane that halts another lane to express a dependency has chosen the wrong seam: the +// pool is a separate desired state with a separate unit file, so it is a separate effect whose refusal costs +// nothing that was already converging. (Parent ruling, warm-tern-755, 2026-08-23.) +// +// WHAT IT ANSWERS TODAY, so this is not read as a description of a working fleet: EnsurePoolTopologyUndeclared, +// on every host, naming gunbc_compile_pool_placement as the cause. That is the correct answer and the landing +// state -- no host has a managed slice, and the flip of that row is the change this converger exists to make +// possible. gunbc_compile_pool_placement's own dissolution trigger says to flip it in the same change that +// converges the managed unit AND ITS SLICE; the unit had a converger and the slice had none. +// +// THE EFFECT CARRIES NO PAYLOAD, AND THAT IS DELIBERATE AFTER THE A5 LESSON. EnsureBuildCacheInstance carries +// its BuildCacheInstance because the desired daemon is a row a realizer would otherwise have to guess. The +// pool's desired state is not a row: the slice NAME comes from the topology and the SIZE is derived by +// gunbc.fleet_host_budget compile_pool_bytes_for from the cited jobserver knobs. Putting either on the effect +// would mint a second answer to a question that already has one authority, so the effect names the target host +// and the derivation happens here from that host's own facts. +// WHICH SLICE IS OBSERVED WHEN THE TOPOLOGY DECLARES NONE. Under CompilePoolInRunnerSlots there is no declared +// slice to read, and the observation is a parameter the admission takes whether or not it consults it -- it +// refuses on topology first. The subject is therefore the slice gunbc.host_layout names, which makes the read +// a true statement about the host (that unit really is what a managed pool would be called there) rather than +// a fabricated subject, and keeps the read unconditional so it cannot become a function of the answer it +// feeds. It also produces a real finding for free: on a host where the topology says no pool is managed and +// the slice nevertheless exists, the reading is there in the observation rather than being unasked. Note the +// layout row and the topology row are two authorities on ONE question -- whether a managed pool is declared -- +// and they disagree today; that disagreement is a standing hazard recorded on this PR, not something this +// function resolves. +fn compile_pool_observation_subject(placement: CompilePoolPlacement) -> NonEmptyStr { + match placement { + CompilePoolSliceDeclared { slice_unit: declared } => declared as NonEmptyStr + CompilePoolInRunnerSlots => gunbc_managed_compile_pool_slice + } +} + +fn compile_pool_ensure_refusal_detail(reason: CompilePoolEnsureRefusal) -> String { + match reason { + EnsurePoolTopologyUndeclared { detail: d } => d as String + EnsurePoolSizingUnresolved { detail: d } => d as String + EnsureCachePlacementUnknown { detail: d } => d as String + EnsurePoolSliceUnobservable { detail: d } => d as String + EnsurePoolSliceNamedDoesNotMatch { declared: dec, observed: obs } => + join( + ["the declared compile-pool slice is ", dec as String, " but the host answers for ", obs as String], + "", + ) + } +} + +fn compile_pool_slice_system_path(slice_unit: NonEmptyStr) -> String { + concat("/etc/systemd/system/", slice_unit as String) +} + +// THE BYTES INSTALLED ARE THE BYTES THE ADMISSION ADMITTED. EnsureCompilePoolAdmitted already carries the +// SystemdUnitFile gunbc.build_cache_unit rendered from the derived size, so this serializes THAT value rather +// than re-rendering from the same inputs. Re-rendering would be a second construction of one artifact, and +// the failure it admits is not hypothetical: the admission could be granted against one size and the file +// written from another with nothing between them to notice. +// +// Install then daemon-reload then START -- never restart. Stopping a slice kills every process in it +// (systemd.slice(5)), so restarting the compile pool would terminate the compilers it exists to hold, which +// is the opposite of converging it. `start` on an already-active slice re-applies the unit's resource +// directives and kills nothing. +fn compile_pool_slice_install_body(slice_unit: NonEmptyStr, slice_unit_file: SystemdUnitFile) -> String { + join([ + "set -euo pipefail", + "_gunbc_stage=\"$(mktemp -d)\"", + "trap 'rm -rf \"$_gunbc_stage\"' EXIT", + join(["cat > \"$_gunbc_stage/", slice_unit as String, "\" <<'__GUNBC_COMPILE_POOL_SLICE__'"], ""), + concat(serialize_systemd_unit_file(file: slice_unit_file), "__GUNBC_COMPILE_POOL_SLICE__"), + join(["sudo -n install -m 0644 \"$_gunbc_stage/", slice_unit as String, "\" ", compile_pool_slice_system_path(slice_unit: slice_unit)], ""), + systemctl_daemon_reload_command(privileged: true), + systemctl_start_command(unit: slice_unit, privileged: true), + "", + ], "\n") +} + +data compile_pool_slice_install_shell_dissolution_trigger: DissolutionCondition = unbound_dissolution(description: "dissolve-on: gunbc.host_effect_realize.compile_pool_slice_install_body -- concat-built systemd slice install shell run over shell_exec_via_bash transport, the same shape and the same debt as build_cache_provision_shell_script beside it. DISSOLVES WHEN bash-emit (#5828) realizes a unit-file install as typed host-effect argv, at which point both bodies retire together rather than this one being converted alone.") + +fn compile_pool_ensure_admission_for( + node: ComputeHost, + transport: HostEffectTransport, +) -> EnsureCompilePoolAdmission { + let placement = gunbc_compile_pool_placement + let instance = ci_cache_instance(host: node.identity) + let survey = survey_build_cache_endpoint( + instance: instance, + reads: build_cache_endpoint_reads(instance: instance, transport: transport), + ) + admit_ensure_compile_pool_slice( + topology: placement, + sizing: compile_pool_sizing_for_host(node: node, placement: placement), + cache_placement: survey.placement, + observed: observe_compile_pool_slice( + transport: transport, + slice_unit: compile_pool_observation_subject(placement: placement), + ), + ) +} + +// The size is DERIVED here from the target host's own facts rather than travelling on the effect, through the +// same compile_pool_bytes_for gunbc.ci_runner_placement uses to shrink the runner pool. The two must agree or +// the host stops conserving, and they agree because there is one derivation, not because two call sites were +// kept in step. +fn compile_pool_sizing_for_host(node: ComputeHost, placement: CompilePoolPlacement) -> CompilePoolResolution { + match compute_host_primary_cpu(host: node) { + Absent => + CompilePoolUnresolved { + reason: "compile pool unresolved: the target ComputeHost declares no primary CPU, so the jobserver token count that sizes the pool cannot be derived; fail-closed, no default size", + } + Present { value: cpu } => + compile_pool_bytes_for( + placement: placement, + nproc: cpu_facts_threads(cpu: cpu), + host: compute_host_ram_bytes_total(host: node), + citation: jobserver_config, + ) + } +} + +// The admission runs TWICE and the second run is not a formality: it is the postcondition, re-derived from a +// FRESH observation of the host rather than from the state we just wrote. A read-back that reused the +// preflight observation would green on evidence it produced -- the exact defect +// provision_observes_before_it_acts_note records from srv3 -- and the whole point is that the pre-state and +// the post-state can differ. Convergence is claimed only when the second admission answers +// EnsureCompilePoolAlreadyConverged, which is the same predicate the preflight uses to skip the write. +fn compile_pool_slice_install_step( + node: ComputeHost, + slice_unit: NonEmptyStr, + sized: ByteSize, + slice_unit_file: SystemdUnitFile, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + let observed = run_shell_transport( + script: retained_srvn( + body: shell_exec_via_bash(body: compile_pool_slice_install_body(slice_unit: slice_unit, slice_unit_file: slice_unit_file)), + reason: "srvN compile-pool slice install (managed cgroup parent for cache-miss compilers)", + ), + transport: transport, + ) + if observed.success == false { + NotConverged { + reason: join( + [ + "host_effect_apply: EnsureCompilePoolSlice ran its install and the shell exited ", + to_string(observed.exit_code), + ": ", + if observed.stderr == "" { observed.stdout } else { observed.stderr }, + ], + "", + ), + applied: effect_intent, + } + } else { + let after = compile_pool_ensure_admission_for(node: node, transport: transport) + if compile_pool_ensure_is_converged(admission: after) { + Converged { + evidence: ShellEffectApplied { + exit_code: observed.exit_code, + stdout: join( + [ + "CompilePoolSliceReceipt: converged unit=", + slice_unit as String, + " memory.max=memory.high=", + to_string(byte_size_count(b: sized)), + " (re-observed after install; the admission answered already-converged)", + ], + "", + ), + stderr: observed.stderr, + }, + applied: effect_intent, + } + } else { + NotConverged { + reason: join( + [ + "host_effect_apply: EnsureCompilePoolSlice installed ", + slice_unit as String, + " and the read-back did not converge -- ", + compile_pool_admission_diagnostic(admission: after), + ], + "", + ), + applied: effect_intent, + } + } + } +} + +fn compile_pool_admission_diagnostic(admission: EnsureCompilePoolAdmission) -> String { + match admission { + EnsureCompilePoolAlreadyConverged { slice_unit: u, sized_bytes: _ } => + join(["the slice ", u as String, " is converged"], "") + EnsureCompilePoolAdmitted { slice_unit: u, sized_bytes: _, slice_unit_file: _ } => + join( + ["the slice ", u as String, " is still admitted for writing, so the installed unit did not take effect on the host"], + "", + ) + EnsureCompilePoolRefused { reason: r } => compile_pool_ensure_refusal_detail(reason: r) + } +} + +fn realize_ensure_compile_pool_slice( + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + match effect_intent.target { + BmcController { node: _ } => + NotConverged { + reason: "host_effect_apply: EnsureCompilePoolSlice requires a HostOs target; fail-closed.", + applied: effect_intent, + } + HostOs { node: node } => + match transport { + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => + NotConverged { + reason: "host_effect_apply: EnsureCompilePoolSlice cannot use EmitArtifactThenThinRun transport; typed mismatch.", + applied: effect_intent, + } + LocalShell => realize_ensure_compile_pool_slice_body(node: node, transport: transport, effect_intent: effect_intent) + SshShell { ssh_host: _ } => realize_ensure_compile_pool_slice_body(node: node, transport: transport, effect_intent: effect_intent) + FleetSsh { target: _, context: _ } => realize_ensure_compile_pool_slice_body(node: node, transport: transport, effect_intent: effect_intent) + } + } +} + +fn realize_ensure_compile_pool_slice_body( + node: ComputeHost, + transport: HostEffectTransport, + effect_intent: HostEffectIntent, +) -> Reconciliation { + match compile_pool_ensure_admission_for(node: node, transport: transport) { + EnsureCompilePoolRefused { reason: r } => + NotConverged { + reason: join( + [ + "host_effect_apply: EnsureCompilePoolSlice refused before writing anything -- ", + compile_pool_ensure_refusal_detail(reason: r), + ], + "", + ), + applied: effect_intent, + } + EnsureCompilePoolAlreadyConverged { slice_unit: u, sized_bytes: b } => + Converged { + evidence: ShellEffectApplied { + exit_code: 0, + stdout: join( + [ + "CompilePoolSliceReceipt: already_converged unit=", + u as String, + " memory.max=memory.high=", + to_string(byte_size_count(b: b)), + " (observed on the host; no unit written, no start issued)", + ], + "", + ), + stderr: "", + }, + applied: effect_intent, + } + EnsureCompilePoolAdmitted { slice_unit: u, sized_bytes: b, slice_unit_file: f } => + compile_pool_slice_install_step( + node: node, + slice_unit: u, + sized: b, + slice_unit_file: f, + transport: transport, + effect_intent: effect_intent, + ) + } +} + fn build_cache_already_converged( unit: NonEmptyStr, effect_intent: HostEffectIntent, diff --git a/dag/gunbc/host_toolchain_ensure.dag b/dag/gunbc/host_toolchain_ensure.dag index c75002d973a..04c20fc22dd 100644 --- a/dag/gunbc/host_toolchain_ensure.dag +++ b/dag/gunbc/host_toolchain_ensure.dag @@ -18,6 +18,7 @@ import gunbc.fleet_intent { ComputeHost } import gunbc.host_effect { HostEffect, EnsureBuildCacheInstance, + EnsureCompilePoolSlice, ProvisionCodexRuntime, OsInstallActuatorToolchainEnsure, HostOs, @@ -29,9 +30,18 @@ import gunbc.os_install_actuator_selection { srv3_os_install_actuator_host } import gunbc.build_cache_instance { ci_cache_instance } import std.realization_reconcile { Converged, NotConverged } -type HostToolchainKind = BuildCache | CodexRuntime | OsInstallActuator +type HostToolchainKind = BuildCache | CodexRuntime | OsInstallActuator | CompilePool -// Compatibility specialization, not the Ensure abstraction: HostToolchainKind is the closed selector for two existing HostEffect adapters in this entrypoint. Generic convergence semantics live in gunbc.ensure and dispatch requirements live in gunbc.roadmap_execution_contract. Cargo, rustc, provider auth, and future task dependencies must not be added to this enum merely because their callers say ensure; they require their own typed observation/plan/readback adapters and may then reuse generic Ensure. +// Compatibility specialization, not the Ensure abstraction: HostToolchainKind is the closed selector for the +// existing HostEffect adapters reachable from this entrypoint. Generic convergence semantics live in +// gunbc.ensure and dispatch requirements live in gunbc.roadmap_execution_contract. Cargo, rustc, provider auth, +// and future task dependencies must not be added to this enum merely because their callers say ensure; they +// require their own typed observation/plan/readback adapters and may then reuse generic Ensure. +// +// CompilePool was admitted against that bar rather than beside it, and the three parts are nameable: the +// observation is gunbc.compile_pool_observe observe_compile_pool_slice, the plan is gunbc.compile_pool_ensure +// admit_ensure_compile_pool_slice, and the read-back is that same admission re-derived from a fresh +// observation after the install. It is listed here because the bar was met, not because its caller says ensure. data host_toolchain_ensure_srv3_fork_scaffold: Disposition = Terminal { reason: "srv3_os_install_actuator_toolchain_ensure thinned to host_toolchain_ensure(host: srv3, kind: OsInstallActuator) — no behavioral fork remains." } @@ -89,6 +99,7 @@ fn host_toolchain_ensure_effect(kind: HostToolchainKind, host: HostIdentity) -> BuildCache => EnsureBuildCacheInstance { desired: ci_cache_instance(host: host) } CodexRuntime => ProvisionCodexRuntime {} OsInstallActuator => OsInstallActuatorToolchainEnsure {} + CompilePool => EnsureCompilePoolSlice {} } } @@ -96,6 +107,7 @@ fn host_toolchain_ensure_target_host(kind: HostToolchainKind, host: HostIdentity match kind { BuildCache => fleet_compute_host_for_identity(identity: host) CodexRuntime => fleet_compute_host_for_identity(identity: host) + CompilePool => fleet_compute_host_for_identity(identity: host) OsInstallActuator => if host == operator_host_srv3 { Present { value: srv3_os_install_actuator_host } diff --git a/dag/gunbc/live_deploy/operations.dag b/dag/gunbc/live_deploy/operations.dag index 5072e671a97..f38cb9cb65f 100644 --- a/dag/gunbc/live_deploy/operations.dag +++ b/dag/gunbc/live_deploy/operations.dag @@ -70,6 +70,14 @@ fn systemctl_enable_command(unit: NonEmptyStr, privileged: Bool) -> String { } } +fn systemctl_start_command(unit: NonEmptyStr, privileged: Bool) -> String { + if privileged { + shell_privileged_command_text_of_argv(argv: systemctl_start_argv(unit: unit)) + } else { + shell_command_text_of_argv(argv: systemctl_start_argv(unit: unit)) + } +} + fn systemctl_restart_command(unit: NonEmptyStr, privileged: Bool) -> String { if privileged { shell_privileged_command_text_of_argv(argv: systemctl_restart_argv(unit: unit)) diff --git a/dag/gunbc/systemctl_show_read.dag b/dag/gunbc/systemctl_show_read.dag index caff1b4bef2..63eb436052f 100644 --- a/dag/gunbc/systemctl_show_read.dag +++ b/dag/gunbc/systemctl_show_read.dag @@ -131,3 +131,78 @@ fn systemctl_show_property_operation_argv_matches_transport( ArgvMaterializationRefused { at: _, cause: _ } => false } } + +// THE LOAD-STATE READ IS A SIBLING OF THE PROPERTY READ, NOT A SPECIAL CASE OF IT. It shares the +// capture type because the shape of the answer is identical -- one `--value` line, or a located +// refusal -- but it cannot share the operation: extdeps.systemd.systemctl's own annotation records +// that SystemdUnitProperty is numeric cgroup evidence with one numeric mock, and `not-found` is not +// a magnitude. Sharing the operation would have meant widening that coproduct with a member whose +// mock lies about every other member's shape. +// +// WHY A CONSUMER NEEDS IT AT ALL: `systemctl show --property=MemoryMax --value` answers +// `infinity` for a unit the manager has never heard of, exactly as it does for a loaded unit with no +// ceiling. Reading only the limits therefore cannot tell an ABSENT slice from an UNBOUNDED one -- +// two states whose remedies are opposite (install it vs refuse and report the drift). LoadState is +// the discriminator, so it is read first and the limits are read only under a loaded unit. +fn systemctl_show_load_state_operation_argv(unit: NonEmptyStr) -> ArgvMaterialization { + shell_materialize_operation_argv( + systemctl_show_property_path, + systemctl_show_property_service, + systemctl_show_load_state_operation, + [operation_argv_bind_text(name: "unit", text: unit as String)] + ) +} + +data systemctl_show_load_state_operation: String = "ShowLoadState" + +fn systemctl_show_load_state_read_local(unit: NonEmptyStr) -> SystemdPropertyCapture { + let result = systemd.Systemctl.ShowLoadState(unit: unit) + systemctl_show_property_value_from_stdout(success: result.success, stdout: result.value) +} + +fn systemctl_show_load_state_read_ssh(host: NonEmptyStr, unit: NonEmptyStr) -> SystemdPropertyCapture { + match systemctl_show_load_state_operation_argv(unit: unit) { + ArgvMaterialized { argv: argv } => + systemctl_show_property_read_ssh_argv(host: host, argv: argv) + ArgvMaterializationRefused { at: _, cause: c } => + SystemdPropertyCaptureRefused { reason: operation_argv_cause_label(cause: c) } + } +} + +fn systemctl_show_load_state_read_fleet_ssh( + target: SshTarget, + context: FleetSshExecutionContext, + unit: NonEmptyStr, +) -> SystemdPropertyCapture { + match systemctl_show_load_state_operation_argv(unit: unit) { + ArgvMaterialized { argv: argv } => + systemd_property_capture_from_outcome( + outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv), + ) + ArgvMaterializationRefused { at: _, cause: c } => + SystemdPropertyCaptureRefused { reason: operation_argv_cause_label(cause: c) } + } +} + +fn systemctl_show_load_state_read( + transport: HostEffectTransport, + unit: NonEmptyStr, +) -> SystemdPropertyCapture { + match transport { + LocalShell => systemctl_show_load_state_read_local(unit: unit) + SshShell { ssh_host: h } => systemctl_show_load_state_read_ssh(host: h, unit: unit) + FleetSsh { target: t, context: c } => systemctl_show_load_state_read_fleet_ssh(target: t, context: c, unit: unit) + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => + SystemdPropertyCaptureRefused { + reason: "systemctl show LoadState refused EmitArtifactThenThinRun transport", + } + } +} + +fn systemctl_show_load_state_operation_argv_matches_transport(unit: NonEmptyStr) -> Bool { + let authority = systemctl_show_load_state_argv(unit: unit) + match systemctl_show_load_state_operation_argv(unit: unit) { + ArgvMaterialized { argv: materialized } => join(materialized, " ") == join(authority, " ") + ArgvMaterializationRefused { at: _, cause: _ } => false + } +} diff --git a/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag new file mode 100644 index 00000000000..25c6a6c9b12 --- /dev/null +++ b/dag/test/claim/compile_pool_ensure_wiring_witness_test.dag @@ -0,0 +1,123 @@ +module test.claim.compile_pool_ensure_wiring_witness + +import gunbc.host_toolchain_ensure { host_toolchain_ensure_effect, CompilePool } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// WHAT THIS MODULE IS FOR: gunbc.compile_pool_ensure was proven over fixtures and computed by nothing -- +// its only caller was its own test module -- so an admission named as the wall between a host and an +// unbounded compile pool was green without ever being asked a question. These witnesses assert the parts of +// the wiring that make it reachable, and the parts of the observation that decide what it is asked. + +// REACHABILITY, not shape: the entrypoint a person or a converge pass actually calls must produce the effect, +// and the effect must resolve to a cell that dispatches to the realizer. Asserting the arm exists on the +// coproduct would prove nothing -- it existed before this change too, with nothing routing to it. +test fn witness_compile_pool_kind_produces_the_effect() -> Bool { + match host_toolchain_ensure_effect(kind: CompilePool, host: "srv1" as HostIdentity) { + EnsureCompilePoolSlice {} => true + _ => false + } +} + +test fn witness_compile_pool_effect_routes_to_the_host_cell() -> Bool { + match resolve_host_effect_cell(target: HostOs { node: srv1_host }, effect: EnsureCompilePoolSlice {}) { + EnsureCompilePoolSliceOnHost { node: _ } => true + _ => false + } +} + +// The BMC arm is the discriminating negative: a slice is an in-band systemd fact, so a Redfish target must +// refuse rather than resolve. Without it the routing witness above would pass on a fold that resolved +// everything to the same cell. +test fn witness_compile_pool_effect_refuses_a_bmc_target() -> Bool { + match resolve_host_effect_cell(target: BmcController { node: srv1_host }, effect: EnsureCompilePoolSlice {}) { + IncompatibleCell { reason: why } => string_contains(s: why, pattern: "EnsureCompilePoolSlice") + _ => false + } +} + +// THE LIVE TOPOLOGY TRIPWIRE, and it reads the REAL gunbc_compile_pool_placement rather than a fixture, so it +// is a claim about this fleet. It asserts a REFUSAL today: the row is CompilePoolInRunnerSlots, so no host has +// a managed slice and every convergence attempt refuses naming that row. When the row flips to +// CompilePoolSliceDeclared this witness reds -- deliberately, in the same change that shrinks runner_pool -- +// and at that point it becomes an assertion that the pool converges rather than being deleted. +test fn witness_live_topology_refuses_the_slice_ensure() -> Bool { + let a = admit_ensure_compile_pool_slice( + topology: gunbc_compile_pool_placement, + sizing: compile_pool_bytes_for( + placement: gunbc_compile_pool_placement, + nproc: hardware_thread_count(count: 128), + host: byte_size(134317047808), + citation: jobserver_config, + ), + cache_placement: ServerPlacementAbsent, + observed: CompilePoolSliceAbsent { slice_unit: gunbc_managed_compile_pool_slice }, + ) + match a { + EnsureCompilePoolRefused { reason: r } => + match r { + EnsurePoolTopologyUndeclared { detail: d } => + string_contains(s: d as String, pattern: "gunbc_compile_pool_placement") + _ => false + } + _ => false + } +} + +// INFINITY IS NOT A SIZE, AND THIS IS THE RED THAT MATTERS MOST IN THE OBSERVATION. `systemctl show +// --property=MemoryMax --value` answers `infinity` both for a unit the manager never heard of and for a +// loaded slice with no ceiling. Reading either as a magnitude would let a slice with NO LIMIT be reported as +// a sized pool, which is the unbounded charge the managed slice exists to replace -- reported as its own +// remedy. The refusal is what keeps the two apart. +test fn witness_unbounded_memory_max_is_not_read_as_a_size() -> Bool { + match compile_pool_limit_of_capture( + slice_unit: gunbc_managed_compile_pool_slice, + property_name: "MemoryMax", + capture: SystemdPropertyCaptured { value: "infinity" }, + ) { + CompilePoolLimitRefused { observation: o } => + match o { + CompilePoolObservationUnavailable { reason: _ } => true + _ => false + } + CompilePoolLimitBytes { bytes: _ } => false + } +} + +// The paired positive control: a real byte count must decode, or the refusal above would be satisfied by a +// decoder that refuses everything. +test fn witness_numeric_memory_max_decodes_to_bytes() -> Bool { + match compile_pool_limit_of_capture( + slice_unit: gunbc_managed_compile_pool_slice, + property_name: "MemoryMax", + capture: SystemdPropertyCaptured { value: "17179869184" }, + ) { + CompilePoolLimitBytes { bytes: b } => byte_size_count(b: b) == 17179869184 + CompilePoolLimitRefused { observation: _ } => false + } +} + +// A read that could not run is UNAVAILABLE, never ABSENT. The two have opposite remedies -- install the slice +// versus fix the reach -- and folding an unreadable manager into "the slice is not there" would send a +// converger to write a unit on a host it cannot see. +test fn witness_unreadable_property_is_unavailable_not_absent() -> Bool { + match compile_pool_limit_of_capture( + slice_unit: gunbc_managed_compile_pool_slice, + property_name: "MemoryMax", + capture: SystemdPropertyCaptureRefused { reason: "systemctl show exited unsuccessfully" }, + ) { + CompilePoolLimitRefused { observation: o } => + match o { + CompilePoolObservationUnavailable { reason: _ } => true + CompilePoolSliceAbsent { slice_unit: _ } => false + CompilePoolSliceObserved { slice_unit: _, limits: _ } => false + } + CompilePoolLimitBytes { bytes: _ } => false + } +} + +// The load-state argv the transport materializes must be the argv extdeps.systemd.systemctl declares. Without +// this the operation could drift from its own authority and every reading above would be of the wrong unit. +test fn witness_load_state_operation_argv_matches_its_authority() -> Bool { + systemctl_show_load_state_operation_argv_matches_transport(unit: gunbc_managed_compile_pool_slice) +}