From 561fa4a89ef320c02a16f7dc0833151c5dd25ce7 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 25 Jul 2026 06:23:39 +0000 Subject: [PATCH 1/6] WIP: srvN subsumption --- dag/gunbc/host_build_cache_provision_script.dag | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/host_build_cache_provision_script.dag b/dag/gunbc/host_build_cache_provision_script.dag index 79730a6be79..fe23d2c5138 100644 --- a/dag/gunbc/host_build_cache_provision_script.dag +++ b/dag/gunbc/host_build_cache_provision_script.dag @@ -37,7 +37,9 @@ fn build_cache_systemd_user_unit_body() -> String { "", "[Service]", "Type=simple", - join(["ExecStart=", sccache_installed_binary_path, " --start-server"], ""), + "Environment=SCCACHE_START_SERVER=1", + "Environment=SCCACHE_NO_DAEMON=1", + join(["ExecStart=", sccache_installed_binary_path], ""), "Restart=on-failure", "", "[Install]", @@ -177,6 +179,16 @@ fn build_cache_unit_execstart_matches_install_path() -> Bool { ) } +data build_cache_unit_foreground_supervision_note: String = "The unit is Type=simple, so systemd tracks the ExecStart process AS the service — it MUST stay in the foreground. `sccache --start-server` DAEMONIZES and returns 0, so under Type=simple systemd sees the ExecStart exit cleanly and marks the unit `inactive (dead)` the instant it starts; the backgrounded server then idle-times-out unsupervised and Restart=on-failure never fires (the exit was success). Proven live on srv3 2026-07-25: the earlier `--start-server` form left the unit inactive with no sccache process, the daemon surviving only via sccache's own per-invocation auto-start — i.e. the unit was a no-op. This is the §5 specification-without-execution trap: the string witness that ExecStart equals the install path was green while the unit did nothing. The foreground form matches the production ctrl-sccache.service invocation (bare ExecStart + SCCACHE_START_SERVER=1 SCCACHE_NO_DAEMON=1 env, verified running month+ on srv1/srv2), reducing the §3 fork between the modeled unit and production rather than inventing a third shape. build_cache_unit_is_foreground_supervised makes the defective combo (Type=simple paired with a backgrounding --start-server) UNWRITABLE; the runtime is-active proof is the live srv3 re-provision (systemd --user is not runnable in the hermetic CI container, so the construction wall carries the CI grain and live execution carries the runtime grain)." + +fn build_cache_unit_is_foreground_supervised() -> Bool { + let body = build_cache_systemd_user_unit_body() + string_contains(s: body, pattern: "Type=simple") + && string_contains(s: body, pattern: "SCCACHE_START_SERVER=1") + && string_contains(s: body, pattern: "SCCACHE_NO_DAEMON=1") + && !string_contains(s: body, pattern: "--start-server") +} + fn build_cache_classifier_covers_every_cited_fragment() -> Bool { fold( sccache_misconfigured_log_fragments, From 73ff9964f516b1c7bae36d345d810a56258f1507 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 25 Jul 2026 06:28:29 +0000 Subject: [PATCH 2/6] sccache unit: foreground-supervise the daemon (Type=simple + --start-server was a no-op) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #7170. Live execution on srv3 caught what the string witnesses missed (§5 specification-without-execution): the modeled unit was Type=simple ExecStart=/usr/local/bin/sccache --start-server # daemonizes, exits 0 so systemd saw the ExecStart exit cleanly and marked the unit `inactive (dead)` the instant it started; the backgrounded server idle-timed-out unsupervised and Restart=on-failure never fired (success exit). The unit was a no-op — sccache survived only via its own per-invocation auto-start. The string witness "ExecStart equals the install path" was green the whole time. Fix (grounded by execution on srv3 first): foreground the server, matching the production ctrl-sccache.service invocation verified running month+ on srv1/srv2: Type=simple Environment=SCCACHE_START_SERVER=1 Environment=SCCACHE_NO_DAEMON=1 ExecStart=/usr/local/bin/sccache This reduces the §3 fork between the modeled unit and production rather than minting a third shape. Closes the execution gap two ways: build_cache_unit_is_foreground_supervised is a construction check that makes the defective combo (Type=simple paired with a backgrounding --start-server) UNWRITABLE (carries the CI grain, since systemd --user is not runnable in the hermetic container), and the live srv3 re-provision carries the runtime is-active grain. Census note accounts the +1-line delta as a correctness swap toward the cited production authority. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/host_build_cache_provision_script.dag | 2 +- .../claim/host_build_cache_provision_design_witness_test.dag | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/host_build_cache_provision_script.dag b/dag/gunbc/host_build_cache_provision_script.dag index fe23d2c5138..c51c09ba063 100644 --- a/dag/gunbc/host_build_cache_provision_script.dag +++ b/dag/gunbc/host_build_cache_provision_script.dag @@ -14,7 +14,7 @@ import extdeps.cache.sccache { data host_build_cache_provision_script_dissolution_trigger: String = "🟡 dissolve-on: host_build_cache_provision_script — concat-built systemd/sccache ensure shell; DISSOLVES WHEN bash-emit (#5828) realizes ProvisionBuildCache observe/ensure/read-back as typed host-effect argv and realize_provision_build_cache retires shell_exec_via_bash transport." -data host_build_cache_provision_script_census_row_note: String = "CENSUS STANDING: docs/plans/shell-to-dag-residual-census-and-arc-completion.md §4.A keys this file as the builder for host_effect_realize.ProvisionBuildCache, action DECOMPOSE, sub-class A5 (srv* cluster, deferred by operator 2026-07-22 — real debt, NOT done; un-defer trigger is the operator). This site is RUNTIME-PRESENT (it realizes through host_effect_apply_gated on LocalShell/SshShell), so per that doc's routing decision its end state is a typed effect-plan with NO bash emitted — the six *_body builders below are the debt, not the design. Standing constraint while the row is deferred, stated precisely (review 42401 sharpened this): NEW emitted shell must be either a DEDUCTION or a cited-de-fork SWAP — never an unbounded, uncited addition. Two exact deltas this PR made, both accounted: (1) the sccache DE-FORK swap — `apt-get install sccache` + an `ExecStart=/usr/bin/sccache` unit went OUT, and the cited-tarball install invocation (sccache_install_script, one authority row) came IN; that is the §3 fork removal this PR exists to make, a swap that de-duplicates rather than grows. (2) `sudo -n loginctl enable-linger \"$USER\"` is genuinely NEW shell, not a swap — acknowledged here as growth rather than hidden: it is the one primitive that makes the already-selected user-scope systemd handler viable at all (a `--user` manager is torn down with the login otherwise), justified in build_cache_user_service_linger_note, and it is bounded (a single fixed leaf, not a corpus-denominated fold). What must NOT grow is the PATH-shadow OBSERVATION: the judgement went to gunbc.host_build_cache_provision (DaemonPathShadowed / RefusalPathShadowed + verdict fold, witnessed against a synthetic observation), and its live shell observation is STAGED, NOT emitted here — an earlier attempt folded `command -v sccache` lines into the two existing bodies, which kept the builder count at six but still grew the shell surface (review 42343 rejected that as a census regression). The observation lands when ProvisionBuildCache realizes as typed argv (dissolve-on #5828). Adding a *_body, or an uncited emitted-shell leaf that is neither a deduction nor a de-fork swap, is a census regression even when the behaviour it adds is correct." +data host_build_cache_provision_script_census_row_note: String = "CENSUS STANDING: docs/plans/shell-to-dag-residual-census-and-arc-completion.md §4.A keys this file as the builder for host_effect_realize.ProvisionBuildCache, action DECOMPOSE, sub-class A5 (srv* cluster, deferred by operator 2026-07-22 — real debt, NOT done; un-defer trigger is the operator). This site is RUNTIME-PRESENT (it realizes through host_effect_apply_gated on LocalShell/SshShell), so per that doc's routing decision its end state is a typed effect-plan with NO bash emitted — the six *_body builders below are the debt, not the design. Standing constraint while the row is deferred, stated precisely (review 42401 sharpened this): NEW emitted shell must be either a DEDUCTION or a cited-de-fork SWAP — never an unbounded, uncited addition. Two exact deltas this PR made, both accounted: (1) the sccache DE-FORK swap — `apt-get install sccache` + an `ExecStart=/usr/bin/sccache` unit went OUT, and the cited-tarball install invocation (sccache_install_script, one authority row) came IN; that is the §3 fork removal this PR exists to make, a swap that de-duplicates rather than grows. (2) `sudo -n loginctl enable-linger \"$USER\"` is genuinely NEW shell, not a swap — acknowledged here as growth rather than hidden: it is the one primitive that makes the already-selected user-scope systemd handler viable at all (a `--user` manager is torn down with the login otherwise), justified in build_cache_user_service_linger_note, and it is bounded (a single fixed leaf, not a corpus-denominated fold). What must NOT grow is the PATH-shadow OBSERVATION: the judgement went to gunbc.host_build_cache_provision (DaemonPathShadowed / RefusalPathShadowed + verdict fold, witnessed against a synthetic observation), and its live shell observation is STAGED, NOT emitted here — an earlier attempt folded `command -v sccache` lines into the two existing bodies, which kept the builder count at six but still grew the shell surface (review 42343 rejected that as a census regression). The observation lands when ProvisionBuildCache realizes as typed argv (dissolve-on #5828). Adding a *_body, or an uncited emitted-shell leaf that is neither a deduction nor a de-fork swap, is a census regression even when the behaviour it adds is correct. FOLLOW-UP #7206 delta (correctness swap, accounted): the unit's `ExecStart=... --start-server` became bare `ExecStart=...` + two `Environment=` lines (SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1) — net +1 line, and NOT uncited growth: it is a swap toward the production ctrl-sccache.service invocation (the single authority the whole lane converges on), replacing a form live execution on srv3 proved is a Type=simple no-op (build_cache_unit_foreground_supervision_note). A correctness swap toward the cited production authority is the sanctioned shape, exactly like delta (1)." data host_build_cache_provision_script_scaffold: Disposition = Scaffold { dissolves_to: SingleAuthority, diff --git a/dag/test/claim/host_build_cache_provision_design_witness_test.dag b/dag/test/claim/host_build_cache_provision_design_witness_test.dag index 609663314d8..335c8abccb3 100644 --- a/dag/test/claim/host_build_cache_provision_design_witness_test.dag +++ b/dag/test/claim/host_build_cache_provision_design_witness_test.dag @@ -168,6 +168,10 @@ fn design_witness_unit_execstart_matches_install_path() -> Bool { build_cache_unit_execstart_matches_install_path() } +fn design_witness_unit_is_foreground_supervised() -> Bool { + build_cache_unit_is_foreground_supervised() +} + fn design_witness_classifier_covers_every_cited_fragment() -> Bool { build_cache_classifier_covers_every_cited_fragment() } @@ -241,6 +245,7 @@ test fn host_build_cache_provision_design_witnesses() -> Bool { && design_witness_path_authority_observation_not_emitted_as_shell() && design_witness_install_is_cited_not_package_manager() && design_witness_unit_execstart_matches_install_path() + && design_witness_unit_is_foreground_supervised() && design_witness_classifier_covers_every_cited_fragment() && design_witness_classifier_is_a_fold_not_an_unrolled_chain() && design_witness_fragment_words_are_shell_safe() From 955fe9d2af5f40812bf3a643d537e08e245e602c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 25 Jul 2026 13:01:22 +0000 Subject: [PATCH 3/6] WIP: srvN subsumption --- dag/extdeps/cache/sccache.dag | 24 ++++++ dag/gunbc/host_build_cache_provision.dag | 26 ++++++ .../host_build_cache_provision_script.dag | 23 ++++-- ...ld_cache_provision_design_witness_test.dag | 82 +++++++++++++++++++ 4 files changed, 150 insertions(+), 5 deletions(-) diff --git a/dag/extdeps/cache/sccache.dag b/dag/extdeps/cache/sccache.dag index abca5a5c62c..0334444934a 100644 --- a/dag/extdeps/cache/sccache.dag +++ b/dag/extdeps/cache/sccache.dag @@ -150,6 +150,30 @@ data sccache_misconfigured_log_fragments: List = [ sccache_failed_to_execute_compile_log_fragment, ] +data sccache_server_lifetime_note: String = "sccache's server EXITS ON ITS OWN. Three environment knobs, all documented by the module's cited authority extdeps_external_authority_anchor (github.com/mozilla/sccache/blob/main/docs/Configuration.md), decide the process lifetime a supervisor sees, and they are modeled here because they are facts about SCCACHE — not about systemd, and not about gunbc. Before 2026-07-25 the P1b unit emitter spelled SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1 as bare literals of its own, which made gunbc.host_build_cache_provision_script a SECOND authority for sccache's environment vocabulary (§3) — and the knob that was missing from that second authority is exactly the one that broke: SCCACHE_IDLE_TIMEOUT. Cited text: 'how long the local daemon process waits for more client requests before exiting, in seconds. Set to 0 to run sccache permanently.' The default is 600 seconds, confirmed by execution rather than read off the doc: on srv3 2026-07-25 the foreground-supervised unit started at 06:28:47 UTC, served no compile request, and EXITED 0 after `Duration: 10min 8.215s`, leaving `inactive (dead)` — Restart=on-failure could not fire because the exit was a success. Consequence for any supervisor: a Type=simple unit whose ExecStart is sccache is only durable when the idle timeout is disabled; otherwise it dies on the first quiet ten minutes and the host reverts to sccache's own unsupervised per-invocation auto-start (the very no-op state the foreground form was introduced to remove). The srv1/srv2 production ctrl-sccache.service does not set this knob either — its month-plus uptime is CI traffic keeping the daemon below the idle threshold, i.e. accidental coverage, not a durable unit; that is a fact about the observed hosts, so this row is a DEDUCTION from the cited doc and the srv3 receipt, not a swap toward production." + +data sccache_start_server_env_var: String = "SCCACHE_START_SERVER" + +data sccache_no_daemon_env_var: String = "SCCACHE_NO_DAEMON" + +data sccache_idle_timeout_env_var: String = "SCCACHE_IDLE_TIMEOUT" + +data sccache_idle_timeout_never_value_note: String = "The value whose MEANING is 'never exit', which is why it is a named row rather than a literal inside the assignment: '0' is not an ordinary duration on this axis (a zero-second idle timeout would mean the opposite), it is the cited doc's sentinel for running permanently. A future consumer that wants a bounded lifetime supplies its own seconds value; a consumer that wants a supervised daemon reads THIS row." + +data sccache_idle_timeout_never_value: String = "0" + +fn sccache_start_server_env_assignment() -> String { + join([sccache_start_server_env_var, "=1"], "") +} + +fn sccache_no_daemon_env_assignment() -> String { + join([sccache_no_daemon_env_var, "=1"], "") +} + +fn sccache_idle_timeout_never_env_assignment() -> String { + join([sccache_idle_timeout_env_var, "=", sccache_idle_timeout_never_value], "") +} + data sccache_local_facts: CacheInterfaceCatalogFacts = { identity: sccache_local_id backing_surface: "sccache-local-disk" diff --git a/dag/gunbc/host_build_cache_provision.dag b/dag/gunbc/host_build_cache_provision.dag index aace34c4402..9e80c7bf5b3 100644 --- a/dag/gunbc/host_build_cache_provision.dag +++ b/dag/gunbc/host_build_cache_provision.dag @@ -39,12 +39,15 @@ data ci_sccache_opportunistic_detect_scaffold: Disposition = Scaffold { } data build_cache_path_authority_observation_note: String = "CI and ctrl-build sessions consume the cache as bare `sccache` on PATH (RUSTC_WRAPPER=sccache), so provisioning the modeled binary is NOT sufficient — the PATH-visible sccache must BE the modeled one. A second sccache earlier in PATH (a distro package, a stale copy) is precisely the two-authority state this lane exists to remove, and it is invisible to a read-back that probes the absolute path: that read-back greens on the binary we provisioned while the compiler calls a different one. DaemonPathShadowed carries BOTH paths so the refusal is located (which binary won, which one was expected) rather than merely typed. This judgement is modeled HERE, in the verdict fold (proven by design_witness_path_shadow_refuses_fail_closed against a synthetic DaemonPathShadowed observation), not decided by the provisioning shell. Its LIVE observation is STAGED, not emitted: per the shell→dag census this module's script side is an A5 `decompose` row that must not grow its emitted shell surface, so the `command -v sccache` resolve waits for the typed-argv dissolve-on (#5828) rather than landing as hand-shell now (review 42343). The model carries the state and its refusal today; only the live producer is deferred." +data build_cache_supervision_observation_note: String = "A stats read-back CANNOT see whether the daemon it just talked to is supervised — and this is not a gap in the probe, it is the probe's semantics: any sccache invocation AUTO-STARTS a server if none is listening, so `--show-stats` manufactures the very evidence it reports. On srv3 2026-07-25 it printed a full stats table while the modeled unit had been `inactive (dead)` for six hours and no sccache process existed beforehand. Read through the verdict fold as it stood, that host observed DaemonStatsOk and therefore converged: the provisioning act would have reported success over a host with NO durable cache — a fabricated plausible output (§5), and the same shape as DaemonPathShadowed (a read-back that greens on evidence it produced rather than on the state the lane exists to establish). DaemonUnsupervised splits that conflation: 'a daemon answered' and 'the modeled supervisor is running it' are different states with different remedies, so they get different names rather than one Ok. The LIVE observation is STAGED exactly as the PATH-shadow one is — the supervisor read (`systemctl --user is-active`) is not emitted as hand-shell here, because this module's script side is an A5 `decompose` census row that may shrink but must not grow (review 42343); it lands with the typed-argv dissolve-on (#5828). The model carries the state and its refusal today; only the live producer is deferred. Construction covers the case this observation would catch on a host provisioned BY THIS LANE — build_cache_unit_daemon_runs_until_stopped makes the self-exiting unit unwritable — so the observation's standing job is the residue construction cannot reach: a host whose unit was stopped, masked, or replaced out of band." + type BuildCacheDaemonObservation = DaemonStatsOk { stats_line: String } | DaemonAbsent | DaemonUnresponsive { detail: NonEmptyStr } | DaemonMisconfigured { matched_fragment: String, detail: NonEmptyStr } | DaemonPathShadowed { resolved: String, expected: String } + | DaemonUnsupervised { unit: String, unit_state: String } type BuildCacheProvisionRefusal = RefusalDaemonAbsent @@ -53,6 +56,7 @@ type BuildCacheProvisionRefusal | RefusalPrivilegeDenied | RefusalCatalogUnknown | RefusalPathShadowed + | RefusalDaemonUnsupervised type BuildCacheProvisionVerdict = ProvisionConverged { catalog_id: CacheInterfaceId, stats_line: String } @@ -82,6 +86,11 @@ fn provision_build_cache_absorbing_fallback_widen_sketch( catalog_id: catalog_id, stats_line: concat("path-shadow-widen-sketch: ", concat(r, concat(" vs ", e))), } + DaemonUnsupervised { unit: u, unit_state: s } => + ProvisionConverged { + catalog_id: catalog_id, + stats_line: concat("unsupervised-widen-sketch: ", concat(u, concat(" is ", s))), + } } } @@ -118,6 +127,12 @@ fn build_cache_provision_gate_accepts( ProvisionConverged { catalog_id: _, stats_line: _ } => false ProvisionRefused { cause: _, reason: _ } => false } + DaemonUnsupervised { unit: _, unit_state: _ } => + match verdict { + ProvisionRefused { cause: RefusalDaemonUnsupervised, reason: _ } => true + ProvisionConverged { catalog_id: _, stats_line: _ } => false + ProvisionRefused { cause: _, reason: _ } => false + } } } @@ -158,6 +173,17 @@ fn provision_build_cache_verdict( ), ) as NonEmptyStr, } + DaemonUnsupervised { unit: u, unit_state: s } => + ProvisionRefused { + cause: RefusalDaemonUnsupervised, + reason: concat( + concat("host_build_cache_provision: sccache answers but its supervisor unit ", u), + concat( + concat(" is ", s), + "; the daemon is only alive because an invocation auto-started it, so it exits on its own idle timeout and nothing restarts it. Fail-closed, no local-build fallback.", + ), + ) as NonEmptyStr, + } } } } diff --git a/dag/gunbc/host_build_cache_provision_script.dag b/dag/gunbc/host_build_cache_provision_script.dag index c51c09ba063..dee206402d0 100644 --- a/dag/gunbc/host_build_cache_provision_script.dag +++ b/dag/gunbc/host_build_cache_provision_script.dag @@ -10,11 +10,14 @@ import extdeps.cache.sccache { sccache_installed_binary_path, sccache_install_script, sccache_release_0_15_0, + sccache_start_server_env_assignment, + sccache_no_daemon_env_assignment, + sccache_idle_timeout_never_env_assignment, } data host_build_cache_provision_script_dissolution_trigger: String = "🟡 dissolve-on: host_build_cache_provision_script — concat-built systemd/sccache ensure shell; DISSOLVES WHEN bash-emit (#5828) realizes ProvisionBuildCache observe/ensure/read-back as typed host-effect argv and realize_provision_build_cache retires shell_exec_via_bash transport." -data host_build_cache_provision_script_census_row_note: String = "CENSUS STANDING: docs/plans/shell-to-dag-residual-census-and-arc-completion.md §4.A keys this file as the builder for host_effect_realize.ProvisionBuildCache, action DECOMPOSE, sub-class A5 (srv* cluster, deferred by operator 2026-07-22 — real debt, NOT done; un-defer trigger is the operator). This site is RUNTIME-PRESENT (it realizes through host_effect_apply_gated on LocalShell/SshShell), so per that doc's routing decision its end state is a typed effect-plan with NO bash emitted — the six *_body builders below are the debt, not the design. Standing constraint while the row is deferred, stated precisely (review 42401 sharpened this): NEW emitted shell must be either a DEDUCTION or a cited-de-fork SWAP — never an unbounded, uncited addition. Two exact deltas this PR made, both accounted: (1) the sccache DE-FORK swap — `apt-get install sccache` + an `ExecStart=/usr/bin/sccache` unit went OUT, and the cited-tarball install invocation (sccache_install_script, one authority row) came IN; that is the §3 fork removal this PR exists to make, a swap that de-duplicates rather than grows. (2) `sudo -n loginctl enable-linger \"$USER\"` is genuinely NEW shell, not a swap — acknowledged here as growth rather than hidden: it is the one primitive that makes the already-selected user-scope systemd handler viable at all (a `--user` manager is torn down with the login otherwise), justified in build_cache_user_service_linger_note, and it is bounded (a single fixed leaf, not a corpus-denominated fold). What must NOT grow is the PATH-shadow OBSERVATION: the judgement went to gunbc.host_build_cache_provision (DaemonPathShadowed / RefusalPathShadowed + verdict fold, witnessed against a synthetic observation), and its live shell observation is STAGED, NOT emitted here — an earlier attempt folded `command -v sccache` lines into the two existing bodies, which kept the builder count at six but still grew the shell surface (review 42343 rejected that as a census regression). The observation lands when ProvisionBuildCache realizes as typed argv (dissolve-on #5828). Adding a *_body, or an uncited emitted-shell leaf that is neither a deduction nor a de-fork swap, is a census regression even when the behaviour it adds is correct. FOLLOW-UP #7206 delta (correctness swap, accounted): the unit's `ExecStart=... --start-server` became bare `ExecStart=...` + two `Environment=` lines (SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1) — net +1 line, and NOT uncited growth: it is a swap toward the production ctrl-sccache.service invocation (the single authority the whole lane converges on), replacing a form live execution on srv3 proved is a Type=simple no-op (build_cache_unit_foreground_supervision_note). A correctness swap toward the cited production authority is the sanctioned shape, exactly like delta (1)." +data host_build_cache_provision_script_census_row_note: String = "CENSUS STANDING: docs/plans/shell-to-dag-residual-census-and-arc-completion.md §4.A keys this file as the builder for host_effect_realize.ProvisionBuildCache, action DECOMPOSE, sub-class A5 (srv* cluster, deferred by operator 2026-07-22 — real debt, NOT done; un-defer trigger is the operator). This site is RUNTIME-PRESENT (it realizes through host_effect_apply_gated on LocalShell/SshShell), so per that doc's routing decision its end state is a typed effect-plan with NO bash emitted — the six *_body builders below are the debt, not the design. Standing constraint while the row is deferred, stated precisely (review 42401 sharpened this): NEW emitted shell must be either a DEDUCTION or a cited-de-fork SWAP — never an unbounded, uncited addition. Two exact deltas this PR made, both accounted: (1) the sccache DE-FORK swap — `apt-get install sccache` + an `ExecStart=/usr/bin/sccache` unit went OUT, and the cited-tarball install invocation (sccache_install_script, one authority row) came IN; that is the §3 fork removal this PR exists to make, a swap that de-duplicates rather than grows. (2) `sudo -n loginctl enable-linger \"$USER\"` is genuinely NEW shell, not a swap — acknowledged here as growth rather than hidden: it is the one primitive that makes the already-selected user-scope systemd handler viable at all (a `--user` manager is torn down with the login otherwise), justified in build_cache_user_service_linger_note, and it is bounded (a single fixed leaf, not a corpus-denominated fold). What must NOT grow is the PATH-shadow OBSERVATION: the judgement went to gunbc.host_build_cache_provision (DaemonPathShadowed / RefusalPathShadowed + verdict fold, witnessed against a synthetic observation), and its live shell observation is STAGED, NOT emitted here — an earlier attempt folded `command -v sccache` lines into the two existing bodies, which kept the builder count at six but still grew the shell surface (review 42343 rejected that as a census regression). The observation lands when ProvisionBuildCache realizes as typed argv (dissolve-on #5828). Adding a *_body, or an uncited emitted-shell leaf that is neither a deduction nor a de-fork swap, is a census regression even when the behaviour it adds is correct. FOLLOW-UP #7206 delta (correctness swap, accounted): the unit's `ExecStart=... --start-server` became bare `ExecStart=...` + two `Environment=` lines (SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1) — net +1 line, and NOT uncited growth: it is a swap toward the production ctrl-sccache.service invocation (the single authority the whole lane converges on), replacing a form live execution on srv3 proved is a Type=simple no-op (build_cache_unit_foreground_supervision_note). A correctness swap toward the cited production authority is the sanctioned shape, exactly like delta (1). FOLLOW-UP #7 delta (DEDUCTION, accounted): one further `Environment=` line, SCCACHE_IDLE_TIMEOUT=0, net +1 emitted line. This one is NOT a swap toward production — srv1/srv2 do not set the knob either — so it is accounted as the other sanctioned shape, a DEDUCTION: it follows from the cited upstream authority (Configuration.md, extdeps.cache.sccache.sccache_server_lifetime_note) plus the srv3 receipt that the foreground unit exits 0 after 10min 8s idle, and it is the minimal line that makes Type=simple supervision hold at all. It also SHRINKS this file's ownership of vocabulary it does not own: all three Environment values now come from extdeps rows (sccache_start_server_env_assignment / sccache_no_daemon_env_assignment / sccache_idle_timeout_never_env_assignment) instead of bare literals spelled here, so gunbc is no longer a second authority for sccache env var names (§3). The supervision OBSERVATION (`systemctl --user is-active`) is STAGED, not emitted, on exactly the PATH-shadow precedent — the judgement lives in gunbc.host_build_cache_provision (DaemonUnsupervised / RefusalDaemonUnsupervised, witnessed against a synthetic observation) and the live producer waits for typed argv (dissolve-on #5828)." data host_build_cache_provision_script_scaffold: Disposition = Scaffold { dissolves_to: SingleAuthority, @@ -37,8 +40,8 @@ fn build_cache_systemd_user_unit_body() -> String { "", "[Service]", "Type=simple", - "Environment=SCCACHE_START_SERVER=1", - "Environment=SCCACHE_NO_DAEMON=1", + join(["Environment=", sccache_start_server_env_assignment()], ""), + join(["Environment=", sccache_no_daemon_env_assignment()], ""), join(["ExecStart=", sccache_installed_binary_path], ""), "Restart=on-failure", "", @@ -184,11 +187,21 @@ data build_cache_unit_foreground_supervision_note: String = "The unit is Type=si fn build_cache_unit_is_foreground_supervised() -> Bool { let body = build_cache_systemd_user_unit_body() string_contains(s: body, pattern: "Type=simple") - && string_contains(s: body, pattern: "SCCACHE_START_SERVER=1") - && string_contains(s: body, pattern: "SCCACHE_NO_DAEMON=1") + && string_contains(s: body, pattern: sccache_start_server_env_assignment()) + && string_contains(s: body, pattern: sccache_no_daemon_env_assignment()) && !string_contains(s: body, pattern: "--start-server") } +data build_cache_unit_daemon_self_exit_note: String = "Foregrounding the server was necessary and NOT sufficient — this is the SAME defect one level deeper, found by the live read-back #7206's own note called for. sccache exits on its own after SCCACHE_IDLE_TIMEOUT seconds of no compile requests (default 600, cited in extdeps.cache.sccache.sccache_server_lifetime_note), so the foreground process under Type=simple terminates with status 0 on the first quiet ten minutes and systemd marks the unit `inactive (dead)`; Restart=on-failure does not fire, because a clean exit is not a failure. Measured on srv3 2026-07-25: `Duration: 10min 8.215s`, `Main PID: 3377939 (code=exited, status=0/SUCCESS)`, no sccache process left. The host then reverts to sccache's own per-invocation auto-start — i.e. back to the unit being a no-op, which is exactly the state the foreground form was introduced to remove. Two notes on the shape of the fix. (1) The fix is the cited env knob, not Restart=always: flipping the restart policy would make a clean exit INVISIBLE by relaunching through it, which is DESIGN §5's absorbing fallback (the arm widens instead of refusing, and the deficit's frequency drops to zero by construction). With the idle timeout disabled, a status-0 exit is unreachable by configuration, so Restart=on-failure is the honest policy and a unit found dead is a real, observable, refusable state. (2) The construction check names each required assignment through its extdeps row INDIVIDUALLY rather than folding a roster: a fold over a shared list would go vacuously green the moment the idle-timeout row were dropped from that list, since the emitter and the check would lose it together — the §5 tell of a check satisfiable by editing the declaration while the realization lies." + +fn build_cache_unit_body_runs_until_stopped(body: String) -> Bool { + string_contains(s: body, pattern: sccache_idle_timeout_never_env_assignment()) +} + +fn build_cache_unit_daemon_runs_until_stopped() -> Bool { + build_cache_unit_body_runs_until_stopped(body: build_cache_systemd_user_unit_body()) +} + fn build_cache_classifier_covers_every_cited_fragment() -> Bool { fold( sccache_misconfigured_log_fragments, diff --git a/dag/test/claim/host_build_cache_provision_design_witness_test.dag b/dag/test/claim/host_build_cache_provision_design_witness_test.dag index 335c8abccb3..bc4fe41a427 100644 --- a/dag/test/claim/host_build_cache_provision_design_witness_test.dag +++ b/dag/test/claim/host_build_cache_provision_design_witness_test.dag @@ -172,6 +172,84 @@ fn design_witness_unit_is_foreground_supervised() -> Bool { build_cache_unit_is_foreground_supervised() } +data unit_self_exiting_sketch_note: String = "RED-control fixture: the unit body EXACTLY as #7206 shipped it — foreground-supervised, and still self-exiting because sccache's default 600s idle timeout is left in place. It is a fixture, never emitted (the authority body is build_cache_systemd_user_unit_body), and it is what makes design_witness_unit_daemon_runs_until_stopped discriminating rather than tautological: the predicate is parameterized over a body, the authority body passes it, and this sketch — the shape live execution on srv3 proved dies after `Duration: 10min 8.215s` with status 0 — fails it. Without the sketch the witness would only assert that a string the same module wrote is present in the string it wrote." + +fn unit_self_exiting_sketch_body() -> String { + join([ + "[Unit]", + "Description=sccache compiler cache (gunbc host standup)", + "After=network.target", + "", + "[Service]", + "Type=simple", + "Environment=SCCACHE_START_SERVER=1", + "Environment=SCCACHE_NO_DAEMON=1", + join(["ExecStart=", sccache_installed_binary_path], ""), + "Restart=on-failure", + "", + "[Install]", + "WantedBy=default.target", + "", + ], "\n") +} + +fn design_witness_unit_daemon_runs_until_stopped() -> Bool { + build_cache_unit_daemon_runs_until_stopped() + && !build_cache_unit_body_runs_until_stopped(body: unit_self_exiting_sketch_body()) + && build_cache_unit_body_runs_until_stopped(body: build_cache_systemd_user_unit_body()) +} + +fn unsupervised_observation_fixture() -> BuildCacheDaemonObservation { + DaemonUnsupervised { unit: "sccache.service", unit_state: "inactive (dead)" } +} + +fn design_witness_unsupervised_refuses_fail_closed() -> Bool { + match provision_build_cache_verdict( + catalog_id: sccache_local_id, + observed: unsupervised_observation_fixture(), + ) { + ProvisionRefused { cause: RefusalDaemonUnsupervised, reason: reason } => + string_contains(s: reason as String, pattern: "sccache.service") + && string_contains(s: reason as String, pattern: "inactive (dead)") + && string_contains(s: reason as String, pattern: "Fail-closed") + ProvisionRefused { cause: _, reason: _ } => false + ProvisionConverged { catalog_id: _, stats_line: _ } => false + } +} + +fn design_witness_red_control_widen_on_unsupervised_rejected_by_gate() -> Bool { + match provision_build_cache_absorbing_fallback_widen_sketch( + catalog_id: sccache_local_id, + observed: unsupervised_observation_fixture(), + ) { + ProvisionConverged { catalog_id: _, stats_line: _ } => true + ProvisionRefused { cause: _, reason: _ } => false + } && !build_cache_provision_gate_accepts( + verdict: provision_build_cache_absorbing_fallback_widen_sketch( + catalog_id: sccache_local_id, + observed: unsupervised_observation_fixture(), + ), + observed: unsupervised_observation_fixture(), + ) + && build_cache_provision_gate_accepts( + verdict: provision_build_cache_verdict( + catalog_id: sccache_local_id, + observed: unsupervised_observation_fixture(), + ), + observed: unsupervised_observation_fixture(), + ) +} + +fn design_witness_supervision_observation_not_emitted_as_shell() -> Bool { + !string_contains( + s: build_cache_observe_stats_body(), + pattern: "is-active", + ) && !string_contains( + s: build_cache_read_back_body(binary: sccache_installed_binary_path), + pattern: "is-active", + ) +} + fn design_witness_classifier_covers_every_cited_fragment() -> Bool { build_cache_classifier_covers_every_cited_fragment() } @@ -246,6 +324,10 @@ test fn host_build_cache_provision_design_witnesses() -> Bool { && design_witness_install_is_cited_not_package_manager() && design_witness_unit_execstart_matches_install_path() && design_witness_unit_is_foreground_supervised() + && design_witness_unit_daemon_runs_until_stopped() + && design_witness_unsupervised_refuses_fail_closed() + && design_witness_red_control_widen_on_unsupervised_rejected_by_gate() + && design_witness_supervision_observation_not_emitted_as_shell() && design_witness_classifier_covers_every_cited_fragment() && design_witness_classifier_is_a_fold_not_an_unrolled_chain() && design_witness_fragment_words_are_shell_safe() From 601ba03af285e574aa4b9bba74f7363af9b57f93 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 25 Jul 2026 13:11:48 +0000 Subject: [PATCH 4/6] WIP: srvN subsumption --- .../host_build_cache_provision_script.dag | 5 ++-- .../srvn-buildcache-provisioning-design.md | 26 ++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/host_build_cache_provision_script.dag b/dag/gunbc/host_build_cache_provision_script.dag index dee206402d0..43ca1743eec 100644 --- a/dag/gunbc/host_build_cache_provision_script.dag +++ b/dag/gunbc/host_build_cache_provision_script.dag @@ -17,7 +17,7 @@ import extdeps.cache.sccache { data host_build_cache_provision_script_dissolution_trigger: String = "🟡 dissolve-on: host_build_cache_provision_script — concat-built systemd/sccache ensure shell; DISSOLVES WHEN bash-emit (#5828) realizes ProvisionBuildCache observe/ensure/read-back as typed host-effect argv and realize_provision_build_cache retires shell_exec_via_bash transport." -data host_build_cache_provision_script_census_row_note: String = "CENSUS STANDING: docs/plans/shell-to-dag-residual-census-and-arc-completion.md §4.A keys this file as the builder for host_effect_realize.ProvisionBuildCache, action DECOMPOSE, sub-class A5 (srv* cluster, deferred by operator 2026-07-22 — real debt, NOT done; un-defer trigger is the operator). This site is RUNTIME-PRESENT (it realizes through host_effect_apply_gated on LocalShell/SshShell), so per that doc's routing decision its end state is a typed effect-plan with NO bash emitted — the six *_body builders below are the debt, not the design. Standing constraint while the row is deferred, stated precisely (review 42401 sharpened this): NEW emitted shell must be either a DEDUCTION or a cited-de-fork SWAP — never an unbounded, uncited addition. Two exact deltas this PR made, both accounted: (1) the sccache DE-FORK swap — `apt-get install sccache` + an `ExecStart=/usr/bin/sccache` unit went OUT, and the cited-tarball install invocation (sccache_install_script, one authority row) came IN; that is the §3 fork removal this PR exists to make, a swap that de-duplicates rather than grows. (2) `sudo -n loginctl enable-linger \"$USER\"` is genuinely NEW shell, not a swap — acknowledged here as growth rather than hidden: it is the one primitive that makes the already-selected user-scope systemd handler viable at all (a `--user` manager is torn down with the login otherwise), justified in build_cache_user_service_linger_note, and it is bounded (a single fixed leaf, not a corpus-denominated fold). What must NOT grow is the PATH-shadow OBSERVATION: the judgement went to gunbc.host_build_cache_provision (DaemonPathShadowed / RefusalPathShadowed + verdict fold, witnessed against a synthetic observation), and its live shell observation is STAGED, NOT emitted here — an earlier attempt folded `command -v sccache` lines into the two existing bodies, which kept the builder count at six but still grew the shell surface (review 42343 rejected that as a census regression). The observation lands when ProvisionBuildCache realizes as typed argv (dissolve-on #5828). Adding a *_body, or an uncited emitted-shell leaf that is neither a deduction nor a de-fork swap, is a census regression even when the behaviour it adds is correct. FOLLOW-UP #7206 delta (correctness swap, accounted): the unit's `ExecStart=... --start-server` became bare `ExecStart=...` + two `Environment=` lines (SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1) — net +1 line, and NOT uncited growth: it is a swap toward the production ctrl-sccache.service invocation (the single authority the whole lane converges on), replacing a form live execution on srv3 proved is a Type=simple no-op (build_cache_unit_foreground_supervision_note). A correctness swap toward the cited production authority is the sanctioned shape, exactly like delta (1). FOLLOW-UP #7 delta (DEDUCTION, accounted): one further `Environment=` line, SCCACHE_IDLE_TIMEOUT=0, net +1 emitted line. This one is NOT a swap toward production — srv1/srv2 do not set the knob either — so it is accounted as the other sanctioned shape, a DEDUCTION: it follows from the cited upstream authority (Configuration.md, extdeps.cache.sccache.sccache_server_lifetime_note) plus the srv3 receipt that the foreground unit exits 0 after 10min 8s idle, and it is the minimal line that makes Type=simple supervision hold at all. It also SHRINKS this file's ownership of vocabulary it does not own: all three Environment values now come from extdeps rows (sccache_start_server_env_assignment / sccache_no_daemon_env_assignment / sccache_idle_timeout_never_env_assignment) instead of bare literals spelled here, so gunbc is no longer a second authority for sccache env var names (§3). The supervision OBSERVATION (`systemctl --user is-active`) is STAGED, not emitted, on exactly the PATH-shadow precedent — the judgement lives in gunbc.host_build_cache_provision (DaemonUnsupervised / RefusalDaemonUnsupervised, witnessed against a synthetic observation) and the live producer waits for typed argv (dissolve-on #5828)." +data host_build_cache_provision_script_census_row_note: String = "CENSUS STANDING: docs/plans/shell-to-dag-residual-census-and-arc-completion.md §4.A keys this file as the builder for host_effect_realize.ProvisionBuildCache, action DECOMPOSE, sub-class A5 (srv* cluster, deferred by operator 2026-07-22 — real debt, NOT done; un-defer trigger is the operator). This site is RUNTIME-PRESENT (it realizes through host_effect_apply_gated on LocalShell/SshShell), so per that doc's routing decision its end state is a typed effect-plan with NO bash emitted — the six *_body builders below are the debt, not the design. Standing constraint while the row is deferred, stated precisely (review 42401 sharpened this): NEW emitted shell must be either a DEDUCTION or a cited-de-fork SWAP — never an unbounded, uncited addition. Two exact deltas this PR made, both accounted: (1) the sccache DE-FORK swap — `apt-get install sccache` + an `ExecStart=/usr/bin/sccache` unit went OUT, and the cited-tarball install invocation (sccache_install_script, one authority row) came IN; that is the §3 fork removal this PR exists to make, a swap that de-duplicates rather than grows. (2) `sudo -n loginctl enable-linger \"$USER\"` is genuinely NEW shell, not a swap — acknowledged here as growth rather than hidden: it is the one primitive that makes the already-selected user-scope systemd handler viable at all (a `--user` manager is torn down with the login otherwise), justified in build_cache_user_service_linger_note, and it is bounded (a single fixed leaf, not a corpus-denominated fold). What must NOT grow is the PATH-shadow OBSERVATION: the judgement went to gunbc.host_build_cache_provision (DaemonPathShadowed / RefusalPathShadowed + verdict fold, witnessed against a synthetic observation), and its live shell observation is STAGED, NOT emitted here — an earlier attempt folded `command -v sccache` lines into the two existing bodies, which kept the builder count at six but still grew the shell surface (review 42343 rejected that as a census regression). The observation lands when ProvisionBuildCache realizes as typed argv (dissolve-on #5828). Adding a *_body, or an uncited emitted-shell leaf that is neither a deduction nor a de-fork swap, is a census regression even when the behaviour it adds is correct. FOLLOW-UP #7206 delta (correctness swap, accounted): the unit's `ExecStart=... --start-server` became bare `ExecStart=...` + two `Environment=` lines (SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1) — net +1 line, and NOT uncited growth: it is a swap toward the production ctrl-sccache.service invocation (the single authority the whole lane converges on), replacing a form live execution on srv3 proved is a Type=simple no-op (build_cache_unit_foreground_supervision_note). A correctness swap toward the cited production authority is the sanctioned shape, exactly like delta (1). FOLLOW-UP #7213 delta (DEDUCTION, accounted): one further `Environment=` line, SCCACHE_IDLE_TIMEOUT=0, net +1 emitted line. This one is NOT a swap toward production — srv1/srv2 do not set the knob either — so it is accounted as the other sanctioned shape, a DEDUCTION: it follows from the cited upstream authority (Configuration.md, extdeps.cache.sccache.sccache_server_lifetime_note) plus the srv3 receipt that the foreground unit exits 0 after 10min 8s idle, and it is the minimal line that makes Type=simple supervision hold at all. It also SHRINKS this file's ownership of vocabulary it does not own: all three Environment values now come from extdeps rows (sccache_start_server_env_assignment / sccache_no_daemon_env_assignment / sccache_idle_timeout_never_env_assignment) instead of bare literals spelled here, so gunbc is no longer a second authority for sccache env var names (§3). The supervision OBSERVATION (`systemctl --user is-active`) is STAGED, not emitted, on exactly the PATH-shadow precedent — the judgement lives in gunbc.host_build_cache_provision (DaemonUnsupervised / RefusalDaemonUnsupervised, witnessed against a synthetic observation) and the live producer waits for typed argv (dissolve-on #5828)." data host_build_cache_provision_script_scaffold: Disposition = Scaffold { dissolves_to: SingleAuthority, @@ -42,6 +42,7 @@ fn build_cache_systemd_user_unit_body() -> String { "Type=simple", join(["Environment=", sccache_start_server_env_assignment()], ""), join(["Environment=", sccache_no_daemon_env_assignment()], ""), + join(["Environment=", sccache_idle_timeout_never_env_assignment()], ""), join(["ExecStart=", sccache_installed_binary_path], ""), "Restart=on-failure", "", @@ -182,7 +183,7 @@ fn build_cache_unit_execstart_matches_install_path() -> Bool { ) } -data build_cache_unit_foreground_supervision_note: String = "The unit is Type=simple, so systemd tracks the ExecStart process AS the service — it MUST stay in the foreground. `sccache --start-server` DAEMONIZES and returns 0, so under Type=simple systemd sees the ExecStart exit cleanly and marks the unit `inactive (dead)` the instant it starts; the backgrounded server then idle-times-out unsupervised and Restart=on-failure never fires (the exit was success). Proven live on srv3 2026-07-25: the earlier `--start-server` form left the unit inactive with no sccache process, the daemon surviving only via sccache's own per-invocation auto-start — i.e. the unit was a no-op. This is the §5 specification-without-execution trap: the string witness that ExecStart equals the install path was green while the unit did nothing. The foreground form matches the production ctrl-sccache.service invocation (bare ExecStart + SCCACHE_START_SERVER=1 SCCACHE_NO_DAEMON=1 env, verified running month+ on srv1/srv2), reducing the §3 fork between the modeled unit and production rather than inventing a third shape. build_cache_unit_is_foreground_supervised makes the defective combo (Type=simple paired with a backgrounding --start-server) UNWRITABLE; the runtime is-active proof is the live srv3 re-provision (systemd --user is not runnable in the hermetic CI container, so the construction wall carries the CI grain and live execution carries the runtime grain)." +data build_cache_unit_foreground_supervision_note: String = "The unit is Type=simple, so systemd tracks the ExecStart process AS the service — it MUST stay in the foreground. `sccache --start-server` DAEMONIZES and returns 0, so under Type=simple systemd sees the ExecStart exit cleanly and marks the unit `inactive (dead)` the instant it starts; the backgrounded server then idle-times-out unsupervised and Restart=on-failure never fires (the exit was success). Proven live on srv3 2026-07-25: the earlier `--start-server` form left the unit inactive with no sccache process, the daemon surviving only via sccache's own per-invocation auto-start — i.e. the unit was a no-op. This is the §5 specification-without-execution trap: the string witness that ExecStart equals the install path was green while the unit did nothing. The foreground form matches the production ctrl-sccache.service invocation (bare ExecStart + SCCACHE_START_SERVER=1 SCCACHE_NO_DAEMON=1 env, verified running month+ on srv1/srv2), reducing the §3 fork between the modeled unit and production rather than inventing a third shape. build_cache_unit_is_foreground_supervised makes the defective combo (Type=simple paired with a backgrounding --start-server) UNWRITABLE; the runtime is-active proof is the live srv3 re-provision (systemd --user is not runnable in the hermetic CI container, so the construction wall carries the CI grain and live execution carries the runtime grain). CORRECTION 2026-07-25 — that runtime claim was an OVERCLAIM and is retracted as stated: the srv3 re-provision observed is-active only in the same minutes it ran, and a read-back six hours later found the unit `inactive (dead)` since 06:38:56 with no sccache process. Foreground supervision was necessary and not sufficient; the residue is sccache's own idle timeout, and the fix, the receipt and the reason Restart=always is the wrong repair are in build_cache_unit_daemon_self_exit_note. The general lesson this lane keeps re-learning: an is-active read taken at provisioning time proves the unit STARTED, never that it STAYS — the durable claim needs a read-back separated from the write by longer than every self-exit timer in the supervised process." fn build_cache_unit_is_foreground_supervised() -> Bool { let body = build_cache_systemd_user_unit_body() diff --git a/docs/plans/srvn-buildcache-provisioning-design.md b/docs/plans/srvn-buildcache-provisioning-design.md index fff1986974d..33defb76aa3 100644 --- a/docs/plans/srvn-buildcache-provisioning-design.md +++ b/docs/plans/srvn-buildcache-provisioning-design.md @@ -22,6 +22,8 @@ Status: **DESIGN ANCHOR** — operator review before implementation (2026-07-17, STEP 2 is **explicitly out of scope** for the first implementation PR. Removing the fallback before provisioning is reliable would red the whole fleet. +**STEP 2 status (2026-07-23): the cache-drop half LANDED.** `ci_retry_escalation_level2` (`-u RUSTC_WRAPPER`) is deleted — the escalation now stops at `CARGO_BUILD_JOBS=1` (keeps sccache) and then fails loud via verify-build-artifacts, so no arm drops the cache any more. The sketch `provision_build_cache_absorbing_fallback_widen_sketch` stays as the RED control guarding the verdict fold against re-introduction. Level 1 was kept as EAGAIN containment rather than converted to a refusal (§4.4's open decision, resolved toward "keep"). Still open under STEP 2: `ci_sccache_opportunistic_detect` — CI's `if sccache --show-stats` is still an opportunistic detect rather than a fail-closed require, and §5.4 Finding 2 sharpens why that flip needs a supervisor read and not a stats read to key on. + ## 2. Reuse map — single authorities, never fork (§3 DFS) | Concern | Single authority (reuse verbatim) | What's NEW | @@ -202,11 +204,33 @@ Walk of `host_standup_spine` phases that could touch compile/cache **before** P1 | **T1** | catalog authority | `catalog_id == sccache_local_id` pins to cited row | wrong id → `CatalogUnknown` | | **T2** | spine consumer | `host_standup_spine` includes `P1b:build-cache-provision` row | remove row → `host_standup_gap_count` witness fails | | **T3** | srv1 dry-run | `host_toolchain_ensure(host: srv1, kind: BuildCache)` dry-run receipt on operator host | — | -| **T4** | live read-back | post-apply `sccache --show-stats` on srv1 **and** srv2; independent of our write | hand-stop daemon → next standup run → `ProvisionRefused`, counted | +| **T4** | live read-back | **superseded — see §5.4.** A post-apply stats read cannot discharge this tier | hand-stop daemon → next standup run → `ProvisionRefused`, counted | | **T5** | CI consumer | STEP 2: build job without `RUSTC_WRAPPER` unset arm stays green for 7d on main | disable daemon on one host → build job typed refuse, not local fallback | Design-anchor PR lands **T1** only (scaffold + type witnesses). T2+ follow implementation PR. +### 5.4 T4 restated — a stats read-back cannot discharge it (2026-07-25) + +T4 was written as "post-apply `sccache --show-stats`, independent of our write." Live execution on srv3 falsified **both** halves of that, and the tier is restated here rather than quietly re-scoped. + +**Finding 1 — the unit still died, for a second reason.** #7206 foregrounded the server (`Type=simple` + bare `ExecStart` + `SCCACHE_START_SERVER=1` / `SCCACHE_NO_DAEMON=1`), which fixed the daemonize-and-exit no-op. A read-back six hours later found the unit `inactive (dead) since 06:38:56`, `Main PID … (code=exited, status=0/SUCCESS)`, `Duration: 10min 8.215s`, and **no sccache process at all**. Root cause: `SCCACHE_IDLE_TIMEOUT` defaults to 600s, so the foreground process exits *cleanly* on the first quiet ten minutes, and `Restart=on-failure` cannot fire on a success exit. Same end state as the bug #7206 fixed, one level deeper — and #7206's own claim that "the live srv3 re-provision carries the runtime is-active grain" was the overclaim that hid it, because that read was taken in the same minutes as the write. + +- **Fix:** `Environment=SCCACHE_IDLE_TIMEOUT=0` — cited to sccache `docs/Configuration.md`: *"how long the local daemon process waits for more client requests before exiting, in seconds. Set to 0 to run sccache permanently."* +- **Not `Restart=always`:** that would relaunch *through* the clean exit and make the deficit invisible — §5's absorbing fallback. With the timeout disabled a status-0 exit is unreachable by configuration, so `on-failure` is the honest policy and a dead unit stays a real, refusable state. +- **Receipt (green):** modeled unit rendered from `build_cache_systemd_user_unit_body()`, installed on srv3, `active (running)`, `NRestarts=0`, same `Main PID`, read back **13 minutes** after the write — past the 600s timer that killed the previous form. +- **Receipt (red control):** the identical unit *without* the knob — i.e. exactly what #7206 shipped — exits 0 after 10min 8s. This is the discriminating pair; the construction check `build_cache_unit_daemon_runs_until_stopped` is parameterized over a unit body so the witness runs both sides. +- **srv1/srv2 are not the counter-example they look like.** The production `ctrl-sccache.service` does not set the knob either; its month-plus uptime is CI traffic keeping the daemon below the idle threshold — accidental coverage, not a durable unit. So this line is a **deduction** from the cited doc plus the srv3 receipt, not a swap toward production. + +**Finding 2 — `--show-stats` is not an independent observation.** Any sccache invocation auto-starts a server when none is listening, so the probe manufactures the evidence it reports. On srv3 it printed a full stats table while the unit had been dead for six hours. Read through the verdict fold as it stood, that host observed `DaemonStatsOk` and **converged** — a provisioning act reporting success over a host with no durable cache (§5 fabricated plausible output, the same shape as `DaemonPathShadowed`). + +- **Model:** `DaemonUnsupervised { unit, unit_state }` → `RefusalDaemonUnsupervised`. "A daemon answered" and "the modeled supervisor is running it" are different states with different remedies, so they get different names rather than one `Ok`. +- **Live observation STAGED**, on the PATH-shadow precedent (review 42343): the `systemctl --user is-active` read is *not* emitted as hand-shell into the A5 census row; it lands with the typed-argv dissolve-on (#5828). The model carries the state and its refusal today. +- Construction covers hosts *this lane* provisions (the self-exiting unit is unwritable); the observation's standing job is the residue construction cannot reach — a unit stopped, masked or replaced out of band. + +**T4 restated.** The tier is discharged by a **delayed supervisor read**, not a post-apply stats read: after provisioning, read `systemctl --user is-active` (and the main PID) **separated from the write by longer than every self-exit timer in the supervised process**, on each subsumed host. An is-active read taken at provisioning time proves the unit *started*, never that it *stays*. + +**T4 status:** srv3 **green** at the restated bar (13-minute delayed read, modeled unit, PID stable). srv1 and srv2 remain **open** — not for a modeling reason: neither host is reachable from a session container with the fleet key today (`Permission denied (publickey)` for `ubuntu`/`briansrls`/`node-orch`), which is the A1/A2/B2 reach-and-identity gap already typed in `gunbc.plans.fleet_subsumption_manual_gaps`. The downstream triggers (`ctrl_sccache_service_retired`, and the `ci_sccache_opportunistic_detect` cutover) stay blocked on those two hosts, and the srv1/srv2 units should be expected to carry the same latent defect until re-provisioned from the model. + ## 6. Dissolution triggers (every scaffold names its exit) | Scaffold | Dissolves to | Trigger | From 1e84e113494e2f8e0594d355aee1964baa87d546 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 25 Jul 2026 13:32:54 +0000 Subject: [PATCH 5/6] WIP: srvN subsumption --- dag/extdeps/cache/sccache.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/extdeps/cache/sccache.dag b/dag/extdeps/cache/sccache.dag index 0334444934a..dbc75a4fcb6 100644 --- a/dag/extdeps/cache/sccache.dag +++ b/dag/extdeps/cache/sccache.dag @@ -150,7 +150,7 @@ data sccache_misconfigured_log_fragments: List = [ sccache_failed_to_execute_compile_log_fragment, ] -data sccache_server_lifetime_note: String = "sccache's server EXITS ON ITS OWN. Three environment knobs, all documented by the module's cited authority extdeps_external_authority_anchor (github.com/mozilla/sccache/blob/main/docs/Configuration.md), decide the process lifetime a supervisor sees, and they are modeled here because they are facts about SCCACHE — not about systemd, and not about gunbc. Before 2026-07-25 the P1b unit emitter spelled SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1 as bare literals of its own, which made gunbc.host_build_cache_provision_script a SECOND authority for sccache's environment vocabulary (§3) — and the knob that was missing from that second authority is exactly the one that broke: SCCACHE_IDLE_TIMEOUT. Cited text: 'how long the local daemon process waits for more client requests before exiting, in seconds. Set to 0 to run sccache permanently.' The default is 600 seconds, confirmed by execution rather than read off the doc: on srv3 2026-07-25 the foreground-supervised unit started at 06:28:47 UTC, served no compile request, and EXITED 0 after `Duration: 10min 8.215s`, leaving `inactive (dead)` — Restart=on-failure could not fire because the exit was a success. Consequence for any supervisor: a Type=simple unit whose ExecStart is sccache is only durable when the idle timeout is disabled; otherwise it dies on the first quiet ten minutes and the host reverts to sccache's own unsupervised per-invocation auto-start (the very no-op state the foreground form was introduced to remove). The srv1/srv2 production ctrl-sccache.service does not set this knob either — its month-plus uptime is CI traffic keeping the daemon below the idle threshold, i.e. accidental coverage, not a durable unit; that is a fact about the observed hosts, so this row is a DEDUCTION from the cited doc and the srv3 receipt, not a swap toward production." +data sccache_server_lifetime_note: String = "sccache's server EXITS ON ITS OWN. Three environment knobs, all documented by the module's cited authority extdeps_external_authority_anchor (github.com/mozilla/sccache/blob/main/docs/Configuration.md), decide the process lifetime a supervisor sees, and they are modeled here because they are facts about SCCACHE — not about systemd, and not about gunbc. Before 2026-07-25 the P1b unit emitter spelled SCCACHE_START_SERVER=1 / SCCACHE_NO_DAEMON=1 as bare literals of its own, which made gunbc.host_build_cache_provision_script a SECOND authority for sccache's environment vocabulary (§3) — and the knob that was missing from that second authority is exactly the one that broke: SCCACHE_IDLE_TIMEOUT. Cited text: 'how long the local daemon process waits for more client requests before exiting, in seconds. Set to 0 to run sccache permanently.' The default is 600 seconds, confirmed by execution rather than read off the doc: on srv3 2026-07-25 the foreground-supervised unit started at 06:28:47 UTC, served no compile request, and EXITED 0 after `Duration: 10min 8.215s`, leaving `inactive (dead)` — Restart=on-failure could not fire because the exit was a success. Consequence for any supervisor: a Type=simple unit whose ExecStart is sccache is only durable when the idle timeout is disabled; otherwise it dies on the first quiet ten minutes and the host reverts to sccache's own unsupervised per-invocation auto-start (the very no-op state the foreground form was introduced to remove). Whether the srv1/srv2 production ctrl-sccache.service sets this knob is UNVERIFIED — neither host is reachable from a session container today, and the only in-tree description of that unit (#7206's note) lists SCCACHE_START_SERVER and SCCACHE_NO_DAEMON and no idle-timeout line, which is an absence in a summary, not evidence of absence in the file. What its month-plus uptime does NOT establish is that the unit is durable: those hosts run CI continuously, so traffic keeps the daemon below the idle threshold whether or not the timer is disabled — the uptime is consistent with both. This row therefore stands on the cited doc plus the srv3 receipt as a DEDUCTION; it is explicitly NOT claimed as a swap toward a production form nobody has read." data sccache_start_server_env_var: String = "SCCACHE_START_SERVER" From b29d4f7ff9e48e37968f546cd89723b6c53f852c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 25 Jul 2026 13:35:50 +0000 Subject: [PATCH 6/6] srvN build-cache: design-doc receipts for the idle-timeout finding --- docs/plans/srvn-buildcache-provisioning-design.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/plans/srvn-buildcache-provisioning-design.md b/docs/plans/srvn-buildcache-provisioning-design.md index 33defb76aa3..00ecfd64839 100644 --- a/docs/plans/srvn-buildcache-provisioning-design.md +++ b/docs/plans/srvn-buildcache-provisioning-design.md @@ -217,9 +217,12 @@ T4 was written as "post-apply `sccache --show-stats`, independent of our write." - **Fix:** `Environment=SCCACHE_IDLE_TIMEOUT=0` — cited to sccache `docs/Configuration.md`: *"how long the local daemon process waits for more client requests before exiting, in seconds. Set to 0 to run sccache permanently."* - **Not `Restart=always`:** that would relaunch *through* the clean exit and make the deficit invisible — §5's absorbing fallback. With the timeout disabled a status-0 exit is unreachable by configuration, so `on-failure` is the honest policy and a dead unit stays a real, refusable state. -- **Receipt (green):** modeled unit rendered from `build_cache_systemd_user_unit_body()`, installed on srv3, `active (running)`, `NRestarts=0`, same `Main PID`, read back **13 minutes** after the write — past the 600s timer that killed the previous form. -- **Receipt (red control):** the identical unit *without* the knob — i.e. exactly what #7206 shipped — exits 0 after 10min 8s. This is the discriminating pair; the construction check `build_cache_unit_daemon_runs_until_stopped` is parameterized over a unit body so the witness runs both sides. -- **srv1/srv2 are not the counter-example they look like.** The production `ctrl-sccache.service` does not set the knob either; its month-plus uptime is CI traffic keeping the daemon below the idle threshold — accidental coverage, not a durable unit. So this line is a **deduction** from the cited doc plus the srv3 receipt, not a swap toward production. +- **Receipt — controlled A/B under guaranteed idleness (srv3, 2026-07-25 13:20:29 → 13:32:36).** Two sccache 0.15.0 servers from the same binary, started in the same second, each with a private `SCCACHE_DIR` and a non-standard `SCCACHE_SERVER_PORT` (4301 / 4302) so no client could reach either — idleness by construction, not by hoping the host is quiet. One variable differs. + - **A**, default idle timeout: **gone** at 12:10 elapsed. Exited on its own. + - **B**, `SCCACHE_IDLE_TIMEOUT=0`: **still running** at 12:10, same PID. +- **Receipt — the modeled unit itself.** `build_cache_systemd_user_unit_body()` rendered from the `.dag`, installed on srv3 with the probe drop-in removed, `active (running)` / `NRestarts=0` / stable `Main PID 3678958` at +13min and +26min. Stated with its limit: srv3 is a live CI runner and a job hit the daemon during that window, so this receipt shows the modeled artifact deploys and stays up — it is **not** an idle test. The A/B pair above is what discharges the idle claim, and it is the one to re-run when the knob changes. +- **Receipt (red control), same host, prior form:** the unit exactly as #7206 shipped it — `Duration: 10min 8.215s`, `code=exited, status=0/SUCCESS`, `inactive (dead)`, no sccache process. The construction check `build_cache_unit_daemon_runs_until_stopped` is parameterized over a unit body so the witness runs both sides of this pair, and dropping the knob from the emitter reds it (verified by perturbation). +- **srv1/srv2 are not the counter-example they look like — and their unit is unread.** Whether production's `ctrl-sccache.service` sets the knob is **unverified**: neither host is reachable from a session container today, and the only in-tree description of that unit (#7206's note) lists two env lines and no idle timeout — an absence in a summary, not evidence of absence in the file. What the month-plus uptime does *not* establish is durability: those hosts run CI continuously, so traffic holds the daemon below the idle threshold whether or not the timer is disabled. The line therefore lands as a **deduction** from the cited doc plus the A/B receipt, and is explicitly *not* claimed as a swap toward a production form nobody has read. **Finding 2 — `--show-stats` is not an independent observation.** Any sccache invocation auto-starts a server when none is listening, so the probe manufactures the evidence it reports. On srv3 it printed a full stats table while the unit had been dead for six hours. Read through the verdict fold as it stood, that host observed `DaemonStatsOk` and **converged** — a provisioning act reporting success over a host with no durable cache (§5 fabricated plausible output, the same shape as `DaemonPathShadowed`).