diff --git a/dag/extdeps/crypto/hash.dag b/dag/extdeps/crypto/hash.dag index 09560c805de..7ae1a193376 100644 --- a/dag/extdeps/crypto/hash.dag +++ b/dag/extdeps/crypto/hash.dag @@ -1,6 +1,7 @@ module extdeps.crypto.hash -import std.types { NonEmptyStr, String } +import std.types { NonEmptyStr, String, Bool, List } +import std.algebra { trim } import std.content_hash { as_content_hash_cryptographic, as_content_hash_sha512, @@ -40,7 +41,7 @@ fn sha512_digest(hex: NonEmptyStr) -> Digest { Digest { algorithm: Sha512, hex: hex } } -data digest_shell_verify_line_legacy_note: NonEmptyStr = "LEGACY SHA-256-only shell-string verifier for the existing sccache consumer. Not a package-delivery realization. SHA-512 intentionally has no production shell arm here — gunbc.package_delivery uses PackageArchiveDigestObservation. dissolve-on: sccache migrates off this helper and the function deletes." +data digest_shell_verify_line_legacy_note: NonEmptyStr = "SHA-256 shell-string verifier: `echo | sha256sum -c -` exits nonzero when the file at path does not hash to the digest, which under set -e stops the line. Two consumers: the sccache pin and the live-deploy executable install, which verifies the SOURCE against the planned digest before copying and the DESTINATION after (gunbc.live_deploy.emit emit_release_member_effects) so an installed executable can never differ from the identity the plan declared converged. Not a package-delivery realization: SHA-512 intentionally has no production shell arm here — gunbc.package_delivery uses PackageArchiveDigestObservation." fn digest_shell_verify_line(digest: Digest, file_path: NonEmptyStr) -> String { match digest.algorithm { @@ -80,3 +81,39 @@ fn sha512_digest_content_hash(digest: Digest) -> ContentHash? { Sha256 => none } } + +// coreutils sha256sum over one file: ` ` on stdout, exit 0. The operation is the LOCAL +// realization; sha256sum_argv is the same invocation spelled for a remote typed-argv transport, and +// sha256sum_line_digest reads the one output line for both arms. The two spellings of the argv are +// the extdeps op-plus-builder seam every shell-transported operation in this tree currently carries +// (extdeps.systemd systemd_parse_label_derivation_debt records the same debt); they fold when the +// transport row can be read from the operation itself. +service crypto.Sha256Sum { + operation File { + input { path: String } + output { + line: String from "stdout" + success: Bool from "exit_success" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["sha256sum", "--", "{path}"] } + exit { + 0 => Unit + nonzero => String "sha256sum failed" + } + mock_response { + 0 => { line: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 {path}", success: true, stderr: "" } "hermetic crypto.Sha256Sum.File: the empty-input digest" + } + } +} + +fn sha256sum_argv(path: String) -> List { + ["sha256sum", "--", path] +} + +// The first field of the output line, admitted only at the digest's exact length. +fn sha256sum_line_digest(line: String) -> Digest? { + let field = fold(split(s: trim(s: line), delimiter: " "), init: "", f: (acc, w) => if acc == "" { w } else { acc }) + if field.length() == 64 { Present { value: sha256_digest(hex: field as NonEmptyStr) } } else { none } +} diff --git a/dag/extdeps/git/inspect.dag b/dag/extdeps/git/inspect.dag index 9963e27ded8..3854323ed25 100644 --- a/dag/extdeps/git/inspect.dag +++ b/dag/extdeps/git/inspect.dag @@ -154,6 +154,13 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { // SP SP TAB (` \t`), which is what makes a path containing a newline, a // quote or a tab-free space unambiguous -- git's own -z contract, not a convention this repository // invents. +// git.Inspect.ReadTreeIntoIndex IS NOT readonly, AND THE THREE NEIGHBOURS AROUND IT ARE. read-tree +// WRITES the index file GIT_INDEX_FILE names; the repository's own index is untouched, which is why +// StatusAgainstIndex and IgnoredAgainstIndex stay readonly, but the scratch index is a file the +// operation creates on the host and a consumer owns its lifetime -- a unique path (extdeps.shell +// shell.Mktemp) and a removal after the reads. Declaring the write readonly was the state this +// module was in when review found a consumer using one fixed path under /tmp per revision, shared by +// every observer of that release and never removed (PR #10696). service git.Inspect { operation Toplevel { input {} @@ -201,6 +208,76 @@ service git.Inspect { } } + operation HeadCommitIn { + input { repository_path: String } + output { + sha: String from "stdout" + success: Bool from "exit_success" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["git", "-C", "{repository_path}", "rev-parse", "HEAD"] } + exit { + 0 => Unit + 128 => String "Not a git repository" + } + mock_response { + 0 => { sha: "0000000000000000000000000000000000000000", success: true, stderr: "" } "hermetic git.Inspect.HeadCommitIn" + } + } + + operation ReadTreeIntoIndex { + input { repository_path: String, revision_hex: String, index_path: String } + output { + success: Bool from "exit_success" + stderr: String from "stderr" + } + transport shell { argv: ["env", "GIT_INDEX_FILE={index_path}", "git", "-C", "{repository_path}", "read-tree", "--no-sparse-checkout", "{revision_hex}"] } + exit { + 0 => Unit + 128 => String "Not a git repository" + } + mock_response { + 0 => { success: true, stderr: "" } "hermetic git.Inspect.ReadTreeIntoIndex" + } + } + + operation StatusAgainstIndex { + input { repository_path: String, index_path: String } + output { + entries_nul: String from "stdout" + success: Bool from "exit_success" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["env", "GIT_INDEX_FILE={index_path}", "git", "-C", "{repository_path}", "--no-optional-locks", "status", "--porcelain=v1", "-z", "--untracked-files=all"] } + exit { + 0 => Unit + 128 => String "Not a git repository" + } + mock_response { + 0 => { entries_nul: "", success: true, stderr: "" } "hermetic git.Inspect.StatusAgainstIndex: clean" + } + } + + operation IgnoredAgainstIndex { + input { repository_path: String, index_path: String } + output { + paths_nul: String from "stdout" + success: Bool from "exit_success" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["env", "GIT_INDEX_FILE={index_path}", "git", "-C", "{repository_path}", "ls-files", "--others", "--ignored", "--exclude-standard", "-z"] } + exit { + 0 => Unit + 128 => String "Not a git repository" + } + mock_response { + 0 => { paths_nul: "", success: true, stderr: "" } "hermetic git.Inspect.IgnoredAgainstIndex: none" + } + } + operation IgnoredFiles { input {} output { diff --git a/dag/extdeps/systemd/systemd.dag b/dag/extdeps/systemd/systemd.dag index b5a28013480..a75580c6904 100644 --- a/dag/extdeps/systemd/systemd.dag +++ b/dag/extdeps/systemd/systemd.dag @@ -71,6 +71,7 @@ type SystemdUnitProperty | NextElapseUSecMonotonic | AccuracyUSec | TimersMonotonic + | NeedDaemonReload fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr { match property { @@ -101,6 +102,7 @@ fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr { NextElapseUSecMonotonic => "NextElapseUSecMonotonic" as NonEmptyStr AccuracyUSec => "AccuracyUSec" as NonEmptyStr TimersMonotonic => "TimersMonotonic" as NonEmptyStr + NeedDaemonReload => "NeedDaemonReload" as NonEmptyStr } } diff --git a/dag/gunbc/host/host_effect.dag b/dag/gunbc/host/host_effect.dag index c23b1eff2fa..3ff9c7c36c2 100644 --- a/dag/gunbc/host/host_effect.dag +++ b/dag/gunbc/host/host_effect.dag @@ -24,6 +24,7 @@ import gunbc.runner_incarnation { LocalRunnerSlotIncarnation } import gunbc.runner_lifecycle { RunnerReplacementCause } import gunbc.auth.github_apps { gunbai_ci_declared } import gunbc.live_deploy.candidate { CandidateRelease } +import gunbc.live_deploy.member_identity { ReleaseEffectPlan } import std.effects { EffectShape, KeySource, @@ -62,7 +63,7 @@ type HostEffect = | Srv3BootOnceReadBackSleep | Srv3ReceiptEmit { lines: List } | HostIdentityShortHostnameRead - | LiveDeployApply { spec: DeploymentSpec, candidate: CandidateRelease } + | LiveDeployApply { spec: DeploymentSpec, candidate: CandidateRelease, plan: ReleaseEffectPlan } | LiveDeployRetract { spec: DeploymentSpec } | LiveDeployEnsureDependency { dep: DeploymentDependencyStep } | LiveDeployDigestReadback { port: Int } diff --git a/dag/gunbc/host/host_effect_realize.dag b/dag/gunbc/host/host_effect_realize.dag index 7b751920334..2adf64ab6c4 100644 --- a/dag/gunbc/host/host_effect_realize.dag +++ b/dag/gunbc/host/host_effect_realize.dag @@ -148,6 +148,7 @@ import gunbc.ci_deploy_access { deploy_access_sudo_probe_effect, } import gunbc.live_deploy.spec { DeploymentSpec, DeploymentDependencyStep } +import gunbc.live_deploy.member_identity { ReleaseEffectPlan } import gunbc.live_deploy.emit { live_deploy_apply_script_for, live_deploy_retract_script_for, @@ -403,7 +404,7 @@ type ResolvedHostEffectCell = | EffectivePrincipalReadOnHost { node: ComputeHost, read: EffectivePosixPrincipalRead } | SudoNopasswdExecuteProbeOnHost { node: ComputeHost, probe: SudoNopasswdExecuteProbeShape } | SudoNopasswdGrantListProbeOnHost { node: ComputeHost, probe: SudoNopasswdGrantListProbeShape } - | LiveDeployApplyOnHost { node: ComputeHost, spec: DeploymentSpec, candidate: CandidateRelease } + | LiveDeployApplyOnHost { node: ComputeHost, spec: DeploymentSpec, candidate: CandidateRelease, plan: ReleaseEffectPlan } | LiveDeployRetractOnHost { node: ComputeHost, spec: DeploymentSpec } | LiveDeployEnsureDependencyOnHost { node: ComputeHost, dep: DeploymentDependencyStep } | LiveDeployDigestReadbackOnHost { node: ComputeHost, port: Int } @@ -468,8 +469,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res BmcwebSessionLoginOnHost { node: n } Srv3BmcwebTokenExtract => BmcwebTokenExtractOnHost { node: n } - LiveDeployApply { spec: s, candidate: c } => - LiveDeployApplyOnHost { node: n, spec: s, candidate: c } + LiveDeployApply { spec: s, candidate: c, plan: p } => + LiveDeployApplyOnHost { node: n, spec: s, candidate: c, plan: p } LiveDeployRetract { spec: s } => LiveDeployRetractOnHost { node: n, spec: s } LiveDeployEnsureDependency { dep: d } => @@ -522,8 +523,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res CodexSupervisedWorkerTurnOnHost { node: n, intent: i, lease_key: k } HostIdentityShortHostnameRead => HostnameReadOnHost { node: n } - LiveDeployApply { spec: s, candidate: c } => - LiveDeployApplyOnHost { node: n, spec: s, candidate: c } + LiveDeployApply { spec: s, candidate: c, plan: p } => + LiveDeployApplyOnHost { node: n, spec: s, candidate: c, plan: p } LiveDeployRetract { spec: s } => LiveDeployRetractOnHost { node: n, spec: s } LiveDeployEnsureDependency { dep: d } => @@ -611,7 +612,7 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res IncompatibleCell { reason: "host_effect: SudoNopasswdGrantListProbe is in-band (typed sudo grant-list probe) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } HostIdentityShortHostnameRead => IncompatibleCell { reason: "host_effect: HostIdentityShortHostnameRead is in-band (shell hostname -s) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } - LiveDeployApply { spec: _, candidate: _ } => + LiveDeployApply { spec: _, candidate: _, plan: _ } => IncompatibleCell { reason: "host_effect: LiveDeployApply is in-band (actuator host shell mutation) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } LiveDeployRetract { spec: _ } => IncompatibleCell { reason: "host_effect: LiveDeployRetract is in-band (actuator host shell mutation) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } @@ -3785,6 +3786,7 @@ fn live_deploy_apply_candidate_locus_refusal(transport: HostEffectTransport) -> fn realize_live_deploy_apply_on_host( spec: DeploymentSpec, candidate: CandidateRelease, + plan: ReleaseEffectPlan, transport: HostEffectTransport, intent: HostEffectIntent, ) -> Reconciliation { @@ -3794,10 +3796,7 @@ fn realize_live_deploy_apply_on_host( } else { realize_shell_on_host( script: retained_srvn( - body: live_deploy_apply_script_for( - spec: spec, - revision: RevisionBoundAtEmission { revision: candidate.revision }, - ), + body: live_deploy_apply_script_for(spec: spec, plan: plan), reason: "live_deploy apply (unit install + start)", ), transport: transport, @@ -3895,8 +3894,8 @@ fn host_effect_apply(target: NodeControlPlane, effect: HostEffect, evidence: Ide realize_sudo_nopasswd_execute_probe_on_host(probe: p, transport: transport, intent: intent) SudoNopasswdGrantListProbeOnHost { node: _, probe: p } => realize_sudo_nopasswd_grant_list_probe_on_host(probe: p, transport: transport, intent: intent) - LiveDeployApplyOnHost { node: _, spec: s, candidate: c } => - realize_live_deploy_apply_on_host(spec: s, candidate: c, transport: transport, intent: intent) + LiveDeployApplyOnHost { node: _, spec: s, candidate: c, plan: p } => + realize_live_deploy_apply_on_host(spec: s, candidate: c, plan: p, transport: transport, intent: intent) LiveDeployRetractOnHost { node: _, spec: s } => realize_live_deploy_retract_on_host(spec: s, transport: transport, intent: intent) LiveDeployEnsureDependencyOnHost { node: _, dep: d } => @@ -4094,8 +4093,8 @@ fn host_effect_apply_gated( realize_sudo_nopasswd_execute_probe_on_host(probe: p, transport: transport, intent: intent) SudoNopasswdGrantListProbeOnHost { node: _, probe: p } => realize_sudo_nopasswd_grant_list_probe_on_host(probe: p, transport: transport, intent: intent) - LiveDeployApplyOnHost { node: _, spec: s, candidate: c } => - realize_live_deploy_apply_on_host(spec: s, candidate: c, transport: transport, intent: intent) + LiveDeployApplyOnHost { node: _, spec: s, candidate: c, plan: p } => + realize_live_deploy_apply_on_host(spec: s, candidate: c, plan: p, transport: transport, intent: intent) LiveDeployRetractOnHost { node: _, spec: s } => realize_live_deploy_retract_on_host(spec: s, transport: transport, intent: intent) LiveDeployEnsureDependencyOnHost { node: _, dep: d } => @@ -4177,7 +4176,7 @@ fn host_effect_drive_directive(r: Reconciliation not_converged_directive(drive: host_effect_drive(intent: intent)) SudoNopasswdExecuteProbeOnHost { node: _, probe: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) SudoNopasswdGrantListProbeOnHost { node: _, probe: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) - LiveDeployApplyOnHost { node: _, spec: _, candidate: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) + LiveDeployApplyOnHost { node: _, spec: _, candidate: _, plan: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) LiveDeployRetractOnHost { node: _, spec: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) LiveDeployEnsureDependencyOnHost { node: _, dep: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) LiveDeployDigestReadbackOnHost { node: _, port: _ } => not_converged_directive(drive: host_effect_drive(intent: intent)) diff --git a/dag/gunbc/live_deploy/apply.dag b/dag/gunbc/live_deploy/apply.dag index 441e2ee0aec..8284cc2ef9f 100644 --- a/dag/gunbc/live_deploy/apply.dag +++ b/dag/gunbc/live_deploy/apply.dag @@ -58,6 +58,8 @@ import gunbc.host_effect { DeployAccessPreflight, } import gunbc.host_effect_realize { host_effect_apply_gated } +import gunbc.live_deploy.member_observe { deployment_effect_plan, deployment_identity_readback, IdentityReadbackConverged, IdentityReadbackDrift, IdentityReadbackRefused } +import gunbc.live_deploy.member_identity { EffectsDerived, EffectsRefused, decision_label } import gunbc.live_deploy.readiness { live_deploy_poll_until_service_ready_for_deploy, live_deploy_healthz_get_for_port, running_release_unidentified_detail, ServiceReadyApplyEffect, @@ -220,20 +222,31 @@ fn live_deploy_ensure_dependency( ) } +// THE PLAN IS DERIVED FROM THE HOST BEFORE THE MUTATION IS BUILT (gunbc.live_deploy.member_observe): +// the same evaluation that decides also acts, and a plan that could not be derived refuses here with +// every member's cause, installing nothing -- never the whole roster as a fallback. fn live_deploy_apply_mutation_via_transport( spec: DeploymentSpec, candidate: CandidateRelease, transport: HostEffectTransport, access: DeployAccess ) -> Reconciliation { - live_deploy_fold( - spec: spec, - effect: LiveDeployApply { spec: spec, candidate: candidate }, - transport: transport, - access: access, - fixture_actor: Absent, - fixture_host: Absent - ) + match deployment_effect_plan(spec: spec, candidate: candidate, transport: transport) { + EffectsRefused { refusals } => + NotConverged { + reason: join(["live_deploy: the deployment plan could not be derived from the host: ", join(map(refusals, d => decision_label(d: d)), "; ")], ""), + applied: live_deploy_preflight_intent(access: access), + } + EffectsDerived { plan } => + live_deploy_fold( + spec: spec, + effect: LiveDeployApply { spec: spec, candidate: candidate, plan: plan }, + transport: transport, + access: access, + fixture_actor: Absent, + fixture_host: Absent + ) + } } fn live_deploy_apply_readiness_gate( @@ -293,6 +306,38 @@ fn live_deploy_apply_digest_readback_via_transport( ) } +// THE MEMBER IDENTITIES ARE READ BACK AFTER READINESS, and readiness is not a substitute for it. +// Readiness proves the unit answers /healthz as the candidate; an equivalent binary reporting the +// same revision, or a unit file rewritten after the copy, would pass it. The plan was derived from +// the host's member identities, so the transaction closes by observing them again +// (gunbc.live_deploy.member_observe deployment_identity_readback) and requiring every member +// AlreadyConverged -- a remaining mutation names which member the apply failed to land, and a refusal +// names which member could not be read afterwards. Both are NotConverged, never a green with a note. +fn live_deploy_apply_identity_readback_gate( + spec: DeploymentSpec, + candidate: CandidateRelease, + transport: HostEffectTransport, + mutation: Reconciliation, +) -> Reconciliation { + let intent = match mutation { + Converged { evidence: _, applied: i } => i + NotConverged { reason: _, applied: i } => i + } + match deployment_identity_readback(spec: spec, candidate: candidate, transport: transport) { + IdentityReadbackConverged => mutation + IdentityReadbackDrift { remaining } => + NotConverged { + reason: join(["live_deploy: the apply completed and readiness held, but the host's member identities still differ from the release: ", join(remaining, "; ")], ""), + applied: intent, + } + IdentityReadbackRefused { refusals } => + NotConverged { + reason: join(["live_deploy: the apply completed but the member identities could not be read back: ", join(map(refusals, d => decision_label(d: d)), "; ")], ""), + applied: intent, + } + } +} + fn live_deploy_apply_via_transport( spec: DeploymentSpec, candidate: CandidateRelease, @@ -321,7 +366,7 @@ fn live_deploy_apply_via_transport( access: access, ) if reconciliation_converged(r: digest) { - gated + live_deploy_apply_identity_readback_gate(spec: spec, candidate: candidate, transport: transport, mutation: gated) } else { match digest { NotConverged { reason: why, applied: _ } => diff --git a/dag/gunbc/live_deploy/emit.dag b/dag/gunbc/live_deploy/emit.dag index cebeccab53d..1b8eea22bfb 100644 --- a/dag/gunbc/live_deploy/emit.dag +++ b/dag/gunbc/live_deploy/emit.dag @@ -57,6 +57,11 @@ import extdeps.systemd.unit_file { SystemdInstallDirective, WantedBy, serialize_systemd_unit_file, } +import gunbc.live_deploy.member_identity { + ReleaseEffectPlan, ConvergedMember, TreePublication, Executable, UnitFile, UnitManager, ServiceProcess, + plan_installs_executable, plan_publishes_tree, plan_writes_unit, plan_reloads_manager, plan_restarts_process, +} +import extdeps.crypto.hash { digest_shell_verify_line } import gunbc.live_deploy.spec { deployment_slice_unit_name, deployment_slice_unit_path, DeploymentSpec, @@ -831,10 +836,34 @@ fn deployment_step_ownership_opt(step: DeploymentStep) -> Ownership? { Present { value: deployment_step_ownership(step: step) } } +// THE RELEASE-CONSUMING MEMBERS LEAVE THE APPLY-ALL ROSTER. Their desired/observed identities are +// decided in gunbc.live_deploy.member_identity and their mutations arrive as a ReleaseEffectPlan the +// caller derived by observing the host (gunbc.live_deploy.member_observe), so an already-converged +// host installs none of them. Every other owned artifact still reconciles against `observed: []` +// below -- the declared frontier, one member kind at a time, never a silent skip. Three deployment +// steps carry the five members: the unit step is where the unit file is written AND where the +// manager reloads and the process restarts, because those two members have no artifact of their own +// on the host and the deployment order already places the unit after the roots its process writes. +fn identity_member_of_step(step: DeploymentStep) -> ConvergedMember? { + match step { + ArtifactStep { step: art } => match art.kind { + GunbcSourceTree => Present { value: TreePublication } + ServeBinary => Present { value: Executable } + SystemdUnit => Present { value: UnitFile } + _ => none + } + Dependency { step: _ } => none + } +} + +fn identity_member_step(step: DeploymentStep) -> Bool { + match identity_member_of_step(step: step) { Present { value: _ } => true Absent => false } +} + fn deployment_apply_plan(spec: DeploymentSpec) -> MembershipPlan { membership_plan_admitting_outcome( outcome: membership_reconcile( - desired: spec.steps, + desired: filter(spec.steps, s => !identity_member_step(step: s)), observed: [], key_of: deployment_step_key, key_eq: deployment_step_key_eq, @@ -1234,6 +1263,120 @@ fn deploy_memory_cap_apply_steps(spec: DeploymentSpec) -> List { // // The in-place-write residue this member leaves is carried by // belt_receipt_in_place_write_residue_dissolution rather than by this paragraph. +// ONE MEMBER, ITS FRAGMENTS, IN THE DEPLOYMENT'S ORDER. These are the SAME fragments the apply-all +// arms below emit for the identity members, regrouped by the member that needs them and emitted at +// that member's position in spec.steps (apply_intent_from_effects), never prepended to the whole +// script: the first cut emitted every release mutation before every remaining membership effect, so +// on a fresh or partially repaired host the new process was started before the roots it writes +// existed and before the ensured packages were present (review on PR #10696). +// +// Publishing the tree is the repository-convergence unit and its oneshot (the rsync-era tree-sync +// unit that GunbcSourceTree also wrote is not re-emitted here -- the convergence unit overwrote it in +// the same script, so it was never the document the host ran); installing the executable is the +// install and the receipts directory; writing the unit is the render and install. The manager and +// process members have no artifact of their own and ride the unit step: daemon-reload once, iff the +// plan says the manager is stale or the file changed, then the restart that closes the transaction. +// +// THE PLANNED IDENTITY BINDS THE ACTUATION. The executable copied is the file at the source path, a +// path that is mutable between planning and copying; so the source is verified against the plan's +// digest immediately before the copy and the destination immediately after, each under set -e. A +// build that changed in between, or a copy that did not land, stops the line -- an installed +// executable can never differ from the identity the plan declared converged. The unit renders from +// the plan's own revision, the one carrier of that fact. +fn emit_release_member_effects(member: ConvergedMember, plan: ReleaseEffectPlan, spec: DeploymentSpec) -> List { + let revision = RevisionBoundAtEmission { revision: plan.target.revision } + match member { + TreePublication => if !plan_publishes_tree(plan: plan) { [] } else { [ + deploy_raw(command: install_directory_command(mode: install_d_shared_mode, path: "/opt/gunbc", privileged: true)), + deploy_raw(command: install_d_owned_command(spec: spec, path: spec.service.repo_root as String)), + apply_tree_sync_unit_write_step(spec: spec, tree_path: spec.service.repo_root as String), + deploy_raw(command: deploy_stage_install_command( + stage_name: "gunbc-tree-sync.service", + mode: install_file_shared_mode, + dest: tree_sync_unit_path(names: spec.names) as String, + )), + deploy_raw(command: deploy_stage_write_expansion_command( + stage_name: "gunbc-tree-sync.env", + format: "GUNBC_TREE_SRC=%s\n", + expansion_var: "PWD", + )), + deploy_raw(command: deploy_stage_install_command( + stage_name: "gunbc-tree-sync.env", + mode: install_file_shared_mode, + dest: spec.names.tree_sync_env_path as String, + )), + deploy_raw(command: deploy_stage_write_command( + stage_name: deployment_slice_unit_name(names: spec.names) as String, + content: render(doc: emit_slice_unit_doc(spec: spec), proto: live_deploy_protocol), + )), + deploy_raw(command: deploy_stage_install_command( + stage_name: deployment_slice_unit_name(names: spec.names) as String, + mode: install_file_shared_mode, + dest: deployment_slice_unit_path(names: spec.names) as String, + )), + deploy_effect(inv: systemctl_daemon_reload_effect()), + tree_sync_restart_step_with_diagnosis(spec: spec), + deploy_raw(command: install_d_owned_command( + spec: spec, + path: instance_receipts_dir(instance_root: spec.service.instance_root as String), + )), + apply_repository_convergence_unit_write_step(spec: spec, revision: revision), + deploy_raw(command: deploy_stage_install_command( + stage_name: spec.names.tree_sync_unit_name as String, + mode: install_file_shared_mode, + dest: tree_sync_unit_path(names: spec.names) as String, + )), + deploy_effect(inv: systemctl_daemon_reload_effect()), + tree_sync_restart_step_with_diagnosis(spec: spec), + deploy_raw(command: repository_convergence_readback_command(spec: spec)), + ] } + Executable => if !plan_installs_executable(plan: plan) { [] } else { [ + deploy_raw(command: digest_shell_verify_line(digest: plan.target.executable, file_path: release_executable_install_source as NonEmptyStr)), + deploy_raw(command: install_directory_command( + mode: install_d_shared_mode, + path: path_parent_dir(path: spec.service.serve_binary as String), + privileged: true, + )), + deploy_raw(command: install_file_owned_command( + mode: install_executable_shared_mode, + owner: spec.service.service_user, + group: spec.service.service_user, + source: release_executable_install_source, + dest: spec.service.serve_binary as String, + )), + deploy_raw(command: digest_shell_verify_line(digest: plan.target.executable, file_path: spec.service.serve_binary)), + deploy_raw(command: install_d_owned_command( + spec: spec, + path: instance_receipts_dir(instance_root: spec.service.instance_root as String), + )), + ] } + UnitFile => concat( + if !plan_writes_unit(plan: plan) { [] } else { [ + apply_systemd_unit_write_step(spec: spec, revision: revision), + deploy_raw(command: deploy_stage_install_command( + stage_name: spec.service.unit_name as String, + mode: install_file_shared_mode, + dest: deployment_systemd_unit_path(names: spec.names) as String, + )), + ] }, + concat( + if !plan_reloads_manager(plan: plan) { [] } else { [deploy_effect(inv: systemctl_daemon_reload_effect())] }, + if !plan_restarts_process(plan: plan) { [] } else { [ + deploy_effect(inv: systemctl_enable_effect(unit: plan.target.unit)), + deploy_effect(inv: systemctl_restart_effect(unit: plan.target.unit)), + ] }, + ), + ) + UnitManager => [] + ServiceProcess => [] + } +} + +// The deploying checkout's build, the one source the install copies from; the same path the planner +// digests (gunbc.live_deploy.member_observe release_executable_source_path) so the verification +// line above and the planned identity name one file. +data release_executable_install_source: String = "target/release/gunbc" + fn emit_artifact_upsert(art: DeploymentArtifactStep, spec: DeploymentSpec, revision: ReleaseRevisionBinding) -> List { match art.kind { GunbcSourceTree => [ @@ -1525,11 +1668,39 @@ fn emit_deploy_member_effect(effect: MemberEffect, spec: Deploym // hand or changes what the trap does. Adding the grammar row is the fix and it is out of this // PR's scope; naming the gap here is what keeps the next reader from reaching for the // approximation. See also the brace-group gap at tree_sync_restart_step_with_diagnosis. +fn member_effect_step(e: MemberEffect) -> DeploymentStep { + match e { + EffectAdd { to } => to + EffectReplace { from: _, to } => to + EffectRemove { from } => from + } +} + +// EVERY STEP OF THE DEPLOYMENT, IN THE DEPLOYMENT'S ORDER, PROJECTED TO ITS EFFECTS. An identity +// member's step projects to the release plan's fragments for that member (possibly none); every other +// step projects to the membership effects the apply-all reconcile derived for its key. The order is +// therefore spec.steps' order -- gunbc.live_deploy.spec deployment_steps_apply_order, the one +// authority -- and this function adds no ordering of its own. +fn apply_step_effects( + step: DeploymentStep, + effects: List>, + spec: DeploymentSpec, + plan: ReleaseEffectPlan, +) -> List { + match identity_member_of_step(step: step) { + Present { value: member } => emit_release_member_effects(member: member, plan: plan, spec: spec) + Absent => flat_map( + filter(effects, e => deployment_step_key_eq(a: deployment_step_key(step: member_effect_step(e: e)), b: deployment_step_key(step: step))), + e => emit_deploy_member_effect(effect: e, spec: spec, revision: RevisionBoundAtEmission { revision: plan.target.revision }), + ) + } +} + fn apply_intent_from_effects( spec: DeploymentSpec, effects: List>, access: DeployAccess, - revision: ReleaseRevisionBinding + plan: ReleaseEffectPlan, ) -> Pipeline { Pipeline { steps: concat( @@ -1547,7 +1718,7 @@ fn apply_intent_from_effects( concat( deploy_memory_cap_apply_steps(spec: spec), concat( - flat_map(effects, e => emit_deploy_member_effect(effect: e, spec: spec, revision: revision)), + flat_map(spec.steps, step => apply_step_effects(step: step, effects: effects, spec: spec, plan: plan)), [ deploy_raw(command: deploy_receipt_command(host: spec.target.host.identity, fold: "apply")), ] @@ -1637,7 +1808,7 @@ fn live_deploy_wholesale_refused_poison(refusals: List String { +fn live_deploy_apply_script_for(spec: DeploymentSpec, plan: ReleaseEffectPlan) -> String { match ci_deploy_srv1_access_or_refusal() { DeployAccessJobPrincipalRefused { refusal: r } => deploy_access_job_principal_refused_emit_poison(refusal: r) @@ -1645,7 +1816,7 @@ fn live_deploy_apply_script_for(spec: DeploymentSpec, revision: ReleaseRevisionB match membership_effects(plan: deployment_apply_plan(spec: spec)) { EffectsReady { effects: es } => emit_pipeline_or_poison( - p: apply_intent_from_effects(spec: spec, effects: es, access: access, revision: revision), + p: apply_intent_from_effects(spec: spec, effects: es, access: access, plan: plan), refused: live_deploy_apply_emit_refused_poison ) ApplyRefused { refusals: rs } => diff --git a/dag/gunbc/live_deploy/member_identity.dag b/dag/gunbc/live_deploy/member_identity.dag new file mode 100644 index 00000000000..a5990b74267 --- /dev/null +++ b/dag/gunbc/live_deploy/member_identity.dag @@ -0,0 +1,456 @@ +module gunbc.live_deploy.member_identity + +import std.types { String, Bool, List, Int, NonEmptyStr, CommitSha } +import extdeps.crypto.hash { Digest, HashAlgorithm, Sha256, Sha512 } +import std.content_hash { Fnv1a64Structural, content_hash_eq_structural } +import extdeps.systemd { SystemdUnitActiveState, Active, Inactive, Activating, Deactivating, Reloading, Failed, systemd_unit_active_state_wire_label } + +// A DEPLOYMENT CONVERGES ON IDENTITIES, NOT ON PATHS. gunbc.live_deploy.emit deployment_apply_plan +// reconciles the desired roster against `observed: []`, so every member is an add on every apply +// and an already-converged host replays its whole installation -- measured 2026-09-06 at nineteen +// minutes, of which the deploy's real mutations were seconds and the rest was five source-compiling +// evaluators started in series by unconditional restarts. docs/plans/deploy-convergence-observed-side.md +// refutes the obvious repair (feed the same path-keyed roster an observed side): a member whose value +// is a function of its own key cannot be seen to be stale, so a non-degenerate diff over paths would +// report CONVERGED over a stale host. The member therefore needs a real identity -- the smallest fact +// whose equality means the host already holds what is desired -- and this module carries that for +// the members that consume the release. Every other owned artifact stays on the apply-all path until +// it earns an identity here; the roster below is the declared frontier, not an oversight. +// +// FIVE MEMBERS, NOT THREE, because the serve unit is three facts with three remedies (review on PR +// #10696): the unit FILE on disk, whose drift is repaired by writing it; the MANAGER's loaded copy, +// whose staleness is repaired by daemon-reload; and the PROCESS, whose absence or stale release is +// repaired by a restart. Fusing them into one identity did both sides wrong at once -- an Active +// process running the PREVIOUS release under a current unit file compared equal and was never +// restarted (an absorbing state: readiness refused, the next plan was empty again), while a Failed +// process under a current file derived a file rewrite and a reload it did not need. +// +// The executable form of the class, from the same record: every reconcile binding must have a +// control that holds the KEY fixed, varies meaningful STATE, and expects a replacement. The +// executable fixture in member_identity_witness_test is exactly that control. +type ConvergedMember + = TreePublication + | Executable + | UnitFile + | UnitManager + | ServiceProcess + +fn converged_member_label(m: ConvergedMember) -> String { + match m { + TreePublication => "tree-publication" + Executable => "executable" + UnitFile => "unit-file" + UnitManager => "unit-manager" + ServiceProcess => "service-process" + } +} + +// THE PUBLISHED TREE'S IDENTITY IS THE REVISION AND WHETHER THE TRACKED PROJECTION MATCHES IT. The +// observed-side record found that revision alone is insufficient: a checkout can sit at the right +// HEAD with a stale or dirty tracked projection, and that is the state the rsync-era sync produced. +// So a tree is converged only when both hold; a clean projection at another revision is a Replace. +type TreePublicationIdentity { + revision: CommitSha + tracked_projection_clean: Bool +} + +// THE UNIT FILE'S IDENTITY IS ITS DOCUMENT: the content hash of the installed bytes against the +// content hash of the rendered document. Both sides are text this evaluator holds, so the +// substrate's own structural hash (std.content_hash) is the identity; the executable, whose bytes we +// never hold, is identified by the host's sha256 instead. +type UnitFileIdentity { + unit: NonEmptyStr + document: Fnv1a64Structural +} + +// THE MANAGER'S STANDING IS WHETHER ITS LOADED COPY IS THE FILE ON DISK, which systemd reports as the +// unit property NeedDaemonReload (extdeps.systemd). A current file over a stale manager is exactly +// the state an interrupted deploy leaves -- written, never reloaded -- and it is decidable only from +// this property, never from the file. +type UnitManagerStanding + = ManagerLoadedCurrent + | ManagerNeedsReload + +// THE PROCESS'S IDENTITY IS THE RELEASE IT IS RUNNING, read from the process itself over /healthz +// (gunbc.running_release_identity), or the fact that no process is running. ActiveState alone is +// not an identity: an Active process may be the previous release, and that is the case a restart +// exists to repair. +type ServiceProcessIdentity + = ProcessRunningRelease { revision: CommitSha } + | ProcessNotRunning { active: SystemdUnitActiveState } + +type MemberIdentity + = TreeIdentity { tree: TreePublicationIdentity } + | ExecutableIdentity { digest: Digest } + | UnitFileAt { file: UnitFileIdentity } + | UnitManagerAt { standing: UnitManagerStanding } + | ProcessAt { process: ServiceProcessIdentity } + +fn member_identity_member(i: MemberIdentity) -> ConvergedMember { + match i { + TreeIdentity { tree: _ } => TreePublication + ExecutableIdentity { digest: _ } => Executable + UnitFileAt { file: _ } => UnitFile + UnitManagerAt { standing: _ } => UnitManager + ProcessAt { process: _ } => ServiceProcess + } +} + +// WHAT THE HOST SAID ABOUT A MEMBER. Absent is an established absence (the observer listed the +// parent, or an ancestor, and the entry was not there, extdeps.filesystem.filesystem_io's rule); +// unobservable is a refused or indeterminate read and is NEVER folded into either of the other two -- +// a member we could not observe is refused below, not created and not skipped. +type ObservedMember + = MemberObserved { identity: MemberIdentity } + | MemberAbsentOnHost { member: ConvergedMember } + | MemberUnobservable { member: ConvergedMember, cause: String } + +fn observed_member_member(o: ObservedMember) -> ConvergedMember { + match o { + MemberObserved { identity } => member_identity_member(i: identity) + MemberAbsentOnHost { member } => member + MemberUnobservable { member, cause: _ } => member + } +} + +// WHY A MEMBER'S DECISION COULD NOT BE MADE, as a closed vocabulary rather than prose, so a +// consumer can branch on the cause and the text is rendered once. +// +// RepositoryBootstrapUnavailable is the honest arm for an ABSENT tree (review on PR #10696): a +// Create for the tree would have to turn an empty directory into a repository at the candidate, and +// no route in this repository does that -- the publication oneshot is git-native repository +// convergence (gunbc.live_deploy.repository_convergence) whose pre-state observation refuses a +// non-repository, and the rsync-era tree sync copies files while excluding .git, so an attempted +// Create left a directory that is not a repository and every later observation refused it. Until a +// bootstrap transition exists (initialize, import the candidate, reset, read back) the deploy +// refuses tree absence by name, installing nothing. +type MemberRefusalCause + = DesiredIdentityIsForAnotherMember { desired: ConvergedMember } + | ObservationIsOfAnotherMember { observed: ConvergedMember } + | HostUnobservable { cause: String } + | RepositoryBootstrapUnavailable + | DecisionFiledUnderAnotherMember { found: ConvergedMember } + | DesiredExecutableUnreadable { cause: String } + | DesiredExecutableAbsent + +fn member_refusal_text(c: MemberRefusalCause) -> String { + match c { + DesiredIdentityIsForAnotherMember { desired } => join(["desired identity is for ", converged_member_label(m: desired)], "") + ObservationIsOfAnotherMember { observed } => join(["observation is of ", converged_member_label(m: observed)], "") + HostUnobservable { cause } => join(["could not observe the host: ", cause], "") + RepositoryBootstrapUnavailable => "the repository root is absent and no bootstrap transition exists: a Create would leave a non-repository directory (rsync excludes .git) that repository convergence then refuses; initialize the repository at the candidate before deploying" + DecisionFiledUnderAnotherMember { found } => join(["decision slot holds a decision about ", converged_member_label(m: found)], "") + DesiredExecutableUnreadable { cause } => join(["the release executable could not be read in the deploying checkout: ", cause], "") + DesiredExecutableAbsent => "the release executable target/release/gunbc is absent from the deploying checkout; build before deploying" + } +} + +type MemberDecision + = AlreadyConverged { member: ConvergedMember } + | Create { member: ConvergedMember, to: MemberIdentity } + | Replace { member: ConvergedMember, from: MemberIdentity, to: MemberIdentity } + | MemberRefused { member: ConvergedMember, cause: MemberRefusalCause } + +fn hash_algorithm_eq(a: HashAlgorithm, b: HashAlgorithm) -> Bool { + match a { + Sha256 => match b { Sha256 => true Sha512 => false } + Sha512 => match b { Sha512 => true Sha256 => false } + } +} + +fn digest_eq(a: Digest, b: Digest) -> Bool { + hash_algorithm_eq(a: a.algorithm, b: b.algorithm) && (a.hex as String) == (b.hex as String) +} + +fn manager_standing_eq(a: UnitManagerStanding, b: UnitManagerStanding) -> Bool { + match a { + ManagerLoadedCurrent => match b { ManagerLoadedCurrent => true ManagerNeedsReload => false } + ManagerNeedsReload => match b { ManagerNeedsReload => true ManagerLoadedCurrent => false } + } +} + +// A PROCESS IS EQUAL TO ANOTHER ONLY WHEN BOTH RUN THE SAME RELEASE. Two not-running observations +// are never "equal" in the sense convergence needs -- the desired side is always a running release, +// so the comparison that matters is running-to-running. +fn process_identity_eq(a: ServiceProcessIdentity, b: ServiceProcessIdentity) -> Bool { + match a { + ProcessRunningRelease { revision: ra } => match b { + ProcessRunningRelease { revision: rb } => (ra as String) == (rb as String) + ProcessNotRunning { active: _ } => false + } + ProcessNotRunning { active: _ } => false + } +} + +fn converged_member_eq(a: ConvergedMember, b: ConvergedMember) -> Bool { + converged_member_label(m: a) == converged_member_label(m: b) +} + +// IDENTITY EQUALITY IS PER MEMBER KIND AND TOTAL; a pair of different kinds is not "unequal", it is +// a wiring defect and the caller refuses it before reaching here (decide_member). +fn member_identity_eq(a: MemberIdentity, b: MemberIdentity) -> Bool { + match a { + TreeIdentity { tree: ta } => match b { + TreeIdentity { tree: tb } => (ta.revision as String) == (tb.revision as String) && ta.tracked_projection_clean && tb.tracked_projection_clean + _ => false + } + ExecutableIdentity { digest: da } => match b { + ExecutableIdentity { digest: db } => digest_eq(a: da, b: db) + _ => false + } + UnitFileAt { file: fa } => match b { + UnitFileAt { file: fb } => (fa.unit as String) == (fb.unit as String) && content_hash_eq_structural(left: fa.document, right: fb.document) + _ => false + } + UnitManagerAt { standing: sa } => match b { + UnitManagerAt { standing: sb } => manager_standing_eq(a: sa, b: sb) + _ => false + } + ProcessAt { process: pa } => match b { + ProcessAt { process: pb } => process_identity_eq(a: pa, b: pb) + _ => false + } + } +} + +// ONE MEMBER, ONE DECISION. Desired is what this release wants; observed is what the host holds. +// The tree's desired identity is always clean by construction (a release is a revision, and the +// projection we publish is its tracked set), so an observed dirty projection at the right revision +// compares unequal and is a Replace -- the stale-projection case the rsync era hid. An absent tree +// is a REFUSAL, not a Create, for the reason RepositoryBootstrapUnavailable states. +fn decide_member(member: ConvergedMember, desired: MemberIdentity, observed: ObservedMember) -> MemberDecision { + if !converged_member_eq(a: member, b: member_identity_member(i: desired)) { + MemberRefused { member: member, cause: DesiredIdentityIsForAnotherMember { desired: member_identity_member(i: desired) } } + } else if !converged_member_eq(a: member, b: observed_member_member(o: observed)) { + MemberRefused { member: member, cause: ObservationIsOfAnotherMember { observed: observed_member_member(o: observed) } } + } else { + match observed { + MemberUnobservable { member: _, cause } => MemberRefused { member: member, cause: HostUnobservable { cause: cause } } + MemberAbsentOnHost { member: _ } => match member { + TreePublication => MemberRefused { member: member, cause: RepositoryBootstrapUnavailable } + _ => Create { member: member, to: desired } + } + MemberObserved { identity } => + if member_identity_eq(a: identity, b: desired) { AlreadyConverged { member: member } } + else { Replace { member: member, from: identity, to: desired } } + } + } +} + +fn decision_is_converged(d: MemberDecision) -> Bool { + match d { AlreadyConverged { member: _ } => true _ => false } +} + +fn decision_changes(d: MemberDecision) -> Bool { + match d { + Create { member: _, to: _ } => true + Replace { member: _, from: _, to: _ } => true + _ => false + } +} + +fn decision_is_refused(d: MemberDecision) -> Bool { + match d { MemberRefused { member: _, cause: _ } => true _ => false } +} + +fn decision_member(d: MemberDecision) -> ConvergedMember { + match d { + AlreadyConverged { member } => member + Create { member, to: _ } => member + Replace { member, from: _, to: _ } => member + MemberRefused { member, cause: _ } => member + } +} + +fn decision_label(d: MemberDecision) -> String { + match d { + AlreadyConverged { member } => join([converged_member_label(m: member), ": already converged"], "") + Create { member, to: _ } => join([converged_member_label(m: member), ": create"], "") + Replace { member, from: _, to: _ } => join([converged_member_label(m: member), ": replace"], "") + MemberRefused { member, cause } => join([converged_member_label(m: member), ": refused -- ", member_refusal_text(c: cause)], "") + } +} + +// ONE DECISION PER MEMBER BY CONSTRUCTION. A list of decisions made completeness and ordering the +// caller's obligation: an empty or partial list derived an accepted empty plan. The record has one +// slot per member, so a plan cannot be derived over a member nobody decided, and the emitter reads +// slots rather than searching a list. Each slot is still checked to hold a decision ABOUT its +// member (derive_effects), because the slot's type admits any MemberDecision. +type ReleaseMemberDecisions { + executable: MemberDecision + tree: MemberDecision + unit_file: MemberDecision + unit_manager: MemberDecision + process: MemberDecision +} + +// In deployment order (gunbc.live_deploy.spec deployment_owned_steps): the executable before the +// tree it publishes, then the unit file, the manager that loads it, the process that runs it. +fn release_member_decisions_list(d: ReleaseMemberDecisions) -> List { + [d.executable, d.tree, d.unit_file, d.unit_manager, d.process] +} + +// THE RELEASE A PLAN INSTALLS, CARRIED ONCE. The plan's effects used to each carry their own operand +// (a revision on PublishTree, a digest on InstallExecutable, a unit name on WriteUnit) beside a +// second revision the emitter received as a separate argument; the emitter then discarded every one +// of them and copied the ambient source path. Two operands for one fact is the §3 nickname, and an +// operand nothing reads is a label. The target lives here, once, and the emitter binds actuation to +// it: the executable it copies is verified against `executable` before and after the copy, and the +// unit it renders names `revision`. +type ReleaseTarget { + revision: CommitSha + executable: Digest + unit: NonEmptyStr +} + +// THE EFFECTS A DEPLOYMENT MAY PERFORM, ONE SWITCH PER MEMBER, so the plan is canonical rather than +// an arbitrary ordered list: there is no spelling of a plan that installs the executable twice or +// restarts before it writes. Mutations are derived along the dependency edges the members have: +// executable changed -> InstallExecutable, and the process runs it so it restarts +// tree changed -> PublishTree, and the process serves it so it restarts +// unit file changed -> WriteUnit, the manager must reload it, and the process restarts +// manager stale -> DaemonReload, and the process restarts into the reloaded definition +// process stale/down -> RestartService +// nothing changed -> no mutation at all +// Readiness is NOT on the plan. It is the parent transaction's policy +// (gunbc.live_deploy.apply live_deploy_apply_via_transport), it runs for every plan including the +// empty one, and a field here that the emitter never read was a second apparent authority for why +// readiness occurs (review on PR #10696). +type ExecutableEffect + = ExecutableKept + | InstallExecutable + +type TreeEffect + = TreeKept + | PublishTree + +type UnitFileEffect + = UnitFileKept + | WriteUnit + +type ManagerEffect + = ManagerKept + | DaemonReload + +type ProcessEffect + = ProcessKept + | RestartService + +type ReleaseEffectPlan { + target: ReleaseTarget + executable: ExecutableEffect + tree: TreeEffect + unit_file: UnitFileEffect + manager: ManagerEffect + process: ProcessEffect +} + +type EffectDerivation + = EffectsDerived { plan: ReleaseEffectPlan } + | EffectsRefused { refusals: List } + +fn slot_refusal(slot: ConvergedMember, d: MemberDecision) -> List { + if converged_member_eq(a: decision_member(d: d), b: slot) { [] } + else { [MemberRefused { member: slot, cause: DecisionFiledUnderAnotherMember { found: decision_member(d: d) } }] } +} + +fn slot_refusals(d: ReleaseMemberDecisions) -> List { + concat( + concat(slot_refusal(slot: Executable, d: d.executable), slot_refusal(slot: TreePublication, d: d.tree)), + concat( + slot_refusal(slot: UnitFile, d: d.unit_file), + concat(slot_refusal(slot: UnitManager, d: d.unit_manager), slot_refusal(slot: ServiceProcess, d: d.process)), + ), + ) +} + +// THE SERVE PROCESS IS THE CONSUMER OF EVERY OTHER MEMBER, so it restarts iff any of them changed, +// and it also restarts when it alone is stale or down. The manager reloads iff the file changed or +// it was already stale. Any refused decision refuses the whole derivation: a plan over a member +// nobody could observe would either reinstall blindly or skip blindly. +fn derive_effects(decisions: ReleaseMemberDecisions, target: ReleaseTarget) -> EffectDerivation { + let refused = concat(slot_refusals(d: decisions), filter(release_member_decisions_list(d: decisions), d => decision_is_refused(d: d))) + if count(refused) > 0 { + EffectsRefused { refusals: refused } + } else { + let file_changed = decision_changes(d: decisions.unit_file) + let any_changed = any(release_member_decisions_list(d: decisions), d => decision_changes(d: d)) + EffectsDerived { plan: ReleaseEffectPlan { + target: target, + executable: if decision_changes(d: decisions.executable) { InstallExecutable } else { ExecutableKept }, + tree: if decision_changes(d: decisions.tree) { PublishTree } else { TreeKept }, + unit_file: if file_changed { WriteUnit } else { UnitFileKept }, + manager: if file_changed || decision_changes(d: decisions.unit_manager) { DaemonReload } else { ManagerKept }, + process: if any_changed { RestartService } else { ProcessKept }, + } } + } +} + +fn plan_installs_executable(plan: ReleaseEffectPlan) -> Bool { + match plan.executable { InstallExecutable => true ExecutableKept => false } +} + +fn plan_publishes_tree(plan: ReleaseEffectPlan) -> Bool { + match plan.tree { PublishTree => true TreeKept => false } +} + +fn plan_writes_unit(plan: ReleaseEffectPlan) -> Bool { + match plan.unit_file { WriteUnit => true UnitFileKept => false } +} + +fn plan_reloads_manager(plan: ReleaseEffectPlan) -> Bool { + match plan.manager { DaemonReload => true ManagerKept => false } +} + +fn plan_restarts_process(plan: ReleaseEffectPlan) -> Bool { + match plan.process { RestartService => true ProcessKept => false } +} + +// The plan rendered as one label per mutation, in deployment order; the witnesses pin these. +fn effect_labels(plan: ReleaseEffectPlan) -> List { + concat( + concat( + if plan_installs_executable(plan: plan) { [join(["install-executable sha256:", plan.target.executable.hex as String], "")] } else { [] }, + if plan_publishes_tree(plan: plan) { [join(["publish-tree ", plan.target.revision as String], "")] } else { [] }, + ), + concat( + concat( + if plan_writes_unit(plan: plan) { [join(["write-unit ", plan.target.unit as String], "")] } else { [] }, + if plan_reloads_manager(plan: plan) { ["daemon-reload"] } else { [] }, + ), + if plan_restarts_process(plan: plan) { [join(["restart ", plan.target.unit as String], "")] } else { [] }, + ), + ) +} + +fn plan_mutation_count(plan: ReleaseEffectPlan) -> Int { + count(effect_labels(plan: plan)) +} + +// THE EMPTY PLAN: every member kept. It is what an apply against an already-converged host derives. +fn converged_effect_plan(target: ReleaseTarget) -> ReleaseEffectPlan { + ReleaseEffectPlan { + target: target, + executable: ExecutableKept, + tree: TreeKept, + unit_file: UnitFileKept, + manager: ManagerKept, + process: ProcessKept, + } +} + +// THE FULL-INSTALL PLAN: every member changed. It is what an apply against a host holding a +// repository at another revision, no executable and no unit derives, and it is the plan the emit +// witnesses hand the emitter so the rendered script they pin is the complete installation. It is +// NOT a fallback for a plan that could not be derived -- a refused observation refuses the deploy +// (gunbc.live_deploy.member_observe deployment_effect_plan), it does not install everything. +fn full_install_plan(target: ReleaseTarget) -> ReleaseEffectPlan { + ReleaseEffectPlan { + target: target, + executable: InstallExecutable, + tree: PublishTree, + unit_file: WriteUnit, + manager: DaemonReload, + process: RestartService, + } +} diff --git a/dag/gunbc/live_deploy/member_observe.dag b/dag/gunbc/live_deploy/member_observe.dag new file mode 100644 index 00000000000..f83cfb89f31 --- /dev/null +++ b/dag/gunbc/live_deploy/member_observe.dag @@ -0,0 +1,515 @@ +module gunbc.live_deploy.member_observe + +import std.types { String, Bool, List, Int, NonEmptyStr, CommitSha, FilePath } +import std.algebra { trim } +import std.content_hash { content_hash_atom } +import extdeps.crypto.hash { Digest, sha256sum_argv, sha256sum_line_digest } +import extdeps.crypto.hash +import extdeps.systemd { parse_systemd_unit_active_state, ActiveState, NeedDaemonReload, Active, SystemdUnitActiveState } +import extdeps.systemd.systemctl +import extdeps.git.inspect +import extdeps.shell +import extdeps.tools.gnu_coreutils { coreutils_rm_force_argv } +import extdeps.filesystem.filesystem_io { + Filesystem, filesystem_listing_observation, filesystem_entry_presence, filesystem_read_outcome, filesystem_file_observation, + FilesystemEntryAbsent, FilesystemEntryListed, FilesystemEntryPresenceIndeterminate, FilesystemEntrySubjectRefused, + FilesystemFileAbsent, FilesystemFileRead, FilesystemFileIndeterminate, FilesystemFileObservationsDisagree, FilesystemFileSubjectRefused, +} +import gunbc.host_effect { HostEffectTransport, LocalShell, SshShell, FleetSsh, EmitArtifactThenThinRun } +import gunbc.host_effect_realize { run_typed_argv_transport } +import gunbc.typed_argv_exec { TypedArgvExecOutcome, TypedArgvExecConverged, TypedArgvExecRefused } +import gunbc.live_deploy.spec { DeploymentSpec, deployment_systemd_unit_path, deployment_plan_listen_port } +import gunbc.live_deploy.candidate { CandidateRelease } +import gunbc.live_deploy.emit { live_deploy_systemd_unit_for, ReleaseRevisionBinding, RevisionBoundAtEmission } +import gunbc.live_deploy.readiness { live_deploy_healthz_get_for_port, healthz_release_identity_refusal, healthz_refusal_reason } +import gunbc.roadmap_site_surface_types { HealthzEffectiveRead, HealthzBodyRead, HealthzProbeRefused } +import gunbc.running_release_identity { observe_running_release_identity, RunningReleaseIdentified, RunningReleaseUnidentified } +import gunbc.live_deploy.deployed_tree_observation { + deployed_tree_read_tree_argv, deployed_tree_status_argv, deployed_tree_ignored_argv, + deployed_tree_content_reading, ContentRead, ContentUnreadable, + DeployedScopeMatchesCommit, DeployedScopeDiverged, +} +import gunbc.live_deploy.member_identity { + ConvergedMember, TreePublication, Executable, UnitFile, UnitManager, ServiceProcess, + TreePublicationIdentity, UnitFileIdentity, UnitManagerStanding, ManagerLoadedCurrent, ManagerNeedsReload, + ServiceProcessIdentity, ProcessRunningRelease, ProcessNotRunning, + MemberIdentity, TreeIdentity, ExecutableIdentity, UnitFileAt, UnitManagerAt, ProcessAt, + ObservedMember, MemberObserved, MemberAbsentOnHost, MemberUnobservable, + MemberDecision, MemberRefused, DesiredExecutableUnreadable, DesiredExecutableAbsent, DesiredIdentityIsForAnotherMember, + decide_member, ReleaseMemberDecisions, ReleaseTarget, + EffectDerivation, EffectsDerived, EffectsRefused, derive_effects, effect_labels, plan_mutation_count, +} + +// THE HOST'S SIDE OF THE RECONCILE, read by the evaluator that plans the deploy so the decision and +// the mutation are one evaluation (operator direction, 2026-09-06: one semantic evaluation per +// convergence transaction; nothing here starts a second evaluator on the host). Each observation is +// ONE interface with TWO realizations, which is DESIGN section 3's rule for transport: when the +// executor is the host (LocalShell) the read is the tool's own typed operation -- extdeps.git.inspect, +// extdeps.crypto.hash, extdeps.systemd.systemctl, extdeps.filesystem, extdeps.shell -- and when the +// host is behind ssh the same invocation goes as argv through gunbc.host_effect_realize +// run_typed_argv_transport, which rightly refuses LocalShell because it has no remote target. The +// argv builders and the operations spell one invocation each; that pair is the extdeps seam every +// shell-transported operation carries today, and it folds when an operation's transport row can be +// read back. +type ObservationArm + = OnTheHostItself + | OverTransport { transport: HostEffectTransport } + +fn observation_arm(transport: HostEffectTransport) -> ObservationArm { + match transport { + LocalShell => OnTheHostItself + SshShell { ssh_host: _ } => OverTransport { transport: transport } + FleetSsh { target: _, context: _ } => OverTransport { transport: transport } + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => OverTransport { transport: transport } + } +} + +// A REMOTE LEG: one argv, its stdout on exit zero, a typed refusal otherwise. +type Leg + = LegOut { stdout: String } + | LegRefused { cause: String } + +fn remote_leg(transport: HostEffectTransport, argv: List, leg: String) -> Leg { + match run_typed_argv_transport(argv: argv, transport: transport) { + TypedArgvExecRefused { reason } => LegRefused { cause: join([leg, ": ", reason], "") } + TypedArgvExecConverged { result: r } => + if r.exit_code != 0 { LegRefused { cause: join([leg, " failed (exit ", to_string(value: r.exit_code), "): ", r.stderr], "") } } + else { LegOut { stdout: r.stdout } } + } +} + +fn leg_of(success: Bool, stdout: String, stderr: String, leg: String) -> Leg { + if success { LegOut { stdout: stdout } } else { LegRefused { cause: join([leg, " failed: ", stderr], "") } } +} + +// ABSENCE IS ESTABLISHED BY A SUCCESSFUL LISTING OF THE PARENT, never by a failed read of the child +// (extdeps.filesystem.filesystem_io's rule). Locally that is the modeled listing observation; over a +// transport it is `ls -1 -A` of the parent and membership of the basename in its lines. +// +// AN ABSENT ANCESTOR ESTABLISHES ABSENCE TOO. On a host that has never been deployed the executable's +// parent directory does not exist, so listing it refuses -- and the first cut read that refusal as +// indeterminate and refused the whole plan before the effect that creates the directory could run +// (review on PR #10696). A listing that refuses is followed by the presence question one level up: +// an ancestor that is itself absent (established by listing ITS parent) means the child is absent; +// an ancestor that is present but cannot be listed is a real refusal and stays indeterminate. The +// walk is bounded by the path's depth and stops at the root, which is listed directly. +type EntryPresence + = EntryPresent + | EntryAbsent + | EntryPresenceIndeterminate { cause: String } + +fn path_parent(path: String) -> String { + let parts = split(s: path, delimiter: "/") + let kept = fold(parts, init: ParentWalk { count: count(parts), at: 0, acc: [] }, f: (acc, seg) => + ParentWalk { count: acc.count, at: acc.at + 1, acc: if acc.at + 1 < acc.count { concat(acc.acc, [seg]) } else { acc.acc } }) + let joined = join(kept.acc, "/") + if joined == "" { "/" } else { joined } +} + +type ParentWalk { + count: Int + at: Int + acc: List +} + +fn path_basename(path: String) -> String { + fold(split(s: path, delimiter: "/"), init: "", f: (acc, seg) => if seg == "" { acc } else { seg }) +} + +fn entry_presence_in_parent(arm: ObservationArm, parent: String, name: String) -> EntryPresence { + match arm { + OnTheHostItself => { + let listing = Filesystem.List(path: parent) + match filesystem_entry_presence( + listing: filesystem_listing_observation(directory: parent, success: listing.success, entries: listing.entries, error: listing.error), + name: name, + ) { + FilesystemEntryAbsent(_) => EntryAbsent + FilesystemEntryListed { directory: _, name: _ } => EntryPresent + FilesystemEntryPresenceIndeterminate { cause } => EntryPresenceIndeterminate { cause: cause } + FilesystemEntrySubjectRefused { directory: _, name: _, cause } => EntryPresenceIndeterminate { cause: cause } + } + } + OverTransport { transport } => match remote_leg(transport: transport, argv: ["ls", "-1", "-A", "--", parent], leg: join(["listing ", parent], "")) { + LegRefused { cause } => EntryPresenceIndeterminate { cause: cause } + LegOut { stdout } => if any(split(s: stdout, delimiter: "\n"), entry => entry == name) { EntryPresent } else { EntryAbsent } + } + } +} + +fn entry_presence(arm: ObservationArm, path: String) -> EntryPresence { + let parent = path_parent(path: path) + let name = path_basename(path: path) + match entry_presence_in_parent(arm: arm, parent: parent, name: name) { + EntryPresent => EntryPresent + EntryAbsent => EntryAbsent + EntryPresenceIndeterminate { cause } => + if parent == "/" { + EntryPresenceIndeterminate { cause: cause } + } else { + match entry_presence(arm: arm, path: parent) { + EntryAbsent => EntryAbsent + EntryPresent => EntryPresenceIndeterminate { cause: cause } + EntryPresenceIndeterminate { cause: ancestor_cause } => EntryPresenceIndeterminate { cause: ancestor_cause } + } + } + } +} + +// THE EXECUTABLE: the host's own sha256 of the installed file (coreutils sha256sum, cited in +// extdeps.crypto.hash), read the same way on both arms and parsed once. +fn sha256_leg(arm: ObservationArm, path: String) -> Leg { + match arm { + OnTheHostItself => { + let r = crypto.Sha256Sum.File(path: path) + leg_of(success: r.success, stdout: r.line, stderr: r.stderr, leg: join(["sha256sum ", path], "")) + } + OverTransport { transport } => remote_leg(transport: transport, argv: sha256sum_argv(path: path), leg: join(["sha256sum ", path], "")) + } +} + +fn observe_executable(arm: ObservationArm, path: String) -> ObservedMember { + match entry_presence(arm: arm, path: path) { + EntryPresenceIndeterminate { cause } => MemberUnobservable { member: Executable, cause: cause } + EntryAbsent => MemberAbsentOnHost { member: Executable } + EntryPresent => match sha256_leg(arm: arm, path: path) { + LegRefused { cause } => MemberUnobservable { member: Executable, cause: cause } + LegOut { stdout } => match sha256sum_line_digest(line: stdout) { + Absent => MemberUnobservable { member: Executable, cause: join(["sha256sum output is not a digest line: ", stdout], "") } + Present { value: digest } => MemberObserved { identity: ExecutableIdentity { digest: digest } } + } + } + } +} + +// The release's executable is the deploying checkout's build, the same source the install step +// copies from; its digest is read where the evaluator runs, whatever transport the host is behind. +// The emitter verifies the source against this digest again immediately before the copy, so a build +// that changes between planning and actuation stops the line rather than installing under a stale +// identity (gunbc.live_deploy.emit emit_release_member_effects). +data release_executable_source_path: String = "target/release/gunbc" + +fn desired_executable() -> ObservedMember { + observe_executable(arm: OnTheHostItself, path: release_executable_source_path) +} + +// THE UNIT FILE: the installed document's content hash. +fn unit_document_leg(arm: ObservationArm, path: String) -> Leg { + match arm { + OnTheHostItself => { + let parent = path_parent(path: path) + let listing = Filesystem.List(path: parent) + let read = Filesystem.Read(path: path) + match filesystem_file_observation( + listing: filesystem_listing_observation(directory: parent, success: listing.success, entries: listing.entries, error: listing.error), + name: path_basename(path: path), + path: path, + read: filesystem_read_outcome(content: read.content, success: read.success, error: read.error), + ) { + FilesystemFileRead { path: _, content } => LegOut { stdout: content } + FilesystemFileAbsent(_) => LegRefused { cause: join([path, " vanished between listing and read"], "") } + FilesystemFileIndeterminate { cause } => LegRefused { cause: cause } + FilesystemFileObservationsDisagree { path: _, cause } => LegRefused { cause: cause } + FilesystemFileSubjectRefused { directory: _, name: _, cause } => LegRefused { cause: cause } + } + } + OverTransport { transport } => remote_leg(transport: transport, argv: ["cat", "--", path], leg: join(["reading ", path], "")) + } +} + +fn observe_unit_file(arm: ObservationArm, spec: DeploymentSpec) -> ObservedMember { + let unit = spec.service.unit_name + let path = deployment_systemd_unit_path(names: spec.names) as String + match entry_presence(arm: arm, path: path) { + EntryPresenceIndeterminate { cause } => MemberUnobservable { member: UnitFile, cause: cause } + EntryAbsent => MemberAbsentOnHost { member: UnitFile } + EntryPresent => match unit_document_leg(arm: arm, path: path) { + LegRefused { cause } => MemberUnobservable { member: UnitFile, cause: cause } + LegOut { stdout: document } => + MemberObserved { identity: UnitFileAt { file: UnitFileIdentity { unit: unit, document: content_hash_atom(value: document as NonEmptyStr) } } } + } + } +} + +fn desired_unit_file(spec: DeploymentSpec, candidate: CandidateRelease) -> MemberIdentity { + UnitFileAt { file: UnitFileIdentity { + unit: spec.service.unit_name, + document: content_hash_atom(value: live_deploy_systemd_unit_for(spec: spec, revision: RevisionBoundAtEmission { revision: candidate.revision }) as NonEmptyStr), + } } +} + +// THE MANAGER: systemd's own answer to whether the loaded unit matches the file on disk. The wire +// values are `yes` and `no` (measured on srv1, 2026-09-06, for a loaded unit and for a not-found +// one, which also answers `no`); anything else is a vocabulary the model does not know and refuses. +fn manager_standing_leg(arm: ObservationArm, unit: NonEmptyStr) -> Leg { + match arm { + OnTheHostItself => { + let r = systemd.Systemctl.ShowProperty(unit: unit, property: NeedDaemonReload) + leg_of(success: r.success, stdout: r.value, stderr: "systemctl show failed", leg: join(["systemctl show NeedDaemonReload ", unit as String], "")) + } + OverTransport { transport } => remote_leg(transport: transport, argv: ["systemctl", "show", unit as String, "--property=NeedDaemonReload", "--value"], leg: join(["systemctl show NeedDaemonReload ", unit as String], "")) + } +} + +fn observe_unit_manager(arm: ObservationArm, unit: NonEmptyStr) -> ObservedMember { + match manager_standing_leg(arm: arm, unit: unit) { + LegRefused { cause } => MemberUnobservable { member: UnitManager, cause: cause } + LegOut { stdout: raw } => { + let answer = trim(s: raw) + if answer == "no" { MemberObserved { identity: UnitManagerAt { standing: ManagerLoadedCurrent } } } + else if answer == "yes" { MemberObserved { identity: UnitManagerAt { standing: ManagerNeedsReload } } } + else { MemberUnobservable { member: UnitManager, cause: join(["NeedDaemonReload is neither yes nor no: ", answer], "") } } + } + } +} + +fn desired_unit_manager() -> MemberIdentity { + UnitManagerAt { standing: ManagerLoadedCurrent } +} + +// THE PROCESS: its ActiveState from the manager and, when Active, the release it reports over +// /healthz through the same probe readiness uses (gunbc.live_deploy.readiness), read through +// gunbc.running_release_identity so a body the identity channel cannot decode refuses here with the +// same cause readiness would give. An Active process that cannot be probed is unobservable, not +// stale: refusing is the honest answer, and a plan that restarted it would be acting on a guess. +fn active_state_leg(arm: ObservationArm, unit: NonEmptyStr) -> Leg { + match arm { + OnTheHostItself => { + let r = systemd.Systemctl.ShowProperty(unit: unit, property: ActiveState) + leg_of(success: r.success, stdout: r.value, stderr: "systemctl show failed", leg: join(["systemctl show ActiveState ", unit as String], "")) + } + OverTransport { transport } => remote_leg(transport: transport, argv: ["systemctl", "show", unit as String, "--property=ActiveState", "--value"], leg: join(["systemctl show ActiveState ", unit as String], "")) + } +} + +fn running_process_identity(spec: DeploymentSpec, transport: HostEffectTransport) -> ObservedMember { + match live_deploy_healthz_get_for_port(port: deployment_plan_listen_port(spec: spec), transport: transport) { + HealthzProbeRefused { probe_error } => MemberUnobservable { member: ServiceProcess, cause: join(["the process is active but /healthz refused: ", probe_error], "") } + HealthzBodyRead { body } => match observe_running_release_identity(healthz_body: body) { + RunningReleaseUnidentified { cause } => + MemberUnobservable { member: ServiceProcess, cause: healthz_refusal_reason(cause: healthz_release_identity_refusal(body: body, cause: cause)) } + RunningReleaseIdentified { observation } => + MemberObserved { identity: ProcessAt { process: ProcessRunningRelease { revision: observation.revision } } } + } + } +} + +fn observe_service_process(spec: DeploymentSpec, transport: HostEffectTransport) -> ObservedMember { + let arm = observation_arm(transport: transport) + match active_state_leg(arm: arm, unit: spec.service.unit_name) { + LegRefused { cause } => MemberUnobservable { member: ServiceProcess, cause: cause } + LegOut { stdout: raw } => match parse_systemd_unit_active_state(raw: trim(s: raw)) { + Absent => MemberUnobservable { member: ServiceProcess, cause: join(["ActiveState not in the systemd vocabulary: ", trim(s: raw)], "") } + Present { value: active } => match active { + Active => running_process_identity(spec: spec, transport: transport) + _ => MemberObserved { identity: ProcessAt { process: ProcessNotRunning { active: active } } } + } + } + } +} + +fn desired_service_process(candidate: CandidateRelease) -> MemberIdentity { + ProcessAt { process: ProcessRunningRelease { revision: candidate.revision } } +} + +// THE PUBLISHED TREE: HEAD, then the three-leg tracked-projection comparison the deployed-tree record +// established as the instrument (read-tree into a private index, status against it, ignored paths), +// reusing its argv builders and its reading so two observers cannot disagree about what clean means. +// +// THE PRIVATE INDEX IS A SCRATCH FILE THE OBSERVATION OWNS: minted by mktemp so two observations -- +// of the same release, under two principals, from the workflow and the operator path at once -- +// never share a path or a lock, and removed after the reads on every arm, including the refusing +// ones. The first cut used one fixed path per revision under /tmp and never removed it, and declared +// the write readonly (review on PR #10696). The scratch's creation and removal are effects and are +// modeled as such (extdeps.shell shell.Mktemp, extdeps.filesystem Filesystem.Delete, coreutils rm -f +// over a transport); a scratch that could not be removed refuses the observation, because an +// observation that leaves state on the host has not finished. +fn head_leg(arm: ObservationArm, repo: String) -> Leg { + match arm { + OnTheHostItself => { + let r = git.Inspect.HeadCommitIn(repository_path: repo) + leg_of(success: r.success, stdout: r.sha, stderr: r.stderr, leg: "head") + } + OverTransport { transport } => remote_leg(transport: transport, argv: ["git", "-C", repo, "rev-parse", "HEAD"], leg: "head") + } +} + +type ScratchPath + = ScratchAt { path: String } + | ScratchRefused { cause: String } + +data scratch_index_directory: String = "/tmp" + +fn scratch_index(arm: ObservationArm) -> ScratchPath { + match arm { + OnTheHostItself => { + let r = shell.Mktemp.FileInDirectory(dir: scratch_index_directory as FilePath) + if r.success { ScratchAt { path: trim(s: r.path as String) } } else { ScratchRefused { cause: "mktemp for the scratch index failed" } } + } + OverTransport { transport } => match remote_leg(transport: transport, argv: ["mktemp", "-p", scratch_index_directory], leg: "scratch index") { + LegRefused { cause } => ScratchRefused { cause: cause } + LegOut { stdout } => ScratchAt { path: trim(s: stdout) } + } + } +} + +fn scratch_release(arm: ObservationArm, path: String) -> Leg { + match arm { + OnTheHostItself => { + let r = Filesystem.Delete(path: path) + leg_of(success: r.success, stdout: "", stderr: r.error, leg: join(["scratch index removal ", path], "")) + } + OverTransport { transport } => remote_leg(transport: transport, argv: coreutils_rm_force_argv(paths: [path]), leg: join(["scratch index removal ", path], "")) + } +} + +fn read_tree_leg(arm: ObservationArm, repo: NonEmptyStr, revision_hex: String, index_path: String) -> Leg { + match arm { + OnTheHostItself => { + let r = git.Inspect.ReadTreeIntoIndex(repository_path: repo as String, revision_hex: revision_hex, index_path: index_path) + leg_of(success: r.success, stdout: "", stderr: r.stderr, leg: "index materialization") + } + OverTransport { transport } => remote_leg(transport: transport, argv: deployed_tree_read_tree_argv(repo: repo, revision_hex: revision_hex, index_path: index_path), leg: "index materialization") + } +} + +fn status_leg(arm: ObservationArm, repo: NonEmptyStr, index_path: String) -> Leg { + match arm { + OnTheHostItself => { + let r = git.Inspect.StatusAgainstIndex(repository_path: repo as String, index_path: index_path) + leg_of(success: r.success, stdout: r.entries_nul, stderr: r.stderr, leg: "worktree comparison") + } + OverTransport { transport } => remote_leg(transport: transport, argv: deployed_tree_status_argv(repo: repo, index_path: index_path), leg: "worktree comparison") + } +} + +fn ignored_leg(arm: ObservationArm, repo: NonEmptyStr, index_path: String) -> Leg { + match arm { + OnTheHostItself => { + let r = git.Inspect.IgnoredAgainstIndex(repository_path: repo as String, index_path: index_path) + leg_of(success: r.success, stdout: r.paths_nul, stderr: r.stderr, leg: "ignored deployed paths") + } + OverTransport { transport } => remote_leg(transport: transport, argv: deployed_tree_ignored_argv(repo: repo, index_path: index_path), leg: "ignored deployed paths") + } +} + +type ProjectionReading + = ProjectionClean + | ProjectionDirty + | ProjectionUnreadable { cause: String } + +fn projection_legs(arm: ObservationArm, repo: NonEmptyStr, revision_hex: String, index_path: String) -> ProjectionReading { + match read_tree_leg(arm: arm, repo: repo, revision_hex: revision_hex, index_path: index_path) { + LegRefused { cause } => ProjectionUnreadable { cause: cause } + LegOut { stdout: _ } => match status_leg(arm: arm, repo: repo, index_path: index_path) { + LegRefused { cause } => ProjectionUnreadable { cause: cause } + LegOut { stdout: status } => match ignored_leg(arm: arm, repo: repo, index_path: index_path) { + LegRefused { cause } => ProjectionUnreadable { cause: cause } + LegOut { stdout: ignored } => match deployed_tree_content_reading(entries_nul: status, ignored_nul: ignored) { + ContentUnreadable { detail } => ProjectionUnreadable { cause: detail } + ContentRead { state: DeployedScopeMatchesCommit } => ProjectionClean + ContentRead { state: DeployedScopeDiverged { changed_paths: _ } } => ProjectionDirty + } + } + } + } +} + +// Mint, read, release, then answer: the reading is computed before the scratch is removed, and the +// removal's outcome is adjudicated before the reading is returned, so a leaked scratch cannot ride +// under a clean answer. +fn tracked_projection(arm: ObservationArm, repo: NonEmptyStr, revision_hex: String) -> ProjectionReading { + match scratch_index(arm: arm) { + ScratchRefused { cause } => ProjectionUnreadable { cause: cause } + ScratchAt { path } => { + let reading = projection_legs(arm: arm, repo: repo, revision_hex: revision_hex, index_path: path) + match scratch_release(arm: arm, path: path) { + LegRefused { cause } => ProjectionUnreadable { cause: join(["the observation left a scratch index behind: ", cause], "") } + LegOut { stdout: _ } => reading + } + } + } +} + +fn tree_at(local: String, clean: Bool) -> ObservedMember { + MemberObserved { identity: TreeIdentity { tree: TreePublicationIdentity { revision: local as CommitSha, tracked_projection_clean: clean } } } +} + +fn observe_tree(arm: ObservationArm, spec: DeploymentSpec, candidate: CandidateRelease) -> ObservedMember { + let repo = spec.service.repo_root + let revision_hex = candidate.revision as String + match entry_presence(arm: arm, path: repo as String) { + EntryPresenceIndeterminate { cause } => MemberUnobservable { member: TreePublication, cause: cause } + EntryAbsent => MemberAbsentOnHost { member: TreePublication } + EntryPresent => match head_leg(arm: arm, repo: repo as String) { + LegRefused { cause } => MemberUnobservable { member: TreePublication, cause: cause } + LegOut { stdout: head } => { + let local = trim(s: head) + if local != revision_hex { + tree_at(local: local, clean: false) + } else { + match tracked_projection(arm: arm, repo: repo, revision_hex: revision_hex) { + ProjectionUnreadable { cause } => MemberUnobservable { member: TreePublication, cause: cause } + ProjectionClean => tree_at(local: local, clean: true) + ProjectionDirty => tree_at(local: local, clean: false) + } + } + } + } + } +} + +fn desired_tree(candidate: CandidateRelease) -> MemberIdentity { + TreeIdentity { tree: TreePublicationIdentity { revision: candidate.revision, tracked_projection_clean: true } } +} + +fn release_member_decisions(spec: DeploymentSpec, candidate: CandidateRelease, transport: HostEffectTransport, desired_bin: MemberIdentity) -> ReleaseMemberDecisions { + let arm = observation_arm(transport: transport) + ReleaseMemberDecisions { + executable: decide_member(member: Executable, desired: desired_bin, observed: observe_executable(arm: arm, path: spec.service.serve_binary as String)), + tree: decide_member(member: TreePublication, desired: desired_tree(candidate: candidate), observed: observe_tree(arm: arm, spec: spec, candidate: candidate)), + unit_file: decide_member(member: UnitFile, desired: desired_unit_file(spec: spec, candidate: candidate), observed: observe_unit_file(arm: arm, spec: spec)), + unit_manager: decide_member(member: UnitManager, desired: desired_unit_manager(), observed: observe_unit_manager(arm: arm, unit: spec.service.unit_name)), + process: decide_member(member: ServiceProcess, desired: desired_service_process(candidate: candidate), observed: observe_service_process(spec: spec, transport: transport)), + } +} + +// THE WHOLE DECISION FOR ONE DEPLOY: five observations, five decisions, one effect derivation. The +// executable's DESIRED side is itself an observation (the local build's digest), so a missing local +// build refuses the plan rather than planning an install of nothing. +fn deployment_effect_plan(spec: DeploymentSpec, candidate: CandidateRelease, transport: HostEffectTransport) -> EffectDerivation { + match desired_executable() { + MemberUnobservable { member: _, cause } => + EffectsRefused { refusals: [MemberRefused { member: Executable, cause: DesiredExecutableUnreadable { cause: cause } }] } + MemberAbsentOnHost { member: _ } => + EffectsRefused { refusals: [MemberRefused { member: Executable, cause: DesiredExecutableAbsent }] } + MemberObserved { identity: desired_bin } => match desired_bin { + ExecutableIdentity { digest } => + derive_effects( + decisions: release_member_decisions(spec: spec, candidate: candidate, transport: transport, desired_bin: desired_bin), + target: ReleaseTarget { revision: candidate.revision, executable: digest, unit: spec.service.unit_name }, + ) + _ => + EffectsRefused { refusals: [MemberRefused { member: Executable, cause: DesiredIdentityIsForAnotherMember { desired: gunbc.live_deploy.member_identity.member_identity_member(i: desired_bin) } }] } + } + } +} + +// THE READBACK AFTER MUTATION IS THE SAME OBSERVATION RUN AGAIN. Process readiness proves the unit +// answers as the candidate; it cannot prove the executable and unit file on disk are the identities +// the plan declared, because an equivalent binary or a rewritten unit would answer the same. So +// after the readiness gate the deploy re-derives its plan and requires it empty: every member is +// AlreadyConverged, or the transaction did not converge and says which members still differ. +type IdentityReadback + = IdentityReadbackConverged + | IdentityReadbackDrift { remaining: List } + | IdentityReadbackRefused { refusals: List } + +fn deployment_identity_readback(spec: DeploymentSpec, candidate: CandidateRelease, transport: HostEffectTransport) -> IdentityReadback { + match deployment_effect_plan(spec: spec, candidate: candidate, transport: transport) { + EffectsRefused { refusals } => IdentityReadbackRefused { refusals: refusals } + EffectsDerived { plan } => + if plan_mutation_count(plan: plan) == 0 { IdentityReadbackConverged } + else { IdentityReadbackDrift { remaining: effect_labels(plan: plan) } } + } +} diff --git a/dag/gunbc/live_deploy/spec.dag b/dag/gunbc/live_deploy/spec.dag index 79e7e586d66..5b44f7a203d 100644 --- a/dag/gunbc/live_deploy/spec.dag +++ b/dag/gunbc/live_deploy/spec.dag @@ -785,8 +785,8 @@ fn deployment_owned_steps( names: DeploymentNames, ) -> List { [ - DeploymentArtifactStep { kind: GunbcSourceTree, path: instance.repo_root as String as NonEmptyStr }, DeploymentArtifactStep { kind: ServeBinary, path: instance.serve_binary as String as NonEmptyStr }, + DeploymentArtifactStep { kind: GunbcSourceTree, path: instance.repo_root as String as NonEmptyStr }, DeploymentArtifactStep { kind: DeployedTreeRemoteUnit, path: deployment_tree_remote_unit_path(names: names), @@ -853,8 +853,12 @@ fn deployment_ensured_steps() -> List { ] } -// Apply order is dependency-honest: the tree lands before the binary (the binary serves that tree), -// both before the two belt-spawn roots (the worktree root and the attempt-state root, which a +// Apply order is dependency-honest: the binary lands before the tree, because the tree is published +// by a service-user oneshot that RUNS the admitted binary (gunbc.live_deploy.repository_convergence_placement +// convergence_unit_doc) -- the order used to read tree-then-binary on the grounds that the binary serves +// the tree, which is true of the running process but not of the installation, and the identity emitter +// had to override it downstream until this authority said it (review on PR #10696); both before the +// two belt-spawn roots (the worktree root and the attempt-state root, which a // dispatch writes in that order — worktree first, then the published attempt evidence), all before // the unit (ExecStart references binary + tree), then the belt timer (its ExecStart references that // same binary and tree, and every tick it fires writes the two roots above — so it is ordered after diff --git a/dag/gunbc/systemd_property_directive_overlap.dag b/dag/gunbc/systemd_property_directive_overlap.dag index 3a94ad4bb09..c7f77d8fbf9 100644 --- a/dag/gunbc/systemd_property_directive_overlap.dag +++ b/dag/gunbc/systemd_property_directive_overlap.dag @@ -126,6 +126,7 @@ fn systemd_property_writability(property: SystemdUnitProperty) -> SystemdPropert NextElapseUSecMonotonic => ObservationOnly AccuracyUSec => ObservationOnly TimersMonotonic => ObservationOnly + NeedDaemonReload => ObservationOnly } } @@ -160,7 +161,7 @@ data systemd_unit_property_members: List = [ MemoryMax, MemorySwapMax, MemoryHigh, CPUWeight, CPUQuota, TasksMax, MemoryCurrent, TasksCurrent, ControlGroup, ActiveEnterTimestampMonotonic, ActiveState, MainPID, LoadState, UnitFileState, SubState, Result, ExecMainStatus, ExecStartProperty, - User, WorkingDirectoryProperty, RuntimeDirectoryProperty, Unit, NextElapseUSecMonotonic, AccuracyUSec, TimersMonotonic, + User, WorkingDirectoryProperty, RuntimeDirectoryProperty, Unit, NextElapseUSecMonotonic, AccuracyUSec, TimersMonotonic, NeedDaemonReload, ] fn settable_property_wires() -> List { diff --git a/dag/test/claim/live_deploy/deploy_release_fixture.dag b/dag/test/claim/live_deploy/deploy_release_fixture.dag index fc155bf3322..69972492b9a 100644 --- a/dag/test/claim/live_deploy/deploy_release_fixture.dag +++ b/dag/test/claim/live_deploy/deploy_release_fixture.dag @@ -1,6 +1,9 @@ module test.claim.live_deploy.deploy_release_fixture -import std.types { CommitSha, String } +import std.types { CommitSha, String, NonEmptyStr } +import extdeps.crypto.hash { Digest, sha256_digest } +import gunbc.live_deploy.spec { DeploymentSpec } +import gunbc.live_deploy.member_identity { ReleaseEffectPlan, ReleaseTarget, full_install_plan } // ONE FIXTURE REVISION FOR EVERY DEPLOY WITNESS, DECLARED HERE AND NOWHERE ELSE. The launch // revision is now an input to the emitted unit, the readiness comparison, and the apply fold, so @@ -20,6 +23,19 @@ import std.types { CommitSha, String } data deploy_witness_release_revision: CommitSha = "0123456789abcdef0123456789abcdef01234567" +// THE FULL-INSTALL PLAN THE EMIT WITNESSES HAND THE EMITTER, so the script they pin is the complete +// installation of an empty host. A converged host derives a smaller plan; that is member_identity's +// witness, not this fixture's. +data deploy_witness_executable_digest: Digest = sha256_digest(hex: "5555555555555555555555555555555555555555555555555555555555555555" as NonEmptyStr) + +fn deploy_witness_release_target(spec: DeploymentSpec) -> ReleaseTarget { + ReleaseTarget { revision: deploy_witness_release_revision, executable: deploy_witness_executable_digest, unit: spec.service.unit_name } +} + +fn deploy_witness_full_install_plan(spec: DeploymentSpec) -> ReleaseEffectPlan { + full_install_plan(target: deploy_witness_release_target(spec: spec)) +} + data deploy_witness_other_release_revision: CommitSha = "89abcdef0123456789abcdef0123456789abcdef" // THREE, BECAUSE THE DISCRIMINATING CASES NEED THREE. Two suffice for 'the process reports a diff --git a/dag/test/claim/live_deploy/emit_test.dag b/dag/test/claim/live_deploy/emit_test.dag index a42abfd0e58..ed358513977 100644 --- a/dag/test/claim/live_deploy/emit_test.dag +++ b/dag/test/claim/live_deploy/emit_test.dag @@ -1,7 +1,11 @@ module test.claim.live_deploy.emit -import test.claim.live_deploy.deploy_release_fixture { deploy_witness_release_revision, deploy_witness_other_release_revision } -import std.types { Bool, String, List, NonEmptyStr } +import gunbc.live_deploy.member_identity { + ReleaseEffectPlan, converged_effect_plan, InstallExecutable, TreeKept, UnitFileKept, ManagerKept, RestartService, + ConvergedMember, TreePublication, Executable, UnitFile, UnitManager, ServiceProcess, +} +import test.claim.live_deploy.deploy_release_fixture { deploy_witness_executable_digest, deploy_witness_release_revision, deploy_witness_other_release_revision, deploy_witness_full_install_plan, deploy_witness_release_target } +import std.types { Bool, String, List, NonEmptyStr, Int } import std.os.types { Apt } import gunbc.live_deploy.spec { DeploymentSpec, @@ -22,9 +26,14 @@ import gunbc.live_deploy.spec { srv1_live_deployment_names, ArtifactStep, Dependency, + DeploymentStep, deployment_tailnet_url, gunbc_roadmap_unit_name, ServeBinary, + OwnedArtifactKind, + GunbcSourceTree, + AttemptStateRoot, + SystemdUnit, FleetConvergeTimerUnit, TailscaleServeMapping, TeardownDisposition, @@ -74,6 +83,7 @@ import gunbc.live_deploy.effect_leaf { dpkg_status_effect, } import gunbc.live_deploy.emit { + emit_release_member_effects, deploy_receipt_command, install_d_shared_mode, path_parent_dir, @@ -85,6 +95,8 @@ import gunbc.live_deploy.emit { live_deploy_wholesale_refused_poison, live_deploy_apply_script_for, live_deploy_retract_script_for, + deploy_apply_preamble_steps, + deploy_publication_principal_ensure_command, live_deploy_emit_shell_dissolution_trigger, live_deploy_remote_mutation_service_op_realization_dissolution_trigger, live_deploy_intent_dependency_graph_emit_dissolution_trigger, @@ -92,6 +104,7 @@ import gunbc.live_deploy.emit { import gunbc.ci_deploy_access { DeployAccessJobPrincipalRefusal, deploy_access_job_principal_refused_emit_poison, + ci_deploy_srv1_access_or_refusal, DeployAccessReady, DeployAccessJobPrincipalRefused, } import gunbc.fleet_posix_accounts { DeployRunnerPrincipal } import gunbc.host_axis_caps { @@ -102,7 +115,7 @@ import gunbc.host_axis_caps { } import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } import std.measure { byte_size_count } -import v2.std.orchestration { Do, Run } +import v2.std.orchestration { Do, Run, PipelineStep, Comment, Exit, If, For, While, Retry, Let, Call } import std.disposition { Scaffold, RealizationDispatch, Terminal } import std.decl_ref { DeclarationRef, WholeDeclaration } import gunbc.generated_artifact { @@ -121,15 +134,29 @@ import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } data witness_script_producers_note: String = "THE COMMITTED SCRIPT ARTIFACTS ARE GONE, SO THESE WITNESSES CALL THE EMITTER DIRECTLY. Nothing executed .github/live-deploy-srv1-apply.sh or -retract.sh — the real deploy runs live_deploy_apply_srv1_wet through gunbc run — and their registry rows named a HostReconciler consumer that does not exist. A committed artifact nobody reads is not coverage; it is a false architectural claim that happened to be convenient as a byte oracle.\\n\\nThe coverage did not need the artifact. A fixture sha is forbidden in a committed PRODUCTION artifact, never in a witness, so these call live_deploy_apply_script_for with RevisionBoundAtEmission at the fixture revision and assert the same bytes the golden used to pin — with the ExecStart line now pinned EXACTLY, which the golden never did." +// ONE RENDER, NOT ONE PER CLAIM. Every claim below that reads the apply script reads the SAME bytes, +// so rendering it inside a fn made the module pay for the emitter once per call site — fifteen full +// renders of one artifact, which is the copied-accumulator cost shape §6 rules is always fixed +// regardless of the realized n. As module-scope data the FIRST claim to touch it pays the render and +// every later one reads it. Measured over the module's 59 witnesses, hoisting moved +// witness_apply_script_contains_systemd_and_tailscale from 5131ms to 1052ms and +// repository_convergence_is_ordered_after_binary_and_before_readback from 5128ms to 0ms. It does +// NOT reduce the claims that render a DIFFERENT spec or plan (the twin, the converged plan): those +// have their own renders and still pay them, which is why the zero-mutation claim below is still +// the expensive one and had to lose its second render outright. +data witness_apply_script_text: String = live_deploy_apply_script_for( + spec: deployment_spec_srv1(), + plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1()), +) + +data witness_retract_script_text: String = live_deploy_retract_script_for(spec: deployment_spec_srv1()) + fn witness_apply_script() -> String { - live_deploy_apply_script_for( - spec: deployment_spec_srv1(), - revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }, - ) + witness_apply_script_text } fn witness_retract_script() -> String { - live_deploy_retract_script_for(spec: deployment_spec_srv1()) + witness_retract_script_text } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -661,13 +688,13 @@ test fn retract_is_the_reverse_of_what_this_spec_applies() -> Bool { data emitted_effect_twin_note: String = "THE PATH-LIST TWIN CLAIM WAS COVERAGE BY ILLUSION AND THIS IS ITS REPLACEMENT. The previous claim compared five declared artifact paths and pinned count == 5, so its proof was bounded by the symbolic footprint — and three destructive effects lived outside it: the systemd unit hardcoded --function roadmap_serve_handle so a twin would IDENTIFY as production, tree-sync installed and deleted one host-wide unit and env file, and teardown ran a node-global `tailscale serve reset`. Every one of those passes a declared-member comparison while the emitted script is unsafe (review, 2026-07-30).\\n\\nSo this reads the SCRIPTS. It asserts over the actual apply and retract text that production's identifying strings — its handler, its unit, its tree-sync files — appear in production's emission and are ABSENT from the twin's, and that the twin's own names are absent from production's. A string a reviewer can see in the emitted output is exactly the grain the previous claim could not reach.\\n\\nThe negative direction is the load-bearing half. Asserting the twin contains its own names would pass even if it ALSO contained production's; only absence proves it cannot touch them. The tailscale conjunct asserts the refusal marker rather than a scoped command, because the CLI offers no per-port removal and the honest emission is a refusal — a claim expecting a scoped teardown would demand a widen that cannot be written safely." test fn production_apply_carries_its_own_handler_and_tree_sync_env() -> Bool { - let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1())) string_contains(s: live, pattern: "--function roadmap_serve_handle ") && string_contains(s: live, pattern: "/etc/gunbc-tree-sync.env") } test fn emitted_twin_apply_touches_none_of_productions_effects() -> Bool { - let twin = live_deploy_apply_script_for(spec: srv1_twin_spec(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let twin = live_deploy_apply_script_for(spec: srv1_twin_spec(), plan: deploy_witness_full_install_plan(spec: srv1_twin_spec())) string_contains(s: twin, pattern: "--function roadmap_serve_handle_srv1_lab ") && !string_contains(s: twin, pattern: "--function roadmap_serve_handle ") && !string_contains(s: twin, pattern: "/etc/gunbc-tree-sync.env") @@ -679,7 +706,7 @@ data belt_spawn_root_witness_note: String = "THE CLAIM THE BELT-TIMER MEMBER MOS test fn twin_belt_unit_carries_the_twins_spawn_root_and_not_productions() -> Bool { let live_instance = srv1_live_dashboard_instance() let lab_instance = srv1_lab_dashboard_instance() - let twin = live_deploy_apply_script_for(spec: srv1_twin_spec(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let twin = live_deploy_apply_script_for(spec: srv1_twin_spec(), plan: deploy_witness_full_install_plan(spec: srv1_twin_spec())) let env = concat(belt_spawn_workdir_env_var as String, "=") string_contains(s: twin, pattern: concat(env, lab_instance.repo_root as String)) && !string_contains(s: twin, pattern: concat(env, live_instance.repo_root as String)) @@ -688,7 +715,7 @@ test fn twin_belt_unit_carries_the_twins_spawn_root_and_not_productions() -> Boo test fn production_belt_unit_carries_productions_spawn_root() -> Bool { let live_instance = srv1_live_dashboard_instance() - let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1())) string_contains(s: live, pattern: concat(concat(belt_spawn_workdir_env_var as String, "="), live_instance.repo_root as String)) } @@ -713,12 +740,12 @@ test fn shell_quote_escapes_an_embedded_apostrophe() -> Bool { data twin_creates_its_own_bin_dir_note: String = "THE PARENT DIRECTORY WAS THE LAST srv1 LITERAL IN THE APPLY (review 45197). serve_binary became instance-derived while the step that creates its containing directory kept production's `/opt/gunbc/bin`, and `install` does not create missing parents — so a twin apply would reach the copy with nowhere to copy to and fail on what reads as a permissions problem.\\n\\nThe claim is stated in BOTH directions because only the negative half proves it. Asserting the twin creates its own bin directory passes even if the emission ALSO still creates production's; asserting production's literal is absent from the twin script is what closes it. The production conjunct pins that the derivation did not move production — /opt/gunbc/bin/gunbc has parent /opt/gunbc/bin, so its emitted command is unchanged, which the committed golden independently confirms." test fn production_apply_creates_productions_binary_parent_directory() -> Bool { - let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1())) string_contains(s: live, pattern: serve_binary_parent_install_command(instance: srv1_live_dashboard_instance())) } test fn twin_apply_creates_its_own_binary_parent_directory_and_not_productions() -> Bool { - let twin = live_deploy_apply_script_for(spec: srv1_twin_spec(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let twin = live_deploy_apply_script_for(spec: srv1_twin_spec(), plan: deploy_witness_full_install_plan(spec: srv1_twin_spec())) !string_contains(s: twin, pattern: serve_binary_parent_install_command(instance: srv1_live_dashboard_instance())) && string_contains(s: twin, pattern: serve_binary_parent_install_command(instance: srv1_lab_dashboard_instance())) } @@ -766,13 +793,13 @@ test fn teardown_disposition_separates_owned_from_ensured() -> Bool { data endpoint_identity_is_the_collision_axis_note: String = "THE CLAIM THE TWIN PROOF WAS MISSING, and its absence is why a green twin suite shipped a production-destroying apply. Every earlier twin claim in this file compares roots, ports, units, handlers and tree-sync files, and all of them passed while BOTH deployments configured the identical tailscale endpoint - because tailscale_serve_path was an invented label the emitter never read, and the emitted command derived from the port alone, which a bare `serve` ignores in favour of the default listener at the root mount.\\n\\nSo this asserts over the axis tailscale actually keys on, in BOTH directions and on BOTH scripts. The positive half - the twin carries its own --set-path - would pass against an emission that also bound production's endpoint, which is exactly the broken state. Only the negative half closes it: production's endpoint spelling must be ABSENT from the twin's apply and from the twin's retract, so the twin can neither steal the route nor remove it.\\n\\nThe retract conjuncts are not a restatement of the apply ones. The upstream requires every original flag on an off command, so apply and off can drift independently - an off missing --set-path targets the ROOT mount, which is production's, and would remove production's routing while exiting zero. Asserting the exact off spelling on each side is what ties the two directions to one endpoint value.\\n\\nONE CLAIM BECAME FOUR, ONE PER EMITTED SCRIPT, AND NOTHING WAS DROPPED (2026-08-23). The seven conjuncts below are the seven that were here; only their grouping changed. The regrouping is lossless because NO CONJUNCT EVER COMPARED TWO EMITTED SCRIPTS: disjointness is asserted by pattern negation against a command spelling derived from the OTHER spec, which is a cheap argv join, never that spec's emitted script. A reader who assumes a disjointness claim needs both scripts by construction will re-fuse these and re-cross the line - this row said exactly that in a PR body and it was wrong, caught by two independent readers of the conjuncts.\\n\\nWHY THE GROUPING MATTERED AT ALL, stated so this is not mistaken for tidiness: emitting a script costs about 6,714 interpreter node-evals per emitted shell WORD, so a claim's cost is the number of SCRIPTS it builds. Four scripts in one claim put this row at ~5009ms against the 5000ms required_floor_claim_cpu_safety_limit_ms and it was INTERRUPTED BEFORE VERDICT - not passing, not failing, unknown - which is a strictly worse outcome than either verdict. One script per claim puts each row where the module's ordinary rows already sit, and a red now locates which script and which direction rather than which of seven conjuncts.\\n\\nTHIS IS A LOCAL UNBLOCK AND NOT THE REPAIR, AND CONFLATING THEM IS THE REAL RISK HERE. Sixteen rows in this module sit at 70-96% of the same limit; this regrouping moves the one row that crossed and leaves the population where it was. The cost is systemic - every emit consumer in the corpus pays the same per-word rate - and its measurement, its falsified hypotheses and its next-rung trigger are docs/probes/live_deploy_emit_witness_cost_attribution_2026-08-23.md. The next fleet runner-slot width increase lengthens the emitted script again and the next-highest row crosses.\\n\\nONE THING THIS REGROUPING MAKES HARDER, DECLARED RATHER THAN DISCOVERED LATER. The claim as written is LITERAL: it negates a spelling. The strictly stronger form is RELATIONAL - assert the two emitted endpoint values differ FROM EACH OTHER, which catches a drift that keeps both spellings absent from each other's script while still colliding. That form genuinely needs both scripts in one claim, so adopting it re-fuses two of these rows and re-crosses the line at today's per-word cost. It is therefore gated on the emit-cost lane rather than on anyone's appetite, and it is named here so the stronger claim is a known deferral and not an idea nobody had." test fn production_apply_serves_its_own_endpoint_and_scopes_no_path() -> Bool { - let live_apply = live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let live_apply = live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1())) string_contains(s: live_apply, pattern: tailscale_enable_command(endpoint: deployment_spec_srv1().names.tailscale_serve_endpoint)) && !string_contains(s: live_apply, pattern: "--set-path") } test fn twin_apply_serves_its_own_endpoint_and_not_productions() -> Bool { - let twin_apply = live_deploy_apply_script_for(spec: srv1_twin_spec(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let twin_apply = live_deploy_apply_script_for(spec: srv1_twin_spec(), plan: deploy_witness_full_install_plan(spec: srv1_twin_spec())) string_contains(s: twin_apply, pattern: tailscale_enable_command(endpoint: srv1_twin_spec().names.tailscale_serve_endpoint)) && !string_contains(s: twin_apply, pattern: tailscale_enable_command(endpoint: deployment_spec_srv1().names.tailscale_serve_endpoint)) } @@ -844,7 +871,7 @@ test fn retract_is_not_emitted_without_an_observed_scoped_removal() -> Bool { // is emitted via fleet-converge.yml instead of a deploy-installed timer. test fn fleet_converge_timer_not_installed_by_deploy_apply() -> Bool { let spec = deployment_spec_srv1() - let live = live_deploy_apply_script_for(spec: spec, revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + let live = live_deploy_apply_script_for(spec: spec, plan: deploy_witness_full_install_plan(spec: spec)) !any( deployment_owned_steps(instance: srv1_live_dashboard_instance(), names: spec.names), s => match s.kind { FleetConvergeTimerUnit => true _ => false } @@ -905,13 +932,147 @@ fn tailscale_off_command(endpoint: TailscaleServeEndpoint) -> String { // deploy through `gunbc converge --mode deploy` on srv1 refused `install: invalid user // 'gunbc-publication'` because the member installed spool directories for a principal nobody had // created. The id-guard is what makes the second deploy a no-op; --system and nologin are what keep -// it a service identity rather than a login. +// it a service identity rather than a login. The claim reads the preamble PRODUCER and the preamble +// step list, not a full apply render: the full render costs more than the floor's per-claim line and +// this claim's subject is two commands, so rendering the whole deployment to find them was a second +// full render whose cost belonged to the sibling claims that need it (required floor on PR #10696). +fn preamble_step_command(step: PipelineStep) -> String { + match step { + Do { run } => run.command + _ => "" + } +} + test fn the_preamble_ensures_the_publication_principal_as_a_nologin_system_account() -> Bool { - let apply = witness_apply_script() - string_contains(s: apply, pattern: "id -u gunbc-publication >/dev/null 2>&1 || ") - && string_contains(s: apply, pattern: "useradd") - && string_contains(s: apply, pattern: "--system") - && string_contains(s: apply, pattern: "--no-create-home") - && string_contains(s: apply, pattern: "/usr/sbin/nologin") - && string_contains(s: apply, pattern: "gunbc-publication") + let spec = deployment_spec_srv1() + let ensure = deploy_publication_principal_ensure_command(spec: spec) + let preamble = match ci_deploy_srv1_access_or_refusal() { + DeployAccessReady { access } => deploy_apply_preamble_steps(access: access, spec: spec) + DeployAccessJobPrincipalRefused { refusal: _ } => [] + } + string_contains(s: ensure, pattern: "id -u gunbc-publication >/dev/null 2>&1 || ") + && string_contains(s: ensure, pattern: "useradd") + && string_contains(s: ensure, pattern: "--system") + && string_contains(s: ensure, pattern: "--no-create-home") + && string_contains(s: ensure, pattern: "/usr/sbin/nologin") + && count(preamble) == 2 + && any(preamble, step => preamble_step_command(step: step) == ensure) +} + + +// A CONVERGED HOST'S APPLY MUTATES NONE OF THE RELEASE-CONSUMING MEMBERS. The plan with no mutations +// renders a script that installs no executable, publishes no tree (so no repository-convergence +// oneshot, the five-minute source-compiling evaluator of 2026-09-06), writes no serve unit and +// restarts no serve process. The other owned artifacts still appear: they are the declared apply-all +// frontier. The positive control -- the full plan renders every one of those -- is +// repository_convergence_is_ordered_after_binary_and_before_readback and +// the_full_plan_lands_its_members_in_the_deployments_order below, over the one full render the module +// shares; rendering it here too put this claim over the floor's per-claim line. +// ── The member projection, read without rendering the whole apply script ──────────────────────── +// +// A CLAIM ABOUT ONE MEMBER'S FRAGMENTS SHOULD NOT COST A WHOLE DEPLOYMENT RENDER, and the floor +// charges per witness, so paying one was not a style question: both claims below were preempted by +// the changed-witness CPU deadline before they reached a verdict, twice. The projection they +// actually assert over is emit_release_member_effects -- the steps ONE member contributes -- and +// reading those directly is both cheaper and a closer statement of the subject than grepping the +// serialized script for their absence. +// +// THE ORDERING HALF IS NOT ASSERTED OVER TEXT EITHER, and it does not need to be: the emitter folds +// flat_map over spec.steps, so the order of the emitted fragments IS the order of the deployment's +// own step list, by construction rather than by coincidence. What is worth witnessing is therefore +// the AUTHORITY -- that the tree, the runtime roots and the unit sit in that order in the +// deployment's steps -- because that is the fact a regression would change. A rendered-text control +// still exists in the sibling claims that hold the shared render. +fn member_fragment_text(member: ConvergedMember, plan: ReleaseEffectPlan) -> String { + join( + map( + emit_release_member_effects(member: member, plan: plan, spec: deployment_spec_srv1()), + st => match st { + Do { run } => run.command + Comment { text: _ } => "" + Exit { status: _ } => "" + If { cond: _, then_: _, else_: _ } => "" + For { binder: _, over: _, body: _ } => "" + While { cond: _, body: _ } => "" + Retry { body: _, escalations: _, on_exhausted: _ } => "" + Let { name: _, value: _ } => "" + Call { callee: _, args: _ } => "" + }, + ), + "\n", + ) +} + +// THE STEP LIST IS TAKEN ONCE AND PASSED, not rebuilt per lookup: reading it from +// deployment_spec_srv1() inside the fold made four ordering comparisons construct the whole +// deployment spec four times. That is the copied-accumulator shape and the standing rule fixes it +// regardless of the realized n -- but THE MEASUREMENT DOES NOT CREDIT IT, and saying so is the +// point of this note. The claim read 469ms before the change and 496ms after, which is noise: the +// spec rebuild was not where its cost was, and the cost that remains is unattributed. What actually +// moved this claim was giving up the full apply render (4830ms -> ~480ms). A reader who takes this +// parameterization as the cost repair will look for the next one in the wrong place. +fn owned_step_index(steps: List, kind: OwnedArtifactKind) -> Int { + fold( + steps, + init: StepScan { at: 0, found: 0 - 1 }, + f: (acc, st) => StepScan { + at: acc.at + 1, + found: match st { + ArtifactStep { step: art } => if art.kind == kind && acc.found < 0 { acc.at } else { acc.found } + Dependency { step: _ } => acc.found + }, + }, + ).found +} + +type StepScan { + at: Int + found: Int +} + +test fn a_zero_mutation_plan_installs_publishes_and_restarts_nothing_of_the_release() -> Bool { + let idle = converged_effect_plan(target: deploy_witness_release_target(spec: deployment_spec_srv1())) + member_fragment_text(member: Executable, plan: idle) == "" + && member_fragment_text(member: TreePublication, plan: idle) == "" + && member_fragment_text(member: UnitFile, plan: idle) == "" + && member_fragment_text(member: UnitManager, plan: idle) == "" + && member_fragment_text(member: ServiceProcess, plan: idle) == "" +} + +// THE FULL PLAN LANDS ITS MEMBERS IN THE DEPLOYMENT'S ORDER, NOT AHEAD OF IT. The repository +// convergence (tree publication) precedes the attempt-state root's creation, which precedes the serve +// restart: the roots the process writes exist before the process starts, and the release mutations +// are interleaved at their steps' positions rather than prepended as a block. The executable's +// install carries its identity: the planned digest is verified against the source before the copy and +// against the destination after it, so the two verify lines bracket the install. +test fn the_full_plan_lands_its_members_in_the_deployments_order() -> Bool { + let spec = deployment_spec_srv1() + let full = deploy_witness_full_install_plan(spec: spec) + let executable = member_fragment_text(member: Executable, plan: full) + let verify_source = "echo '5555555555555555555555555555555555555555555555555555555555555555 target/release/gunbc' | sha256sum -c -" + let verify_dest = join(["echo '5555555555555555555555555555555555555555555555555555555555555555 ", spec.service.serve_binary as String, "' | sha256sum -c -"], "") + let steps = spec.steps + three_markers_in_order(text: executable, first: verify_source, second: "target/release/gunbc' '", third: verify_dest) + && owned_step_index(steps: steps, kind: ServeBinary) < owned_step_index(steps: steps, kind: GunbcSourceTree) + && owned_step_index(steps: steps, kind: GunbcSourceTree) < owned_step_index(steps: steps, kind: AttemptStateRoot) + && owned_step_index(steps: steps, kind: AttemptStateRoot) < owned_step_index(steps: steps, kind: SystemdUnit) + && member_fragment_text(member: ServiceProcess, plan: full) == "" +} + +// A BINARY-ONLY PLAN INSTALLS THE EXECUTABLE AND RESTARTS, AND DOES NOT PUBLISH THE TREE: the +// emitter honours the plan's population, so the repository-convergence oneshot is not started for a +// change that touched no tree. +test fn a_binary_only_plan_installs_and_restarts_without_publishing_the_tree() -> Bool { + let spec = deployment_spec_srv1() + let sh = live_deploy_apply_script_for(spec: spec, plan: ReleaseEffectPlan { + target: deploy_witness_release_target(spec: spec), + executable: InstallExecutable, + tree: TreeKept, + unit_file: UnitFileKept, + manager: ManagerKept, + process: RestartService, + }) + string_contains(s: sh, pattern: "target/release/gunbc") + && string_contains(s: sh, pattern: deploy_effect_command(inv: systemctl_restart_effect(unit: spec.service.unit_name))) + && !string_contains(s: sh, pattern: "--function repository_converge_srv1_owner_wet") } diff --git a/dag/test/claim/live_deploy/member_identity_witness_test.dag b/dag/test/claim/live_deploy/member_identity_witness_test.dag new file mode 100644 index 00000000000..cca10c75da2 --- /dev/null +++ b/dag/test/claim/live_deploy/member_identity_witness_test.dag @@ -0,0 +1,210 @@ +module test.claim.live_deploy.member_identity_witness_test + +import std.types { Bool, String, List, NonEmptyStr, CommitSha } +import extdeps.crypto.hash { Digest, sha256_digest } +import std.content_hash { Fnv1a64Structural, content_hash_atom } +import extdeps.systemd { Active, Failed, Inactive } +import gunbc.live_deploy.member_identity { + ConvergedMember, TreePublication, Executable, UnitFile, UnitManager, ServiceProcess, + TreePublicationIdentity, UnitFileIdentity, UnitManagerStanding, ManagerLoadedCurrent, ManagerNeedsReload, + ServiceProcessIdentity, ProcessRunningRelease, ProcessNotRunning, + MemberIdentity, TreeIdentity, ExecutableIdentity, UnitFileAt, UnitManagerAt, ProcessAt, + ObservedMember, MemberObserved, MemberAbsentOnHost, MemberUnobservable, + MemberDecision, AlreadyConverged, Create, Replace, MemberRefused, + MemberRefusalCause, RepositoryBootstrapUnavailable, ObservationIsOfAnotherMember, DecisionFiledUnderAnotherMember, + decide_member, derive_effects, ReleaseMemberDecisions, ReleaseTarget, + EffectDerivation, EffectsDerived, EffectsRefused, effect_labels, +} + +fn rev_a() -> CommitSha { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" as CommitSha } +fn rev_b() -> CommitSha { "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" as CommitSha } +fn bin_a() -> Digest { sha256_digest(hex: "1111111111111111111111111111111111111111111111111111111111111111" as NonEmptyStr) } +fn bin_b() -> Digest { sha256_digest(hex: "2222222222222222222222222222222222222222222222222222222222222222" as NonEmptyStr) } +fn unit_a() -> Fnv1a64Structural { content_hash_atom(value: "[Unit]\nDescription=a\n" as NonEmptyStr) } +fn unit_b() -> Fnv1a64Structural { content_hash_atom(value: "[Unit]\nDescription=b\n" as NonEmptyStr) } +fn unit_name() -> NonEmptyStr { "gunbc-roadmap.service" as NonEmptyStr } + +fn release_target() -> ReleaseTarget { ReleaseTarget { revision: rev_a(), executable: bin_a(), unit: unit_name() } } + +fn desired_tree() -> MemberIdentity { TreeIdentity { tree: TreePublicationIdentity { revision: rev_a(), tracked_projection_clean: true } } } +fn desired_bin() -> MemberIdentity { ExecutableIdentity { digest: bin_a() } } +fn desired_file() -> MemberIdentity { UnitFileAt { file: UnitFileIdentity { unit: unit_name(), document: unit_a() } } } +fn desired_manager() -> MemberIdentity { UnitManagerAt { standing: ManagerLoadedCurrent } } +fn desired_process() -> MemberIdentity { ProcessAt { process: ProcessRunningRelease { revision: rev_a() } } } + +fn converged_observations() -> ReleaseMemberDecisions { + decisions( + observed_bin: MemberObserved { identity: desired_bin() }, + observed_tree: MemberObserved { identity: desired_tree() }, + observed_file: MemberObserved { identity: desired_file() }, + observed_manager: MemberObserved { identity: desired_manager() }, + observed_process: MemberObserved { identity: desired_process() }, + ) +} + +fn decisions( + observed_bin: ObservedMember, + observed_tree: ObservedMember, + observed_file: ObservedMember, + observed_manager: ObservedMember, + observed_process: ObservedMember, +) -> ReleaseMemberDecisions { + ReleaseMemberDecisions { + executable: decide_member(member: Executable, desired: desired_bin(), observed: observed_bin), + tree: decide_member(member: TreePublication, desired: desired_tree(), observed: observed_tree), + unit_file: decide_member(member: UnitFile, desired: desired_file(), observed: observed_file), + unit_manager: decide_member(member: UnitManager, desired: desired_manager(), observed: observed_manager), + process: decide_member(member: ServiceProcess, desired: desired_process(), observed: observed_process), + } +} + +fn labels_of(d: EffectDerivation) -> List { + match d { EffectsDerived { plan } => effect_labels(plan: plan) EffectsRefused { refusals: _ } => ["REFUSED"] } +} + +fn labels(d: ReleaseMemberDecisions) -> List { + labels_of(d: derive_effects(decisions: d, target: release_target())) +} + +fn one_drift(member: ConvergedMember, observed: ObservedMember) -> ReleaseMemberDecisions { + let c = converged_observations() + match member { + Executable => ReleaseMemberDecisions { executable: decide_member(member: Executable, desired: desired_bin(), observed: observed), tree: c.tree, unit_file: c.unit_file, unit_manager: c.unit_manager, process: c.process } + TreePublication => ReleaseMemberDecisions { executable: c.executable, tree: decide_member(member: TreePublication, desired: desired_tree(), observed: observed), unit_file: c.unit_file, unit_manager: c.unit_manager, process: c.process } + UnitFile => ReleaseMemberDecisions { executable: c.executable, tree: c.tree, unit_file: decide_member(member: UnitFile, desired: desired_file(), observed: observed), unit_manager: c.unit_manager, process: c.process } + UnitManager => ReleaseMemberDecisions { executable: c.executable, tree: c.tree, unit_file: c.unit_file, unit_manager: decide_member(member: UnitManager, desired: desired_manager(), observed: observed), process: c.process } + ServiceProcess => ReleaseMemberDecisions { executable: c.executable, tree: c.tree, unit_file: c.unit_file, unit_manager: c.unit_manager, process: decide_member(member: ServiceProcess, desired: desired_process(), observed: observed) } + } +} + +// FIXTURE 1: ALREADY CONVERGED -> ZERO MUTATIONS. The host holds exactly the desired identities, so +// the plan carries no mutation. This is the property the apply-all path could not have: with +// observed = [] every member was an add on every apply. +test fn an_already_converged_host_produces_zero_mutations() -> Bool { + count(labels(d: converged_observations())) == 0 +} + +// FIXTURE 2: EXECUTABLE-ONLY DRIFT -> INSTALL + RESTART, NOTHING ELSE. The key (the binary's path) +// is fixed and only its content differs -- the control the observed-side record demands of every +// reconcile binding, and the one the path-keyed roster could never express. +test fn executable_drift_installs_the_executable_and_restarts_its_consumer_only() -> Bool { + labels(d: one_drift(member: Executable, observed: MemberObserved { identity: ExecutableIdentity { digest: bin_b() } })) + == ["install-executable sha256:1111111111111111111111111111111111111111111111111111111111111111", "restart gunbc-roadmap.service"] +} + +// FIXTURE 3: UNIT-FILE DRIFT -> WRITE + DAEMON-RELOAD + RESTART. The reload appears because the file +// changed, and appears once. +test fn unit_file_drift_writes_the_unit_reloads_the_manager_and_restarts() -> Bool { + labels(d: one_drift(member: UnitFile, observed: MemberObserved { identity: UnitFileAt { file: UnitFileIdentity { unit: unit_name(), document: unit_b() } } })) + == ["write-unit gunbc-roadmap.service", "daemon-reload", "restart gunbc-roadmap.service"] +} + +// THE THREE UNIT MEMBERS ARE THREE REMEDIES, AND THIS IS THE PAIR THAT PROVES IT. A stale MANAGER +// under a current file reloads and restarts -- and does NOT rewrite the file, which the fused +// identity did (it derived a write for any unit-side difference). This is the interrupted deploy: +// file written, never reloaded. +test fn a_stale_manager_reloads_and_restarts_without_rewriting_the_file() -> Bool { + labels(d: one_drift(member: UnitManager, observed: MemberObserved { identity: UnitManagerAt { standing: ManagerNeedsReload } })) + == ["daemon-reload", "restart gunbc-roadmap.service"] +} + +// A DOWN PROCESS UNDER A CURRENT FILE AND A CURRENT MANAGER RESTARTS, AND ONLY RESTARTS. The fused +// identity wrote the unit and reloaded the manager for this state too; both were work the host did +// not need. +test fn a_failed_process_restarts_and_nothing_else() -> Bool { + labels(d: one_drift(member: ServiceProcess, observed: MemberObserved { identity: ProcessAt { process: ProcessNotRunning { active: Failed } } })) + == ["restart gunbc-roadmap.service"] +} + +// AN ACTIVE PROCESS RUNNING THE PREVIOUS RELEASE IS NOT CONVERGED. This is the state the fused +// identity could not express at all: unit file current, manager current, ActiveState Active, so +// every member compared equal and the plan was EMPTY -- while readiness refused with +// ProcessRevisionMismatch and no later plan ever restarted it. An absorbing refusal, repaired here +// by making the running release part of the process's identity. +test fn an_active_process_running_the_previous_release_restarts() -> Bool { + labels(d: one_drift(member: ServiceProcess, observed: MemberObserved { identity: ProcessAt { process: ProcessRunningRelease { revision: rev_b() } } })) + == ["restart gunbc-roadmap.service"] +} + +// A DIRTY TRACKED PROJECTION AT THE RIGHT REVISION IS A REPLACE, and a right revision is not enough: +// this is the stale-projection state the observed-side record found revision-only checks blind to. +test fn a_dirty_projection_at_the_desired_revision_republishes_the_tree() -> Bool { + labels(d: one_drift(member: TreePublication, observed: MemberObserved { identity: TreeIdentity { tree: TreePublicationIdentity { revision: rev_a(), tracked_projection_clean: false } } })) + == ["publish-tree aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "restart gunbc-roadmap.service"] +} + +// AN ABSENT EXECUTABLE IS A CREATE; AN UNOBSERVABLE MEMBER REFUSES THE WHOLE PLAN. Absence is an +// established fact and produces the install; a read that could not be made is neither absence nor +// convergence, and a plan built over it would either reinstall blindly or skip blindly. +test fn absence_creates_and_unobservability_refuses() -> Bool { + labels(d: one_drift(member: Executable, observed: MemberAbsentOnHost { member: Executable })) + == ["install-executable sha256:1111111111111111111111111111111111111111111111111111111111111111", "restart gunbc-roadmap.service"] + && (match derive_effects(decisions: one_drift(member: Executable, observed: MemberUnobservable { member: Executable, cause: "sha256sum refused" }), target: release_target()) { + EffectsRefused { refusals } => count(refusals) == 1 + EffectsDerived { plan: _ } => false + }) +} + +// AN ABSENT REPOSITORY REFUSES BY NAME RATHER THAN PLANNING A CREATE IT CANNOT PERFORM. The tree's +// Create would publish into a directory that is not a repository (the sync excludes .git and the +// convergence oneshot refuses a non-repository pre-state), leaving a state every later observation +// refuses -- a recoverable absence turned into a permanent one. Until a bootstrap transition exists +// the deploy says so and installs nothing. +test fn an_absent_repository_refuses_with_the_bootstrap_cause() -> Bool { + match derive_effects(decisions: one_drift(member: TreePublication, observed: MemberAbsentOnHost { member: TreePublication }), target: release_target()) { + EffectsDerived { plan: _ } => false + EffectsRefused { refusals } => count(refusals) == 1 && any(refusals, d => match d { + MemberRefused { member: TreePublication, cause: RepositoryBootstrapUnavailable } => true + _ => false + }) + } +} + +// A DECISION ASKED OF THE WRONG MEMBER IS A REFUSAL, NOT A COMPARISON: the tree's decision handed an +// executable observation cannot be converged or changed, only wrong. +test fn an_observation_of_another_member_is_refused() -> Bool { + match decide_member(member: TreePublication, desired: desired_tree(), observed: MemberObserved { identity: desired_bin() }) { + MemberRefused { member: TreePublication, cause: ObservationIsOfAnotherMember { observed: Executable } } => true + _ => false + } +} + +// A DECISION FILED IN THE WRONG SLOT IS REFUSED BY THE DERIVATION. The record gives one slot per +// member, so a member cannot be missing; what it cannot type-check is that a slot holds a decision +// ABOUT its member, and that is checked here rather than assumed. +test fn a_decision_in_the_wrong_slot_refuses_the_plan() -> Bool { + let c = converged_observations() + let swapped = ReleaseMemberDecisions { + executable: c.tree, + tree: c.executable, + unit_file: c.unit_file, + unit_manager: c.unit_manager, + process: c.process, + } + match derive_effects(decisions: swapped, target: release_target()) { + EffectsDerived { plan: _ } => false + EffectsRefused { refusals } => count(refusals) == 2 && any(refusals, d => match d { + MemberRefused { member: Executable, cause: DecisionFiledUnderAnotherMember { found: TreePublication } } => true + _ => false + }) + } +} + +// A FULL DRIFT DERIVES EVERY MUTATION ONCE, IN THE DEPLOYMENT'S ORDER, and the reload appears once +// even though both the file and the manager changed. +test fn a_fully_drifted_host_derives_each_mutation_once_in_order() -> Bool { + let d = decisions( + observed_bin: MemberObserved { identity: ExecutableIdentity { digest: bin_b() } }, + observed_tree: MemberObserved { identity: TreeIdentity { tree: TreePublicationIdentity { revision: rev_b(), tracked_projection_clean: true } } }, + observed_file: MemberObserved { identity: UnitFileAt { file: UnitFileIdentity { unit: unit_name(), document: unit_b() } } }, + observed_manager: MemberObserved { identity: UnitManagerAt { standing: ManagerNeedsReload } }, + observed_process: MemberObserved { identity: ProcessAt { process: ProcessNotRunning { active: Inactive } } }, + ) + labels(d: d) == [ + "install-executable sha256:1111111111111111111111111111111111111111111111111111111111111111", + "publish-tree aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "write-unit gunbc-roadmap.service", + "daemon-reload", + "restart gunbc-roadmap.service", + ] +} diff --git a/dag/test/claim/shell_exec_run_argv_embed_witness_test.dag b/dag/test/claim/shell_exec_run_argv_embed_witness_test.dag index 3e3892ba59f..65687e70ede 100644 --- a/dag/test/claim/shell_exec_run_argv_embed_witness_test.dag +++ b/dag/test/claim/shell_exec_run_argv_embed_witness_test.dag @@ -1,6 +1,6 @@ module test.claim.shell_exec_run_argv_embed_witness -import test.claim.live_deploy.deploy_release_fixture { deploy_witness_release_revision } +import test.claim.live_deploy.deploy_release_fixture { deploy_witness_release_revision, deploy_witness_full_install_plan } import std.types { Bool, String } import std.measure { byte_size_count } import gunbc.live_deploy.emit { live_deploy_apply_script_for, RevisionBoundAtEmission } @@ -29,7 +29,7 @@ fn src_has(path: String, needle: String) -> Bool { // host_exec_arg_max_strlen authority). test fn witness_apply_script_stays_under_argv_embed_budget() -> Bool { - let script_len = length(live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision })) + let script_len = length(live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1()))) let budget = byte_size_count(b: host_exec_arg_max_strlen) script_len > 0 && script_len < budget } @@ -39,7 +39,7 @@ test fn witness_shell_exec_run_uses_stdin_transport() -> Bool { } test fn witness_apply_script_requires_bash_receiver() -> Bool { - string_contains(s: live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }), pattern: "pipefail") + string_contains(s: live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1())), pattern: "pipefail") } test fn witness_shell_exec_run_uses_bash_receiver() -> Bool { diff --git a/dag/test/claim/ssh_transport_witness_test.dag b/dag/test/claim/ssh_transport_witness_test.dag index 86c265e3377..f7647941007 100644 --- a/dag/test/claim/ssh_transport_witness_test.dag +++ b/dag/test/claim/ssh_transport_witness_test.dag @@ -1,6 +1,6 @@ module test.claim.ssh_transport_witness -import test.claim.live_deploy.deploy_release_fixture { deploy_witness_release_revision } +import test.claim.live_deploy.deploy_release_fixture { deploy_witness_release_revision, deploy_witness_full_install_plan } import std.types { Bool, NonEmptyStr } import std.measure { byte_size_count } import gunbc.live_deploy.emit { live_deploy_apply_script_for, RevisionBoundAtEmission } @@ -20,7 +20,7 @@ data ssh_exec_script_remote_receiver: NonEmptyStr = "bash -s" // the sibling witnesses regardless of script size. test fn witness_apply_script_stays_under_argv_embed_budget() -> Bool { - let script_len = length(live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision })) + let script_len = length(live_deploy_apply_script_for(spec: deployment_spec_srv1(), plan: deploy_witness_full_install_plan(spec: deployment_spec_srv1()))) let budget = byte_size_count(b: host_exec_arg_max_strlen) script_len > 0 && script_len < budget }