A deployment converges on member identities: tree, executable and serve unit mutate only when they differ

dag/extdeps/crypto/hash.dag

@@ -1,6 +1,7 @@
 module extdeps.crypto.hash
 
-import std.types { NonEmptyStr, String }
+import std.types { NonEmptyStr, String, Bool, List }
+import std.algebra { trim }
 import std.content_hash {
   as_content_hash_cryptographic,
   as_content_hash_sha512,
@@ -40,7 +41,7 @@ fn sha512_digest(hex: NonEmptyStr) -> Digest {
   Digest { algorithm: Sha512, hex: hex }
 }
 
-data digest_shell_verify_line_legacy_note: NonEmptyStr = "LEGACY SHA-256-only shell-string verifier for the existing sccache consumer. Not a package-delivery realization. SHA-512 intentionally has no production shell arm here — gunbc.package_delivery uses PackageArchiveDigestObservation. dissolve-on: sccache migrates off this helper and the function deletes."
+data digest_shell_verify_line_legacy_note: NonEmptyStr = "SHA-256 shell-string verifier: `echo <hex>  <path> | sha256sum -c -` exits nonzero when the file at path does not hash to the digest, which under set -e stops the line. Two consumers: the sccache pin and the live-deploy executable install, which verifies the SOURCE against the planned digest before copying and the DESTINATION after (gunbc.live_deploy.emit emit_release_member_effects) so an installed executable can never differ from the identity the plan declared converged. Not a package-delivery realization: SHA-512 intentionally has no production shell arm here — gunbc.package_delivery uses PackageArchiveDigestObservation."
 
 fn digest_shell_verify_line(digest: Digest, file_path: NonEmptyStr) -> String {
   match digest.algorithm {
@@ -80,3 +81,39 @@ fn sha512_digest_content_hash(digest: Digest) -> ContentHash? {
     Sha256 => none
   }
 }
+
+// coreutils sha256sum over one file: `<hex>  <path>` on stdout, exit 0. The operation is the LOCAL
+// realization; sha256sum_argv is the same invocation spelled for a remote typed-argv transport, and
+// sha256sum_line_digest reads the one output line for both arms. The two spellings of the argv are
+// the extdeps op-plus-builder seam every shell-transported operation in this tree currently carries
+// (extdeps.systemd systemd_parse_label_derivation_debt records the same debt); they fold when the
+// transport row can be read from the operation itself.
+service crypto.Sha256Sum {
+  operation File {
+    input { path: String }
+    output {
+      line: String from "stdout"
+      success: Bool from "exit_success"
+      stderr: String from "stderr"
+    }
+    readonly
+    transport shell { argv: ["sha256sum", "--", "{path}"] }
+    exit {
+      0 => Unit
+      nonzero => String "sha256sum failed"
+    }
+    mock_response {
+      0 => { line: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  {path}", success: true, stderr: "" } "hermetic crypto.Sha256Sum.File: the empty-input digest"
+    }
+  }
+}
+
+fn sha256sum_argv(path: String) -> List<String> {
+  ["sha256sum", "--", path]
+}
+
+// The first field of the output line, admitted only at the digest's exact length.
+fn sha256sum_line_digest(line: String) -> Digest? {
+  let field = fold(split(s: trim(s: line), delimiter: " "), init: "", f: (acc, w) => if acc == "" { w } else { acc })
+  if field.length() == 64 { Present { value: sha256_digest(hex: field as NonEmptyStr) } } else { none }
+}

dag/extdeps/git/inspect.dag

@@ -154,6 +154,13 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
 // SP SP TAB (`<mode> <type> <object>\t<path>`), which is what makes a path containing a newline, a
 // quote or a tab-free space unambiguous -- git's own -z contract, not a convention this repository
 // invents.
+// git.Inspect.ReadTreeIntoIndex IS NOT readonly, AND THE THREE NEIGHBOURS AROUND IT ARE. read-tree
+// WRITES the index file GIT_INDEX_FILE names; the repository's own index is untouched, which is why
+// StatusAgainstIndex and IgnoredAgainstIndex stay readonly, but the scratch index is a file the
+// operation creates on the host and a consumer owns its lifetime -- a unique path (extdeps.shell
+// shell.Mktemp) and a removal after the reads. Declaring the write readonly was the state this
+// module was in when review found a consumer using one fixed path under /tmp per revision, shared by
+// every observer of that release and never removed (PR #10696).
 service git.Inspect {
   operation Toplevel {
     input {}
@@ -201,6 +208,76 @@ service git.Inspect {
     }
   }
 
+  operation HeadCommitIn {
+    input { repository_path: String }
+    output {
+      sha: String from "stdout"
+      success: Bool from "exit_success"
+      stderr: String from "stderr"
+    }
+    readonly
+    transport shell { argv: ["git", "-C", "{repository_path}", "rev-parse", "HEAD"] }
+    exit {
+      0 => Unit
+      128 => String "Not a git repository"
+    }
+    mock_response {
+      0 => { sha: "0000000000000000000000000000000000000000", success: true, stderr: "" } "hermetic git.Inspect.HeadCommitIn"
+    }
+  }
+
+  operation ReadTreeIntoIndex {
+    input { repository_path: String, revision_hex: String, index_path: String }
+    output {
+      success: Bool from "exit_success"
+      stderr: String from "stderr"
+    }
+    transport shell { argv: ["env", "GIT_INDEX_FILE={index_path}", "git", "-C", "{repository_path}", "read-tree", "--no-sparse-checkout", "{revision_hex}"] }
+    exit {
+      0 => Unit
+      128 => String "Not a git repository"
+    }
+    mock_response {
+      0 => { success: true, stderr: "" } "hermetic git.Inspect.ReadTreeIntoIndex"
+    }
+  }
+
+  operation StatusAgainstIndex {
+    input { repository_path: String, index_path: String }
+    output {
+      entries_nul: String from "stdout"
+      success: Bool from "exit_success"
+      stderr: String from "stderr"
+    }
+    readonly
+    transport shell { argv: ["env", "GIT_INDEX_FILE={index_path}", "git", "-C", "{repository_path}", "--no-optional-locks", "status", "--porcelain=v1", "-z", "--untracked-files=all"] }
+    exit {
+      0 => Unit
+      128 => String "Not a git repository"
+    }
+    mock_response {
+      0 => { entries_nul: "", success: true, stderr: "" } "hermetic git.Inspect.StatusAgainstIndex: clean"
+    }
+  }
+
+  operation IgnoredAgainstIndex {
+    input { repository_path: String, index_path: String }
+    output {
+      paths_nul: String from "stdout"
+      success: Bool from "exit_success"
+      stderr: String from "stderr"
+    }
+    readonly
+    transport shell { argv: ["env", "GIT_INDEX_FILE={index_path}", "git", "-C", "{repository_path}", "ls-files", "--others", "--ignored", "--exclude-standard", "-z"] }
+    exit {
+      0 => Unit
+      128 => String "Not a git repository"
+    }
+    mock_response {
+      0 => { paths_nul: "", success: true, stderr: "" } "hermetic git.Inspect.IgnoredAgainstIndex: none"
+    }
+  }
+
   operation IgnoredFiles {
     input {}
     output {

dag/extdeps/systemd/systemd.dag

@@ -71,6 +71,7 @@ type SystemdUnitProperty
   | NextElapseUSecMonotonic
   | AccuracyUSec
   | TimersMonotonic
+  | NeedDaemonReload
 
 fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr {
   match property {
@@ -101,6 +102,7 @@ fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr {
     NextElapseUSecMonotonic => "NextElapseUSecMonotonic" as NonEmptyStr
     AccuracyUSec => "AccuracyUSec" as NonEmptyStr
     TimersMonotonic => "TimersMonotonic" as NonEmptyStr
+    NeedDaemonReload => "NeedDaemonReload" as NonEmptyStr
   }
 }
 

dag/gunbc/host/host_effect.dag

@@ -24,6 +24,7 @@ import gunbc.runner_incarnation { LocalRunnerSlotIncarnation }
 import gunbc.runner_lifecycle { RunnerReplacementCause }
 import gunbc.auth.github_apps { gunbai_ci_declared }
 import gunbc.live_deploy.candidate { CandidateRelease }
+import gunbc.live_deploy.member_identity { ReleaseEffectPlan }
 import std.effects {
   EffectShape,
   KeySource,
@@ -62,7 +63,7 @@ type HostEffect =
   | Srv3BootOnceReadBackSleep
   | Srv3ReceiptEmit { lines: List<String> }
   | HostIdentityShortHostnameRead
-  | LiveDeployApply { spec: DeploymentSpec, candidate: CandidateRelease }
+  | LiveDeployApply { spec: DeploymentSpec, candidate: CandidateRelease, plan: ReleaseEffectPlan }
   | LiveDeployRetract { spec: DeploymentSpec }
   | LiveDeployEnsureDependency { dep: DeploymentDependencyStep }
   | LiveDeployDigestReadback { port: Int }

dag/gunbc/host/host_effect_realize.dag

@@ -148,6 +148,7 @@ import gunbc.ci_deploy_access {
   deploy_access_sudo_probe_effect,
 }
 import gunbc.live_deploy.spec { DeploymentSpec, DeploymentDependencyStep }
+import gunbc.live_deploy.member_identity { ReleaseEffectPlan }
 import gunbc.live_deploy.emit {
   live_deploy_apply_script_for,
   live_deploy_retract_script_for,
@@ -403,7 +404,7 @@ type ResolvedHostEffectCell =
   | EffectivePrincipalReadOnHost { node: ComputeHost, read: EffectivePosixPrincipalRead }
   | SudoNopasswdExecuteProbeOnHost { node: ComputeHost, probe: SudoNopasswdExecuteProbeShape }
   | SudoNopasswdGrantListProbeOnHost { node: ComputeHost, probe: SudoNopasswdGrantListProbeShape }
-  | LiveDeployApplyOnHost { node: ComputeHost, spec: DeploymentSpec, candidate: CandidateRelease }
+  | LiveDeployApplyOnHost { node: ComputeHost, spec: DeploymentSpec, candidate: CandidateRelease, plan: ReleaseEffectPlan }
   | LiveDeployRetractOnHost { node: ComputeHost, spec: DeploymentSpec }
   | LiveDeployEnsureDependencyOnHost { node: ComputeHost, dep: DeploymentDependencyStep }
   | LiveDeployDigestReadbackOnHost { node: ComputeHost, port: Int }
@@ -468,8 +469,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res
           BmcwebSessionLoginOnHost { node: n }
         Srv3BmcwebTokenExtract =>
           BmcwebTokenExtractOnHost { node: n }
-        LiveDeployApply { spec: s, candidate: c } =>
-          LiveDeployApplyOnHost { node: n, spec: s, candidate: c }
+        LiveDeployApply { spec: s, candidate: c, plan: p } =>
+          LiveDeployApplyOnHost { node: n, spec: s, candidate: c, plan: p }
         LiveDeployRetract { spec: s } =>
           LiveDeployRetractOnHost { node: n, spec: s }
         LiveDeployEnsureDependency { dep: d } =>
@@ -522,8 +523,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res
               CodexSupervisedWorkerTurnOnHost { node: n, intent: i, lease_key: k }
             HostIdentityShortHostnameRead =>
               HostnameReadOnHost { node: n }
-            LiveDeployApply { spec: s, candidate: c } =>
-              LiveDeployApplyOnHost { node: n, spec: s, candidate: c }
+            LiveDeployApply { spec: s, candidate: c, plan: p } =>
+              LiveDeployApplyOnHost { node: n, spec: s, candidate: c, plan: p }
             LiveDeployRetract { spec: s } =>
               LiveDeployRetractOnHost { node: n, spec: s }
             LiveDeployEnsureDependency { dep: d } =>
@@ -611,7 +612,7 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res
               IncompatibleCell { reason: "host_effect: SudoNopasswdGrantListProbe is in-band (typed sudo grant-list probe) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." }
             HostIdentityShortHostnameRead =>
               IncompatibleCell { reason: "host_effect: HostIdentityShortHostnameRead is in-band (shell hostname -s) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." }
-            LiveDeployApply { spec: _, candidate: _ } =>
+            LiveDeployApply { spec: _, candidate: _, plan: _ } =>
               IncompatibleCell { reason: "host_effect: LiveDeployApply is in-band (actuator host shell mutation) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." }
             LiveDeployRetract { spec: _ } =>
               IncompatibleCell { reason: "host_effect: LiveDeployRetract is in-band (actuator host shell mutation) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." }
@@ -3785,6 +3786,7 @@ fn live_deploy_apply_candidate_locus_refusal(transport: HostEffectTransport) ->
 fn realize_live_deploy_apply_on_host(
   spec: DeploymentSpec,
   candidate: CandidateRelease,
+  plan: ReleaseEffectPlan,
   transport: HostEffectTransport,
   intent: HostEffectIntent,
 ) -> Reconciliation<HostEffectIntent, HostEffectEvidence> {
@@ -3794,10 +3796,7 @@ fn realize_live_deploy_apply_on_host(
   } else {
     realize_shell_on_host(
       script: retained_srvn(
-        body: live_deploy_apply_script_for(
-          spec: spec,
-          revision: RevisionBoundAtEmission { revision: candidate.revision },
-        ),
+        body: live_deploy_apply_script_for(spec: spec, plan: plan),
         reason: "live_deploy apply (unit install + start)",
       ),
       transport: transport,
@@ -3895,8 +3894,8 @@ fn host_effect_apply(target: NodeControlPlane, effect: HostEffect, evidence: Ide
       realize_sudo_nopasswd_execute_probe_on_host(probe: p, transport: transport, intent: intent)
     SudoNopasswdGrantListProbeOnHost { node: _, probe: p } =>
       realize_sudo_nopasswd_grant_list_probe_on_host(probe: p, transport: transport, intent: intent)
-    LiveDeployApplyOnHost { node: _, spec: s, candidate: c } =>
-      realize_live_deploy_apply_on_host(spec: s, candidate: c, transport: transport, intent: intent)
+    LiveDeployApplyOnHost { node: _, spec: s, candidate: c, plan: p } =>
+      realize_live_deploy_apply_on_host(spec: s, candidate: c, plan: p, transport: transport, intent: intent)
     LiveDeployRetractOnHost { node: _, spec: s } =>
       realize_live_deploy_retract_on_host(spec: s, transport: transport, intent: intent)
     LiveDeployEnsureDependencyOnHost { node: _, dep: d } =>
@@ -4094,8 +4093,8 @@ fn host_effect_apply_gated(
           realize_sudo_nopasswd_execute_probe_on_host(probe: p, transport: transport, intent: intent)
         SudoNopasswdGrantListProbeOnHost { node: _, probe: p } =>
           realize_sudo_nopasswd_grant_list_probe_on_host(probe: p, transport: transport, intent: intent)
-        LiveDeployApplyOnHost { node: _, spec: s, candidate: c } =>
-          realize_live_deploy_apply_on_host(spec: s, candidate: c, transport: transport, intent: intent)
+        LiveDeployApplyOnHost { node: _, spec: s, candidate: c, plan: p } =>
+          realize_live_deploy_apply_on_host(spec: s, candidate: c, plan: p, transport: transport, intent: intent)
         LiveDeployRetractOnHost { node: _, spec: s } =>
           realize_live_deploy_retract_on_host(spec: s, transport: transport, intent: intent)
         LiveDeployEnsureDependencyOnHost { node: _, dep: d } =>
@@ -4177,7 +4176,7 @@ fn host_effect_drive_directive(r: Reconciliation<HostEffectIntent, HostEffectEvi
             EffectivePrincipalReadOnHost { node: _, read: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
             SudoNopasswdExecuteProbeOnHost { node: _, probe: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
             SudoNopasswdGrantListProbeOnHost { node: _, probe: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
-            LiveDeployApplyOnHost { node: _, spec: _, candidate: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
+            LiveDeployApplyOnHost { node: _, spec: _, candidate: _, plan: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
             LiveDeployRetractOnHost { node: _, spec: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
             LiveDeployEnsureDependencyOnHost { node: _, dep: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))
             LiveDeployDigestReadbackOnHost { node: _, port: _ } => not_converged_directive(drive: host_effect_drive(intent: intent))

dag/gunbc/live_deploy/apply.dag

@@ -58,6 +58,8 @@ import gunbc.host_effect {
   DeployAccessPreflight,
 }
 import gunbc.host_effect_realize { host_effect_apply_gated }
+import gunbc.live_deploy.member_observe { deployment_effect_plan, deployment_identity_readback, IdentityReadbackConverged, IdentityReadbackDrift, IdentityReadbackRefused }
+import gunbc.live_deploy.member_identity { EffectsDerived, EffectsRefused, decision_label }
 import gunbc.live_deploy.readiness {
   live_deploy_poll_until_service_ready_for_deploy, live_deploy_healthz_get_for_port, running_release_unidentified_detail,
   ServiceReadyApplyEffect,
@@ -220,20 +222,31 @@ fn live_deploy_ensure_dependency(
   )
 }
 
+// THE PLAN IS DERIVED FROM THE HOST BEFORE THE MUTATION IS BUILT (gunbc.live_deploy.member_observe):
+// the same evaluation that decides also acts, and a plan that could not be derived refuses here with
+// every member's cause, installing nothing -- never the whole roster as a fallback.
 fn live_deploy_apply_mutation_via_transport(
   spec: DeploymentSpec,
   candidate: CandidateRelease,
   transport: HostEffectTransport,
   access: DeployAccess
 ) -> Reconciliation<HostEffectIntent, HostEffectEvidence> {
-  live_deploy_fold(
-    spec: spec,
-    effect: LiveDeployApply { spec: spec, candidate: candidate },
-    transport: transport,
-    access: access,
-    fixture_actor: Absent,
-    fixture_host: Absent
-  )
+  match deployment_effect_plan(spec: spec, candidate: candidate, transport: transport) {
+    EffectsRefused { refusals } =>
+      NotConverged {
+        reason: join(["live_deploy: the deployment plan could not be derived from the host: ", join(map(refusals, d => decision_label(d: d)), "; ")], ""),
+        applied: live_deploy_preflight_intent(access: access),
+      }
+    EffectsDerived { plan } =>
+      live_deploy_fold(
+        spec: spec,
+        effect: LiveDeployApply { spec: spec, candidate: candidate, plan: plan },
+        transport: transport,
+        access: access,
+        fixture_actor: Absent,
+        fixture_host: Absent
+      )
+  }
 }
 
 fn live_deploy_apply_readiness_gate(
@@ -293,6 +306,38 @@ fn live_deploy_apply_digest_readback_via_transport(
   )
 }
 
+// THE MEMBER IDENTITIES ARE READ BACK AFTER READINESS, and readiness is not a substitute for it.
+// Readiness proves the unit answers /healthz as the candidate; an equivalent binary reporting the
+// same revision, or a unit file rewritten after the copy, would pass it. The plan was derived from
+// the host's member identities, so the transaction closes by observing them again
+// (gunbc.live_deploy.member_observe deployment_identity_readback) and requiring every member
+// AlreadyConverged -- a remaining mutation names which member the apply failed to land, and a refusal
+// names which member could not be read afterwards. Both are NotConverged, never a green with a note.
+fn live_deploy_apply_identity_readback_gate(
+  spec: DeploymentSpec,
+  candidate: CandidateRelease,
+  transport: HostEffectTransport,
+  mutation: Reconciliation<HostEffectIntent, HostEffectEvidence>,
+) -> Reconciliation<HostEffectIntent, HostEffectEvidence> {
+  let intent = match mutation {
+    Converged { evidence: _, applied: i } => i
+    NotConverged { reason: _, applied: i } => i
+  }
+  match deployment_identity_readback(spec: spec, candidate: candidate, transport: transport) {
+    IdentityReadbackConverged => mutation
+    IdentityReadbackDrift { remaining } =>
+      NotConverged {
+        reason: join(["live_deploy: the apply completed and readiness held, but the host's member identities still differ from the release: ", join(remaining, "; ")], ""),
+        applied: intent,
+      }
+    IdentityReadbackRefused { refusals } =>
+      NotConverged {
+        reason: join(["live_deploy: the apply completed but the member identities could not be read back: ", join(map(refusals, d => decision_label(d: d)), "; ")], ""),
+        applied: intent,
+      }
+  }
+}
+
 fn live_deploy_apply_via_transport(
   spec: DeploymentSpec,
   candidate: CandidateRelease,
@@ -321,7 +366,7 @@ fn live_deploy_apply_via_transport(
       access: access,
     )
     if reconciliation_converged(r: digest) {
-      gated
+      live_deploy_apply_identity_readback_gate(spec: spec, candidate: candidate, transport: transport, mutation: gated)
     } else {
       match digest {
         NotConverged { reason: why, applied: _ } =>