ntfy channel readback: observed binary path; nested helper gets its checkout root (stacked on #12561)

dag/extdeps/systemd/journalctl.dag

@@ -1,6 +1,7 @@
 module extdeps.systemd.journalctl
 
-import std.types { NonEmptyStr, String, Bool }
+import std.types { NonEmptyStr, Bool, Unit }
+import std.string_type { String }
 import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef }
 import std.decl_ref { DeclarationRef, WholeDeclaration }
 import extdeps.uri { Uri, Https }

dag/gunbc/auth/approval_ntfy_deployment.dag

@@ -62,9 +62,21 @@ data approval_ntfy_unit_name: NonEmptyStr = "gunbc-ntfy.service"
 // never by guessing which user is which.
 data approval_ntfy_publisher_user: NonEmptyStr = "gunbc-approval-publisher"
 data approval_ntfy_operator_user: NonEmptyStr = "gunbc-approval-operator"
-// The packaged server's CLI (the upstream .deb installs /usr/bin/ntfy). Absolute because it runs
-// under sudo as the server's principal; an absent binary is a failed readback, which refuses.
-data approval_ntfy_binary_path: NonEmptyStr = "/usr/bin/ntfy"
+// THE SERVER BINARY THE HOST ACTUALLY RUNS. Absolute because it runs under sudo as the server's
+// principal; an absent binary is a failed readback, which refuses. The readback compares the running
+// process's executable against THIS row, and that comparison is only a check because the row is not
+// read from the host it checks: deriving it from the unit's ExecStart would admit a swapped unit.
+//
+// THE PREVIOUS VALUE WAS A BET ABOUT THE PACKAGING, and the host falsified it. It read "/usr/bin/ntfy"
+// on the ground that the upstream .deb installs there. But no model member installs this binary
+// (gunbc.auth.approval_ntfy_converge names it as outside its roster), and srv1's server was installed
+// 2026-09-07 at /usr/local/bin/ntfy. gunbc-ntfy.service ExecStart names that path, as read on the host
+// by proud-deer-538 on 2026-09-28 while running issue_device_enrolment_code by hand. The readback then
+// refused before minting by stat-ing a path the server does not run. So the row is the observed install,
+// and it stays a declaration: an install moved elsewhere refuses loudly here until the row moves with
+// it, which is the intended wall. Its dissolution is an installer member that places the binary, at
+// which point this row is that member's destination rather than a transcription of the host.
+data approval_ntfy_binary_path: NonEmptyStr = "/usr/local/bin/ntfy"
 
 // LOOPBACK, AND THAT IS THE SECURITY BOUNDARY RATHER THAN A DEFAULT. The server binds 127.0.0.1
 // and reaches the tailnet only through tailscale serve, which is what makes the tailnet identity

dag/gunbc/executor_privileged_operation.dag

@@ -23,6 +23,9 @@ import std.disposition { Disposition, Scaffold, SingleAuthority, RealizationDisp
 import std.decl_ref { DeclarationRef, WholeDeclaration }
 import extdeps.sudo.elevation { ElevatedInvocation, sudo_elevate, sudo_elevate_as, sudo_elevate_argv, sudo_elevate_command, sudo_elevate_as_command }
 import extdeps.exec.command { ArgvCommand }
+import extdeps.tools.env { env_prefixed_argv }
+import v2.std.orchestration { EnvSet }
+import gunbc.cli_run_workspace_root_scaffold { gunbc_workspace_root_env_name }
 import gunbc.cli_invoke { gunbc_run_invocation_words_with_functions, invocation_words_bare }
 import extdeps.posix.shell_command_language { posix_single_quote }
 import v2.std.algebra { fold_list }
@@ -84,6 +87,7 @@ type ExecutorPrivilegedOperation
   | ReadUserspaceConntrackHelpers
   | RunGunbcEntryAs {
       run_as: NonEmptyStr
+      workspace_root: NonEmptyStr
       binary: NonEmptyStr
       source_roots: List<NonEmptyStr>
       entry: NonEmptyStr
@@ -106,14 +110,23 @@ type OperationPrincipal
 // match (which is this argv) leaves no value open for the grantee to choose. The binary and every
 // path are absolute and name one release directory, so the grant authorizes one build over one
 // tree, and a sudoers wildcard is never needed to name it.
+//
+// IT RUNS WITH ITS LOCUS AS ITS CHECKOUT ROOT. The seed resolves a workspace root before any verb,
+// from gunbc.cli_run_workspace_root_scaffold gunbc_workspace_root_env_name or by walking up from the
+// working directory. sudo keeps the caller's working directory, which is no checkout, so without the
+// binding the entry refuses before it reads anything (found running the approval broker's ntfy
+// observation helper by hand on srv1, 2026-09-28). The binding is part of the argv, so the sudoers
+// match covers it and the grantee cannot point the root elsewhere. It goes through env(1) because
+// sudo's env_reset drops anything set outside the command.
 fn gunbc_entry_run_argv(
+  workspace_root: NonEmptyStr,
   binary: NonEmptyStr,
   source_roots: List<NonEmptyStr>,
   entry: NonEmptyStr,
   function: NonEmptyStr,
 ) -> List<NonEmptyStr> {
   concat(
-    [binary],
+    map(env_prefixed_argv(bindings: [EnvSet { name: gunbc_workspace_root_env_name, value: workspace_root as String }], command_argv: [binary as String]), w => w as NonEmptyStr),
     map(
       invocation_words_bare(words: gunbc_run_invocation_words_with_functions(
         source_roots: map(source_roots, r => r as String),
@@ -207,15 +220,15 @@ fn executor_privileged_operation_argv(op: ExecutorPrivilegedOperation) -> List<N
       u,
       a
     ]
-    RunGunbcEntryAs { run_as: _, binary: b, source_roots: rs, entry: e, function: f } =>
-      gunbc_entry_run_argv(binary: b, source_roots: rs, entry: e, function: f)
+    RunGunbcEntryAs { run_as: _, workspace_root: w, binary: b, source_roots: rs, entry: e, function: f } =>
+      gunbc_entry_run_argv(workspace_root: w, binary: b, source_roots: rs, entry: e, function: f)
   }
 }
 
 // EXHAUSTIVE, NO WILDCARD: a new operation must say whom it runs as instead of inheriting root.
 fn executor_privileged_operation_principal(op: ExecutorPrivilegedOperation) -> OperationPrincipal {
   match op {
-    RunGunbcEntryAs { run_as: u, binary: _, source_roots: _, entry: _, function: _ } => RunsAsAccount { login: u }
+    RunGunbcEntryAs { run_as: u, workspace_root: _, binary: _, source_roots: _, entry: _, function: _ } => RunsAsAccount { login: u }
     EnsureOwnedDirectory { path: _, owner: _, group: _, mode: _ } => RunsAsRoot
     RemoveDirectoryTree { path: _ } => RunsAsRoot
     SystemdDaemonReload => RunsAsRoot
@@ -291,9 +304,26 @@ fn executor_privileged_operation_elevated_command(op: ExecutorPrivilegedOperatio
 // operations name their paths: a sudoers wildcard over unit names would authorize activating units
 // this fleet does not model, and `systemctl enable` accepts a path as a unit argument.
 
-// THE SUDOERS COMMAND MATCH IS THE ARGV. Derived, never authored beside it.
+// THE SUDOERS COMMAND MATCH IS THE ARGV. Derived, never authored beside it, and spelled in sudoers'
+// own grammar. sudoers(5), "Command line arguments": the characters `,` `:` `=` and `\` must be
+// escaped with a backslash when used in a command's arguments, because unescaped they are Cmnd_Spec
+// grammar (list separators, a run-as/tag delimiter, an assignment, the escape itself). The escaped
+// spelling is the one the grammar documents, and it matches the same argv word. A bare `=` in these
+// arguments is evidently accepted by the sudo installed today: the systemctl set-property grants below
+// were rendered bare and passed the visudo gate. So this is the documented spelling replacing a
+// tolerated one, not a repair of grants known to mis-match. The backslash is escaped first so the
+// escapes added for the other three are not escaped again.
+// First consumers of the escape: the env(1) binding in a gunbc entry run (`GUNBC_WORKSPACE_ROOT=<dir>`,
+// see RunGunbcEntryAs) and a systemctl set-property assignment (`<Property>=<value>`,
+// gunbc.runner_host_grants).
+data sudoers_argument_escaped_characters: List<String> = ["\\", ",", ":", "="]
+
+fn sudoers_argument_word(w: String) -> String {
+  fold(sudoers_argument_escaped_characters, init: w, f: (acc, c) => join(split(s: acc, delimiter: c), join(["\\", c], "")))
+}
+
 fn executor_privileged_operation_sudoers_command(op: ExecutorPrivilegedOperation) -> NonEmptyStr {
-  join(map(executor_privileged_operation_argv(op: op), w => w as String), " ") as NonEmptyStr
+  join(map(executor_privileged_operation_argv(op: op), w => sudoers_argument_word(w: w as String)), " ") as NonEmptyStr
 }
 
 // RUN-AS IS root, NOT ALL. The executor previously rendered ALL=(ALL), which authorizes the
@@ -447,7 +477,7 @@ fn executor_privileged_operation_precondition(op: ExecutorPrivilegedOperation) -
     ReadConntrackExpectations => NoPrecondition
     ReadLegacyXtablesRules => NoPrecondition
     ReadUserspaceConntrackHelpers => NoPrecondition
-    RunGunbcEntryAs { run_as: _, binary: _, source_roots: _, entry: _, function: _ } => NoPrecondition
+    RunGunbcEntryAs { run_as: _, workspace_root: _, binary: _, source_roots: _, entry: _, function: _ } => NoPrecondition
   }
 }
 

dag/gunbc/fleet/fleet_release_bins_key.dag

@@ -124,6 +124,23 @@ data release_bins_key_prefix: String = "release-bins-"
 data release_bins_preimage_name: String = "release-bins.preimage"
 data release_bins_manifest_name: String = "release-bins.manifest"
 data release_bins_staging_dir: String = ".release-bins-staging"
+
+// WHERE A CONSUMER JOB RECEIVES THE PACK: a directory under the runner's temp root, never the checkout.
+// The consumer's checkout IS a deploy source. The approval broker's dark install ships its working tree,
+// and candidate admission refuses an untracked path that would ship. #12422 made the consumer keep all
+// three entry files (preimage, manifest, archive) for verification, and the download step, naming no
+// path, dropped them in the checkout root. So the dark install refused CandidateCheckoutNotClean on the
+// job's own pack (fleet-converge run 36473122990). Before #12422 the only download was the archive,
+// deleted after unpack, so the same misplacement was cleaned up rather than prevented. Placing the pack
+// outside the tree makes it unshippable by construction. Excluding the three names from the deployed
+// scope would be the widened scope. The download step's `path:` and every consumer read below derive
+// from this one row, as ${{ runner.temp }}/<dir> and $RUNNER_TEMP/<dir>: one location, two spellings of
+// the same runner variable.
+data release_bins_download_dir: String = "release-bins-download"
+
+fn release_bins_download_path_expression() -> String {
+  join(["${{ runner.temp }}/", release_bins_download_dir], "")
+}
 data release_bins_key_kind: String = "gunbc.fleet_release_bins"
 data release_bins_admitted_producer_ref: String = "refs/heads/main"
 
@@ -193,6 +210,7 @@ fn test_words(ws: List<Node>) -> Node { bash_build_test_bracket(words: ws) }
 fn staged(file: String) -> Node { words(ws: [var(n: "STAGE"), lit(t: join(["/", file], ""))]) }
 fn rooted(file: String) -> Node { words(ws: [var(n: "ROOT"), lit(t: join(["/", file], ""))]) }
 fn temp(file: String) -> Node { words(ws: [var(n: "RUNNER_TEMP"), lit(t: join(["/", file], ""))]) }
+fn downloaded(file: String) -> Node { temp(file: join([release_bins_download_dir, "/", file], "")) }
 
 fn append_line(file_var: String, line: Node) -> Node {
   bash_build_with_redir_to_file(
@@ -1021,25 +1039,25 @@ fn release_bins_consumer_statements() -> List<Node> {
         root_statement(),
         or_refuse(check: test_words(ws: [lit(t: "-n"), var(n: "RELEASE_BINS_KEY")]), r: ConsumerKeyMissing),
       ],
-      map(entry_files(), f => or_refuse(check: test_words(ws: [lit(t: "-f"), rooted(file: f)]), r: ConsumerFileMissing { file: f })),
+      map(entry_files(), f => or_refuse(check: test_words(ws: [lit(t: "-f"), downloaded(file: f)]), r: ConsumerFileMissing { file: f })),
     ),
     concat(
       concat(
         concat(
-          digest_of(name: "DOWNLOADED_DIGEST", path: rooted(file: release_bins_preimage_name)),
+          digest_of(name: "DOWNLOADED_DIGEST", path: downloaded(file: release_bins_preimage_name)),
           [or_refuse(check: key_matches(digest_var: "DOWNLOADED_DIGEST"), r: ConsumerPreimageKeyMismatch)],
         ),
         source_closure_statements(),
       ),
       seq(parts: [[
         or_refuse(
-          check: cmd(ws: [lit(t: "grep"), lit(t: "-qxF"), words(ws: [lit(t: "source_closure="), var(n: "SOURCE_CLOSURE")]), rooted(file: release_bins_preimage_name)]),
+          check: cmd(ws: [lit(t: "grep"), lit(t: "-qxF"), words(ws: [lit(t: "source_closure="), var(n: "SOURCE_CLOSURE")]), downloaded(file: release_bins_preimage_name)]),
           r: ConsumerSourceTreeMismatch,
         ),
       ], release_bins_verify_pack(
         site: ConsumerSite,
-        archive: rooted(file: ci_release_bins_archive_name),
-        manifest: rooted(file: release_bins_manifest_name),
+        archive: downloaded(file: ci_release_bins_archive_name),
+        manifest: downloaded(file: release_bins_manifest_name),
         dir: temp(file: "release-bins-check"),
       ), [
         cmd(ws: concat(

dag/gunbc/fleet/fleet_workflow_steps.dag

@@ -48,7 +48,7 @@ import gunbc.fleet_release_bins_key {
   release_bins_cache_hit_expression,
   release_bins_build_condition, release_bins_key_step_id, release_bins_lookup_step_id,
   release_bins_cache_step_id, release_bins_outcome_step_id, release_bins_preimage_name,
-  release_bins_manifest_name, release_bins_staging_dir,
+  release_bins_manifest_name, release_bins_staging_dir, release_bins_download_path_expression,
 }
 import gunbc.ci_spec {
   ci_release_bins_archive_name,
@@ -442,7 +442,10 @@ fn ci_release_bins_download_step() -> Step {
     id: none,
     uses: download_artifact_action,
     with: Present {
-      value: [kv(key: "name", value: yaml_string(s: ci_release_bins_artifact_name))]
+      value: [
+        kv(key: "name", value: yaml_string(s: ci_release_bins_artifact_name)),
+        kv(key: "path", value: yaml_string(s: release_bins_download_path_expression())),
+      ]
     },
     env: none,
     if_condition: none,

dag/gunbc/live_deploy/emit.dag

@@ -100,7 +100,7 @@ import extdeps.tools.curl { curl_download_command }
 import extdeps.tools.tar { tar_extract_command }
 import extdeps.tools.mkdir { mkdir_parents_command }
 import extdeps.exec.command { argv_words }
-import gunbc.cli_run_workspace_root_scaffold { gunbc_workspace_root_env_name }
+import gunbc.cli_run_workspace_root_scaffold { gunbc_workspace_root_env_name, release_locus_tree_receipt_name }
 import gunbc.runner_microvm_slot_unit {
   microvm_controller_root, microvm_controller_owner, microvm_controller_firecracker_release,
   microvm_controller_vmm_binary, microvm_controller_jailer_binary,
@@ -1987,12 +1987,21 @@ fn release_locus_install_steps(owner: NonEmptyStr, base_root: String, revision:
 // THE RECEIPT IS WRITTEN BY THE PRINCIPAL THAT OWNS THE LOCUS: `install -m 0644 /dev/stdin`
 // into a root-owned directory needs root, and the digest pipeline feeding it is the same either way.
 fn release_locus_tree_receipt_step(owner: NonEmptyStr, base_root: String, revision: ReleaseRevisionBinding) -> PipelineStep {
+  locus_tree_receipt_step_at(owner: owner, dir: approval_broker_release_dir(root: base_root, revision: revision), revision: revision)
+}
+
+// ONE RECEIPT SHAPE FOR EVERY LOCUS A gunbc BINARY IS SPAWNED FROM. The seed admits a spawn root
+// (gunbc.cli_run_workspace_root_scaffold gunbc_workspace_root_env_name) only when it holds dag/ and
+// the tree receipt, so a locus without one cannot be run from at all. The approval broker's release
+// locus and the root-owned helper snapshot both write it, with the same content: the candidate
+// revision and the digest of the installed sources.
+fn locus_tree_receipt_step_at(owner: NonEmptyStr, dir: String, revision: ReleaseRevisionBinding) -> PipelineStep {
   deploy_raw(command: join([
     printf_program as String, " 'candidate_revision=", release_revision_execstart_text(binding: revision),
     "\\ninstalled_digest=%s\\n' ",
-    "\"$(", approval_broker_tree_digest_expr(root: approval_broker_release_dir(root: base_root, revision: revision)), ")\"",
+    "\"$(", approval_broker_tree_digest_expr(root: dir), ")\"",
     " | ", join(sudo_elevate_argv(command: [install_program as String, "-m", "0644", "-o", owner as String, "-g", owner as String, "/dev/stdin",
-      approval_broker_tree_receipt_path_for(root: base_root, revision: revision)]), " "),
+      join([dir, "/", release_locus_tree_receipt_name], "")]), " "),
   ], ""))
 }
 
@@ -2219,13 +2228,19 @@ fn approval_broker_helper_sudoers_content(revision: ReleaseRevisionBinding) -> S
 }
 
 // THE GRANT'S WORDS MUST BE PLAIN, OR THE GRANT IS NOT EMITTED. A sudoers Cmnd_Spec treats these
-// characters specially: `,` and `:` separate list items, `=`, `!`, `(` and `)` are grammar, `\`
+// characters specially: `,` and `:` separate list items, `!`, `(` and `)` are grammar, `\`
 // escapes, whitespace splits arguments, `#` starts a comment, and `*` `?` `[` `]` are glob
 // metacharacters that would turn an exact match into a pattern. The poisoned RevisionNotInstalled
 // path contains spaces, so an unbound revision is refused here too. The rest are shell quoting and
 // expansion characters, which no path this module names verbatim contains.
+//
+// `=` IS NOT REFUSED, because it is escaped rather than forbidden. The sudoers renderer
+// (gunbc.executor_privileged_operation sudoers_argument_word) spells it `\=`, as sudoers(5) requires,
+// and the helper's argv carries one legitimately: the env(1) binding that gives the helper its
+// checkout root. `,` `:` and `\` are escaped by that same renderer but stay refused here, since
+// no helper word needs them and refusing is the narrower grant.
 data sudoers_command_word_refused_characters: List<String> = [
-  ",", ":", "=", "!", "(", ")", "\\", " ", "\t", "\n", "#", "*", "?", "[", "]", "\"", "'", "$", "`",
+  ",", ":", "!", "(", ")", "\\", " ", "\t", "\n", "#", "*", "?", "[", "]", "\"", "'", "$", "`",
 ]
 
 type HelperGrantAdmission
@@ -2411,6 +2426,7 @@ fn approval_broker_helper_snapshot_steps(revision: ReleaseRevisionBinding) -> Li
           group: approval_broker_helper_root_principal,
           dest: join([dir, "/", gunbc_release_bin_name], ""),
         )),
+        locus_tree_receipt_step_at(owner: approval_broker_helper_root_principal, dir: dir, revision: revision),
         deploy_raw(command: approval_broker_helper_snapshot_readback_command(dir: dir)),
       ],
     ),

dag/gunbc/live_deploy/release_locus.dag

@@ -115,6 +115,7 @@ fn approval_broker_ntfy_runtime_observe_operation_at(locus: HelperLocus, revisio
   let dir = helper_locus_dir(locus: locus, revision: revision)
   RunGunbcEntryAs {
     run_as: approval_ntfy_runtime_observe_run_as(),
+    workspace_root: dir as NonEmptyStr,
     binary: join([dir, "/", gunbc_release_bin_name], "") as NonEmptyStr,
     source_roots: map(approval_broker_tree_subdirs, sub => join([dir, "/", sub], "") as NonEmptyStr),
     entry: join([dir, "/", approval_ntfy_runtime_observe_entry_rel as String], "") as NonEmptyStr,