Converge the runner teardown and a needrestart deferral onto runner hosts

.github/workflows/fleet-converge.yml

@@ -11,9 +11,9 @@ on:
         options: [srv1, srv2, srv3, srv4]
         type: choice
       mode:
-        description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject
+        description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown
         required: true
-        options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return]
+        options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge]
         type: choice
       target:
         description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact"
@@ -572,6 +572,36 @@ jobs:
           WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }}
         if: github.event.inputs.mode == 'spark_runtime_image_probe'
         timeout-minutes: 60
+      - name: "Runner host files: observe the teardown drop-in, the needrestart deferral and the loaded teardown (no writes)"
+        id: runner_host_file_observe
+        run: |
+          ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
+          "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_host_file_converge.dag --function runner_host_file_observe_ci_wet
+          cat "$ROOT/target/runner-host-file-converge-receipt.txt"
+        env:
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
+        if: github.event.inputs.mode == 'runner_host_file_observe'
+        timeout-minutes: 15
+      - name: "Runner host files: write what differs as the administrator, reload if the drop-in changed, read the loaded teardown back"
+        id: runner_host_file_converge
+        run: |
+          ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
+          "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_host_file_converge.dag --function runner_host_file_converge_ci_wet
+          cat "$ROOT/target/runner-host-file-converge-receipt.txt"
+        env:
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
+        if: github.event.inputs.mode == 'runner_host_file_converge'
+        timeout-minutes: 15
+      - name: Upload runner host file converge receipt
+        id: runner_host_file_receipt_upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: runner-host-file-converge-receipt
+          path: target/runner-host-file-converge-receipt.txt
+          if-no-files-found: error
+          retention-days: 30
+        if: always() && (github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge')
+        timeout-minutes: 10
       - name: Kill the in-run ssh-agent (key never outlives the job)
         run: |
           if [ -n "${SSH_AGENT_PID:-}" ]; then ssh-agent -k || true; fi

dag/extdeps/needrestart/needrestart.dag

@@ -0,0 +1,152 @@
+module extdeps.needrestart
+
+import std.types { String, NonEmptyStr, List, Bool, Int }
+import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef }
+import extdeps.uri { Uri, Https }
+import std.decl_ref { DeclarationRef, WholeDeclaration }
+
+// needrestart (Thomas Liske) runs after every dpkg transaction on Ubuntu, finds daemons still mapping
+// replaced libraries, and restarts them. Ubuntu wires it into apt through a dpkg hook, so an
+// unattended security upgrade of libc restarts every service whose process image still maps the old
+// libc -- which on a self-hosted CI host includes every busy runner slot.
+data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
+  uri: Uri {
+    scheme: Https
+    locator: "github.com/liske/needrestart"
+  }
+}
+
+// The configuration surface this module models, at the upstream tag Ubuntu noble packages. The shipped
+// needrestart.conf documents override_rc as "Override service default selection (hash of regex)" and
+// ends by evaluating every /etc/needrestart/conf.d/*.conf in Perl sort order, dying on a parse error.
+data needrestart_config_authority: ExternalAuthority = ExternalAuthority {
+  uri: Uri {
+    scheme: Https
+    locator: "github.com/liske/needrestart/blob/v3.6/ex/needrestart/needrestart.conf"
+  }
+}
+
+// The consumer of override_rc: for each service needing a restart, the first key whose regex matches
+// the unit name decides whether it is restarted (true) or deferred and only reported (false).
+data needrestart_override_rc_consumer_authority: ExternalAuthority = ExternalAuthority {
+  uri: Uri {
+    scheme: Https
+    locator: "github.com/liske/needrestart/blob/v3.6/needrestart"
+  }
+}
+
+// The distribution package the fleet runs, read off srv1 with dpkg-query on 2026-09-11.
+data needrestart_ubuntu_noble_package_authority: ExternalAuthority = ExternalAuthority {
+  uri: Uri {
+    scheme: Https
+    locator: "launchpad.net/ubuntu/+source/needrestart/3.6-7ubuntu4.5"
+  }
+}
+
+data extdeps_model_scope: ExternalModelScope = ExternalModelScope {
+  subject: ExternalSubjectRef {
+    declaration: DeclarationRef {
+      module_path: "extdeps.needrestart",
+      decl_name: "NeedrestartServiceSelectionOverride",
+      field: WholeDeclaration
+    }
+  },
+  first_citation: extdeps_external_authority_anchor,
+  further_citations: [
+    needrestart_config_authority,
+    needrestart_override_rc_consumer_authority,
+    needrestart_ubuntu_noble_package_authority,
+  ]
+}
+
+data needrestart_upstream_version: NonEmptyStr = "3.6" as NonEmptyStr
+
+data needrestart_ubuntu_noble_package_version: NonEmptyStr = "3.6-7ubuntu4.5" as NonEmptyStr
+
+data needrestart_conf_d_dir: NonEmptyStr = "/etc/needrestart/conf.d" as NonEmptyStr
+
+// Only files ending in `.conf` are evaluated, so the suffix is part of the file's identity rather than
+// a naming habit: a snippet without it is inert, and nothing reports that.
+data needrestart_conf_d_suffix: String = ".conf"
+
+fn needrestart_conf_d_path(stem: NonEmptyStr) -> NonEmptyStr {
+  join([needrestart_conf_d_dir as String, "/", stem as String, needrestart_conf_d_suffix], "") as NonEmptyStr
+}
+
+// The value side of one override_rc entry. upstream stores it as a Perl boolean and reads it into
+// `$restart`; the two states have opposite consequences for a running process, so they are named.
+type NeedrestartDefaultSelection
+  = RestartSelectedByDefault
+  | RestartDeferredByDefault
+
+fn needrestart_default_selection_wire(selection: NeedrestartDefaultSelection) -> String {
+  match selection {
+    RestartSelectedByDefault => "1"
+    RestartDeferredByDefault => "0"
+  }
+}
+
+// THE KEY SIDE IS A UNIT-NAME PREFIX, NOT A REGEX, because a caller-authored regex is a second
+// language with its own failure modes and needrestart dies on a malformed one. The module owns the
+// translation: anchor at the start, quote every character the way Perl's quotemeta does, so the key
+// can only ever mean "unit names beginning with exactly these characters".
+//
+// The accepted alphabet is systemd.unit(5)'s unit-name characters plus the template separator `@`,
+// with the backslash excluded: systemd admits it only as an escape introducer, and a prefix carrying
+// one would make the prefix's meaning depend on a second escaping layer this module does not model.
+type NeedrestartUnitNamePrefix sole_constructor {
+  prefix: NonEmptyStr
+}
+
+type NeedrestartUnitNamePrefixAdmission
+  = NeedrestartUnitNamePrefixAdmitted { prefix: NeedrestartUnitNamePrefix }
+  | NeedrestartUnitNamePrefixRefused { cause: NonEmptyStr }
+
+fn needrestart_unit_name_char_allowed(cp: Int) -> Bool {
+  (cp >= 97 && cp <= 122) || (cp >= 65 && cp <= 90) || (cp >= 48 && cp <= 57)
+    || cp == 58 || cp == 45 || cp == 95 || cp == 46 || cp == 64
+}
+
+fn needrestart_unit_name_prefix(raw: String) -> NeedrestartUnitNamePrefixAdmission {
+  if raw == "" {
+    NeedrestartUnitNamePrefixRefused { cause: "needrestart unit-name prefix is empty, which would match every service" as NonEmptyStr }
+  } else if fold(chars(s: raw), init: true, f: (ok, cp) => ok && needrestart_unit_name_char_allowed(cp: cp)) {
+    NeedrestartUnitNamePrefixAdmitted { prefix: NeedrestartUnitNamePrefix { prefix: raw as NonEmptyStr } }
+  } else {
+    NeedrestartUnitNamePrefixRefused {
+      cause: join(["needrestart unit-name prefix `", raw, "` carries a character outside systemd's unit-name alphabet"], "") as NonEmptyStr,
+    }
+  }
+}
+
+// perlfunc quotemeta: every ASCII character that is not a word character is preceded by a backslash.
+fn needrestart_perl_quotemeta_code_point(cp: Int) -> String {
+  if (cp >= 97 && cp <= 122) || (cp >= 65 && cp <= 90) || (cp >= 48 && cp <= 57) || cp == 95 {
+    from_code_point(cp: cp)
+  } else {
+    concat("\\", from_code_point(cp: cp))
+  }
+}
+
+fn needrestart_unit_name_prefix_regex(prefix: NeedrestartUnitNamePrefix) -> String {
+  join(["qr(^", join(map(chars(s: prefix.prefix as String), cp => needrestart_perl_quotemeta_code_point(cp: cp)), ""), ")"], "")
+}
+
+type NeedrestartServiceSelectionOverride {
+  units: NeedrestartUnitNamePrefix
+  selection: NeedrestartDefaultSelection
+}
+
+// One assignment into the hash needrestart.conf has already created. Assigning a key rather than
+// replacing the hash is what lets a snippet add one policy without erasing the distribution's own
+// entries (dbus, display managers, getty) that the shipped file declares.
+fn needrestart_override_rc_line(o: NeedrestartServiceSelectionOverride) -> String {
+  join([
+    "$nrconf\{override_rc\}->\{", needrestart_unit_name_prefix_regex(prefix: o.units), "\} = ",
+    needrestart_default_selection_wire(selection: o.selection), ";",
+  ], "")
+}
+
+fn needrestart_conf_d_text(overrides: List<NeedrestartServiceSelectionOverride>) -> String {
+  join(map(overrides, o => concat(needrestart_override_rc_line(o: o), "\n")), "")
+}

dag/extdeps/systemd/systemctl.dag

@@ -212,6 +212,31 @@ fn systemctl_show_property_argv(unit: NonEmptyStr, property: SystemdUnitProperty
   ]
 }
 
+// SEVERAL PROPERTIES OF SEVERAL UNITS IN ONE READ. systemctl show accepts a comma-separated
+// --property list and any number of units, and prints one block of `Name=value` lines per unit with a
+// blank line between blocks. Without --value the property names ride along, so a consumer can key each
+// value by name rather than trusting an output order systemctl does not document; asking for Id makes
+// every block say which unit it describes. A unit the manager has not loaded still gets a block --
+// LoadState=not-found beside the manager's DEFAULTS for every other property -- so a consumer must read
+// LoadState before believing any other value in that block.
+fn systemctl_show_properties_argv(units: List<NonEmptyStr>, properties: List<SystemdUnitProperty>) -> List<String> {
+  concat(
+    [
+      "systemctl",
+      "show",
+      concat("--property=", join(map(properties, p => systemd_unit_property_wire(property: p) as String), ",")),
+    ],
+    map(units, u => u as String),
+  )
+}
+
+// Every service unit the manager holds, loaded or not, as plain rows whose first field is the unit
+// name. No pattern: a glob is a shell word the fleet's far-side re-parse refuses, so the caller filters
+// by name in the model instead.
+fn systemctl_list_all_service_units_argv() -> List<String> {
+  ["systemctl", "list-units", "--type=service", "--all", "--no-legend", "--plain"]
+}
+
 // LoadState is a non-numeric unit lifecycle observation, so it has its own argv authority rather than joining SystemdUnitProperty, whose shared ShowProperty mock is numeric cgroup evidence. systemctl show exits nonzero on an unreadable manager; the load-state wire value not-found is the only observation that proves an absent unit.
 // `is-enabled` answers the INSTALL state (whether the unit is wanted by a target), which is a
 // different question from `is-active` (whether it is running now) and from LoadState (whether the

dag/extdeps/systemd/systemd.dag

@@ -78,6 +78,9 @@ type SystemdUnitProperty
   | AccuracyUSec
   | TimersMonotonic
   | NeedDaemonReload
+  | IdProperty
+  | KillModeProperty
+  | ExitTypeProperty
 
 fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr {
   match property {
@@ -110,6 +113,9 @@ fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr {
     AccuracyUSec => "AccuracyUSec" as NonEmptyStr
     TimersMonotonic => "TimersMonotonic" as NonEmptyStr
     NeedDaemonReload => "NeedDaemonReload" as NonEmptyStr
+    IdProperty => "Id" as NonEmptyStr
+    KillModeProperty => "KillMode" as NonEmptyStr
+    ExitTypeProperty => "ExitType" as NonEmptyStr
   }
 }
 

dag/gunbc/ci/ci_spec.dag

@@ -770,6 +770,16 @@ data gunbc_ci_runner_microvm_host_converge_target: GunbcRunStepTarget = GunbcRun
   function: "runner_microvm_host_converge_wet",
 }
 
+data gunbc_ci_runner_host_file_observe_target: GunbcRunStepTarget = GunbcRunStepTarget {
+  entry: "dag/gunbc/runner/runner_host_file_converge.dag",
+  function: "runner_host_file_observe_ci_wet",
+}
+
+data gunbc_ci_runner_host_file_converge_target: GunbcRunStepTarget = GunbcRunStepTarget {
+  entry: "dag/gunbc/runner/runner_host_file_converge.dag",
+  function: "runner_host_file_converge_ci_wet",
+}
+
 // THE OBSERVER SIDE OF ONE RESET ATTEMPT. It takes no arguments: the dispatch names the OBSERVER, the
 // step hands that label over by environment, and the subject is derived from the label through the
 // modeled assignment. There is no input of any kind through which a caller could name a subject,
@@ -951,6 +961,8 @@ fn gunbc_run_step_targets() -> List<GunbcRunStepTarget> {
     gunbc_ci_runner_guest_image_observe_target,
     gunbc_ci_runner_guest_image_converge_target,
     gunbc_ci_runner_microvm_host_converge_target,
+    gunbc_ci_runner_host_file_observe_target,
+    gunbc_ci_runner_host_file_converge_target,
     gunbc_ci_runner_microvm_boot_probe_target,
     gunbc_ci_host_reset_return_target,
     gunbc_ci_host_reset_dispatch_admission_target,
@@ -1722,6 +1734,26 @@ fn gunbc_ci_runner_microvm_host_converge_invoke() -> String {
   )
 }
 
+fn gunbc_ci_runner_host_file_observe_invoke() -> String {
+  gunbc_run_step_script(
+    source_roots: witness_layer_roots,
+    entry: gunbc_ci_runner_host_file_observe_target.entry,
+    function: gunbc_ci_runner_host_file_observe_target.function,
+    claim_run: false,
+    receipt_rel: "target/runner-host-file-converge-receipt.txt"
+  )
+}
+
+fn gunbc_ci_runner_host_file_converge_invoke() -> String {
+  gunbc_run_step_script(
+    source_roots: witness_layer_roots,
+    entry: gunbc_ci_runner_host_file_converge_target.entry,
+    function: gunbc_ci_runner_host_file_converge_target.function,
+    claim_run: false,
+    receipt_rel: "target/runner-host-file-converge-receipt.txt"
+  )
+}
+
 fn gunbc_ci_host_reset_return_invoke() -> String {
   gunbc_run_step_script(
     source_roots: witness_layer_roots,

dag/gunbc/extdeps_scope_frontier.dag

@@ -276,6 +276,7 @@ data scope_carrier_paths: List<String> = [
   "dag/extdeps/automation/playwright.dag",
   "dag/extdeps/package.dag",
   "dag/extdeps/npm.dag",
+  "dag/extdeps/needrestart/needrestart.dag",
   "dag/extdeps/browser/chromium.dag",
   "dag/extdeps/container/oci/digest.dag",
   "dag/extdeps/git/publication_transport.dag",