DSV41-6 enroll sshpass as a host_cli_dependency for password-session runners (srv3/srv4 lack it)

.github/workflows/fleet-converge.yml

@@ -13,7 +13,7 @@ on:
       mode:
         description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown
         required: true
-        options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge]
+        options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge]
         type: choice
       target:
         description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact"
@@ -592,6 +592,26 @@ jobs:
           FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
         if: github.event.inputs.mode == 'runner_host_file_converge'
         timeout-minutes: 15
+      - name: "Password-session tools: ensure every tool password_session_host_cli_requirements enrolls is present on this runner, installing only what is measured absent"
+        id: runner_password_session_tool_converge
+        run: |
+          ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
+          "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_password_session_tool_converge.dag --function runner_password_session_tool_converge_ci_wet
+          cat "$ROOT/target/runner-password-session-tool-converge-receipt.txt"
+        env:
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
+        if: github.event.inputs.mode == 'runner_password_session_tool_converge'
+        timeout-minutes: 15
+      - name: Upload password-session tool converge receipt
+        id: runner_password_session_tool_receipt_upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: runner-password-session-tool-converge-receipt
+          path: target/runner-password-session-tool-converge-receipt.txt
+          if-no-files-found: error
+          retention-days: 30
+        if: always() && (github.event.inputs.mode == 'runner_password_session_tool_converge')
+        timeout-minutes: 10
       - name: Upload runner host file converge receipt
         id: runner_host_file_receipt_upload
         uses: actions/upload-artifact@v4

dag/extdeps/package_managers/apt.dag

@@ -15,6 +15,13 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
 data apt_bin_dir: FilePath = "/usr/bin"
 data apt_local_bin_dir: FilePath = "/usr/local/bin"
 
+// Debian/Ubuntu apt packages in this corpus install their binaries under apt_bin_dir. The
+// path is derived from that directory plus the binary name, the same FHS location
+// apt_get_binary_path already names for apt-get.
+fn apt_installed_bin_path(binary: NonEmptyStr) -> FilePath {
+  concat(apt_bin_dir as String, concat("/", binary as String)) as FilePath
+}
+
 data package_coreutils: NonEmptyStr = "coreutils"     
 data package_grep: NonEmptyStr = "grep"               
 data package_diffutils: NonEmptyStr = "diffutils"     

dag/extdeps/tools.dag

@@ -64,6 +64,47 @@ fn install_hint(sources: List<InstallSource>) -> String {
   sources |> map(s => one_hint(source: s)) |> join(separator: " | ")
 }
 
+// THE APT PACKAGE, STRUCTURALLY, BESIDE THE SENTENCE. one_hint renders every InstallSource as
+// PROSE for a human, and until now that was the only thing that read the variants -- so a caller
+// that wanted to ACT on an install source had nothing to read and would have re-authored the
+// package name beside the tool. This is the same fold, asked for the value rather than the wording,
+// and it is what lets a convergence consume `installable_via` instead of carrying a second package
+// row that could disagree with it (§3).
+//
+// IT IS OPTIONAL AND NOT A DEFAULT. installable_via is a LIST and a tool may legitimately carry no
+// apt source at all -- websocat is a GitHub release, rustfmt comes from rustup, the coreutils tools
+// come with the system -- so "no apt package" is a real and correct answer, not a failure of this
+// function. Absent is returned rather than a fabricated package name, and the CALLER names the tool
+// in its refusal, because the caller is what knows why it wanted one. A default here would be the
+// §5 fabricated-plausible-output arm: `apt install <binary name>` is right often enough to look
+// correct and wrong exactly where the package and the binary differ (mosquitto_pub from
+// mosquitto-clients, busybox from busybox-static -- both already in this corpus).
+//
+// FIRST MATCH WINS and the order is the roster's. No tool in the corpus declares two apt sources;
+// if one ever does, the row is expressing a choice this fold cannot make and the roster is where
+// that belongs.
+fn apt_package_of(tool: CliTool) -> NonEmptyStr? {
+  fold_list(
+    xs: tool.installable_via,
+    empty: none,
+    cons: fn(acc, source) {
+      match acc {
+        Present { value: found } => Present { value: found }
+        Absent =>
+          match source {
+            SourceApt { package: p } => Present { value: p }
+            SourceCoreutils => none
+            SourceRustup => none
+            SourceBrew { formula: _ } => none
+            SourceGitHubRelease {
+              repo: _, tag: _, asset_aarch64: _, asset_x86_64: _, install_path: _
+            } => none
+          }
+      }
+    },
+  )
+}
+
 // Migrated off shell.Which.Check to shell.PosixCommandV.Check (crisp-wren-896, PR #8590): no fleet
 // host's OperatingSystemSurface carries a package-set authority, our own CI/falsifier runner
 // included, so `which`'s external-binary presence was never an established fact here either, and

dag/gunbc/ci/ci_spec.dag

@@ -780,6 +780,11 @@ data gunbc_ci_runner_host_file_converge_target: GunbcRunStepTarget = GunbcRunSte
   function: "runner_host_file_converge_ci_wet",
 }
 
+data gunbc_ci_runner_password_session_tool_converge_target: GunbcRunStepTarget = GunbcRunStepTarget {
+  entry: "dag/gunbc/runner/runner_password_session_tool_converge.dag",
+  function: "runner_password_session_tool_converge_ci_wet",
+}
+
 // THE OBSERVER SIDE OF ONE RESET ATTEMPT. It takes no arguments: the dispatch names the OBSERVER, the
 // step hands that label over by environment, and the subject is derived from the label through the
 // modeled assignment. There is no input of any kind through which a caller could name a subject,
@@ -963,6 +968,7 @@ fn gunbc_run_step_targets() -> List<GunbcRunStepTarget> {
     gunbc_ci_runner_microvm_host_converge_target,
     gunbc_ci_runner_host_file_observe_target,
     gunbc_ci_runner_host_file_converge_target,
+    gunbc_ci_runner_password_session_tool_converge_target,
     gunbc_ci_runner_microvm_boot_probe_target,
     gunbc_ci_host_reset_return_target,
     gunbc_ci_host_reset_dispatch_admission_target,
@@ -1754,6 +1760,16 @@ fn gunbc_ci_runner_host_file_converge_invoke() -> String {
   )
 }
 
+fn gunbc_ci_runner_password_session_tool_converge_invoke() -> String {
+  gunbc_run_step_script(
+    source_roots: witness_layer_roots,
+    entry: gunbc_ci_runner_password_session_tool_converge_target.entry,
+    function: gunbc_ci_runner_password_session_tool_converge_target.function,
+    claim_run: false,
+    receipt_rel: "target/runner-password-session-tool-converge-receipt.txt"
+  )
+}
+
 fn gunbc_ci_host_reset_return_invoke() -> String {
   gunbc_run_step_script(
     source_roots: witness_layer_roots,

dag/gunbc/fleet/fleet_converge_workflow.dag

@@ -168,6 +168,7 @@ type FleetConvergeWorkflowMode
   | HostResetReturn
   | RunnerHostFileObserve
   | RunnerHostFileConverge
+  | RunnerPasswordSessionToolConverge
 
 fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String {
   match mode {
@@ -190,6 +191,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String
     HostResetReturn => "host_reset_return"
     RunnerHostFileObserve => "runner_host_file_observe"
     RunnerHostFileConverge => "runner_host_file_converge"
+    RunnerPasswordSessionToolConverge => "runner_password_session_tool_converge"
   }
 }
 
@@ -214,7 +216,7 @@ fn fleet_converge_spark_target_description() -> String {
   ], "")
 }
 
-data fleet_converge_workflow_modes: List<FleetConvergeWorkflowMode> = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge]
+data fleet_converge_workflow_modes: List<FleetConvergeWorkflowMode> = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, RunnerPasswordSessionToolConverge]
 
 // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL.
 //
@@ -260,6 +262,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc
     HostResetReturn => none
     RunnerHostFileObserve => none
     RunnerHostFileConverge => none
+    RunnerPasswordSessionToolConverge => none
   }
 }
 
@@ -651,6 +654,35 @@ data fleet_converge_runner_host_file_converge_step_if: String = fleet_converge_m
 
 data fleet_converge_runner_host_file_any_step_if: String = fleet_converge_any_mode_step_if(modes: [RunnerHostFileObserve, RunnerHostFileConverge])
 
+// THE MODE THAT CLOSES #11061'S REFUSAL. That change made a password-session leg refuse with a
+// typed cause naming sshpass; this mode is the route that makes the refusal stop being true. It is
+// a STEP in the existing host-pinned fleet-converge job rather than a job of its own, because the
+// two own-job precedents here each state a reason that does not apply to it: host reset kills its
+// own runner, and dashboard deploy needs the mutation concurrency domain. An apt ensure needs
+// neither.
+data fleet_converge_runner_password_session_tool_converge_step_if: String = fleet_converge_mode_step_if(mode: RunnerPasswordSessionToolConverge)
+
+// THE RECEIPT IS UPLOADED ON FAILURE TOO. FailFast means the step's cat never runs when the
+// ensure refuses, which is exactly when the body names the cause. always() is what keeps that
+// body reachable after the job has already failed.
+fn fleet_converge_runner_password_session_tool_receipt_upload_step() -> Step {
+  UsesStep {
+    name: Present { value: "Upload password-session tool converge receipt" },
+    id: Present { value: "runner_password_session_tool_receipt_upload" },
+    uses: upload_artifact_action,
+    with: Present { value: [
+      kv(key: "name", value: yaml_string(s: "runner-password-session-tool-converge-receipt")),
+      kv(key: "path", value: yaml_string(s: "target/runner-password-session-tool-converge-receipt.txt")),
+      kv(key: "if-no-files-found", value: yaml_string(s: "error")),
+      kv(key: "retention-days", value: yaml_int(n: 30)),
+    ] },
+    env: none,
+    if_condition: Present { value: concat("always() && (", concat(fleet_converge_runner_password_session_tool_converge_step_if, ")")) },
+    continue_on_error: none,
+    timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes }
+  }
+}
+
 data fleet_converge_spark_grants_step_if: String = fleet_converge_mode_step_if(mode: SparkGrants)
 
 data fleet_converge_spark_bootstrap_step_if: String = fleet_converge_mode_step_if(mode: SparkBootstrap)
@@ -931,6 +963,18 @@ fn fleet_converge_runner_host_file_converge_step() -> Step {
   )
 }
 
+// IT REUSES THE RUNNER-HOST STEP SHAPER rather than spelling a fifth Step literal: same population
+// (a rostered runner named by the host input), same env binding, same timeout authority. What
+// differs is only the id, the sentence and the mode gate, which is exactly what the shaper takes.
+fn fleet_converge_runner_password_session_tool_converge_step() -> Step {
+  fleet_converge_runner_host_file_step(
+    id: "runner_password_session_tool_converge",
+    name: "Password-session tools: ensure every tool password_session_host_cli_requirements enrolls is present on this runner, installing only what is measured absent",
+    invoke: gunbc_ci_runner_password_session_tool_converge_invoke(),
+    if_condition: fleet_converge_runner_password_session_tool_converge_step_if,
+  )
+}
+
 fn fleet_converge_runner_host_file_receipt_upload_step() -> Step {
   UsesStep {
     name: Present { value: "Upload runner host file converge receipt" },
@@ -1355,6 +1399,8 @@ fn fleet_converge_job() -> Job {
         fleet_converge_spark_runtime_image_probe_step(),
         fleet_converge_runner_host_file_observe_step(),
         fleet_converge_runner_host_file_converge_step(),
+        fleet_converge_runner_password_session_tool_converge_step(),
+        fleet_converge_runner_password_session_tool_receipt_upload_step(),
         fleet_converge_runner_host_file_receipt_upload_step(),
         ci_fleet_key_agent_cleanup_step(),
       ]