diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 1ee5b8ebedc..387c2490fc9 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -592,6 +592,26 @@ jobs: FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} if: github.event.inputs.mode == 'runner_host_file_converge' timeout-minutes: 15 + - name: "Password-session tools: ensure every tool password_session_host_cli_requirements enrolls is present on this runner, installing only what is measured absent" + id: runner_password_session_tool_converge + run: | + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_password_session_tool_converge.dag --function runner_password_session_tool_converge_ci_wet + cat "$ROOT/target/runner-password-session-tool-converge-receipt.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} + if: github.event.inputs.mode == 'runner_password_session_tool_converge' + timeout-minutes: 15 + - name: Upload password-session tool converge receipt + id: runner_password_session_tool_receipt_upload + uses: actions/upload-artifact@v4 + with: + name: runner-password-session-tool-converge-receipt + path: target/runner-password-session-tool-converge-receipt.txt + if-no-files-found: error + retention-days: 30 + if: always() && (github.event.inputs.mode == 'runner_password_session_tool_converge') + timeout-minutes: 10 - name: Upload runner host file converge receipt id: runner_host_file_receipt_upload uses: actions/upload-artifact@v4 diff --git a/dag/extdeps/package_managers/apt.dag b/dag/extdeps/package_managers/apt.dag index 7915d3e3648..45c0e4c600c 100644 --- a/dag/extdeps/package_managers/apt.dag +++ b/dag/extdeps/package_managers/apt.dag @@ -15,6 +15,13 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { data apt_bin_dir: FilePath = "/usr/bin" data apt_local_bin_dir: FilePath = "/usr/local/bin" +// Debian/Ubuntu apt packages in this corpus install their binaries under apt_bin_dir. The +// path is derived from that directory plus the binary name, the same FHS location +// apt_get_binary_path already names for apt-get. +fn apt_installed_bin_path(binary: NonEmptyStr) -> FilePath { + concat(apt_bin_dir as String, concat("/", binary as String)) as FilePath +} + data package_coreutils: NonEmptyStr = "coreutils" data package_grep: NonEmptyStr = "grep" data package_diffutils: NonEmptyStr = "diffutils" diff --git a/dag/extdeps/tools.dag b/dag/extdeps/tools.dag index a4da19909d5..36a85ba4145 100644 --- a/dag/extdeps/tools.dag +++ b/dag/extdeps/tools.dag @@ -64,6 +64,47 @@ fn install_hint(sources: List) -> String { sources |> map(s => one_hint(source: s)) |> join(separator: " | ") } +// THE APT PACKAGE, STRUCTURALLY, BESIDE THE SENTENCE. one_hint renders every InstallSource as +// PROSE for a human, and until now that was the only thing that read the variants -- so a caller +// that wanted to ACT on an install source had nothing to read and would have re-authored the +// package name beside the tool. This is the same fold, asked for the value rather than the wording, +// and it is what lets a convergence consume `installable_via` instead of carrying a second package +// row that could disagree with it (§3). +// +// IT IS OPTIONAL AND NOT A DEFAULT. installable_via is a LIST and a tool may legitimately carry no +// apt source at all -- websocat is a GitHub release, rustfmt comes from rustup, the coreutils tools +// come with the system -- so "no apt package" is a real and correct answer, not a failure of this +// function. Absent is returned rather than a fabricated package name, and the CALLER names the tool +// in its refusal, because the caller is what knows why it wanted one. A default here would be the +// §5 fabricated-plausible-output arm: `apt install ` is right often enough to look +// correct and wrong exactly where the package and the binary differ (mosquitto_pub from +// mosquitto-clients, busybox from busybox-static -- both already in this corpus). +// +// FIRST MATCH WINS and the order is the roster's. No tool in the corpus declares two apt sources; +// if one ever does, the row is expressing a choice this fold cannot make and the roster is where +// that belongs. +fn apt_package_of(tool: CliTool) -> NonEmptyStr? { + fold_list( + xs: tool.installable_via, + empty: none, + cons: fn(acc, source) { + match acc { + Present { value: found } => Present { value: found } + Absent => + match source { + SourceApt { package: p } => Present { value: p } + SourceCoreutils => none + SourceRustup => none + SourceBrew { formula: _ } => none + SourceGitHubRelease { + repo: _, tag: _, asset_aarch64: _, asset_x86_64: _, install_path: _ + } => none + } + } + }, + ) +} + // Migrated off shell.Which.Check to shell.PosixCommandV.Check (crisp-wren-896, PR #8590): no fleet // host's OperatingSystemSurface carries a package-set authority, our own CI/falsifier runner // included, so `which`'s external-binary presence was never an established fact here either, and diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 83f916a498c..f006ae7adf7 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -780,6 +780,11 @@ data gunbc_ci_runner_host_file_converge_target: GunbcRunStepTarget = GunbcRunSte function: "runner_host_file_converge_ci_wet", } +data gunbc_ci_runner_password_session_tool_converge_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/runner/runner_password_session_tool_converge.dag", + function: "runner_password_session_tool_converge_ci_wet", +} + // THE OBSERVER SIDE OF ONE RESET ATTEMPT. It takes no arguments: the dispatch names the OBSERVER, the // step hands that label over by environment, and the subject is derived from the label through the // modeled assignment. There is no input of any kind through which a caller could name a subject, @@ -963,6 +968,7 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_runner_microvm_host_converge_target, gunbc_ci_runner_host_file_observe_target, gunbc_ci_runner_host_file_converge_target, + gunbc_ci_runner_password_session_tool_converge_target, gunbc_ci_runner_microvm_boot_probe_target, gunbc_ci_host_reset_return_target, gunbc_ci_host_reset_dispatch_admission_target, @@ -1754,6 +1760,16 @@ fn gunbc_ci_runner_host_file_converge_invoke() -> String { ) } +fn gunbc_ci_runner_password_session_tool_converge_invoke() -> String { + gunbc_run_step_script( + source_roots: witness_layer_roots, + entry: gunbc_ci_runner_password_session_tool_converge_target.entry, + function: gunbc_ci_runner_password_session_tool_converge_target.function, + claim_run: false, + receipt_rel: "target/runner-password-session-tool-converge-receipt.txt" + ) +} + fn gunbc_ci_host_reset_return_invoke() -> String { gunbc_run_step_script( source_roots: witness_layer_roots, diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index fe7d651d411..7ca8b286f19 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -168,6 +168,7 @@ type FleetConvergeWorkflowMode | HostResetReturn | RunnerHostFileObserve | RunnerHostFileConverge + | RunnerPasswordSessionToolConverge fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String { match mode { @@ -190,6 +191,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String HostResetReturn => "host_reset_return" RunnerHostFileObserve => "runner_host_file_observe" RunnerHostFileConverge => "runner_host_file_converge" + RunnerPasswordSessionToolConverge => "runner_password_session_tool_converge" } } @@ -214,7 +216,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, RunnerPasswordSessionToolConverge] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // @@ -260,6 +262,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc HostResetReturn => none RunnerHostFileObserve => none RunnerHostFileConverge => none + RunnerPasswordSessionToolConverge => none } } @@ -651,6 +654,35 @@ data fleet_converge_runner_host_file_converge_step_if: String = fleet_converge_m data fleet_converge_runner_host_file_any_step_if: String = fleet_converge_any_mode_step_if(modes: [RunnerHostFileObserve, RunnerHostFileConverge]) +// THE MODE THAT CLOSES #11061'S REFUSAL. That change made a password-session leg refuse with a +// typed cause naming sshpass; this mode is the route that makes the refusal stop being true. It is +// a STEP in the existing host-pinned fleet-converge job rather than a job of its own, because the +// two own-job precedents here each state a reason that does not apply to it: host reset kills its +// own runner, and dashboard deploy needs the mutation concurrency domain. An apt ensure needs +// neither. +data fleet_converge_runner_password_session_tool_converge_step_if: String = fleet_converge_mode_step_if(mode: RunnerPasswordSessionToolConverge) + +// THE RECEIPT IS UPLOADED ON FAILURE TOO. FailFast means the step's cat never runs when the +// ensure refuses, which is exactly when the body names the cause. always() is what keeps that +// body reachable after the job has already failed. +fn fleet_converge_runner_password_session_tool_receipt_upload_step() -> Step { + UsesStep { + name: Present { value: "Upload password-session tool converge receipt" }, + id: Present { value: "runner_password_session_tool_receipt_upload" }, + uses: upload_artifact_action, + with: Present { value: [ + kv(key: "name", value: yaml_string(s: "runner-password-session-tool-converge-receipt")), + kv(key: "path", value: yaml_string(s: "target/runner-password-session-tool-converge-receipt.txt")), + kv(key: "if-no-files-found", value: yaml_string(s: "error")), + kv(key: "retention-days", value: yaml_int(n: 30)), + ] }, + env: none, + if_condition: Present { value: concat("always() && (", concat(fleet_converge_runner_password_session_tool_converge_step_if, ")")) }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } + } +} + data fleet_converge_spark_grants_step_if: String = fleet_converge_mode_step_if(mode: SparkGrants) data fleet_converge_spark_bootstrap_step_if: String = fleet_converge_mode_step_if(mode: SparkBootstrap) @@ -931,6 +963,18 @@ fn fleet_converge_runner_host_file_converge_step() -> Step { ) } +// IT REUSES THE RUNNER-HOST STEP SHAPER rather than spelling a fifth Step literal: same population +// (a rostered runner named by the host input), same env binding, same timeout authority. What +// differs is only the id, the sentence and the mode gate, which is exactly what the shaper takes. +fn fleet_converge_runner_password_session_tool_converge_step() -> Step { + fleet_converge_runner_host_file_step( + id: "runner_password_session_tool_converge", + name: "Password-session tools: ensure every tool password_session_host_cli_requirements enrolls is present on this runner, installing only what is measured absent", + invoke: gunbc_ci_runner_password_session_tool_converge_invoke(), + if_condition: fleet_converge_runner_password_session_tool_converge_step_if, + ) +} + fn fleet_converge_runner_host_file_receipt_upload_step() -> Step { UsesStep { name: Present { value: "Upload runner host file converge receipt" }, @@ -1355,6 +1399,8 @@ fn fleet_converge_job() -> Job { fleet_converge_spark_runtime_image_probe_step(), fleet_converge_runner_host_file_observe_step(), fleet_converge_runner_host_file_converge_step(), + fleet_converge_runner_password_session_tool_converge_step(), + fleet_converge_runner_password_session_tool_receipt_upload_step(), fleet_converge_runner_host_file_receipt_upload_step(), ci_fleet_key_agent_cleanup_step(), ] diff --git a/dag/gunbc/host/host_effect_realize.dag b/dag/gunbc/host/host_effect_realize.dag index b4e8d14e0f4..fe99597d6d4 100644 --- a/dag/gunbc/host/host_effect_realize.dag +++ b/dag/gunbc/host/host_effect_realize.dag @@ -115,7 +115,14 @@ import product.placement_supply { HostIdentity } import extdeps.toolchain.types { Architecture, architecture_diagnostic_name } import gunbc.host_converge { HostConverge, FreshStandup, ExistingHostQuiescentReload, gunbc_apply_mode_for_host } import extdeps.gunbc -import extdeps.apt { package_curl, package_nbdkit, package_socat, package_xorriso, apt_get_binary_path } +import extdeps.apt { + package_curl, + package_nbdkit, + package_socat, + package_xorriso, + apt_get_binary_path, + apt_installed_bin_path, +} import extdeps.tools.curl { curl_cli_tool } import extdeps.tools.xorriso { xorriso_cli_tool } import extdeps.tools.nbdkit { nbdkit_cli_tool } @@ -1616,6 +1623,22 @@ fn srv3_transport_test_executable(path: String, transport: HostEffectTransport) } } +// TRANSPORT-CONDITIONED PRESENCE, not a silent fork of one question. LocalShell and SshShell keep +// posix_command_v_check_argv / shell.PosixCommandV.Check: PRESENT ON PATH. FleetSsh cannot carry that +// argv -- the -c payload is not a PortableRemoteWord (spaces, quotes, `$`) -- so it answers a +// different, named question: shell.Test.IsExecutable over extdeps.apt apt_installed_bin_path, +// PRESENT AT THIS PATH (`apt_bin_dir` + binary). A tool installed outside apt_bin_dir (diverted +// path, /usr/sbin, a local build) now reads absent and the ensure takes the install arm; apt then +// no-ops if the package is already recorded, so the host is not harmed, but the receipt can say +// the install ran for a host that already had the binary elsewhere. +// +// Path-presence is the right question for an apt-acquired tool: the installer places the binary +// at that FHS path, and the password-session exec is that argv0. PATH search would accept a +// shadow that apt does not own. srv3_toolchain_rows consumers are unaffected: host_toolchain_ensure +// passes host_identity_ssh_access transport (SshShell), and seeded-install-media toolchain +// ensure uses ci_deploy_srv1_access (LocalShell). Neither production caller uses FleetSsh; both +// still rely on PATH presence. The realize entry gate admits FleetSsh so a later caller would +// inherit this path-presence arm -- that is the caveat, not a current production change. fn srv3_apt_tool_present(tool_name: NonEmptyStr, transport: HostEffectTransport) -> Srv3HostPredicate { match transport { LocalShell => @@ -1628,13 +1651,10 @@ fn srv3_apt_tool_present(tool_name: NonEmptyStr, transport: HostEffectTransport) srv3_predicate_from_typed_argv( outcome: typed_argv_exec_over_ssh(host: h, argv: posix_command_v_check_argv(command: tool_name)), ) - FleetSsh { target: t, context: c } => - srv3_predicate_from_typed_argv( - outcome: typed_argv_exec_over_fleet_ssh( - target: t, - context: c, - argv: posix_command_v_check_argv(command: tool_name), - ), + FleetSsh { target: _, context: _ } => + srv3_transport_test_executable( + path: apt_installed_bin_path(binary: tool_name) as String, + transport: transport, ) EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => Srv3PredicateUnobserved { @@ -1643,6 +1663,12 @@ fn srv3_apt_tool_present(tool_name: NonEmptyStr, transport: HostEffectTransport) } } +// FleetSsh resolved path is the same question as srv3_apt_tool_present: executable at +// apt_installed_bin_path (FHS /usr/bin/), not PATH stdout from command -v. LocalShell and +// SshShell still resolve via command -v. This arm is in scope because srv3_apt_tool_version_ok +// (and srv3_run_tool_by_name) read the path from here; leaving FleetSsh on posix_command_v_check_argv +// would make version_ok Unobserved after a successful path-presence probe -- the install arm would +// never be able to prove the version on FleetSsh. A binary only elsewhere still reads absent. fn srv3_tool_bin_path(tool_name: NonEmptyStr, transport: HostEffectTransport) -> Srv3ObservedToken { match transport { LocalShell => { @@ -1660,14 +1686,16 @@ fn srv3_tool_bin_path(tool_name: NonEmptyStr, transport: HostEffectTransport) -> srv3_observed_token_from_typed_argv( outcome: typed_argv_exec_over_ssh(host: h, argv: posix_command_v_check_argv(command: tool_name)), ) - FleetSsh { target: t, context: c } => - srv3_observed_token_from_typed_argv( - outcome: typed_argv_exec_over_fleet_ssh( - target: t, - context: c, - argv: posix_command_v_check_argv(command: tool_name), - ), - ) + FleetSsh { target: _, context: _ } => + match srv3_transport_test_executable( + path: apt_installed_bin_path(binary: tool_name) as String, + transport: transport, + ) { + Srv3PredicateHolds => + Srv3TokenObserved { value: apt_installed_bin_path(binary: tool_name) as String } + Srv3PredicateFails => Srv3TokenAbsent + Srv3PredicateUnobserved { reason: why } => Srv3TokenUnobserved { reason: why } + } EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => Srv3TokenUnobserved { reason: "EmitArtifactThenThinRun transport has no arm that can resolve a tool path", diff --git a/dag/gunbc/runner/runner_password_session_tool_converge.dag b/dag/gunbc/runner/runner_password_session_tool_converge.dag new file mode 100644 index 00000000000..0a42a23efc4 --- /dev/null +++ b/dag/gunbc/runner/runner_password_session_tool_converge.dag @@ -0,0 +1,267 @@ +module gunbc.runner_password_session_tool_converge + +import std.types { Bool, List, NonEmptyStr, String } +import v2.std.optional { Present, Absent } +import extdeps.tools { CliTool, apt_package_of } +import gunbc.host_cli_dependency { password_session_host_cli_requirements } +import gunbc.host_effect { HostEffectTransport, FleetSsh } +import gunbc.host_effect_realize { + Srv3HostPredicate, + Srv3PredicateHolds, + Srv3PredicateFails, + Srv3PredicateUnobserved, + srv3_ensure_apt_tool, +} +import product.placement_supply { HostIdentity } +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } +import gunbc.fleet_known_hosts_anchor { SshTarget, FleetSshExecutionContext } +import gunbc.runner_host_deploy { RunnerHostDeployFound, RunnerHostDeployUnmodeled, runner_host_deploy_for } +import gunbc.runner_host_grants { executor_bootstrap_principal } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.resources { Network } +import extdeps.filesystem.filesystem_io { Filesystem } +import gunbc.clock_read { clock_now_probed_at, probed_at_word } +import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent } +import gunbc.fleet_converge_receipt { fleet_converge_expected_host_env_name } +import gunbc.fleet_ssh_locus { prepare_fleet_ssh_agent_context, FleetSshContextReady, FleetSshContextRefused } + +// THE OTHER HALF OF #11061, AND THE ONLY HALF THAT CHANGES A HOST. That change enrolled sshpass as +// a host_cli_dependency and made a password-session leg refuse, before exec, with a typed cause +// naming the binary -- which turned `failed to execute 'sshpass': No such file or directory` into a +// legible refusal but left srv3 and srv4 exactly as unable to run the leg as before. A refusal with +// no route to a fix is the defect docs/plans/dependency-acquisition-gap-analysis.md names: it hands +// a person a sentence, and no modeled thing acts on it. This module is that route. +// +// THE ROSTER IS THE SUBJECT, NOT A SECOND LIST. Which tools must be present is +// gunbc.host_cli_dependency password_session_host_cli_requirements, the same declaration the +// REFUSAL reads. So the thing that converges a runner and the thing that refuses on it cannot +// disagree about what is required: adding a tool to that roster extends both at once, and there is +// no second place to forget. The apt package likewise is not re-authored here -- it is read off the +// CliTool's own installable_via through extdeps.tools apt_package_of, so "which package provides +// this binary" stays one fact (§3). +// +// WHICH HOSTS: the runner roster, gunbc.runner_host_deploy fleet_runner_host_deploys, resolved per +// host through runner_host_deploy_for, which refuses a machine with no deploy row rather than +// borrowing a neighbour's. "Executes a password-session leg" is a property of the TRANSPORT a +// runner must be able to spawn, not of the fleet the far side belongs to -- so the population is +// every runner host, which is exactly the measured incident: spark_grants died on srv3 and srv4 +// while srv1 and srv2 carried the binary. + +type PasswordSessionToolSubjectAdmission + = PasswordSessionToolSubjectAdmitted { host: HostIdentity, target: SshTarget } + | PasswordSessionToolSubjectRefused { cause: String } + +// A HOST WITH NO RUNNER DEPLOY ROW IS UNKNOWN, NOT TYPICAL. It refuses rather than converging a +// machine this repository does not model, for the reason runner_host_deploy states in its own +// resolution type: an unclassified machine has no declared administrator principal, so there is no +// account this convergence could honestly act as. +fn password_session_tool_subject(host: HostIdentity) -> PasswordSessionToolSubjectAdmission { + match runner_host_deploy_for(host: host) { + RunnerHostDeployUnmodeled { host: h } => + PasswordSessionToolSubjectRefused { + cause: join([ + "runner_password_session_tool_converge: ", + h as String, + " has no runner deploy row, so it is not a runner host and carries no declared administrator principal", + ], ""), + } + RunnerHostDeployFound { deploy: _ } => + PasswordSessionToolSubjectAdmitted { + host: host, + target: SshTarget { + endpoint: fleet_probe_endpoint_for(host: host as String) as NonEmptyStr, + principal: executor_bootstrap_principal.login, + }, + } + } +} + +// A TOOL WITH NO APT SOURCE REFUSES RATHER THAN GUESSING ITS PACKAGE. apt_package_of returns +// Optional precisely so this decision is made here, by the caller that knows why it wanted a +// package: a roster tool installed from a GitHub release or shipped with coreutils has no apt +// package, and inventing `apt install ` would be right often enough to look correct +// and wrong exactly where the two differ -- mosquitto_pub comes from mosquitto-clients, busybox +// from busybox-static, both already in this corpus (§5, no fabricated plausible output). +type PasswordSessionToolOutcome { + tool: NonEmptyStr + verdict: Srv3HostPredicate +} + +type PasswordSessionToolPlan + = PasswordSessionToolAptPackage { tool: NonEmptyStr, package: NonEmptyStr } + | PasswordSessionToolUnacquirable { tool: NonEmptyStr } + +fn password_session_tool_plan(tool: CliTool) -> PasswordSessionToolPlan { + match apt_package_of(tool: tool) { + Present { value: p } => PasswordSessionToolAptPackage { tool: tool.name, package: p } + Absent => PasswordSessionToolUnacquirable { tool: tool.name } + } +} + +// require_version_probe IS FALSE, AND THAT IS DERIVED RATHER THAN CHOSEN. sshpass_cli_tool declares +// min_version: none, so there is no version constraint to gate on and a version probe would be +// asserting against nothing. It is also what makes the second run reach no install: with the probe +// off, srv3_ensure_apt_tool returns Srv3PredicateHolds the moment the tool is observed present and +// the install arm is never evaluated. Zero mutation on a converged host is therefore a property of +// the fold's shape, not an assertion laid over it. +// +// THE INSTALLER IS REUSED, NOT RE-AUTHORED, and srv3_ensure_apt_tool is the one this corpus already +// paid for twice: its annotation records that the install must never run on an UNOBSERVED probe (a +// transport that cannot answer must refuse, not install blind) and that the install must sit in a +// BRANCH rather than an argument position (a past regression evaluated the privileged apt install +// on every ensure, including of already-present, correctly-versioned tools). Calling it is how this +// module inherits both walls instead of re-deriving them. Its `srv3_` prefix is now misleading -- +// the function is generic over tool and transport -- and renaming it is a follow-up, deliberately +// not this change. +fn converge_one_password_session_tool( + tool: CliTool, + transport: HostEffectTransport, +) -> PasswordSessionToolOutcome { + match password_session_tool_plan(tool: tool) { + PasswordSessionToolUnacquirable { tool: t } => + PasswordSessionToolOutcome { + tool: t, + verdict: Srv3PredicateUnobserved { + reason: join([ + "runner_password_session_tool_converge: ", + t as String, + " is enrolled in password_session_host_cli_requirements but its CliTool declares no apt install source, so this convergence has no modeled way to acquire it", + ], ""), + }, + } + PasswordSessionToolAptPackage { tool: t, package: p } => + PasswordSessionToolOutcome { + tool: t, + verdict: srv3_ensure_apt_tool( + tool_name: t, + package: p, + require_version_probe: false, + transport: transport, + ), + } + } +} + +fn converge_password_session_tools( + transport: HostEffectTransport, +) -> List { + map( + password_session_host_cli_requirements, + t => converge_one_password_session_tool(tool: t, transport: transport), + ) +} + +// THE FLEET EDGE IS THE ONLY ROUTE, and that is a security decision this corpus already made for +// this exact population. apt-get install is root-evaluated, and gunbc.runner_host_file_converge +// refuses to obtain that as the CI job user: "A sudo grant letting the fleet-converge job user +// install them would let every CI job on the host, pull requests included, stage its own content +// and become root. So the writes run over the fleet SSH edge as the declared administrator +// principal." The same reasoning binds here -- a job-user apt grant is a root escalation available +// to every pull request that runs on the host -- so this convergence reaches the runner as the +// administrator principal over FleetSsh even though the step executes ON the target. +fn password_session_tool_transport( + target: SshTarget, + context: FleetSshExecutionContext, +) -> HostEffectTransport { + FleetSsh { target: target, context: context } +} + +fn password_session_tool_outcome_holds(outcome: PasswordSessionToolOutcome) -> Bool { + match outcome.verdict { + Srv3PredicateHolds => true + Srv3PredicateFails => false + Srv3PredicateUnobserved { reason: _ } => false + } +} + +fn password_session_tools_all_hold(outcomes: List) -> Bool { + fold_list( + xs: outcomes, + empty: true, + cons: fn(acc, o) { acc && password_session_tool_outcome_holds(outcome: o) }, + ) +} + +// THE RECEIPT IS DERIVED FROM THE SAME OUTCOMES THE CONVERGENCE PRODUCED, so a tool cannot appear +// in one and not the other and its label cannot disagree with its verdict. Unobserved is rendered +// distinctly from fails because they have different remedies: fails means the host answered and the +// tool is still absent, unobserved means nothing was established and an operator must not read it +// as either. The unobserved arm carries the typed cause after the colon so the receipt and the +// failing diagnostic name why, not only that the probe was unobserved. +fn password_session_tool_receipt_entry(outcome: PasswordSessionToolOutcome) -> String { + concat( + concat(outcome.tool as String, "="), + match outcome.verdict { + Srv3PredicateHolds => "present" + Srv3PredicateFails => "absent" + Srv3PredicateUnobserved { reason: r } => concat("unobserved:", r) + }, + ) +} + +fn password_session_tool_receipt(outcomes: List) -> String { + concat( + "PasswordSessionToolConvergeReceipt: ", + join(map(outcomes, o => password_session_tool_receipt_entry(outcome: o)), " "), + ) +} + +data runner_password_session_tool_converge_attempt_raw: String = "runner-password-session-tool-converge-ci" + +data runner_password_session_tool_converge_receipt_path: String = "target/runner-password-session-tool-converge-receipt.txt" + +// ONE HOST PER RUN, NAMED BY THE DISPATCH, which is why the host arrives through the environment +// rather than by folding the roster here. The fleet-converge job is pinned to a runner ON the named +// host, so a run that converged every rostered runner would be reaching three machines from a +// fourth for no reason and would report one verdict over four independent outcomes. The roster +// still decides WHICH TOOLS; the dispatch decides WHICH HOST. +// +// THE RECEIPT IS WRITTEN BEFORE THE VERDICT IS RETURNED, and on the failing path too. A convergence +// whose receipt only survives success is unreadable exactly when it is needed: the operator asking +// "what happened on srv4" is usually asking after a failure. +func runner_password_session_tool_converge_ci_wet() -> ProcessExit + uses net: Network +{ + match actions_variable_read(name: fleet_converge_expected_host_env_name) { + ActionsVariableAbsent { variable: v, detail: d } => + exit_failure(reason: join([ + "runner_password_session_tool_converge: ", v, " ", d, + "; one host per run, named by the dispatch", + ], "")) + ActionsVariablePresent { value: raw_host } => + match password_session_tool_subject(host: raw_host as HostIdentity) { + PasswordSessionToolSubjectRefused { cause: c } => exit_failure(reason: c) + PasswordSessionToolSubjectAdmitted { host: h, target: target } => + match prepare_fleet_ssh_agent_context( + attempt_raw: runner_password_session_tool_converge_attempt_raw, + ) { + FleetSshContextRefused { cause: c } => exit_failure(reason: c) + FleetSshContextReady { context: context, receipt: context_receipt } => { + let outcomes = converge_password_session_tools( + transport: password_session_tool_transport(target: target, context: context), + ) + let body = join([ + "runner_password_session_tool_converge host=", h as String, + " probed_at=", probed_at_word(reading: clock_now_probed_at()) as String, "\n", + context_receipt, + password_session_tool_receipt(outcomes: outcomes), + "\n", + ], "") + let w = Filesystem.Write( + path: runner_password_session_tool_converge_receipt_path, + content: body, + ) + if w.success == false { + exit_failure(reason: join([ + "runner_password_session_tool_converge: receipt write failed: ", w.error, "\n", body, + ], "")) + } else if password_session_tools_all_hold(outcomes: outcomes) == false { + exit_failure(reason: body) + } else { + ExitSuccess + } + } + } + } + } +} diff --git a/dag/test/claim/extdeps/apt_package_of_witness_test.dag b/dag/test/claim/extdeps/apt_package_of_witness_test.dag new file mode 100644 index 00000000000..f42cb247549 --- /dev/null +++ b/dag/test/claim/extdeps/apt_package_of_witness_test.dag @@ -0,0 +1,56 @@ +module test.claim.apt_package_of_witness + +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import v2.std.optional { Present, Absent } +import std.types { Bool, String } +import extdeps.tools { apt_package_of } +import extdeps.tools.sshpass { sshpass_cli_tool } +import extdeps.tools.websocat { websocat_cli_tool } +import extdeps.tools.mosquitto_pub { mosquitto_pub_cli_tool } +import extdeps.apt { package_sshpass, apt_installed_bin_path, apt_bin_dir } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// THE PAIR IS THE POINT, and it is a pair of REAL rows rather than fixtures. apt_package_of is a +// fold that answers Present for a tool carrying SourceApt and Absent for one that does not, so a +// witness asserting only the Present side would pass against a function that returned the first +// source unconditionally, and one asserting only Absent would pass against a function that always +// returned none. Both live tools are enrolled here because the corpus already contains both shapes +// and a fixture would prove less: sshpass is the subject of this lane, websocat is installed from a +// GitHub release and has no apt source at all. +// Hermetic controls for extdeps.tools apt_package_of: an apt-sourced tool resolves to its declared +// package, a release-sourced tool resolves to Absent rather than a fabricated package, and the +// resolved package is the declaration rather than the binary name. +test fn apt_package_of_resolves_the_declared_package() -> Bool { + match apt_package_of(tool: sshpass_cli_tool) { + Present { value: p } => (p as String) == (package_sshpass as String) + Absent => false + } +} + +// THE DISCRIMINATING HALF. websocat_cli_tool is installed from a GitHub release, so "no apt +// package" is the CORRECT answer and not a failure to resolve one. This is the row that would go +// red if apt_package_of ever grew a default -- the §5 fabricated-plausible-output arm, which here +// would most naturally be `apt install ` and would look right for most tools. +test fn a_tool_with_no_apt_source_resolves_to_absent() -> Bool { + match apt_package_of(tool: websocat_cli_tool) { + Present { value: _ } => false + Absent => true + } +} + +// THE PACKAGE IS NOT THE BINARY NAME, witnessed against a live row where the two genuinely differ: +// mosquitto_pub is the executable, mosquitto-clients is the package that provides it. A default of +// "apt install " passes both rows above and fails this one, which is why it is enrolled +// separately rather than folded into the first assertion. +test fn the_resolved_package_is_not_the_binary_name() -> Bool { + match apt_package_of(tool: mosquitto_pub_cli_tool) { + Present { value: p } => (p as String) != (mosquitto_pub_cli_tool.name as String) + Absent => false + } +} + +test fn apt_installed_bin_path_is_apt_bin_dir_plus_the_binary_name() -> Bool { + (apt_installed_bin_path(binary: sshpass_cli_tool.name) as String) + == concat(apt_bin_dir as String, concat("/", sshpass_cli_tool.name as String)) +} diff --git a/dag/test/claim/runner/runner_password_session_tool_converge_witness_test.dag b/dag/test/claim/runner/runner_password_session_tool_converge_witness_test.dag new file mode 100644 index 00000000000..1fe20701f78 --- /dev/null +++ b/dag/test/claim/runner/runner_password_session_tool_converge_witness_test.dag @@ -0,0 +1,157 @@ +module test.claim.runner_password_session_tool_converge_witness + +import std.logic { Bool } +import std.types { String } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import gunbc.host_effect { + HostEffectTransport, + EmitArtifactThenThinRun, + BootstrapFragment, + ThinInvocation, +} +import gunbc.host_effect_realize { + Srv3HostPredicate, + Srv3PredicateHolds, + Srv3PredicateFails, + Srv3PredicateUnobserved, +} +import gunbc.runner_host_grants { executor_bootstrap_principal } +import gunbc.runner_password_session_tool_converge { + PasswordSessionToolAptPackage, + PasswordSessionToolUnacquirable, + PasswordSessionToolOutcome, + PasswordSessionToolSubjectAdmitted, + PasswordSessionToolSubjectRefused, + password_session_tool_plan, + password_session_tool_subject, + converge_one_password_session_tool, + password_session_tool_receipt, +} +import extdeps.tools.sshpass { sshpass_cli_tool } +import extdeps.tools.websocat { websocat_cli_tool } +import extdeps.apt { package_sshpass } +import product.placement_supply { HostIdentity } +import gunbc.fleet_intent_network { operator_host_srv4 } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// THE TRANSPORT THAT CANNOT PROBE IS THE WALL THIS FILE EXISTS FOR, and it is inherited rather +// than re-derived. srv3_apt_tool_present once returned Bool, so the EmitArtifactThenThinRun arm -- +// which cannot run a probe at all -- answered `false`, which is also what a host says when the tool +// is genuinely missing; srv3_ensure_apt_tool read that and ran a PRIVILEGED apt install against a +// host whose state had never been established. This module reuses that ensure precisely so it +// inherits the repair, and the row below is what proves the inheritance is real rather than +// asserted by the annotation. +data unprobeable_transport: HostEffectTransport = EmitArtifactThenThinRun { + bootstrap: BootstrapFragment { script: "true" }, + invocation: ThinInvocation { argv: ["true"] }, +} + +fn verdict_is_unobserved(p: Srv3HostPredicate) -> Bool { + match p { + Srv3PredicateUnobserved { reason: _ } => true + Srv3PredicateHolds => false + Srv3PredicateFails => false + } +} + +// ── The package is read off the tool, never re-authored ────────────────────────────────────────── + +test fn the_plan_reads_the_package_off_the_enrolled_tool() -> Bool { + match password_session_tool_plan(tool: sshpass_cli_tool) { + PasswordSessionToolAptPackage { tool: t, package: p } => + (t as String) == (sshpass_cli_tool.name as String) + && (p as String) == (package_sshpass as String) + PasswordSessionToolUnacquirable { tool: _ } => false + } +} + +// THE DISCRIMINATING HALF OF THE PLAN. websocat is installed from a GitHub release, so it has no +// apt package and the plan must say so rather than inventing one. This is the row that reddens if +// apt_package_of ever grows a default, and it is written against a LIVE tool row rather than a +// fixture so it also reddens if websocat's own declaration changes out from under the claim. +test fn a_tool_with_no_apt_source_is_unacquirable_not_guessed() -> Bool { + match password_session_tool_plan(tool: websocat_cli_tool) { + PasswordSessionToolUnacquirable { tool: t } => + (t as String) == (websocat_cli_tool.name as String) + PasswordSessionToolAptPackage { tool: _, package: _ } => false + } +} + +// ── The unprobeable transport refuses instead of installing ────────────────────────────────────── + +// THE CLASS THIS MODULE COULD MOST EASILY HAVE RE-INTRODUCED. A convergence that read a probe as +// "absent" when the probe could not run would install blind, on a privileged path, against an +// unestablished host. The assertion is Unobserved specifically -- not merely "not Holds" -- because +// Fails would mean the host answered, and the whole point is that it did not. +test fn an_unprobeable_transport_refuses_rather_than_installing() -> Bool { + verdict_is_unobserved( + p: converge_one_password_session_tool( + tool: sshpass_cli_tool, + transport: unprobeable_transport, + ).verdict, + ) +} + +// AN UNACQUIRABLE TOOL IS ALSO UNOBSERVED, and for a different reason worth keeping distinct: the +// transport here is irrelevant because the refusal happens before any host is reached. A roster +// tool with no apt source is an authoring gap in the roster, and this convergence says so rather +// than reporting it as a fact about the runner. +test fn an_unacquirable_tool_refuses_before_reaching_a_host() -> Bool { + let outcome = converge_one_password_session_tool( + tool: websocat_cli_tool, + transport: unprobeable_transport, + ) + verdict_is_unobserved(p: outcome.verdict) + && (outcome.tool as String) == (websocat_cli_tool.name as String) +} + +// ── Subject admission ──────────────────────────────────────────────────────────────────────────── + +// A MACHINE WITH NO RUNNER DEPLOY ROW IS UNKNOWN, NOT TYPICAL. It must refuse rather than borrow a +// neighbour's row, because an unmodeled machine carries no declared administrator principal and so +// there is no account this convergence could honestly act as. +test fn an_unmodeled_host_is_refused_as_a_subject() -> Bool { + match password_session_tool_subject(host: "srv-not-a-runner" as HostIdentity) { + PasswordSessionToolSubjectRefused { cause: c } => + string_contains(c as String, "srv-not-a-runner") + && string_contains(c as String, "no runner deploy row") + PasswordSessionToolSubjectAdmitted { host: _, target: _ } => false + } +} + +// THE ADMITTED SIDE, so the refusal above is discriminating rather than a one-armed assertion: a +// real runner host resolves and its target names the administrator principal, not the job user. +test fn a_rostered_runner_is_admitted_with_the_administrator_principal() -> Bool { + match password_session_tool_subject(host: operator_host_srv4) { + PasswordSessionToolSubjectAdmitted { host: h, target: t } => + (h as String) == (operator_host_srv4 as String) + && (t.principal as String) == (executor_bootstrap_principal.login as String) + PasswordSessionToolSubjectRefused { cause: _ } => false + } +} + +// ── The receipt is derived from the outcomes ───────────────────────────────────────────────────── + +// UNOBSERVED IS RENDERED DISTINCTLY FROM ABSENT because the two have different remedies: absent +// means the host answered and the tool is still missing, unobserved means nothing was established. +// An operator who reads one as the other either chases a host that is fine or trusts a host that +// was never checked. +test fn the_receipt_distinguishes_unobserved_from_absent() -> Bool { + let unobserved = password_session_tool_receipt(outcomes: [ + PasswordSessionToolOutcome { + tool: sshpass_cli_tool.name, + verdict: Srv3PredicateUnobserved { reason: "probe could not run" }, + }, + ]) + let absent = password_session_tool_receipt(outcomes: [ + PasswordSessionToolOutcome { tool: sshpass_cli_tool.name, verdict: Srv3PredicateFails }, + ]) + let present = password_session_tool_receipt(outcomes: [ + PasswordSessionToolOutcome { tool: sshpass_cli_tool.name, verdict: Srv3PredicateHolds }, + ]) + string_contains(unobserved, "sshpass=unobserved:probe could not run") + && string_contains(absent, "sshpass=absent") + && string_contains(present, "sshpass=present") + && unobserved != absent +} diff --git a/dag/test/claim/srv3/srv3_host_effect_realize_witness_test.dag b/dag/test/claim/srv3/srv3_host_effect_realize_witness_test.dag index 04881246b53..c97b8420e06 100644 --- a/dag/test/claim/srv3/srv3_host_effect_realize_witness_test.dag +++ b/dag/test/claim/srv3/srv3_host_effect_realize_witness_test.dag @@ -20,6 +20,9 @@ import gunbc.fleet_intent { srv1_host } import std.realization_reconcile { NotConverged } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.shell { posix_command_v_check_argv, shell_test_is_executable_argv } +import extdeps.apt { apt_installed_bin_path } +import gunbc.fleet_known_hosts_anchor { portable_remote_words } +import v2.std.optional { Present, Absent } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -70,3 +73,19 @@ test fn posix_command_v_check_argv_shapes_a_command_v_invocation() -> Bool { test fn shell_test_is_executable_argv_shapes_a_test_dash_x_invocation() -> Bool { shell_test_is_executable_argv(path: "/usr/bin/curl" as FilePath) == ["test", "-x", "/usr/bin/curl"] } + +test fn fleet_ssh_apt_tool_presence_argv_is_portable() -> Bool { + match portable_remote_words( + raws: shell_test_is_executable_argv(path: apt_installed_bin_path(binary: "sshpass" as NonEmptyStr)), + ) { + Present { value: _ } => true + Absent => false + } +} + +test fn posix_command_v_check_argv_is_not_portable() -> Bool { + match portable_remote_words(raws: posix_command_v_check_argv(command: "sshpass" as NonEmptyStr)) { + Present { value: _ } => false + Absent => true + } +} diff --git a/dag/test/claim/workflow_dispatch_input_witness_test.dag b/dag/test/claim/workflow_dispatch_input_witness_test.dag index 525a49b2781..268199f754c 100644 --- a/dag/test/claim/workflow_dispatch_input_witness_test.dag +++ b/dag/test/claim/workflow_dispatch_input_witness_test.dag @@ -7,6 +7,7 @@ import gunbc.fleet_converge_workflow { FleetConvergeWorkflowMode, fleet_converge_workflow_modes, fleet_converge_workflow_mode_wire, fleet_converge_mode_step_if, fleet_converge_mode_scope, FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, DashboardDeploy, + RunnerPasswordSessionToolConverge, fleet_converge_spark_bootstrap_step_if, fleet_converge_mode_options, fleet_converge_reset_observer_dispatch_admission_step, fleet_converge_build_job_steps, fleet_converge_reset_subject_mutation_concurrency_group_expression, fleet_converge_reset_subject_map_json, @@ -346,30 +347,27 @@ test fn a_reset_subject_map_escapes_a_label_that_would_break_the_json() -> Bool // THE VOCABULARY IS THE ONLY SPELLING AUTHORITY, and these are the facts that make that a claim // rather than a hope. // -// COUNT AND DISTINCTNESS TOGETHER. The count alone is not enough: two members returning the same -// wire string would still meet it, and the dispatch menu would silently offer one option twice while -// one mode became unselectable. Counting the DISTINCT wire values is what refuses that. -// -// THE LITERAL MOVED FROM FIFTEEN TO SIXTEEN when HostResetReturn was added, and to NINETEEN when -// SparkRuntimeImageProbe, RunnerHostFileObserve and RunnerHostFileConverge were added, and the roster -// annotation in gunbc.fleet_converge_workflow says why a literal is here at all: nothing reflects -// over a closed vocabulary to produce its members, so fleet_converge_workflow_modes is hand-ordered -// and this count is what makes a member DROPPED from that roster visible. The count is grounded in -// the coproduct, not copied from a measurement of itself -- adding a variant without rostering it -// leaves the type and the roster disagreeing, and this is the only place that disagreement is loud. -test fn the_mode_vocabulary_members_have_that_many_distinct_wire_values() -> Bool { +// DISTINCTNESS, NOT A CENSUS LITERAL. Two members returning the same wire would still populate the +// menu twice while one mode became unselectable; unique wires refuse that. A count equality +// against a hand-updated N is a change detector: updating N when the roster grows collapses the +// assertion to measure() == measure() (DESIGN §5). The coproduct-vs-roster drop is still the +// known stall (roster_re_enumerates_its_own_rows_stall); this witness cannot mint that join, and +// a literal does not substitute for it. What it CAN join is identities: the mode this lane added +// is on the roster, every dispatch option is some rostered mode's wire, and every rostered wire is +// a dispatch option. +test fn the_password_session_tool_mode_is_on_the_roster_with_distinct_wires() -> Bool { let wires = map(fleet_converge_workflow_modes, m => fleet_converge_workflow_mode_wire(mode: m)) - count(fleet_converge_workflow_modes) == 19 - && count(wires) == 19 + any(wires, w => w == fleet_converge_workflow_mode_wire(mode: RunnerPasswordSessionToolConverge)) && unique(xs: wires, eq: (a, b) => a == b) } // EVERY DISPATCH OPTION IS A WIRE VALUE, tying the menu GitHub renders to the vocabulary rather than -// to a hand-kept list that merely happens to agree with it today. +// to a hand-kept list that merely happens to agree with it today. The join is both directions, so +// a roster row with no option and an option with no row are both red. test fn every_dispatch_option_is_a_wire_value_of_the_vocabulary() -> Bool { let wires = map(fleet_converge_workflow_modes, m => fleet_converge_workflow_mode_wire(mode: m)) - count(fleet_converge_mode_options) == 19 - && all(fleet_converge_mode_options, o => any(wires, w => w == o)) + all(fleet_converge_mode_options, o => any(wires, w => w == o)) + && all(wires, w => any(fleet_converge_mode_options, o => o == w)) } // THE STEP CONDITION IS DERIVED, not spelled beside the option. The discriminating half is the