diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index fb0916ae903..1ee5b8ebedc 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -11,9 +11,9 @@ on: options: [srv1, srv2, srv3, srv4] type: choice mode: - description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject + description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -572,6 +572,36 @@ jobs: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} if: github.event.inputs.mode == 'spark_runtime_image_probe' timeout-minutes: 60 + - name: "Runner host files: observe the teardown drop-in, the needrestart deferral and the loaded teardown (no writes)" + id: runner_host_file_observe + run: | + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_host_file_converge.dag --function runner_host_file_observe_ci_wet + cat "$ROOT/target/runner-host-file-converge-receipt.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} + if: github.event.inputs.mode == 'runner_host_file_observe' + timeout-minutes: 15 + - name: "Runner host files: write what differs as the administrator, reload if the drop-in changed, read the loaded teardown back" + id: runner_host_file_converge + run: | + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_host_file_converge.dag --function runner_host_file_converge_ci_wet + cat "$ROOT/target/runner-host-file-converge-receipt.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} + if: github.event.inputs.mode == 'runner_host_file_converge' + timeout-minutes: 15 + - name: Upload runner host file converge receipt + id: runner_host_file_receipt_upload + uses: actions/upload-artifact@v4 + with: + name: runner-host-file-converge-receipt + path: target/runner-host-file-converge-receipt.txt + if-no-files-found: error + retention-days: 30 + if: always() && (github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge') + timeout-minutes: 10 - name: Kill the in-run ssh-agent (key never outlives the job) run: | if [ -n "${SSH_AGENT_PID:-}" ]; then ssh-agent -k || true; fi diff --git a/dag/extdeps/needrestart/needrestart.dag b/dag/extdeps/needrestart/needrestart.dag new file mode 100644 index 00000000000..9a2b9ce049e --- /dev/null +++ b/dag/extdeps/needrestart/needrestart.dag @@ -0,0 +1,152 @@ +module extdeps.needrestart + +import std.types { String, NonEmptyStr, List, Bool, Int } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } +import std.decl_ref { DeclarationRef, WholeDeclaration } + +// needrestart (Thomas Liske) runs after every dpkg transaction on Ubuntu, finds daemons still mapping +// replaced libraries, and restarts them. Ubuntu wires it into apt through a dpkg hook, so an +// unattended security upgrade of libc restarts every service whose process image still maps the old +// libc -- which on a self-hosted CI host includes every busy runner slot. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/liske/needrestart" + } +} + +// The configuration surface this module models, at the upstream tag Ubuntu noble packages. The shipped +// needrestart.conf documents override_rc as "Override service default selection (hash of regex)" and +// ends by evaluating every /etc/needrestart/conf.d/*.conf in Perl sort order, dying on a parse error. +data needrestart_config_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/liske/needrestart/blob/v3.6/ex/needrestart/needrestart.conf" + } +} + +// The consumer of override_rc: for each service needing a restart, the first key whose regex matches +// the unit name decides whether it is restarted (true) or deferred and only reported (false). +data needrestart_override_rc_consumer_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/liske/needrestart/blob/v3.6/needrestart" + } +} + +// The distribution package the fleet runs, read off srv1 with dpkg-query on 2026-09-11. +data needrestart_ubuntu_noble_package_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "launchpad.net/ubuntu/+source/needrestart/3.6-7ubuntu4.5" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.needrestart", + decl_name: "NeedrestartServiceSelectionOverride", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [ + needrestart_config_authority, + needrestart_override_rc_consumer_authority, + needrestart_ubuntu_noble_package_authority, + ] +} + +data needrestart_upstream_version: NonEmptyStr = "3.6" as NonEmptyStr + +data needrestart_ubuntu_noble_package_version: NonEmptyStr = "3.6-7ubuntu4.5" as NonEmptyStr + +data needrestart_conf_d_dir: NonEmptyStr = "/etc/needrestart/conf.d" as NonEmptyStr + +// Only files ending in `.conf` are evaluated, so the suffix is part of the file's identity rather than +// a naming habit: a snippet without it is inert, and nothing reports that. +data needrestart_conf_d_suffix: String = ".conf" + +fn needrestart_conf_d_path(stem: NonEmptyStr) -> NonEmptyStr { + join([needrestart_conf_d_dir as String, "/", stem as String, needrestart_conf_d_suffix], "") as NonEmptyStr +} + +// The value side of one override_rc entry. upstream stores it as a Perl boolean and reads it into +// `$restart`; the two states have opposite consequences for a running process, so they are named. +type NeedrestartDefaultSelection + = RestartSelectedByDefault + | RestartDeferredByDefault + +fn needrestart_default_selection_wire(selection: NeedrestartDefaultSelection) -> String { + match selection { + RestartSelectedByDefault => "1" + RestartDeferredByDefault => "0" + } +} + +// THE KEY SIDE IS A UNIT-NAME PREFIX, NOT A REGEX, because a caller-authored regex is a second +// language with its own failure modes and needrestart dies on a malformed one. The module owns the +// translation: anchor at the start, quote every character the way Perl's quotemeta does, so the key +// can only ever mean "unit names beginning with exactly these characters". +// +// The accepted alphabet is systemd.unit(5)'s unit-name characters plus the template separator `@`, +// with the backslash excluded: systemd admits it only as an escape introducer, and a prefix carrying +// one would make the prefix's meaning depend on a second escaping layer this module does not model. +type NeedrestartUnitNamePrefix sole_constructor { + prefix: NonEmptyStr +} + +type NeedrestartUnitNamePrefixAdmission + = NeedrestartUnitNamePrefixAdmitted { prefix: NeedrestartUnitNamePrefix } + | NeedrestartUnitNamePrefixRefused { cause: NonEmptyStr } + +fn needrestart_unit_name_char_allowed(cp: Int) -> Bool { + (cp >= 97 && cp <= 122) || (cp >= 65 && cp <= 90) || (cp >= 48 && cp <= 57) + || cp == 58 || cp == 45 || cp == 95 || cp == 46 || cp == 64 +} + +fn needrestart_unit_name_prefix(raw: String) -> NeedrestartUnitNamePrefixAdmission { + if raw == "" { + NeedrestartUnitNamePrefixRefused { cause: "needrestart unit-name prefix is empty, which would match every service" as NonEmptyStr } + } else if fold(chars(s: raw), init: true, f: (ok, cp) => ok && needrestart_unit_name_char_allowed(cp: cp)) { + NeedrestartUnitNamePrefixAdmitted { prefix: NeedrestartUnitNamePrefix { prefix: raw as NonEmptyStr } } + } else { + NeedrestartUnitNamePrefixRefused { + cause: join(["needrestart unit-name prefix `", raw, "` carries a character outside systemd's unit-name alphabet"], "") as NonEmptyStr, + } + } +} + +// perlfunc quotemeta: every ASCII character that is not a word character is preceded by a backslash. +fn needrestart_perl_quotemeta_code_point(cp: Int) -> String { + if (cp >= 97 && cp <= 122) || (cp >= 65 && cp <= 90) || (cp >= 48 && cp <= 57) || cp == 95 { + from_code_point(cp: cp) + } else { + concat("\\", from_code_point(cp: cp)) + } +} + +fn needrestart_unit_name_prefix_regex(prefix: NeedrestartUnitNamePrefix) -> String { + join(["qr(^", join(map(chars(s: prefix.prefix as String), cp => needrestart_perl_quotemeta_code_point(cp: cp)), ""), ")"], "") +} + +type NeedrestartServiceSelectionOverride { + units: NeedrestartUnitNamePrefix + selection: NeedrestartDefaultSelection +} + +// One assignment into the hash needrestart.conf has already created. Assigning a key rather than +// replacing the hash is what lets a snippet add one policy without erasing the distribution's own +// entries (dbus, display managers, getty) that the shipped file declares. +fn needrestart_override_rc_line(o: NeedrestartServiceSelectionOverride) -> String { + join([ + "$nrconf\{override_rc\}->\{", needrestart_unit_name_prefix_regex(prefix: o.units), "\} = ", + needrestart_default_selection_wire(selection: o.selection), ";", + ], "") +} + +fn needrestart_conf_d_text(overrides: List) -> String { + join(map(overrides, o => concat(needrestart_override_rc_line(o: o), "\n")), "") +} diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index 052825fec0f..f8caa4f8cfb 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -212,6 +212,31 @@ fn systemctl_show_property_argv(unit: NonEmptyStr, property: SystemdUnitProperty ] } +// SEVERAL PROPERTIES OF SEVERAL UNITS IN ONE READ. systemctl show accepts a comma-separated +// --property list and any number of units, and prints one block of `Name=value` lines per unit with a +// blank line between blocks. Without --value the property names ride along, so a consumer can key each +// value by name rather than trusting an output order systemctl does not document; asking for Id makes +// every block say which unit it describes. A unit the manager has not loaded still gets a block -- +// LoadState=not-found beside the manager's DEFAULTS for every other property -- so a consumer must read +// LoadState before believing any other value in that block. +fn systemctl_show_properties_argv(units: List, properties: List) -> List { + concat( + [ + "systemctl", + "show", + concat("--property=", join(map(properties, p => systemd_unit_property_wire(property: p) as String), ",")), + ], + map(units, u => u as String), + ) +} + +// Every service unit the manager holds, loaded or not, as plain rows whose first field is the unit +// name. No pattern: a glob is a shell word the fleet's far-side re-parse refuses, so the caller filters +// by name in the model instead. +fn systemctl_list_all_service_units_argv() -> List { + ["systemctl", "list-units", "--type=service", "--all", "--no-legend", "--plain"] +} + // LoadState is a non-numeric unit lifecycle observation, so it has its own argv authority rather than joining SystemdUnitProperty, whose shared ShowProperty mock is numeric cgroup evidence. systemctl show exits nonzero on an unreadable manager; the load-state wire value not-found is the only observation that proves an absent unit. // `is-enabled` answers the INSTALL state (whether the unit is wanted by a target), which is a // different question from `is-active` (whether it is running now) and from LoadState (whether the diff --git a/dag/extdeps/systemd/systemd.dag b/dag/extdeps/systemd/systemd.dag index 6c36fa8e1e2..1ef4b2b3af5 100644 --- a/dag/extdeps/systemd/systemd.dag +++ b/dag/extdeps/systemd/systemd.dag @@ -78,6 +78,9 @@ type SystemdUnitProperty | AccuracyUSec | TimersMonotonic | NeedDaemonReload + | IdProperty + | KillModeProperty + | ExitTypeProperty fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr { match property { @@ -110,6 +113,9 @@ fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr { AccuracyUSec => "AccuracyUSec" as NonEmptyStr TimersMonotonic => "TimersMonotonic" as NonEmptyStr NeedDaemonReload => "NeedDaemonReload" as NonEmptyStr + IdProperty => "Id" as NonEmptyStr + KillModeProperty => "KillMode" as NonEmptyStr + ExitTypeProperty => "ExitType" as NonEmptyStr } } diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 4df3cf39ae8..83f916a498c 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -770,6 +770,16 @@ data gunbc_ci_runner_microvm_host_converge_target: GunbcRunStepTarget = GunbcRun function: "runner_microvm_host_converge_wet", } +data gunbc_ci_runner_host_file_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/runner/runner_host_file_converge.dag", + function: "runner_host_file_observe_ci_wet", +} + +data gunbc_ci_runner_host_file_converge_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/runner/runner_host_file_converge.dag", + function: "runner_host_file_converge_ci_wet", +} + // THE OBSERVER SIDE OF ONE RESET ATTEMPT. It takes no arguments: the dispatch names the OBSERVER, the // step hands that label over by environment, and the subject is derived from the label through the // modeled assignment. There is no input of any kind through which a caller could name a subject, @@ -951,6 +961,8 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_runner_guest_image_observe_target, gunbc_ci_runner_guest_image_converge_target, gunbc_ci_runner_microvm_host_converge_target, + gunbc_ci_runner_host_file_observe_target, + gunbc_ci_runner_host_file_converge_target, gunbc_ci_runner_microvm_boot_probe_target, gunbc_ci_host_reset_return_target, gunbc_ci_host_reset_dispatch_admission_target, @@ -1722,6 +1734,26 @@ fn gunbc_ci_runner_microvm_host_converge_invoke() -> String { ) } +fn gunbc_ci_runner_host_file_observe_invoke() -> String { + gunbc_run_step_script( + source_roots: witness_layer_roots, + entry: gunbc_ci_runner_host_file_observe_target.entry, + function: gunbc_ci_runner_host_file_observe_target.function, + claim_run: false, + receipt_rel: "target/runner-host-file-converge-receipt.txt" + ) +} + +fn gunbc_ci_runner_host_file_converge_invoke() -> String { + gunbc_run_step_script( + source_roots: witness_layer_roots, + entry: gunbc_ci_runner_host_file_converge_target.entry, + function: gunbc_ci_runner_host_file_converge_target.function, + claim_run: false, + receipt_rel: "target/runner-host-file-converge-receipt.txt" + ) +} + fn gunbc_ci_host_reset_return_invoke() -> String { gunbc_run_step_script( source_roots: witness_layer_roots, diff --git a/dag/gunbc/extdeps_scope_frontier.dag b/dag/gunbc/extdeps_scope_frontier.dag index 35eef224c38..e6f1fb55a3f 100644 --- a/dag/gunbc/extdeps_scope_frontier.dag +++ b/dag/gunbc/extdeps_scope_frontier.dag @@ -276,6 +276,7 @@ data scope_carrier_paths: List = [ "dag/extdeps/automation/playwright.dag", "dag/extdeps/package.dag", "dag/extdeps/npm.dag", + "dag/extdeps/needrestart/needrestart.dag", "dag/extdeps/browser/chromium.dag", "dag/extdeps/container/oci/digest.dag", "dag/extdeps/git/publication_transport.dag", diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 9c9c41b2edd..fe7d651d411 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -39,6 +39,7 @@ import gunbc.fleet_workflow_steps { ci_fleet_key_agent_cleanup_step, gunbc_ci_aux_step_timeout_minutes, gunbc_ci_runner_guest_image_step_timeout_minutes, + gunbc_ci_runner_host_file_step_timeout_minutes, gunbc_ci_spark_serving_step_timeout_minutes, gunbc_ci_artifact_transfer_step_timeout_minutes, gunbc_ci_fleet_job_backstop_timeout_minutes, @@ -64,6 +65,8 @@ import gunbc.ci_spec { gunbc_ci_runner_guest_image_observe_invoke, gunbc_ci_runner_guest_image_converge_invoke, gunbc_ci_runner_microvm_host_converge_invoke, + gunbc_ci_runner_host_file_observe_invoke, + gunbc_ci_runner_host_file_converge_invoke, gunbc_ci_runner_microvm_boot_probe_invoke, gunbc_ci_live_deploy_apply_srv1_invoke, gunbc_ci_spark_grant_install_invoke, @@ -163,6 +166,8 @@ type FleetConvergeWorkflowMode | DashboardDeploy | RlmLaunchDeploymentReceipt | HostResetReturn + | RunnerHostFileObserve + | RunnerHostFileConverge fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String { match mode { @@ -183,6 +188,8 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String DashboardDeploy => "dashboard_deploy" RlmLaunchDeploymentReceipt => "rlm_launch_deployment_receipt" HostResetReturn => "host_reset_return" + RunnerHostFileObserve => "runner_host_file_observe" + RunnerHostFileConverge => "runner_host_file_converge" } } @@ -207,7 +214,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // @@ -251,6 +258,8 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc DashboardDeploy => none RlmLaunchDeploymentReceipt => none HostResetReturn => none + RunnerHostFileObserve => none + RunnerHostFileConverge => none } } @@ -636,6 +645,12 @@ data fleet_converge_microvm_host_converge_step_if: String = fleet_converge_mode_ +data fleet_converge_runner_host_file_observe_step_if: String = fleet_converge_mode_step_if(mode: RunnerHostFileObserve) + +data fleet_converge_runner_host_file_converge_step_if: String = fleet_converge_mode_step_if(mode: RunnerHostFileConverge) + +data fleet_converge_runner_host_file_any_step_if: String = fleet_converge_any_mode_step_if(modes: [RunnerHostFileObserve, RunnerHostFileConverge]) + data fleet_converge_spark_grants_step_if: String = fleet_converge_mode_step_if(mode: SparkGrants) data fleet_converge_spark_bootstrap_step_if: String = fleet_converge_mode_step_if(mode: SparkBootstrap) @@ -876,6 +891,64 @@ fn fleet_converge_microvm_host_standing_upload_step() -> Step { } } +// THE RUNNER HOST FILES: the teardown drop-in and the needrestart deferral (gunbc.runner_host_file_converge). +// Observe and converge are two modes rather than one flag so an operator can dispatch the read alone +// and see what a converge would write before any root-evaluated file changes. Both reach the target +// over the job's fleet-key agent as the declared administrator; the job user gains no grant. +fn fleet_converge_runner_host_file_step(id: String, name: String, invoke: String, if_condition: String) -> Step { + RunStep { + name: Present { value: name }, + id: Present { value: id }, + run: invoke, + shell: none, + env: Present { + value: [ + kv(key: fleet_converge_expected_host_env_name, value: yaml_string(s: "${{ github.event.inputs.host }}")), + ] + }, + working_directory: none, + if_condition: Present { value: if_condition }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_runner_host_file_step_timeout_minutes } + } +} + +fn fleet_converge_runner_host_file_observe_step() -> Step { + fleet_converge_runner_host_file_step( + id: "runner_host_file_observe", + name: "Runner host files: observe the teardown drop-in, the needrestart deferral and the loaded teardown (no writes)", + invoke: gunbc_ci_runner_host_file_observe_invoke(), + if_condition: fleet_converge_runner_host_file_observe_step_if, + ) +} + +fn fleet_converge_runner_host_file_converge_step() -> Step { + fleet_converge_runner_host_file_step( + id: "runner_host_file_converge", + name: "Runner host files: write what differs as the administrator, reload if the drop-in changed, read the loaded teardown back", + invoke: gunbc_ci_runner_host_file_converge_invoke(), + if_condition: fleet_converge_runner_host_file_converge_step_if, + ) +} + +fn fleet_converge_runner_host_file_receipt_upload_step() -> Step { + UsesStep { + name: Present { value: "Upload runner host file converge receipt" }, + id: Present { value: "runner_host_file_receipt_upload" }, + uses: upload_artifact_action, + with: Present { value: [ + kv(key: "name", value: yaml_string(s: "runner-host-file-converge-receipt")), + kv(key: "path", value: yaml_string(s: "target/runner-host-file-converge-receipt.txt")), + kv(key: "if-no-files-found", value: yaml_string(s: "error")), + kv(key: "retention-days", value: yaml_int(n: 30)), + ] }, + env: none, + if_condition: Present { value: concat("always() && (", concat(fleet_converge_runner_host_file_any_step_if, ")")) }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } + } +} + fn fleet_converge_microvm_boot_probe_step() -> Step { RunStep { name: Present { value: "Runner micro-VM: boot the guest image and read the serial console" }, @@ -1280,6 +1353,9 @@ fn fleet_converge_job() -> Job { fleet_converge_spark_bootstrap_step(), fleet_converge_spark_serving_apply_step(), fleet_converge_spark_runtime_image_probe_step(), + fleet_converge_runner_host_file_observe_step(), + fleet_converge_runner_host_file_converge_step(), + fleet_converge_runner_host_file_receipt_upload_step(), ci_fleet_key_agent_cleanup_step(), ] ), @@ -1423,7 +1499,7 @@ data fleet_converge_workflow: Workflow = Workflow { }, DispatchInput { name: "mode", - description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject" }, + description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown" }, required: true, default: none, type: InputChoice { options: fleet_converge_mode_options }, diff --git a/dag/gunbc/fleet/fleet_workflow_steps.dag b/dag/gunbc/fleet/fleet_workflow_steps.dag index 4221183bdcc..080a5fd9a35 100644 --- a/dag/gunbc/fleet/fleet_workflow_steps.dag +++ b/dag/gunbc/fleet/fleet_workflow_steps.dag @@ -453,6 +453,14 @@ STEADY STATE IS SECONDS, NOT MINUTES. A host that already holds the base artifac DISSOLVE-ON: the first successful guest_image_converge on a cold host produces the measurement this envelope stands in for -- replace the number with that receipt, dated and cited to its run id, per the rule stated one constant down that raising a budget needs a measurement rather than a round number." +// Runner host file observe/converge step budget, raised from the shared aux 5m for a measured reason: +// the step is one `gunbc run` whose closure resolve dominates, and the fleet plan step -- the nearest +// resolve of comparable size -- took 3m55s to 4m18s on runs of 2026-09-04 against that same 5m, so the +// aux budget would make a slow resolve read as a converge failure. The remote legs are a dozen short +// ssh execs. Its own constant for the reason the Spark envelope records: the aux value is multiplied by +// the step count in the fleet job backstop. +data gunbc_ci_runner_host_file_step_timeout_minutes: Duration = 15 + data gunbc_ci_v1_compiler_tests_compile_gate_timeout_minutes: Duration = 15 // v1-compiler-tests retained-kernel execution gate step budget. Originally a compile-contract gate @@ -615,7 +623,10 @@ data gunbc_ci_fleet_job_worst_case_mode_spark_step_count: Int = 1 // mode (RLM-2a) is, like dashboard_deploy, its OWN job (fleet_converge_workflow // fleet_converge_rlm_launch_deployment_receipt_job) with its own backstop // (gunbc_ci_rlm_launch_deployment_receipt_job_backstop_timeout_minutes), so it does not move this -// figure either. +// figure either. The runner_host_file_observe and runner_host_file_converge modes DO add steps to +// this job -- one gated run step each plus one shared receipt upload -- and still do not move the +// bound: a dispatch of either runs one step at gunbc_ci_runner_host_file_step_timeout_minutes and one +// transfer-tier upload, which is below the spark_grants branch that sets the worst case. type JobTimeoutBudget = JobTimeoutWithinCeiling { minutes: Int } | JobTimeoutExceedsCeiling { minutes: Int, ceiling: Int } diff --git a/dag/gunbc/runner/runner_host_file_converge.dag b/dag/gunbc/runner/runner_host_file_converge.dag new file mode 100644 index 00000000000..0e42cb3fd37 --- /dev/null +++ b/dag/gunbc/runner/runner_host_file_converge.dag @@ -0,0 +1,988 @@ +module gunbc.runner_host_file_converge + +import std.types { String, NonEmptyStr, List, Bool, Int, FilePath } +import std.algebra { trim } +import std.resources { Network } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import v2.std.algebra { list_head, HeadFound, HeadAbsent } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.systemd.unit_file { SystemdKillMode, SystemdExitType, systemd_kill_mode_wire, systemd_exit_type_wire } +import extdeps.systemd { IdProperty, LoadState, NeedDaemonReload, KillModeProperty, ExitTypeProperty, systemd_unit_property_wire } +import extdeps.systemd.systemctl { systemctl_show_properties_argv, systemctl_list_all_service_units_argv, systemctl_daemon_reload_argv } +import extdeps.sudo.elevation { ElevatedInvocation, sudo_elevate } +import extdeps.needrestart { + NeedrestartServiceSelectionOverride, + RestartDeferredByDefault, + NeedrestartUnitNamePrefixAdmitted, + NeedrestartUnitNamePrefixRefused, + needrestart_unit_name_prefix, + needrestart_conf_d_path, + needrestart_conf_d_text, +} +import product.placement_supply { HostIdentity } +import gunbc.clock_read { clock_now_probed_at, probed_at_word } +import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent } +import gunbc.fleet_converge_receipt { fleet_converge_expected_host_env_name } +import gunbc.fleet_ssh_locus { prepare_fleet_ssh_agent_context, FleetSshContextReady, FleetSshContextRefused } +import gunbc.fleet_known_hosts_anchor { FleetSshExecutionContext, SshTarget } +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } +import gunbc.file_access { FileTreeScope } +import gunbc.typed_argv_exec { typed_argv_exec_over_fleet_ssh, TypedArgvExecOutcome, TypedArgvExecConverged, TypedArgvExecRefused } +import gunbc.typed_remote_file_write { + RemoteFileMode, + RemoteFileModeOwnerReadWriteWorldRead, + RemoteFileViaNonInteractiveSudo, + RemoteFileConverged, + TypedRemoteFileWriteAdmission, + TypedRemoteFileWriteAdmitted, + TypedRemoteFileWriteAdmissionRefused, + TypedRemoteFileConverged, + TypedRemoteFileConvergenceRefused, + RemoteFileComparisonReport, + ComparisonIdentical, + ComparisonDiffers, + ComparisonUnavailable, + typed_remote_file_write_seal, + typed_remote_file_step, + TypedRemoteFileStepRan, + TypedRemoteFileStepArgvRefused, + TypedRemoteFileStepLegRefused, + typed_remote_file_compare_argv, + classify_typed_remote_file_comparison, + converge_typed_remote_file, +} +import gunbc.runner_unit { + RunnerUnitLifecycleIntent, + gunbc_runner_lifecycle_intent, + runner_unit_prefix, + runner_unit_suffix, + runner_unit_name_of_registration, +} +import gunbc.runner_unit_file { runner_unit_teardown_dropin_path, gunbc_runner_unit_teardown_dropin_text } +import gunbc.runner_host_deploy { RunnerHostDeploy, RunnerHostDeployFound, RunnerHostDeployUnmodeled, runner_host_deploy_for } +import gunbc.runner_slot_provision { desired_runner_slot_members } +import gunbc.runner_host_grants { executor_bootstrap_principal, bootstrap_principal_is_not_the_grantee } + +// THE HOST FILES A RUNNER HOST MUST CARRY FOR ITS SLOTS TO BE TORN DOWN WHOLE, CONVERGED OVER THE +// FLEET EDGE AS THE HOST'S ADMINISTRATOR. +// +// WHAT IT FIXES, MEASURED. apt-daily-upgrade installs security updates every morning; when one of them +// replaces a library a runner maps (libc6 on 2026-09-11), needrestart runs `systemctl restart` over +// every actions-runner@ unit. The live unit is `KillMode=process`, so the restart kills the listener +// and nothing else: srv1's journal records `Unit process ... (Runner.Worker) remains running after unit +// stopped` for the in-flight job's worker, claim_executor and gunbc, then `Found left-over process ... +// Ignoring`, and thirteen seconds later a new listener takes a new job into the SAME _work directory. +// The two jobs then delete each other's toolchain and checkout and share one slot's memory ceiling -- +// the vanished rustc, registry sources and toolchain components the CI floor reported that morning. +// srv1's dpkg log shows needrestart restarting runner units on ten separate days since June. +// +// TWO FILES, TWO HALVES OF ONE CLASS, NEITHER SUFFICIENT ALONE. +// - the teardown drop-in makes ANY stop of a slot reach its whole cgroup, whoever issues it; +// - the needrestart deferral stops the daily upgrade from issuing that stop at all. An ephemeral +// runner already begins a fresh process after every job, so the upgraded library is picked up at +// the next job boundary without anyone restarting a slot mid-job. +// Without the first, any other restart (an operator, a converge, the liveness reconciler) still +// orphans. Without the second, the daily upgrade would still stop every busy slot -- loudly and +// cleanly after the first lands, but as a wave of cancelled jobs rather than a silent corruption. +// +// WHO WRITES, AND WHY NOT THE CI JOB USER. Both files are evaluated by root: systemd merges the drop-in +// and needrestart `eval`s its conf.d snippet as Perl. A sudo grant letting the fleet-converge job user +// install them would let every CI job on the host, pull requests included, stage its own content and +// become root. So the writes run over the fleet SSH edge as the declared administrator principal +// (gunbc.runner_host_grants executor_bootstrap_principal), and the job user gains no grant. +// +// NOTHING HERE RESTARTS A SLOT. The drop-in takes effect at daemon-reload for every loaded unit, and a +// running job keeps running; the next stop of that slot, from any source, reaches the whole cgroup. +// +// EVERY DECISION IS DERIVED FROM WHAT THE HOST REPORTS, NOT FROM WHAT THIS RUN DID, so a run that stops +// partway is finished by the next one. Whether a reload is owed is the manager's own NeedDaemonReload, +// not "this invocation wrote a file" -- the latter is false on the retry after a reload failed, which +// would leave the bytes right and the manager stale forever. Whether the teardown holds is read from +// EVERY slot unit, desired or merely loaded, not from a representative one. And a file is only left in +// place when its bytes AND its ownership are right: root evaluates both files, so an identical file a +// CI user could rewrite is not converged. + +type RunnerHostFileDesired { + subject: NonEmptyStr + path: FilePath + content: String + mode: RemoteFileMode +} + +type RunnerHostFileDesiredPopulation + = RunnerHostFilesDesired { files: List } + | RunnerHostFilesDesiredRefused { cause: NonEmptyStr } + +// Every desired path lives under /etc, and typed_remote_file_write refuses a path outside the scope it +// is sealed against, so this row is the whole of what the converge may write. +data runner_host_file_scope: FileTreeScope = FileTreeScope { root_path: "/etc" as NonEmptyStr } + +data runner_needrestart_conf_d_stem: NonEmptyStr = "50-gunbc-actions-runner" as NonEmptyStr + +fn runner_host_file_desired_population() -> RunnerHostFileDesiredPopulation { + match needrestart_unit_name_prefix(raw: runner_unit_prefix) { + NeedrestartUnitNamePrefixRefused { cause: c } => RunnerHostFilesDesiredRefused { cause: c } + NeedrestartUnitNamePrefixAdmitted { prefix: p } => + RunnerHostFilesDesired { + files: [ + RunnerHostFileDesired { + subject: "runner-teardown-dropin" as NonEmptyStr, + path: runner_unit_teardown_dropin_path() as FilePath, + content: gunbc_runner_unit_teardown_dropin_text(), + mode: RemoteFileModeOwnerReadWriteWorldRead, + }, + RunnerHostFileDesired { + subject: "needrestart-runner-deferral" as NonEmptyStr, + path: needrestart_conf_d_path(stem: runner_needrestart_conf_d_stem) as FilePath, + content: needrestart_conf_d_text(overrides: [ + NeedrestartServiceSelectionOverride { units: p, selection: RestartDeferredByDefault }, + ]), + mode: RemoteFileModeOwnerReadWriteWorldRead, + }, + ], + } + } +} + +// ── The subject host ───────────────────────────────────────────────────────────────────────────── + +type RunnerHostFileSubject { + host: HostIdentity + target: SshTarget + desired_units: List +} + +type RunnerHostFileSubjectAdmission + = RunnerHostFileSubjectAdmitted { subject: RunnerHostFileSubject } + | RunnerHostFileSubjectRefused { cause: String } + +// A host with no runner deploy row has no slots to protect and no declared administrator, and a host +// whose deploy row names no desired slot has no unit the teardown could be verified on; both refuse +// rather than converging files nothing would consume. +fn runner_host_file_subject(host: HostIdentity) -> RunnerHostFileSubjectAdmission { + match runner_host_deploy_for(host: host) { + RunnerHostDeployUnmodeled { host: h } => + RunnerHostFileSubjectRefused { + cause: join(["runner_host_file_converge: ", h as String, " has no runner deploy row, so it is not a runner host"], ""), + } + RunnerHostDeployFound { deploy: d } => + if !bootstrap_principal_is_not_the_grantee(bootstrap: executor_bootstrap_principal, deploy: d) { + RunnerHostFileSubjectRefused { + cause: join(["runner_host_file_converge: the administrator principal ", executor_bootstrap_principal.login as String, " is the job user on ", host as String, ", so the account that writes root-evaluated files would be the one CI jobs run as"], ""), + } + } else { + let units = map(desired_runner_slot_members(deploy: d), m => runner_unit_name_of_registration(registration_name: m.instance_name)) + if length(units) == 0 { + RunnerHostFileSubjectRefused { + cause: join(["runner_host_file_converge: ", host as String, " has no desired runner slot, so no unit exists to verify the teardown on"], ""), + } + } else { + RunnerHostFileSubjectAdmitted { + subject: RunnerHostFileSubject { + host: host, + target: SshTarget { + endpoint: fleet_probe_endpoint_for(host: host as String) as NonEmptyStr, + principal: executor_bootstrap_principal.login, + }, + desired_units: units, + }, + } + } + } + } +} + +// ── File observation and plan (pure) ───────────────────────────────────────────────────────────── + +type RunnerHostFileStanding + = HostFileIdentical + | HostFileDiffers + | HostFileAbsent + | HostFileUnobservable { reason: String } + +// `test -e` answers 0 or 1 and nothing else on its own; any other exit means the test did not run. +fn classify_host_file_presence(exit_code: Int, stderr: String) -> RunnerHostFileStanding? { + if exit_code == 0 { + none + } else if exit_code == 1 { + Present { value: HostFileAbsent } + } else { + Present { value: HostFileUnobservable { reason: join(["presence test exit=", to_string(exit_code), " ", trim(s: stderr)], "") } } + } +} + +// WHO CAN CHANGE A FILE IS PART OF WHAT THE FILE IS, when root evaluates it. A byte-exact needrestart +// snippet that the CI user owns, or that sits in a directory the CI user can write, is a root-code +// injection point that merely happens to hold the right bytes today. So every path the file depends on +// -- the file and each ancestor directory up to the write scope -- must be owned by root and writable by +// no group and no other user. +type RunnerHostPathMetadata + = PathsRootOnly + | PathsNotRootOnly { paths: List, detail: String } + | PathsUnobservable { reason: String } + | PathsNotApplicable + +// The directories a file's safety rests on: every ancestor from the write scope's root down to the +// file's own directory. Each is a prefix of the path ending just before one of its separators. +fn runner_host_path_ancestors(path: String) -> List { + filter( + map( + filter(enumerate(chars(s: path)), pair => pair.second == 47 && pair.first > 0), + pair => substring(s: path, start: 0, end: pair.first), + ), + a => a == (runner_host_file_scope.root_path as String) + || starts_with(s: a, prefix: concat(runner_host_file_scope.root_path as String, "/")), + ) +} + +// The predicate is evaluated FAR-SIDE by find, which prints exactly the named paths that are of the +// given type, owned by root:root, and carry neither the group- nor the other-write bit; `-maxdepth 0` +// keeps it to the named paths themselves. What comes back is therefore the SAFE set, and anything +// requested and not printed failed at least one clause -- including not existing, and a symlink where a +// file or directory was expected, since find does not follow links here. +fn runner_host_root_only_find_argv(paths: List, kind: String) -> List { + concat(["find"], concat(paths, ["-maxdepth", "0", "-type", kind, "-user", "root", "-group", "root", "-not", "-perm", "/022"])) +} + +fn classify_root_only_paths(requested: List, exit_code: Int, stdout: String, stderr: String) -> RunnerHostPathMetadata { + let printed = filter(map(split(s: stdout, delimiter: "\n"), l => trim(s: l)), l => l != "") + let unsafe = filter(requested, r => !any(printed, p => p == r)) + if length(unsafe) == 0 { + PathsRootOnly + } else { + PathsNotRootOnly { + paths: unsafe, + detail: join(["not printed by find as root:root with no group/other write (missing, wrong owner, writable, or wrong type); find exit=", to_string(exit_code), " ", trim(s: stderr)], ""), + } + } +} + +fn runner_host_path_metadata_wire(m: RunnerHostPathMetadata) -> String { + match m { + PathsRootOnly => "root-only" + PathsNotRootOnly { paths: ps, detail: d } => join(["NOT-ROOT-ONLY ", join(ps, ","), " ", d], "") + PathsUnobservable { reason: r } => concat("UNOBSERVABLE ", r) + PathsNotApplicable => "n/a" + } +} + +type RunnerHostFileObservation { + content: RunnerHostFileStanding + file_metadata: RunnerHostPathMetadata + ancestors: RunnerHostPathMetadata +} + +type RunnerHostFileAction + = HostFileLeaveInPlace + | HostFileWrite + | HostFileActionRefused { reason: String } + +// The ancestors are decided first because the converge does not own them: a writable directory above +// the file cannot be repaired by writing the file, and writing into it would publish root-evaluated +// bytes where a CI user can replace them. Unsafe FILE metadata, by contrast, IS repaired by a write -- +// the staged file is created by root and renamed over the old one, replacing its owner and mode. +fn runner_host_file_action(observation: RunnerHostFileObservation) -> RunnerHostFileAction { + match observation.ancestors { + PathsNotRootOnly { paths: ps, detail: d } => + HostFileActionRefused { reason: join(["ancestor directory not root-only: ", join(ps, ","), " ", d], "") } + PathsUnobservable { reason: r } => HostFileActionRefused { reason: concat("ancestor metadata unobservable: ", r) } + PathsNotApplicable => HostFileActionRefused { reason: "ancestor metadata was not read" } + PathsRootOnly => + match observation.content { + HostFileUnobservable { reason: r } => HostFileActionRefused { reason: r } + HostFileAbsent => HostFileWrite + HostFileDiffers => HostFileWrite + HostFileIdentical => + match observation.file_metadata { + PathsRootOnly => HostFileLeaveInPlace + PathsNotRootOnly { paths: _, detail: _ } => HostFileWrite + PathsUnobservable { reason: r } => HostFileActionRefused { reason: concat("file metadata unobservable: ", r) } + PathsNotApplicable => HostFileActionRefused { reason: "identical content but file metadata was not read" } + } + } + } +} + +fn runner_host_file_standing_wire(standing: RunnerHostFileStanding) -> String { + match standing { + HostFileIdentical => "identical" + HostFileDiffers => "differs" + HostFileAbsent => "absent" + HostFileUnobservable { reason: r } => concat("unobservable ", r) + } +} + +fn runner_host_file_action_wire(action: RunnerHostFileAction) -> String { + match action { + HostFileLeaveInPlace => "leave-in-place" + HostFileWrite => "write" + HostFileActionRefused { reason: r } => concat("REFUSED ", r) + } +} + +type RunnerHostFilePlanned { + file: RunnerHostFileDesired + observation: RunnerHostFileObservation + action: RunnerHostFileAction +} + +// ── Slot population and teardown (pure) ────────────────────────────────────────────────────────── + +// One slot unit as the manager reports it. Every value is carried as the manager spells it; the +// judgment below decides what each means. +type RunnerSlotUnitReading { + unit: NonEmptyStr + load_state: String + need_daemon_reload: String + kill_mode: String + exit_type: String +} + +type RunnerSlotUnitStanding + = SlotTeardownEffective + | SlotTeardownNotEffective { kill_mode: String, exit_type: String } + | SlotReloadPending + | SlotUnitNotLoaded { load_state: String } + | SlotUnitUnread + +// ORDER MATTERS AND IS THE POINT. A unit the manager has not loaded still gets a show block, carrying +// the manager's DEFAULTS -- and the default KillMode is control-group, so reading KillMode first would +// certify a slot that does not exist. A loaded unit whose configuration changed on disk reports the +// OLD values until a reload, so NeedDaemonReload is read before the pair it would otherwise vouch for. +fn classify_runner_slot_unit(reading: RunnerSlotUnitReading, intent: RunnerUnitLifecycleIntent) -> RunnerSlotUnitStanding { + if trim(s: reading.load_state) != "loaded" { + SlotUnitNotLoaded { load_state: trim(s: reading.load_state) } + } else if trim(s: reading.need_daemon_reload) != "no" { + SlotReloadPending + } else if trim(s: reading.kill_mode) == systemd_kill_mode_wire(mode: intent.kill_mode) + && trim(s: reading.exit_type) == systemd_exit_type_wire(exit_type: intent.exit_type) { + SlotTeardownEffective + } else { + SlotTeardownNotEffective { kill_mode: trim(s: reading.kill_mode), exit_type: trim(s: reading.exit_type) } + } +} + +fn runner_slot_unit_standing_wire(standing: RunnerSlotUnitStanding) -> String { + match standing { + SlotTeardownEffective => "effective" + SlotTeardownNotEffective { kill_mode: k, exit_type: e } => join(["NOT-EFFECTIVE KillMode=", k, " ExitType=", e], "") + SlotReloadPending => "RELOAD-PENDING" + SlotUnitNotLoaded { load_state: l } => concat("NOT-LOADED LoadState=", l) + SlotUnitUnread => "UNREAD" + } +} + +type RunnerSlotUnitResult { + unit: NonEmptyStr + standing: RunnerSlotUnitStanding +} + +type RunnerSlotPopulation + = SlotPopulationObserved { units: List } + | SlotPopulationUnobservable { reason: String } + +data runner_slot_show_properties: List = [IdProperty, LoadState, NeedDaemonReload, KillModeProperty, ExitTypeProperty] + +fn runner_unit_name_is_slot(name: String) -> Bool { + starts_with(s: name, prefix: runner_unit_prefix) && ends_with(s: name, suffix: runner_unit_suffix) +} + +// Plain list-units rows carry the unit name as the first field; every other field is presentation. +fn runner_slot_units_from_list_units(stdout: String) -> List { + filter( + map(filter(split(s: stdout, delimiter: "\n"), l => trim(s: l) != ""), l => fold(split(s: trim(s: l), delimiter: " "), init: "", f: (acc, w) => if acc == "" { w } else { acc })), + n => runner_unit_name_is_slot(name: n), + ) +} + +// THE POPULATION IS THE UNION, DESIRED FIRST. A desired slot the manager does not hold must be seen as +// NOT-LOADED rather than skipped, and a loaded slot outside the desired roster -- a leftover +// incarnation, a slot added by hand -- is still a unit a restart can orphan from. list-units names each +// unit the manager holds once, so the loaded half needs no de-duplication of its own. +fn runner_slot_population_units(desired: List, loaded: List) -> List { + concat(desired, map(filter(loaded, l => !any(desired, d => (d as String) == l)), l => l as NonEmptyStr)) +} + +fn show_block_value(lines: List, property: SystemdUnitProperty) -> String? { + let key = concat(systemd_unit_property_wire(property: property) as String, "=") + fold(lines, init: none, f: (acc, l) => + if starts_with(s: l, prefix: key) { + Present { value: substring(s: l, start: string_length(s: key), end: string_length(s: l)) } + } else { + acc + }) +} + +fn parse_runner_slot_show_block(block: String) -> RunnerSlotUnitReading? { + let lines = map(split(s: block, delimiter: "\n"), l => trim(s: l)) + match show_block_value(lines: lines, property: IdProperty) { + Absent => none + Present { value: id } => + match show_block_value(lines: lines, property: LoadState) { + Absent => none + Present { value: load } => + match show_block_value(lines: lines, property: NeedDaemonReload) { + Absent => none + Present { value: reload } => + match show_block_value(lines: lines, property: KillModeProperty) { + Absent => none + Present { value: kill } => + match show_block_value(lines: lines, property: ExitTypeProperty) { + Absent => none + Present { value: exit } => + if id == "" { none } else { + Present { value: RunnerSlotUnitReading { unit: id as NonEmptyStr, load_state: load, need_daemon_reload: reload, kill_mode: kill, exit_type: exit } } + } + } + } + } + } + } +} + +// Each requested unit is joined to the block that NAMES it; a unit with no block of its own is UNREAD, +// never borrowed from a neighbour's. +fn classify_runner_slot_population(units: List, show_stdout: String, intent: RunnerUnitLifecycleIntent) -> List { + let readings = flat_map(split(s: show_stdout, delimiter: "\n\n"), b => match parse_runner_slot_show_block(block: b) { + Present { value: r } => [r] + Absent => [] + }) + map(units, u => + RunnerSlotUnitResult { + unit: u, + standing: fold(readings, init: SlotUnitUnread, f: (acc, r) => + if (r.unit as String) == (u as String) { classify_runner_slot_unit(reading: r, intent: intent) } else { acc }), + }) +} + +fn runner_slot_population_owes_reload(population: RunnerSlotPopulation) -> Bool { + match population { + SlotPopulationUnobservable { reason: _ } => false + SlotPopulationObserved { units: us } => + any(us, r => match r.standing { + SlotReloadPending => true + SlotTeardownEffective => false + SlotTeardownNotEffective { kill_mode: _, exit_type: _ } => false + SlotUnitNotLoaded { load_state: _ } => false + SlotUnitUnread => false + }) + } +} + +fn runner_slot_population_effective(population: RunnerSlotPopulation) -> Bool { + match population { + SlotPopulationUnobservable { reason: _ } => false + SlotPopulationObserved { units: us } => + length(us) > 0 && all(us, r => match r.standing { + SlotTeardownEffective => true + SlotReloadPending => false + SlotTeardownNotEffective { kill_mode: _, exit_type: _ } => false + SlotUnitNotLoaded { load_state: _ } => false + SlotUnitUnread => false + }) + } +} + +fn runner_slot_population_read_completely(population: RunnerSlotPopulation) -> Bool { + match population { + SlotPopulationUnobservable { reason: _ } => false + SlotPopulationObserved { units: us } => + length(us) > 0 && all(us, r => match r.standing { + SlotUnitUnread => false + SlotTeardownEffective => true + SlotReloadPending => true + SlotTeardownNotEffective { kill_mode: _, exit_type: _ } => true + SlotUnitNotLoaded { load_state: _ } => true + }) + } +} + +// ── Remote legs ────────────────────────────────────────────────────────────────────────────────── + +fn elevated_argv(command: List) -> List { + let inv = sudo_elevate(command: command) + concat([inv.bin_path], inv.args) +} + +type RunnerHostPrivilegeStanding + = AdministratorPrivilegeAvailable + | AdministratorPrivilegeRefused { reason: String } + +func observe_runner_host_administrator_privilege( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, +) -> RunnerHostPrivilegeStanding + uses net: Network +{ + match typed_argv_exec_over_fleet_ssh(target: subject.target, context: context, argv: elevated_argv(command: ["true"])) { + TypedArgvExecRefused { reason: why } => AdministratorPrivilegeRefused { reason: why } + TypedArgvExecConverged { result: r } => + if r.success { + AdministratorPrivilegeAvailable + } else { + AdministratorPrivilegeRefused { + reason: join(["non-interactive sudo as ", subject.target.principal as String, " exit=", to_string(r.exit_code), " ", trim(s: r.stderr)], ""), + } + } + } +} + +func observe_runner_host_paths( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + paths: List, + kind: String, +) -> RunnerHostPathMetadata + uses net: Network +{ + match typed_argv_exec_over_fleet_ssh(target: subject.target, context: context, argv: runner_host_root_only_find_argv(paths: paths, kind: kind)) { + TypedArgvExecRefused { reason: why } => PathsUnobservable { reason: why } + TypedArgvExecConverged { result: r } => + classify_root_only_paths(requested: paths, exit_code: r.exit_code, stdout: r.stdout, stderr: r.stderr) + } +} + +fn runner_host_file_write_admission(file: RunnerHostFileDesired) -> TypedRemoteFileWriteAdmission { + typed_remote_file_write_seal(path: file.path, content: file.content, allowed_scope: runner_host_file_scope) +} + +fn runner_host_file_standing_of_report(report: RemoteFileComparisonReport) -> RunnerHostFileStanding { + match report { + ComparisonIdentical => HostFileIdentical + ComparisonDiffers { reason: _ } => HostFileDiffers + ComparisonUnavailable { reason: r } => HostFileUnobservable { reason: r } + } +} + +// THE PLAN'S "IDENTICAL" AND THE APPLY'S "CONVERGED" ARE ONE JUDGMENT OVER ONE LEG. The comparison is +// typed_remote_file_write's own far-side `cmp -s -` with the declared bytes on stdin, classified by +// its own exit vocabulary, so an observation cannot call a file converged that the write's read-back +// would call different. +func observe_runner_host_file_content( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + file: RunnerHostFileDesired, +) -> RunnerHostFileStanding + uses net: Network +{ + match typed_argv_exec_over_fleet_ssh(target: subject.target, context: context, argv: ["test", "-e", file.path as String]) { + TypedArgvExecRefused { reason: why } => HostFileUnobservable { reason: why } + TypedArgvExecConverged { result: presence } => + match classify_host_file_presence(exit_code: presence.exit_code, stderr: presence.stderr) { + Present { value: settled } => settled + Absent => + match runner_host_file_write_admission(file: file) { + TypedRemoteFileWriteAdmissionRefused { cause: c } => HostFileUnobservable { reason: c as String } + TypedRemoteFileWriteAdmitted { write: w } => + match typed_remote_file_step( + target: subject.target, + context: context, + raws: typed_remote_file_compare_argv(write: w), + stdin_payload: w.content, + ) { + TypedRemoteFileStepArgvRefused => HostFileUnobservable { reason: "comparison argv refused as non-portable under the far-side shell re-parse allowlist" } + TypedRemoteFileStepLegRefused { cause: why } => HostFileUnobservable { reason: why as String } + TypedRemoteFileStepRan { result: back } => + runner_host_file_standing_of_report( + report: classify_typed_remote_file_comparison(write: w, compare_exit_code: back.exit_code, compare_stderr: back.stderr), + ) + } + } + } + } +} + +func observe_runner_host_file( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + file: RunnerHostFileDesired, +) -> RunnerHostFileObservation + uses net: Network +{ + let content = observe_runner_host_file_content(subject: subject, context: context, file: file) + RunnerHostFileObservation { + content: content, + file_metadata: match content { + HostFileAbsent => PathsNotApplicable + HostFileUnobservable { reason: _ } => PathsNotApplicable + HostFileIdentical => observe_runner_host_paths(subject: subject, context: context, paths: [file.path as String], kind: "f") + HostFileDiffers => observe_runner_host_paths(subject: subject, context: context, paths: [file.path as String], kind: "f") + }, + ancestors: observe_runner_host_paths(subject: subject, context: context, paths: runner_host_path_ancestors(path: file.path as String), kind: "d"), + } +} + +func observe_runner_slot_population( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, +) -> RunnerSlotPopulation + uses net: Network +{ + match typed_argv_exec_over_fleet_ssh(target: subject.target, context: context, argv: systemctl_list_all_service_units_argv()) { + TypedArgvExecRefused { reason: why } => SlotPopulationUnobservable { reason: why } + TypedArgvExecConverged { result: listed } => + if !listed.success { + SlotPopulationUnobservable { reason: join(["systemctl list-units exit=", to_string(listed.exit_code), " ", trim(s: listed.stderr)], "") } + } else { + let units = runner_slot_population_units(desired: subject.desired_units, loaded: runner_slot_units_from_list_units(stdout: listed.stdout)) + match typed_argv_exec_over_fleet_ssh(target: subject.target, context: context, argv: systemctl_show_properties_argv(units: units, properties: runner_slot_show_properties)) { + TypedArgvExecRefused { reason: why } => SlotPopulationUnobservable { reason: why } + TypedArgvExecConverged { result: shown } => + if !shown.success { + SlotPopulationUnobservable { reason: join(["systemctl show exit=", to_string(shown.exit_code), " ", trim(s: shown.stderr)], "") } + } else { + SlotPopulationObserved { units: classify_runner_slot_population(units: units, show_stdout: shown.stdout, intent: gunbc_runner_lifecycle_intent) } + } + } + } + } +} + +// ── Apply ──────────────────────────────────────────────────────────────────────────────────────── + +type RunnerHostFileConvergeMode + = RunnerHostFilesObserveOnly + | RunnerHostFilesApply + +type RunnerHostFileOutcome + = HostFileLeftInPlace + | HostFileConvergedByWrite { converged: RemoteFileConverged } + | HostFileWriteRefused { step: String, reason: String } + | HostFileNotAttempted { reason: String } + | HostFileNotAppliedObserveOnly + +type RunnerHostFileResult { + planned: RunnerHostFilePlanned + outcome: RunnerHostFileOutcome +} + +// A write is converged only when its bytes read back AND the published file is root-only. The byte +// read-back is typed_remote_file_write's; the ownership read-back is here, because that carrier +// deliberately claims nothing about metadata it never reads. +func apply_runner_host_file( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + planned: RunnerHostFilePlanned, +) -> RunnerHostFileOutcome + uses net: Network +{ + match planned.action { + HostFileLeaveInPlace => HostFileLeftInPlace + HostFileActionRefused { reason: r } => HostFileNotAttempted { reason: r } + HostFileWrite => + match converge_typed_remote_file( + target: subject.target, + context: context, + admission: runner_host_file_write_admission(file: planned.file), + mode: planned.file.mode, + privilege: RemoteFileViaNonInteractiveSudo, + ) { + TypedRemoteFileConvergenceRefused { step: st, reason: why } => HostFileWriteRefused { step: st, reason: why } + TypedRemoteFileConverged { converged: c } => + match observe_runner_host_paths(subject: subject, context: context, paths: [planned.file.path as String], kind: "f") { + PathsRootOnly => HostFileConvergedByWrite { converged: c } + PathsNotRootOnly { paths: _, detail: d } => HostFileWriteRefused { step: "metadata-readback", reason: d } + PathsUnobservable { reason: r } => HostFileWriteRefused { step: "metadata-readback", reason: r } + PathsNotApplicable => HostFileWriteRefused { step: "metadata-readback", reason: "file metadata was not read after the write" } + } + } + } +} + +fn runner_host_file_outcome_settled(outcome: RunnerHostFileOutcome) -> Bool { + match outcome { + HostFileLeftInPlace => true + HostFileConvergedByWrite { converged: _ } => true + HostFileWriteRefused { step: _, reason: _ } => false + HostFileNotAttempted { reason: _ } => false + HostFileNotAppliedObserveOnly => false + } +} + +fn runner_host_file_outcome_wire(outcome: RunnerHostFileOutcome) -> String { + match outcome { + HostFileLeftInPlace => "left-in-place" + HostFileConvergedByWrite { converged: _ } => "written-and-read-back" + HostFileWriteRefused { step: st, reason: why } => join(["REFUSED step=", st, " ", why], "") + HostFileNotAttempted { reason: why } => concat("NOT-ATTEMPTED ", why) + HostFileNotAppliedObserveOnly => "observe-only" + } +} + +type RunnerHostManagerReload + = ManagerReloadNotOwed + | ManagerReloaded + | ManagerReloadRefused { reason: String } + | ManagerReloadOwedNotAttempted { reason: String } + +func reload_runner_host_manager( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, +) -> RunnerHostManagerReload + uses net: Network +{ + match typed_argv_exec_over_fleet_ssh(target: subject.target, context: context, argv: elevated_argv(command: systemctl_daemon_reload_argv())) { + TypedArgvExecRefused { reason: why } => ManagerReloadRefused { reason: why } + TypedArgvExecConverged { result: r } => + if r.success { + ManagerReloaded + } else { + ManagerReloadRefused { reason: join(["daemon-reload exit=", to_string(r.exit_code), " ", trim(s: r.stderr)], "") } + } + } +} + +fn runner_host_manager_reload_wire(reload: RunnerHostManagerReload) -> String { + match reload { + ManagerReloadNotOwed => "not-owed" + ManagerReloaded => "reloaded" + ManagerReloadRefused { reason: why } => concat("REFUSED ", why) + ManagerReloadOwedNotAttempted { reason: why } => concat("OWED-NOT-ATTEMPTED ", why) + } +} + +fn runner_host_privilege_wire(privilege: RunnerHostPrivilegeStanding) -> String { + match privilege { + AdministratorPrivilegeAvailable => "available" + AdministratorPrivilegeRefused { reason: why } => concat("REFUSED ", why) + } +} + +// ── The run ────────────────────────────────────────────────────────────────────────────────────── + +type RunnerHostFileRun { + subject: RunnerHostFileSubject + mode: RunnerHostFileConvergeMode + privilege: RunnerHostPrivilegeStanding + results: List + reload: RunnerHostManagerReload + slots: RunnerSlotPopulation +} + +func plan_runner_host_file( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + file: RunnerHostFileDesired, +) -> RunnerHostFilePlanned + uses net: Network +{ + let observation = observe_runner_host_file(subject: subject, context: context, file: file) + RunnerHostFilePlanned { file: file, observation: observation, action: runner_host_file_action(observation: observation) } +} + +// The reload decision reads the manager AFTER the writes, so it covers this run's writes and any +// earlier run's write whose reload never happened; the slots are then read again, and that second +// reading is the one the terminal is judged on. +func runner_host_manager_reload_step( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + mode: RunnerHostFileConvergeMode, + privilege: RunnerHostPrivilegeStanding, + before: RunnerSlotPopulation, +) -> RunnerHostManagerReload + uses net: Network +{ + if !runner_slot_population_owes_reload(population: before) { + ManagerReloadNotOwed + } else { + match mode { + RunnerHostFilesObserveOnly => ManagerReloadOwedNotAttempted { reason: "observe-only" } + RunnerHostFilesApply => + match privilege { + AdministratorPrivilegeRefused { reason: why } => ManagerReloadOwedNotAttempted { reason: concat("administrator privilege refused: ", why) } + AdministratorPrivilegeAvailable => reload_runner_host_manager(subject: subject, context: context) + } + } + } +} + +// ORDER: privilege, observe every file, write the ones that need it, read every slot, reload if the +// manager owes one, read every slot again. The privilege leg runs FIRST and gates every write, so a host +// where the declared administrator cannot sudo reports that by name. Observation does not need +// privilege and still runs, so an observe-only dispatch against such a host still says what WOULD be +// written and what the slots hold. +func run_runner_host_files( + subject: RunnerHostFileSubject, + context: FleetSshExecutionContext, + files: List, + mode: RunnerHostFileConvergeMode, +) -> RunnerHostFileRun + uses net: Network +{ + let privilege = observe_runner_host_administrator_privilege(subject: subject, context: context) + let planned = map(files, f => plan_runner_host_file(subject: subject, context: context, file: f)) + let results = map(planned, p => + RunnerHostFileResult { + planned: p, + outcome: match mode { + RunnerHostFilesObserveOnly => HostFileNotAppliedObserveOnly + RunnerHostFilesApply => + match privilege { + AdministratorPrivilegeRefused { reason: why } => HostFileNotAttempted { reason: concat("administrator privilege refused: ", why) } + AdministratorPrivilegeAvailable => apply_runner_host_file(subject: subject, context: context, planned: p) + } + }, + }) + let before = observe_runner_slot_population(subject: subject, context: context) + let reload = runner_host_manager_reload_step(subject: subject, context: context, mode: mode, privilege: privilege, before: before) + RunnerHostFileRun { + subject: subject, + mode: mode, + privilege: privilege, + results: results, + reload: reload, + slots: match reload { + ManagerReloaded => observe_runner_slot_population(subject: subject, context: context) + ManagerReloadNotOwed => before + ManagerReloadRefused { reason: _ } => before + ManagerReloadOwedNotAttempted { reason: _ } => before + }, + } +} + +fn runner_host_file_run_mode_wire(mode: RunnerHostFileConvergeMode) -> String { + match mode { + RunnerHostFilesObserveOnly => "observe" + RunnerHostFilesApply => "apply" + } +} + +fn runner_host_file_planned_write(p: RunnerHostFilePlanned) -> Bool { + match p.action { + HostFileWrite => true + HostFileLeaveInPlace => false + HostFileActionRefused { reason: _ } => false + } +} + +fn runner_host_file_planned_refused(p: RunnerHostFilePlanned) -> Bool { + match p.action { + HostFileActionRefused { reason: _ } => true + HostFileWrite => false + HostFileLeaveInPlace => false + } +} + +// WHAT EACH MODE MAY CALL SUCCESS. Observe succeeds when it could SEE everything -- privilege probed, +// every file classified, every slot unit read -- whatever it saw; drift is its finding, not its failure. +// Apply succeeds only when the host now holds every declared file, root-only, and EVERY slot unit +// reports the declared teardown with nothing pending a reload. +fn runner_host_file_run_succeeded(run: RunnerHostFileRun) -> Bool { + match run.mode { + RunnerHostFilesObserveOnly => + (match run.privilege { + AdministratorPrivilegeAvailable => true + AdministratorPrivilegeRefused { reason: _ } => false + }) + && !any(run.results, r => runner_host_file_planned_refused(p: r.planned)) + && runner_slot_population_read_completely(population: run.slots) + RunnerHostFilesApply => + all(run.results, r => runner_host_file_outcome_settled(outcome: r.outcome)) + && (match run.reload { + ManagerReloadNotOwed => true + ManagerReloaded => true + ManagerReloadRefused { reason: _ } => false + ManagerReloadOwedNotAttempted { reason: _ } => false + }) + && runner_slot_population_effective(population: run.slots) + } +} + +fn runner_host_file_result_line(r: RunnerHostFileResult) -> String { + join([ + "file ", r.planned.file.subject as String, + " path=", r.planned.file.path as String, + " content=", runner_host_file_standing_wire(standing: r.planned.observation.content), + " file-metadata=", runner_host_path_metadata_wire(m: r.planned.observation.file_metadata), + " ancestors=", runner_host_path_metadata_wire(m: r.planned.observation.ancestors), + " action=", runner_host_file_action_wire(action: r.planned.action), + " outcome=", runner_host_file_outcome_wire(outcome: r.outcome), + "\n", + ], "") +} + +fn runner_slot_population_text(population: RunnerSlotPopulation) -> String { + match population { + SlotPopulationUnobservable { reason: r } => concat("slots UNOBSERVABLE ", concat(r, "\n")) + SlotPopulationObserved { units: us } => + join([ + "slots observed=", to_string(length(us)), + " effective=", to_string(length(filter(us, r => match r.standing { SlotTeardownEffective => true SlotReloadPending => false SlotTeardownNotEffective { kill_mode: _, exit_type: _ } => false SlotUnitNotLoaded { load_state: _ } => false SlotUnitUnread => false }))), + "\n", + join(map(us, r => join(["slot ", r.unit as String, " ", runner_slot_unit_standing_wire(standing: r.standing), "\n"], "")), ""), + ], "") + } +} + +fn runner_host_file_run_text(run: RunnerHostFileRun) -> String { + join([ + "host=", run.subject.host as String, + " mode=", runner_host_file_run_mode_wire(mode: run.mode), + " principal=", run.subject.target.principal as String, + " endpoint=", run.subject.target.endpoint as String, "\n", + "administrator-privilege=", runner_host_privilege_wire(privilege: run.privilege), "\n", + join(map(run.results, r => runner_host_file_result_line(r: r)), ""), + "planned-writes=", to_string(length(filter(run.results, r => runner_host_file_planned_write(p: r.planned)))), "\n", + "manager-reload=", runner_host_manager_reload_wire(reload: run.reload), "\n", + runner_slot_population_text(population: run.slots), + "terminal=", if runner_host_file_run_succeeded(run: run) { "succeeded" } else { "REFUSED" }, "\n", + ], "") +} + +// ── The CI entries ─────────────────────────────────────────────────────────────────────────────── + +data runner_host_file_converge_attempt_raw: String = "runner-host-file-converge-ci" + +data runner_host_file_converge_receipt_path: String = "target/runner-host-file-converge-receipt.txt" + +// ONE HOST PER RUN, NAMED BY THE DISPATCH. The fleet-converge workflow already carries the operator's +// chosen host in FLEET_CONVERGE_EXPECTED_HOST; an unset host refuses rather than fanning a root-file +// write across the fleet from an empty variable. +func runner_host_file_ci_wet(mode: RunnerHostFileConvergeMode) -> ProcessExit + uses net: Network +{ + match actions_variable_read(name: fleet_converge_expected_host_env_name) { + ActionsVariableAbsent { variable: v, detail: d } => + exit_failure(reason: join(["runner_host_file_converge: ", v, " ", d, "; one host per run, named by the dispatch"], "")) + ActionsVariablePresent { value: raw_host } => + match runner_host_file_subject(host: raw_host as HostIdentity) { + RunnerHostFileSubjectRefused { cause: c } => exit_failure(reason: c) + RunnerHostFileSubjectAdmitted { subject: subject } => + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: c } => exit_failure(reason: c as String) + RunnerHostFilesDesired { files: files } => + match prepare_fleet_ssh_agent_context(attempt_raw: runner_host_file_converge_attempt_raw) { + FleetSshContextRefused { cause: c } => exit_failure(reason: c) + FleetSshContextReady { context: context, receipt: context_receipt } => { + let run = run_runner_host_files(subject: subject, context: context, files: files, mode: mode) + let body = join([ + "runner_host_file_converge probed_at=", probed_at_word(reading: clock_now_probed_at()) as String, "\n", + context_receipt, + runner_host_file_run_text(run: run), + ], "") + let w = Filesystem.Write(path: runner_host_file_converge_receipt_path, content: body) + if w.success == false { + exit_failure(reason: join(["runner_host_file_converge: receipt write failed: ", w.error, "\n", body], "")) + } else if runner_host_file_run_succeeded(run: run) == false { + exit_failure(reason: body) + } else { + ExitSuccess + } + } + } + } + } + } +} + +func runner_host_file_observe_ci_wet() -> ProcessExit + uses net: Network +{ + runner_host_file_ci_wet(mode: RunnerHostFilesObserveOnly) +} + +func runner_host_file_converge_ci_wet() -> ProcessExit + uses net: Network +{ + runner_host_file_ci_wet(mode: RunnerHostFilesApply) +} diff --git a/dag/gunbc/runner/runner_host_grants.dag b/dag/gunbc/runner/runner_host_grants.dag index 6106dd40fe4..80a2228c1e5 100644 --- a/dag/gunbc/runner/runner_host_grants.dag +++ b/dag/gunbc/runner/runner_host_grants.dag @@ -382,7 +382,14 @@ type ExecutorBootstrapPrincipal { reachable_with_secret: NonEmptyStr } -data srv3_executor_bootstrap_principal: ExecutorBootstrapPrincipal = ExecutorBootstrapPrincipal { +// ONE ROW FOR THE FLEET, NOT ONE PER HOST, AND WHAT IS MEASURED IS STATED SEPARATELY FROM WHAT IS +// DECLARED. It was named for srv3 because srv3 is where the probe above ran. Two further facts arrived +// later and neither is a measurement of reach: srv1 carries an `ubuntu` account with a NOPASSWD sudoers +// file (read on srv1, 2026-09-11), and srv1-srv4 were imaged the same way. So the row is the DECLARED +// administrator for every runner host, measured only on srv3; gunbc.runner_host_file_converge probes +// `sudo -n` as this principal on every run and refuses by name on a host where the declaration is +// false, before any write. Per-host rows would be the right shape only once a host is known to differ. +data executor_bootstrap_principal: ExecutorBootstrapPrincipal = ExecutorBootstrapPrincipal { login: "ubuntu" as NonEmptyStr, reachable_with_secret: "fleet-automation-ssh-key" as NonEmptyStr, } diff --git a/dag/gunbc/runner/runner_unit.dag b/dag/gunbc/runner/runner_unit.dag index fa7c80ca04c..607a16a301c 100644 --- a/dag/gunbc/runner/runner_unit.dag +++ b/dag/gunbc/runner/runner_unit.dag @@ -222,6 +222,17 @@ fn lifecycle_intent_cannot_orphan(intent: RunnerUnitLifecycleIntent, listener_is // was asserted because one decisive-feeling measurement made it FEEL closed, while a measurement // already taken contradicted it. A sufficient mechanism is not the operating mechanism. // +// THE OPERATING MECHANISM, FOUND 2026-09-11, AND WHY NRestarts COULD NOT SEE IT. NRestarts counts +// only the restarts systemd schedules itself under Restart=; an explicit `systemctl restart` issued +// by another process is not counted. needrestart issues exactly that after an unattended upgrade +// replaces a library the runners map: srv1's unattended-upgrades dpkg log records `systemctl restart +// actions-runner@srv1-01.service ...` on ten separate days since June, and on 2026-09-11 the journal +// shows every slot stopped at 06:36:58 with its in-flight Runner.Worker left running and a new +// listener taking a new job into the same directory thirteen seconds later. So srv4-03's detached +// generations at NRestarts=0 are what an externally issued restart looks like, not a contradiction +// of one. gunbc.runner_host_file_converge carries both halves of the repair -- this intent as a +// drop-in, and a needrestart deferral so the upgrade stops issuing the restart. +// // THE DECLARED INTENT DOES NOT DEPEND ON THAT CHAIN, WHICH IS WHY THE RETRACTION DOES NOT MOVE IT. // ExitTypeCgroup plus KillModeControlGroup is warranted by what a teardown must reach, and by the // executed fact that today's teardown reaches only the main process. Whether the ordinary path diff --git a/dag/gunbc/runner/runner_unit_file.dag b/dag/gunbc/runner/runner_unit_file.dag index 1d30bec4db6..7dcabef9535 100644 --- a/dag/gunbc/runner/runner_unit_file.dag +++ b/dag/gunbc/runner/runner_unit_file.dag @@ -35,11 +35,15 @@ import extdeps.systemd.unit_file { WantedBy, systemd_instance_name_specifier, serialize_systemd_unit_file, + SystemdDropIn, + DropInServiceSection, + serialize_systemd_drop_in, } import gunbc.runner_unit { RunnerUnitLifecycleIntent, gunbc_runner_lifecycle_intent, runner_unit_template_name, + runner_unit_dropin_path, } import gunbc.runner_host_deploy { RunnerHostDeploy, GithubAppRunnerConfig, github_app_runner_org } import gunbc.runner_lifecycle { runner_lifecycle_jit_wrapper_path } @@ -347,3 +351,36 @@ fn runner_unit_text(rendering: RunnerUnitFileRendering) -> RunnerUnitTextRenderi } data runner_unit_system_path: String = join(["/etc/systemd/system/", runner_unit_template_name as String], "") + +// THE TEARDOWN REACHES THE FLEET AS A DROP-IN, AND THE WHOLE UNIT STILL DOES NOT, for a reason that is +// about the other drop-ins rather than about this one. The live unit on every runner host is ctrl's +// fragment plus a hand-placed `depriv.conf` that carries `User=root`, `CTRL_RUNNER_JOB_USER`, the +// shared sccache environment and `DOCKER_HOST` -- facts no module in this repository declares. Installing +// runner_unit_text as the fragment would keep those facts in force only because a drop-in the model has +// never read happens to override the render, so the modeled unit would be correct by accident on some +// hosts and wrong on any host missing that file. Modeling depriv's facts is the cut that lets the +// fragment be delivered; the teardown does not wait on it, because systemd merges drop-ins over the +// fragment and no present drop-in sets either directive (read on srv1, 2026-09-11). +// +// ONE AUTHORITY, TWO ARTIFACTS. The directive pair comes from runner_unit_teardown_directives, the same +// function the fragment render uses, so the drop-in and the eventual fragment cannot disagree about the +// teardown. When the fragment is delivered the drop-in becomes a restatement of it and deletes with that +// change. +// +// The `70-` prefix sorts after every numbered drop-in the fleet carries (10- through 60-) so a later +// numbered file cannot silently re-set the pair; `depriv.conf` sorts later still and sets neither. +data runner_unit_teardown_dropin_file_name: NonEmptyStr = "70-fleet-teardown.conf" as NonEmptyStr + +fn runner_unit_teardown_dropin(intent: RunnerUnitLifecycleIntent) -> SystemdDropIn { + SystemdDropIn { + sections: [DropInServiceSection { directives: runner_unit_teardown_directives(intent: intent) }], + } +} + +fn runner_unit_teardown_dropin_path() -> NonEmptyStr { + runner_unit_dropin_path(file_name: runner_unit_teardown_dropin_file_name) +} + +fn gunbc_runner_unit_teardown_dropin_text() -> String { + serialize_systemd_drop_in(drop_in: runner_unit_teardown_dropin(intent: gunbc_runner_lifecycle_intent)) +} diff --git a/dag/gunbc/spark/remote_step_sequence.dag b/dag/gunbc/spark/remote_step_sequence.dag index 8ca523bdb20..0f396e02234 100644 --- a/dag/gunbc/spark/remote_step_sequence.dag +++ b/dag/gunbc/spark/remote_step_sequence.dag @@ -29,6 +29,7 @@ import gunbc.remote_operation { import gunbc.typed_remote_file_write { TypedRemoteFileWriteAdmitted, TypedRemoteFileConvergenceRefused, + RemoteFileAsSessionPrincipal, converge_typed_remote_file, } @@ -56,6 +57,7 @@ func run_remote_operation( context: ctx, admission: TypedRemoteFileWriteAdmitted { write: w }, mode: m, + privilege: RemoteFileAsSessionPrincipal, ), } LocalShell => diff --git a/dag/gunbc/systemd_property_directive_overlap.dag b/dag/gunbc/systemd_property_directive_overlap.dag index 160a84b31eb..71cbb477663 100644 --- a/dag/gunbc/systemd_property_directive_overlap.dag +++ b/dag/gunbc/systemd_property_directive_overlap.dag @@ -99,6 +99,11 @@ type SystemdPropertyWritability // WorkingDirectory and Unit are accepted unit-file directives with the same spelling carried by // the show surface. The other additions are manager-derived state, including AccuracyUSec: its // writable timer directive is AccuracySec=, not the reported microsecond property. +// Slice is the unit-file directive systemd.resource-control(5) specifies for placing a unit in a slice. +// It was absent from this classifier on main, which left the match non-exhaustive and the module +// unresolvable; it is added here because this change is the first to touch the classifier since. +// KillMode and ExitType are the [Service] directives of the same spelling (systemd.kill(5), +// systemd.service(5)); Id is the manager's name for the loaded unit and no unit file assigns it. fn systemd_property_writability(property: SystemdUnitProperty) -> SystemdPropertyWritability { match property { MemoryMax => SettableDirective @@ -111,6 +116,7 @@ fn systemd_property_writability(property: SystemdUnitProperty) -> SystemdPropert MemoryCurrent => ObservationOnly TasksCurrent => ObservationOnly ControlGroup => ObservationOnly + SliceProperty => SettableDirective ActiveEnterTimestampMonotonic => ObservationOnly ActiveState => ObservationOnly MainPID => ObservationOnly @@ -128,6 +134,9 @@ fn systemd_property_writability(property: SystemdUnitProperty) -> SystemdPropert AccuracyUSec => ObservationOnly TimersMonotonic => ObservationOnly NeedDaemonReload => ObservationOnly + IdProperty => ObservationOnly + KillModeProperty => SettableDirective + ExitTypeProperty => SettableDirective } } @@ -160,9 +169,10 @@ fn systemd_property_is_settable(property: SystemdUnitProperty) -> Bool { // corpus rather than computed from this list. data systemd_unit_property_members: List = [ MemoryMax, MemorySwapMax, MemoryHigh, CPUWeight, CPUQuota, CPUQuotaPerSecUSec, TasksMax, - MemoryCurrent, TasksCurrent, ControlGroup, ActiveEnterTimestampMonotonic, ActiveState, MainPID, + MemoryCurrent, TasksCurrent, ControlGroup, SliceProperty, ActiveEnterTimestampMonotonic, ActiveState, MainPID, LoadState, UnitFileState, SubState, Result, ExecMainStatus, ExecStartProperty, User, WorkingDirectoryProperty, RuntimeDirectoryProperty, Unit, NextElapseUSecMonotonic, AccuracyUSec, TimersMonotonic, NeedDaemonReload, + IdProperty, KillModeProperty, ExitTypeProperty, ] fn settable_property_wires() -> List { diff --git a/dag/gunbc/typed_remote_file_write.dag b/dag/gunbc/typed_remote_file_write.dag index 9ae1815c8b5..2f8c1106965 100644 --- a/dag/gunbc/typed_remote_file_write.dag +++ b/dag/gunbc/typed_remote_file_write.dag @@ -1,6 +1,7 @@ module gunbc.typed_remote_file_write import extdeps.tools.chmod { chmod_path_resolved_program } +import extdeps.sudo.elevation { ElevatedInvocation, sudo_elevate } import std.types { String, Bool, List, NonEmptyStr, FilePath, Int } import std.algebra { trim } import extdeps.ssh.session { SshSessionExecResult } @@ -102,6 +103,35 @@ fn remote_file_mode_octal(mode: RemoteFileMode) -> String { } } +// WHO RUNS THE FOUR LEGS. A file under a directory the session principal owns needs nothing more; a +// file under /etc needs root, and the fleet's administrator principals reach root through +// non-interactive sudo rather than by logging in as root. So privilege is a property of THIS write +// against THIS target, not of the transport, and it is spelled once here rather than by each consumer +// prefixing words onto argv it did not build. +// +// THE SECOND ARM IS extdeps.sudo.elevation sudo_elevate, which mints only the non-interactive form. +// sudo without -n prompts for a password on a channel with no +// terminal, which hangs until the step times out; with it, a missing grant is an immediate nonzero exit +// carrying sudo's own diagnostic, located at the leg that needed the privilege. +// +// The prefix goes on EVERY leg, the read-back included, so the principal that compared the bytes is the +// principal that wrote them: an unprivileged cmp against a file the writer made unreadable would report +// a comparison failure that was really a permission boundary. +type RemoteFilePrivilege + = RemoteFileAsSessionPrincipal + | RemoteFileViaNonInteractiveSudo + +fn remote_file_privileged_argv(privilege: RemoteFilePrivilege, argv: List) -> List { + match privilege { + RemoteFileAsSessionPrincipal => argv + RemoteFileViaNonInteractiveSudo => elevated_invocation_argv(invocation: sudo_elevate(command: argv)) + } +} + +fn elevated_invocation_argv(invocation: ElevatedInvocation) -> List { + concat([invocation.bin_path], invocation.args) +} + // The converged verdict is SEALED and the refusal is not, and the asymmetry is the whole point -- // the same one gunbc.readback_independence argues for on probes: only the POSITIVE is poisoned. A // caller minting a refusal has declined to claim anything, which is always safe and is sometimes the @@ -501,6 +531,7 @@ func converge_typed_remote_file( context: FleetSshExecutionContext, admission: TypedRemoteFileWriteAdmission, mode: RemoteFileMode, + privilege: RemoteFilePrivilege, ) -> TypedRemoteFileConvergence { match admission { TypedRemoteFileWriteAdmissionRefused { cause: cause } => @@ -508,28 +539,28 @@ func converge_typed_remote_file( TypedRemoteFileWriteAdmitted { write: write } => match typed_remote_file_step_outcome( target: target, context: context, step: "stage", - raws: typed_remote_file_stage_argv(write: write), + raws: remote_file_privileged_argv(privilege: privilege, argv: typed_remote_file_stage_argv(write: write)), stdin_payload: typed_remote_file_write_stdin_payload(write: write), ) { Present { value: refusal } => refusal Absent => match typed_remote_file_step_outcome( target: target, context: context, step: "mode", - raws: typed_remote_file_chmod_argv(write: write, mode: mode), + raws: remote_file_privileged_argv(privilege: privilege, argv: typed_remote_file_chmod_argv(write: write, mode: mode)), stdin_payload: "", ) { Present { value: refusal } => refusal Absent => match typed_remote_file_step_outcome( target: target, context: context, step: "rename", - raws: typed_remote_file_rename_argv(write: write), + raws: remote_file_privileged_argv(privilege: privilege, argv: typed_remote_file_rename_argv(write: write)), stdin_payload: "", ) { Present { value: refusal } => refusal Absent => match typed_remote_file_step( target: target, context: context, - raws: typed_remote_file_compare_argv(write: write), + raws: remote_file_privileged_argv(privilege: privilege, argv: typed_remote_file_compare_argv(write: write)), stdin_payload: write.content, ) { TypedRemoteFileStepArgvRefused => diff --git a/dag/test/claim/executor_privileged_operation_witness_test.dag b/dag/test/claim/executor_privileged_operation_witness_test.dag index 3fb122d8e81..dbbf6f03c08 100644 --- a/dag/test/claim/executor_privileged_operation_witness_test.dag +++ b/dag/test/claim/executor_privileged_operation_witness_test.dag @@ -288,11 +288,11 @@ test fn witness_grant_population_covers_the_provisioned_slot() -> Bool { // THE PRIVILEGE LAW, WALLED. A grantee that is also its own bootstrap principal is an account that // authorizes itself, which is the one shape the whole staged-install transaction exists to prevent. -// RED IS AUTHORABLE: point srv3_executor_bootstrap_principal.login at the deploy row's job_user and +// RED IS AUTHORABLE: point executor_bootstrap_principal.login at the deploy row's job_user and // this goes false. test fn witness_bootstrap_principal_is_not_the_grantee() -> Bool { bootstrap_principal_is_not_the_grantee( - bootstrap: srv3_executor_bootstrap_principal, + bootstrap: executor_bootstrap_principal, deploy: srv3_runner_host_deploy, ) } diff --git a/dag/test/claim/runner/runner_host_file_converge_witness_test.dag b/dag/test/claim/runner/runner_host_file_converge_witness_test.dag new file mode 100644 index 00000000000..6ae058172d2 --- /dev/null +++ b/dag/test/claim/runner/runner_host_file_converge_witness_test.dag @@ -0,0 +1,540 @@ +module test.claim.runner_host_file_converge_witness_test + +import std.types { Bool, String, NonEmptyStr } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import extdeps.systemd.unit_file { ExitTypeMain, KillModeProcess, serialize_systemd_drop_in } +import extdeps.needrestart { + NeedrestartUnitNamePrefixAdmitted, + NeedrestartUnitNamePrefixRefused, + needrestart_unit_name_prefix, + needrestart_unit_name_prefix_regex, +} +import product.placement_supply { HostIdentity } +import gunbc.runner_unit { + RunnerUnitLifecycleIntent, + gunbc_runner_lifecycle_intent, + gunbc_runner_listener_is_main_process, + lifecycle_intent_cannot_orphan, + runner_unit_prefix, + runner_unit_name_of_registration, +} +import gunbc.runner_unit_file { + runner_unit_teardown_dropin, + runner_unit_teardown_dropin_path, + gunbc_runner_unit_teardown_dropin_text, +} +import gunbc.typed_remote_file_write { + TypedRemoteFileWriteAdmitted, + TypedRemoteFileWriteAdmissionRefused, +} +import test.claim.typed_remote_file_write_fleet_ssh_fixture { fleet_ssh_execution_context_hermetic_witness_fixture } +import gunbc.runner_host_file_converge { + RunnerHostFileDesired, + RunnerHostFilesDesired, + RunnerHostFilesDesiredRefused, + runner_host_file_desired_population, + RunnerHostFileSubject, + RunnerHostFileSubjectAdmitted, + RunnerHostFileSubjectRefused, + runner_host_file_subject, + HostFileIdentical, + HostFileDiffers, + HostFileAbsent, + HostFileUnobservable, + classify_host_file_presence, + RunnerHostFileStanding, + RunnerHostPathMetadata, + PathsRootOnly, + PathsNotRootOnly, + PathsUnobservable, + PathsNotApplicable, + runner_host_path_ancestors, + runner_host_root_only_find_argv, + classify_root_only_paths, + RunnerHostFileObservation, + HostFileLeaveInPlace, + HostFileWrite, + HostFileActionRefused, + RunnerHostFileAction, + runner_host_file_action, + RunnerSlotUnitReading, + RunnerSlotUnitStanding, + SlotTeardownEffective, + SlotTeardownNotEffective, + SlotReloadPending, + SlotUnitNotLoaded, + SlotUnitUnread, + classify_runner_slot_unit, + RunnerSlotUnitResult, + RunnerSlotPopulation, + SlotPopulationObserved, + SlotPopulationUnobservable, + runner_slot_units_from_list_units, + runner_slot_population_units, + classify_runner_slot_population, + runner_slot_population_owes_reload, + runner_slot_population_effective, + runner_slot_population_read_completely, + RunnerHostFilePlanned, + RunnerHostFileResult, + HostFileLeftInPlace, + HostFileConvergedByWrite, + HostFileWriteRefused, + HostFileNotAttempted, + HostFileNotAppliedObserveOnly, + RunnerHostFileRun, + RunnerHostFileConvergeMode, + RunnerHostFilesObserveOnly, + RunnerHostFilesApply, + AdministratorPrivilegeAvailable, + AdministratorPrivilegeRefused, + RunnerHostManagerReload, + ManagerReloadNotOwed, + ManagerReloaded, + ManagerReloadRefused, + ManagerReloadOwedNotAttempted, + runner_host_file_run_succeeded, + runner_host_file_write_admission, + run_runner_host_files, +} + +// Every claim here is a function of declared rows except the route test, whose every remote leg is +// answered by the hermetic ssh mock; nothing reads the live tree. +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// ── The bytes, against independently authored oracles ───────────────────────────────────────────── + +// THE ORACLE IS WRITTEN HERE, NOT DERIVED. systemd.unit(5) drop-in grammar: a section header, one +// directive per line, a terminating newline. If the renderer and this string ever agree because both +// moved, the witness has become a restatement, so the literal stays literal. +data expected_teardown_dropin_text: String = "[Service]\nExitType=cgroup\nKillMode=control-group\n" + +test fn the_teardown_dropin_carries_the_declared_pair_and_nothing_else() -> Bool { + gunbc_runner_unit_teardown_dropin_text() == expected_teardown_dropin_text +} + +test fn the_teardown_dropin_lands_in_the_runner_template_dropin_directory() -> Bool { + (runner_unit_teardown_dropin_path() as String) == "/etc/systemd/system/actions-runner@.service.d/70-fleet-teardown.conf" +} + +// THE DISCRIMINATING RED, AS THE LIVE FLEET CARRIES IT. `KillMode=process` with systemd's default +// `ExitType=main` is what srv1 reported on 2026-09-11. Rendering that pair through the same function +// must produce different bytes, and the orphan predicate must refuse it, or the drop-in is a +// decoration that would read green whatever the fleet did. +test fn the_live_teardown_pair_renders_differently_and_can_orphan() -> Bool { + let live = RunnerUnitLifecycleIntent { exit_type: ExitTypeMain, kill_mode: KillModeProcess } + serialize_systemd_drop_in(drop_in: runner_unit_teardown_dropin(intent: live)) != expected_teardown_dropin_text + && !lifecycle_intent_cannot_orphan(intent: live, listener_is_main_process: gunbc_runner_listener_is_main_process) + && lifecycle_intent_cannot_orphan(intent: gunbc_runner_lifecycle_intent, listener_is_main_process: gunbc_runner_listener_is_main_process) +} + +// The needrestart snippet, against upstream's syntax: an assignment into the override_rc hashref, +// keyed by a start-anchored quotemeta'd prefix, valued false so the restart is deferred. Perl `die`s +// on a malformed conf.d file, so these bytes are not cosmetic. +data expected_needrestart_text: String = "$nrconf\{override_rc\}->\{qr(^actions\\-runner\\@)\} = 0;\n" + +fn desired_content(subject: String) -> String { + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: _ } => "" + RunnerHostFilesDesired { files: fs } => + fold(fs, init: "", f: (acc, f) => if (f.subject as String) == subject { f.content } else { acc }) + } +} + +fn desired_path(subject: String) -> String { + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: _ } => "" + RunnerHostFilesDesired { files: fs } => + fold(fs, init: "", f: (acc, f) => if (f.subject as String) == subject { f.path as String } else { acc }) + } +} + +test fn the_needrestart_snippet_defers_runner_units_by_upstream_syntax() -> Bool { + desired_content(subject: "needrestart-runner-deferral") == expected_needrestart_text + && desired_path(subject: "needrestart-runner-deferral") == "/etc/needrestart/conf.d/50-gunbc-actions-runner.conf" +} + +test fn the_desired_population_is_the_two_files_and_the_dropin_is_the_rendered_one() -> Bool { + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: _ } => false + RunnerHostFilesDesired { files: fs } => + length(fs) == 2 + && desired_content(subject: "runner-teardown-dropin") == expected_teardown_dropin_text + } +} + +// The deferral key is derived from the SAME prefix every slot unit name is derived from, so a rename +// of the unit family moves both or neither. +test fn every_slot_unit_name_begins_with_the_deferred_prefix() -> Bool { + starts_with(s: runner_unit_name_of_registration(registration_name: "srv1-01" as NonEmptyStr) as String, prefix: runner_unit_prefix) + && starts_with(s: runner_unit_name_of_registration(registration_name: "srv4-21" as NonEmptyStr) as String, prefix: runner_unit_prefix) +} + +test fn quotemeta_escapes_every_non_word_character_of_the_prefix() -> Bool { + match needrestart_unit_name_prefix(raw: "a.b-c:d_e@f") { + NeedrestartUnitNamePrefixRefused { cause: _ } => false + NeedrestartUnitNamePrefixAdmitted { prefix: p } => + needrestart_unit_name_prefix_regex(prefix: p) == "qr(^a\\.b\\-c\\:d_e\\@f)" + } +} + +// An empty prefix would match every service, and a character outside systemd's unit-name alphabet +// could only come from a caller who meant something other than a unit name. Both refuse. +test fn a_prefix_that_is_empty_or_outside_the_unit_alphabet_refuses() -> Bool { + let empty = match needrestart_unit_name_prefix(raw: "") { + NeedrestartUnitNamePrefixRefused { cause: _ } => true + NeedrestartUnitNamePrefixAdmitted { prefix: _ } => false + } + let spaced = match needrestart_unit_name_prefix(raw: "actions runner") { + NeedrestartUnitNamePrefixRefused { cause: _ } => true + NeedrestartUnitNamePrefixAdmitted { prefix: _ } => false + } + let paren = match needrestart_unit_name_prefix(raw: "a)b") { + NeedrestartUnitNamePrefixRefused { cause: _ } => true + NeedrestartUnitNamePrefixAdmitted { prefix: _ } => false + } + empty && spaced && paren +} + +// Every desired path is inside the sealed /etc scope, so the write seal admits it; a path outside +// would refuse at the seal rather than reaching a sudo leg. +test fn every_desired_file_is_admitted_by_the_etc_write_seal() -> Bool { + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: _ } => false + RunnerHostFilesDesired { files: fs } => + fold(fs, init: true, f: (ok, f) => ok && match runner_host_file_write_admission(file: f) { + TypedRemoteFileWriteAdmitted { write: _ } => true + TypedRemoteFileWriteAdmissionRefused { cause: _ } => false + }) + } +} + +// ── The subject ────────────────────────────────────────────────────────────────────────────────── + +// The subject carries EVERY desired slot unit, not a representative, and the declared administrator. +test fn a_runner_host_admits_as_the_declared_administrator_with_every_desired_slot_unit() -> Bool { + match runner_host_file_subject(host: "srv1" as HostIdentity) { + RunnerHostFileSubjectRefused { cause: _ } => false + RunnerHostFileSubjectAdmitted { subject: s } => + (s.target.principal as String) == "ubuntu" + && (s.target.endpoint as String) == "srv1" + && length(s.desired_units) > 1 + && any(s.desired_units, u => (u as String) == "actions-runner@srv1-01.service") + && any(s.desired_units, u => (u as String) == "actions-runner@srv1-02.service") + } +} + +test fn a_host_with_no_runner_deploy_row_refuses() -> Bool { + match runner_host_file_subject(host: "srv9" as HostIdentity) { + RunnerHostFileSubjectRefused { cause: _ } => true + RunnerHostFileSubjectAdmitted { subject: _ } => false + } +} + +// ── File content and metadata judgments ────────────────────────────────────────────────────────── + +fn standing_is_absent(standing: RunnerHostFileStanding) -> Bool { + match standing { + HostFileAbsent => true + HostFileIdentical => false + HostFileDiffers => false + HostFileUnobservable { reason: _ } => false + } +} + +fn standing_is_unobservable(standing: RunnerHostFileStanding) -> Bool { + match standing { + HostFileUnobservable { reason: _ } => true + HostFileAbsent => false + HostFileIdentical => false + HostFileDiffers => false + } +} + +test fn presence_zero_continues_one_is_absent_other_is_unobservable() -> Bool { + let continues = match classify_host_file_presence(exit_code: 0, stderr: "") { + Absent => true + Present { value: _ } => false + } + let absent = match classify_host_file_presence(exit_code: 1, stderr: "") { + Present { value: st } => standing_is_absent(standing: st) + Absent => false + } + let unobservable = match classify_host_file_presence(exit_code: 2, stderr: "test: extra argument") { + Present { value: st } => standing_is_unobservable(standing: st) + Absent => false + } + continues && absent && unobservable +} + +// Every directory from the write scope down to the file's own, in order, and nothing above /etc. +test fn the_ancestors_of_a_file_are_every_directory_from_the_scope_down() -> Bool { + runner_host_path_ancestors(path: "/etc/needrestart/conf.d/50-gunbc-actions-runner.conf") + == ["/etc", "/etc/needrestart", "/etc/needrestart/conf.d"] + && runner_host_path_ancestors(path: "/etc/systemd/system/actions-runner@.service.d/70-fleet-teardown.conf") + == ["/etc", "/etc/systemd", "/etc/systemd/system", "/etc/systemd/system/actions-runner@.service.d"] +} + +test fn the_root_only_predicate_is_evaluated_far_side_by_find() -> Bool { + runner_host_root_only_find_argv(paths: ["/etc", "/etc/needrestart"], kind: "d") + == ["find", "/etc", "/etc/needrestart", "-maxdepth", "0", "-type", "d", "-user", "root", "-group", "root", "-not", "-perm", "/022"] +} + +// find prints the SAFE set; anything requested and not printed failed a clause. +test fn a_path_find_did_not_print_is_not_root_only() -> Bool { + let all_safe = match classify_root_only_paths(requested: ["/etc", "/etc/needrestart"], exit_code: 0, stdout: "/etc\n/etc/needrestart\n", stderr: "") { + PathsRootOnly => true + PathsNotRootOnly { paths: _, detail: _ } => false + PathsUnobservable { reason: _ } => false + PathsNotApplicable => false + } + let one_unsafe = match classify_root_only_paths(requested: ["/etc", "/etc/needrestart"], exit_code: 0, stdout: "/etc\n", stderr: "") { + PathsNotRootOnly { paths: ps, detail: _ } => ps == ["/etc/needrestart"] + PathsRootOnly => false + PathsUnobservable { reason: _ } => false + PathsNotApplicable => false + } + let missing = match classify_root_only_paths(requested: ["/etc/needrestart/conf.d"], exit_code: 1, stdout: "", stderr: "find: '/etc/needrestart/conf.d': No such file or directory") { + PathsNotRootOnly { paths: ps, detail: _ } => ps == ["/etc/needrestart/conf.d"] + PathsRootOnly => false + PathsUnobservable { reason: _ } => false + PathsNotApplicable => false + } + all_safe && one_unsafe && missing +} + +fn action_of(content: RunnerHostFileStanding, file_metadata: RunnerHostPathMetadata, ancestors: RunnerHostPathMetadata) -> RunnerHostFileAction { + runner_host_file_action(observation: RunnerHostFileObservation { content: content, file_metadata: file_metadata, ancestors: ancestors }) +} + +fn is_leave(a: RunnerHostFileAction) -> Bool { match a { HostFileLeaveInPlace => true HostFileWrite => false HostFileActionRefused { reason: _ } => false } } +fn is_write(a: RunnerHostFileAction) -> Bool { match a { HostFileWrite => true HostFileLeaveInPlace => false HostFileActionRefused { reason: _ } => false } } +fn is_refused(a: RunnerHostFileAction) -> Bool { match a { HostFileActionRefused { reason: _ } => true HostFileWrite => false HostFileLeaveInPlace => false } } + +data unsafe_file_metadata: RunnerHostPathMetadata = PathsNotRootOnly { paths: ["/etc/needrestart/conf.d/50-gunbc-actions-runner.conf"], detail: "owned by ghrunner" } + +// THE METADATA RED. Byte-identical content that the CI user can rewrite is NOT left in place: root +// evaluates it, so it is rewritten, which replaces its owner and mode with root's. +test fn identical_bytes_with_unsafe_file_metadata_are_rewritten_not_left() -> Bool { + is_leave(a: action_of(content: HostFileIdentical, file_metadata: PathsRootOnly, ancestors: PathsRootOnly)) + && is_write(a: action_of(content: HostFileIdentical, file_metadata: unsafe_file_metadata, ancestors: PathsRootOnly)) + && is_refused(a: action_of(content: HostFileIdentical, file_metadata: PathsUnobservable { reason: "find exit=2" }, ancestors: PathsRootOnly)) +} + +// A writable ancestor cannot be repaired by writing the file, and writing into it would publish +// root-evaluated bytes where a CI user can replace them, so it refuses whatever the content is. +test fn an_ancestor_that_is_not_root_only_refuses_every_action() -> Bool { + let unsafe_dir = PathsNotRootOnly { paths: ["/etc/needrestart/conf.d"], detail: "mode 0777" } + is_refused(a: action_of(content: HostFileIdentical, file_metadata: PathsRootOnly, ancestors: unsafe_dir)) + && is_refused(a: action_of(content: HostFileAbsent, file_metadata: PathsNotApplicable, ancestors: unsafe_dir)) + && is_refused(a: action_of(content: HostFileDiffers, file_metadata: PathsRootOnly, ancestors: PathsUnobservable { reason: "ssh refused" })) +} + +// A failure to look plans no write: an unobservable file is refused, never overwritten on the strength +// of not having been read. +test fn only_a_seen_difference_or_absence_plans_a_write() -> Bool { + is_write(a: action_of(content: HostFileDiffers, file_metadata: PathsRootOnly, ancestors: PathsRootOnly)) + && is_write(a: action_of(content: HostFileAbsent, file_metadata: PathsNotApplicable, ancestors: PathsRootOnly)) + && is_refused(a: action_of(content: HostFileUnobservable { reason: "permission denied" }, file_metadata: PathsNotApplicable, ancestors: PathsRootOnly)) +} + +// ── Slot unit judgments ────────────────────────────────────────────────────────────────────────── + +fn reading(load: String, reload: String, kill: String, exit: String) -> RunnerSlotUnitReading { + RunnerSlotUnitReading { unit: "actions-runner@srv1-01.service" as NonEmptyStr, load_state: load, need_daemon_reload: reload, kill_mode: kill, exit_type: exit } +} + +fn standing_wire_of(st: RunnerSlotUnitStanding) -> String { + match st { + SlotTeardownEffective => "effective" + SlotTeardownNotEffective { kill_mode: k, exit_type: e } => concat(concat("not-effective:", k), concat(":", e)) + SlotReloadPending => "reload-pending" + SlotUnitNotLoaded { load_state: l } => concat("not-loaded:", l) + SlotUnitUnread => "unread" + } +} + +test fn the_declared_pair_on_a_loaded_current_unit_is_effective() -> Bool { + standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "loaded", reload: "no", kill: "control-group", exit: "cgroup"), intent: gunbc_runner_lifecycle_intent)) == "effective" +} + +// THE FALSE-POSITIVE RED. systemctl shows a unit it has not loaded with the manager's DEFAULTS, and the +// default KillMode is control-group; a check reading KillMode first would certify a slot that does not +// exist. LoadState is read first. +test fn a_not_loaded_unit_reporting_default_values_is_not_certified() -> Bool { + standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "not-found", reload: "no", kill: "control-group", exit: "cgroup"), intent: gunbc_runner_lifecycle_intent)) == "not-loaded:not-found" +} + +// A loaded unit whose files changed on disk still reports the OLD values; it is reload-pending, not a +// verdict on the teardown in either direction. +test fn a_unit_needing_reload_is_pending_whatever_it_reports() -> Bool { + standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "loaded", reload: "yes", kill: "control-group", exit: "cgroup"), intent: gunbc_runner_lifecycle_intent)) == "reload-pending" + && standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "loaded", reload: "yes", kill: "process", exit: "main"), intent: gunbc_runner_lifecycle_intent)) == "reload-pending" +} + +// srv1's and srv2's live reading (2026-09-11), and each half alone. +test fn the_live_reading_and_each_half_alone_are_not_effective() -> Bool { + standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "loaded", reload: "no", kill: "process", exit: "main"), intent: gunbc_runner_lifecycle_intent)) == "not-effective:process:main" + && standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "loaded", reload: "no", kill: "control-group", exit: "main"), intent: gunbc_runner_lifecycle_intent)) == "not-effective:control-group:main" + && standing_wire_of(st: classify_runner_slot_unit(reading: reading(load: "loaded", reload: "no", kill: "mixed", exit: "cgroup"), intent: gunbc_runner_lifecycle_intent)) == "not-effective:mixed:cgroup" +} + +// ── The slot population ────────────────────────────────────────────────────────────────────────── + +data list_units_fixture: String = "actions-runner@srv1-01.service loaded active running Ephemeral GitHub Actions Runner (srv1-01)\nactions-runner@srv1-02.service loaded active running Ephemeral GitHub Actions Runner (srv1-02)\nactions-runner@srv1-38.service loaded inactive dead Ephemeral GitHub Actions Runner (srv1-38)\nssh.service loaded active running OpenBSD Secure Shell server\nxactions-runner@srv1-01.service loaded active running decoy\n" + +test fn list_units_yields_only_runner_slot_unit_names() -> Bool { + runner_slot_units_from_list_units(stdout: list_units_fixture) + == ["actions-runner@srv1-01.service", "actions-runner@srv1-02.service", "actions-runner@srv1-38.service"] +} + +// A loaded slot outside the desired roster is part of the population; a desired slot the manager does +// not hold is kept, to be read as not-loaded rather than skipped. +test fn the_population_is_desired_units_then_loaded_extras() -> Bool { + map( + runner_slot_population_units( + desired: ["actions-runner@srv1-01.service" as NonEmptyStr, "actions-runner@srv1-03.service" as NonEmptyStr], + loaded: runner_slot_units_from_list_units(stdout: list_units_fixture), + ), + u => u as String, + ) == ["actions-runner@srv1-01.service", "actions-runner@srv1-03.service", "actions-runner@srv1-02.service", "actions-runner@srv1-38.service"] +} + +data show_fixture: String = "Id=actions-runner@srv1-01.service\nLoadState=loaded\nNeedDaemonReload=no\nExitType=cgroup\nKillMode=control-group\n\nId=actions-runner@srv1-02.service\nLoadState=loaded\nNeedDaemonReload=no\nExitType=main\nKillMode=process\n\nId=actions-runner@srv1-03.service\nLoadState=not-found\nNeedDaemonReload=no\nExitType=main\nKillMode=control-group" + +fn population_wires(units: List) -> List { + map( + classify_runner_slot_population(units: map(units, u => u as NonEmptyStr), show_stdout: show_fixture, intent: gunbc_runner_lifecycle_intent), + r => standing_wire_of(st: r.standing), + ) +} + +// Each unit is joined to the block that NAMES it -- property order within a block is not relied on -- +// and a unit with no block of its own is unread, never borrowed from a neighbour. +test fn each_unit_is_judged_on_its_own_block_and_a_missing_block_is_unread() -> Bool { + population_wires(units: ["actions-runner@srv1-02.service", "actions-runner@srv1-01.service", "actions-runner@srv1-03.service", "actions-runner@srv1-09.service"]) + == ["not-effective:process:main", "effective", "not-loaded:not-found", "unread"] +} + +fn population_of(standings: List) -> RunnerSlotPopulation { + SlotPopulationObserved { units: map(standings, st => RunnerSlotUnitResult { unit: "actions-runner@srv1-01.service" as NonEmptyStr, standing: st }) } +} + +// THE HOST-WIDE VERDICT IS OVER EVERY UNIT. One not-effective, not-loaded, pending or unread unit among +// effective ones is enough to deny it, and an empty population proves nothing. +test fn the_host_is_effective_only_when_every_unit_is() -> Bool { + runner_slot_population_effective(population: population_of(standings: [SlotTeardownEffective, SlotTeardownEffective])) + && !runner_slot_population_effective(population: population_of(standings: [SlotTeardownEffective, SlotTeardownNotEffective { kill_mode: "process", exit_type: "main" }])) + && !runner_slot_population_effective(population: population_of(standings: [SlotTeardownEffective, SlotUnitNotLoaded { load_state: "not-found" }])) + && !runner_slot_population_effective(population: population_of(standings: [SlotTeardownEffective, SlotReloadPending])) + && !runner_slot_population_effective(population: population_of(standings: [SlotTeardownEffective, SlotUnitUnread])) + && !runner_slot_population_effective(population: population_of(standings: [])) + && !runner_slot_population_effective(population: SlotPopulationUnobservable { reason: "ssh refused" }) +} + +// THE RETRY RED. A reload is owed whenever the manager says a unit is stale, whether or not this run +// wrote anything -- which is exactly the state a run finds after an earlier run landed the file and its +// reload failed. +test fn a_reload_is_owed_by_a_stale_unit_even_when_nothing_was_written() -> Bool { + runner_slot_population_owes_reload(population: population_of(standings: [SlotTeardownNotEffective { kill_mode: "process", exit_type: "main" }, SlotReloadPending])) + && !runner_slot_population_owes_reload(population: population_of(standings: [SlotTeardownNotEffective { kill_mode: "process", exit_type: "main" }])) + && !runner_slot_population_owes_reload(population: SlotPopulationUnobservable { reason: "ssh refused" }) +} + +// ── Terminal ───────────────────────────────────────────────────────────────────────────────────── + +fn teardown_desired() -> RunnerHostFileDesired? { + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: _ } => none + RunnerHostFilesDesired { files: fs } => + fold(fs, init: none, f: (acc, f) => if (f.subject as String) == "runner-teardown-dropin" { Present { value: f } } else { acc }) + } +} + +fn witness_subject() -> RunnerHostFileSubject? { + match runner_host_file_subject(host: "srv1" as HostIdentity) { + RunnerHostFileSubjectAdmitted { subject: s } => Present { value: s } + RunnerHostFileSubjectRefused { cause: _ } => none + } +} + +fn settled_run(mode: RunnerHostFileConvergeMode, reload: RunnerHostManagerReload, slots: RunnerSlotPopulation) -> Bool { + match witness_subject() { + Absent => false + Present { value: s } => + match teardown_desired() { + Absent => false + Present { value: f } => + runner_host_file_run_succeeded(run: RunnerHostFileRun { + subject: s, + mode: mode, + privilege: AdministratorPrivilegeAvailable, + results: [RunnerHostFileResult { + planned: RunnerHostFilePlanned { + file: f, + observation: RunnerHostFileObservation { content: HostFileIdentical, file_metadata: PathsRootOnly, ancestors: PathsRootOnly }, + action: HostFileLeaveInPlace, + }, + outcome: match mode { RunnerHostFilesObserveOnly => HostFileNotAppliedObserveOnly RunnerHostFilesApply => HostFileLeftInPlace }, + }], + reload: reload, + slots: slots, + }) + } + } +} + +// Observe succeeds on a host whose teardown is NOT yet effective -- that is what it exists to report -- +// and apply does not. Apply also refuses a reload it owed and did not perform, and a population with +// any unit it could not read. +test fn observe_reports_what_apply_refuses() -> Bool { + let old = population_of(standings: [SlotTeardownNotEffective { kill_mode: "process", exit_type: "main" }]) + let good = population_of(standings: [SlotTeardownEffective, SlotTeardownEffective]) + settled_run(mode: RunnerHostFilesObserveOnly, reload: ManagerReloadNotOwed, slots: old) + && !settled_run(mode: RunnerHostFilesApply, reload: ManagerReloadNotOwed, slots: old) + && settled_run(mode: RunnerHostFilesApply, reload: ManagerReloaded, slots: good) + && !settled_run(mode: RunnerHostFilesApply, reload: ManagerReloadRefused { reason: "exit=1" }, slots: good) + && !settled_run(mode: RunnerHostFilesApply, reload: ManagerReloadOwedNotAttempted { reason: "privilege refused" }, slots: good) + && !settled_run(mode: RunnerHostFilesObserveOnly, reload: ManagerReloadNotOwed, slots: population_of(standings: [SlotUnitUnread])) +} + +// ── The route, over the hermetic ssh fixture ───────────────────────────────────────────────────── + +// THE WHOLE APPLY ROUTE RUNS: privilege, content and metadata observation for both files, the slot +// listing and show, the reload decision. The mock answers every leg with exit 0 and a stdout that +// names no path and no unit, so find certifies no ancestor as root-only -- both files are REFUSED +// before any write -- and no slot unit has a block of its own -- every desired unit reads as UNREAD. +// Nothing is written or reloaded, and the run ends refused. A route that reported success here would +// be certifying ownership and a teardown that no host reported. +test fn the_apply_route_refuses_what_the_host_did_not_report() -> Bool { + match witness_subject() { + Absent => false + Present { value: s } => + match fleet_ssh_execution_context_hermetic_witness_fixture() { + Absent => false + Present { value: context } => + match runner_host_file_desired_population() { + RunnerHostFilesDesiredRefused { cause: _ } => false + RunnerHostFilesDesired { files: fs } => { + let run = run_runner_host_files(subject: s, context: context, files: fs, mode: RunnerHostFilesApply) + let privileged = match run.privilege { AdministratorPrivilegeAvailable => true AdministratorPrivilegeRefused { reason: _ } => false } + let refused_before_write = all(run.results, r => is_refused(a: r.planned.action) && match r.outcome { + HostFileNotAttempted { reason: _ } => true + HostFileLeftInPlace => false + HostFileNotAppliedObserveOnly => false + HostFileWriteRefused { step: _, reason: _ } => false + HostFileConvergedByWrite { converged: _ } => false + }) + let not_reloaded = match run.reload { ManagerReloadNotOwed => true ManagerReloaded => false ManagerReloadRefused { reason: _ } => false ManagerReloadOwedNotAttempted { reason: _ } => false } + let every_desired_unread = match run.slots { + SlotPopulationUnobservable { reason: _ } => false + SlotPopulationObserved { units: us } => + length(us) == length(s.desired_units) && all(us, r => standing_wire_of(st: r.standing) == "unread") + } + privileged && length(run.results) == 2 && refused_before_write && not_reloaded && every_desired_unread + && !runner_host_file_run_succeeded(run: run) + } + } + } + } +} diff --git a/dag/test/claim/typed_remote_file_converge_witness_test.dag b/dag/test/claim/typed_remote_file_converge_witness_test.dag index 0cb1d941199..b16bb0d47db 100644 --- a/dag/test/claim/typed_remote_file_converge_witness_test.dag +++ b/dag/test/claim/typed_remote_file_converge_witness_test.dag @@ -2,6 +2,7 @@ module test.claim.typed_remote_file_converge_witness_test import std.types { Bool, String, NonEmptyStr, FilePath } import gunbc.file_access { FileTreeScope } +import std.materialization_ladder { string_list_contains } import gunbc.fleet_known_hosts_anchor { SshTarget } import test.claim.typed_remote_file_write_fleet_ssh_fixture { fleet_ssh_execution_context_hermetic_witness_fixture, @@ -14,6 +15,9 @@ import gunbc.typed_remote_file_write { RemoteFileMode, RemoteFileModeOwnerReadWrite, RemoteFileModeOwnerReadWriteWorldRead, + RemoteFileAsSessionPrincipal, + RemoteFileViaNonInteractiveSudo, + remote_file_privileged_argv, remote_file_mode_octal, typed_remote_file_staging_path, typed_remote_file_stage_argv, @@ -135,6 +139,40 @@ test fn the_comparison_takes_the_declared_side_from_stdin() -> Bool { } } +// ── Privilege ─────────────────────────────────────────────────────────────────────────────────── + +// The session arm is the identity on argv, so every consumer that wrote as its own principal before +// the privilege axis existed runs exactly the words it ran then. +test fn the_session_principal_leaves_every_leg_as_built() -> Bool { + match subject_write() { + Absent => false + Present { value: w } => + remote_file_privileged_argv(privilege: RemoteFileAsSessionPrincipal, argv: typed_remote_file_stage_argv(write: w)) + == typed_remote_file_stage_argv(write: w) + && remote_file_privileged_argv(privilege: RemoteFileAsSessionPrincipal, argv: typed_remote_file_compare_argv(write: w)) + == typed_remote_file_compare_argv(write: w) + } +} + +// The sudo arm is non-interactive on every leg. A bare `sudo` would prompt on a channel with no +// terminal and hang to the step timeout instead of refusing at the leg that lacked the grant. +test fn the_sudo_arm_prefixes_every_leg_non_interactively() -> Bool { + match subject_write() { + Absent => false + Present { value: w } => + remote_file_privileged_argv(privilege: RemoteFileViaNonInteractiveSudo, argv: typed_remote_file_stage_argv(write: w)) + == concat(["/usr/bin/sudo", "-n"], typed_remote_file_stage_argv(write: w)) + && remote_file_privileged_argv(privilege: RemoteFileViaNonInteractiveSudo, argv: typed_remote_file_rename_argv(write: w)) + == concat(["/usr/bin/sudo", "-n"], typed_remote_file_rename_argv(write: w)) + && remote_file_privileged_argv(privilege: RemoteFileViaNonInteractiveSudo, argv: typed_remote_file_compare_argv(write: w)) + == concat(["/usr/bin/sudo", "-n"], typed_remote_file_compare_argv(write: w)) + && !string_list_contains( + xs: remote_file_privileged_argv(privilege: RemoteFileViaNonInteractiveSudo, argv: typed_remote_file_stage_argv(write: w)), + wanted: "-S", + ) + } +} + test fn the_two_modes_render_their_octals() -> Bool { remote_file_mode_octal(mode: RemoteFileModeOwnerReadWrite) == "600" && remote_file_mode_octal(mode: RemoteFileModeOwnerReadWriteWorldRead) == "644" @@ -258,6 +296,7 @@ test fn the_converge_route_yields_a_receipt_naming_the_exact_write_and_target() context: context, admission: TypedRemoteFileWriteAdmitted { write: w }, mode: RemoteFileModeOwnerReadWrite, + privilege: RemoteFileAsSessionPrincipal, ) { TypedRemoteFileConverged { converged: c } => ((c.write.path as String) == (subject_path as String)) diff --git a/dag/test/claim/typed_remote_file_write_fleet_ssh_fixture.dag b/dag/test/claim/typed_remote_file_write_fleet_ssh_fixture.dag index d43d999ccb3..93495be0f8c 100644 --- a/dag/test/claim/typed_remote_file_write_fleet_ssh_fixture.dag +++ b/dag/test/claim/typed_remote_file_write_fleet_ssh_fixture.dag @@ -74,6 +74,7 @@ fn fleet_ssh_execution_context_hermetic_witness_fixture() -> FleetSshExecutionCo admit_callers: [ decl_ref(module_path: "test.claim.typed_remote_file_write_witness", decl_name: "w_over_fleet_ssh_hermetic_mock_exec_accepted"), decl_ref(module_path: "test.claim.typed_remote_file_converge_witness_test", decl_name: "the_converge_route_yields_a_receipt_naming_the_exact_write_and_target"), + decl_ref(module_path: "test.claim.runner_host_file_converge_witness_test", decl_name: "the_apply_route_refuses_what_the_host_did_not_report"), decl_ref(module_path: "test.claim.fleet_bootstrap_principal_session_witness", decl_name: "witness_leg_stdout"), ] { diff --git a/dag/test/claim/workflow_dispatch_input_witness_test.dag b/dag/test/claim/workflow_dispatch_input_witness_test.dag index c3a3f172a8d..525a49b2781 100644 --- a/dag/test/claim/workflow_dispatch_input_witness_test.dag +++ b/dag/test/claim/workflow_dispatch_input_witness_test.dag @@ -350,7 +350,8 @@ test fn a_reset_subject_map_escapes_a_label_that_would_break_the_json() -> Bool // wire string would still meet it, and the dispatch menu would silently offer one option twice while // one mode became unselectable. Counting the DISTINCT wire values is what refuses that. // -// THE LITERAL MOVED FROM FIFTEEN TO SIXTEEN when HostResetReturn was added, and the roster +// THE LITERAL MOVED FROM FIFTEEN TO SIXTEEN when HostResetReturn was added, and to NINETEEN when +// SparkRuntimeImageProbe, RunnerHostFileObserve and RunnerHostFileConverge were added, and the roster // annotation in gunbc.fleet_converge_workflow says why a literal is here at all: nothing reflects // over a closed vocabulary to produce its members, so fleet_converge_workflow_modes is hand-ordered // and this count is what makes a member DROPPED from that roster visible. The count is grounded in @@ -358,8 +359,8 @@ test fn a_reset_subject_map_escapes_a_label_that_would_break_the_json() -> Bool // leaves the type and the roster disagreeing, and this is the only place that disagreement is loud. test fn the_mode_vocabulary_members_have_that_many_distinct_wire_values() -> Bool { let wires = map(fleet_converge_workflow_modes, m => fleet_converge_workflow_mode_wire(mode: m)) - count(fleet_converge_workflow_modes) == 16 - && count(wires) == 16 + count(fleet_converge_workflow_modes) == 19 + && count(wires) == 19 && unique(xs: wires, eq: (a, b) => a == b) } @@ -367,7 +368,7 @@ test fn the_mode_vocabulary_members_have_that_many_distinct_wire_values() -> Boo // to a hand-kept list that merely happens to agree with it today. test fn every_dispatch_option_is_a_wire_value_of_the_vocabulary() -> Bool { let wires = map(fleet_converge_workflow_modes, m => fleet_converge_workflow_mode_wire(mode: m)) - count(fleet_converge_mode_options) == 16 + count(fleet_converge_mode_options) == 19 && all(fleet_converge_mode_options, o => any(wires, w => w == o)) }