diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5b008e9a7c..2cd782ba1e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -603,3 +603,241 @@ jobs: compression-level: 0 if: always() && steps.heal_commit_push.outputs.heal_author_commit_required == '1' timeout-minutes: 10 + observe_srv1_tasks_preapply: + runs-on: [self-hosted, linux, arm64, srv1] + needs: [build] + timeout-minutes: 10 + if: github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ inputs.expected_healed_sha || github.sha }} + - name: Refuse a heal revalidation whose run subject or checkout does not name the expected healed SHA + run: | + EXPECTED_HEALED_SHA="${{ inputs.expected_healed_sha }}" + RUN_SUBJECT_HEAD="${{ github.sha }}" + if [ -n "$EXPECTED_HEALED_SHA" ]; then + ACTUAL_HEAD="$(git rev-parse HEAD)" + if ! [ "$RUN_SUBJECT_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: workflow run subject $RUN_SUBJECT_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + if ! [ "$ACTUAL_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: checkout head $ACTUAL_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + echo "heal revalidation preflight: checkout names expected healed head $EXPECTED_HEALED_SHA" + fi + - name: Download release-bins artifact + uses: actions/download-artifact@v4 + with: + name: release-bins + timeout-minutes: 10 + - name: Unpack + verify release bins (claim_executor --verify-build-artifacts; fail-closed) + id: release_bins + run: | + # 🟡 dissolve-on: ci_release_bins_unpack_verify_script — concat-built foreign-executor (GitHub Actions run:) release-bins unpack + claim_executor --verify-build-artifacts runner; membership of verified paths is derived from gunbc.ci_release_bins, but the unpack/verify transport itself remains hand-shell; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) realizes the unpack/verify runner through orchestration emit or typed host_effect_apply without a medium-as-string concat scaffold + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + mkdir -p "$ROOT/target/release" + tar -xzf "$ROOT/release-bins.tgz" -C "$ROOT/target/release" + rm -f "$ROOT/release-bins.tgz" + "$ROOT/target/release/claim_executor" --verify-build-artifacts "$ROOT/target/release/claim_executor" "$ROOT/target/release/gunbc" "$ROOT/target/release/regen_floor_skip_witness" "$ROOT/target/release/floor_skip_discovery_witness" "$ROOT/target/release/selection_control_skip_witness" "$ROOT/target/release/discover_source_root_ingest" "$ROOT/target/release/claim_batch" "$ROOT/target/release/regen_stage0" "$ROOT/target/release/interp_recorded_fixture_witness" "$ROOT/target/release/v1_src_dag_parse" "$ROOT/target/release/auth_declared_but_unwired_witness" "$ROOT/target/release/bootstrap_witness" "$ROOT/target/release/dag_collect_fingerprint_witness" "$ROOT/target/release/diagnostics_witness" "$ROOT/target/release/effects_rest_transport_witness" "$ROOT/target/release/infer_semantics_witness" "$ROOT/target/release/parse_witness" "$ROOT/target/release/resolution_divergence_census" "$ROOT/target/release/cssl_assemble" "$ROOT/target/release/frontier_probe_survey" "$ROOT/target/release/namespace_structural_root_exposure_generated_witness" + timeout-minutes: 5 + - name: Read-only srv1 tasks pre-apply observation (observe_srv1_tasks_preapply_wet) + run: | + # 🟡 dissolve-on: gunbc_ci_srv1_tasks_observation_invoke — feature:task-count-measure-carrier sibling transport; concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.srv1_tasks_preapply_observation observe_srv1_tasks_preapply_wet plus a cat of the typed receipt file; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the observation step to orchestration emit over the typed gunbc.WitnessBin.Run route without a medium-as-string concat scaffold + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/srv1_tasks_preapply_observation.dag --function observe_srv1_tasks_preapply_wet + cat "$ROOT/target/srv1-tasks-preapply-receipt.txt" + timeout-minutes: 5 + probe_fleet_reach: + runs-on: [self-hosted, linux, arm64, srv1] + needs: [build] + timeout-minutes: 10 + if: github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ inputs.expected_healed_sha || github.sha }} + - name: Refuse a heal revalidation whose run subject or checkout does not name the expected healed SHA + run: | + EXPECTED_HEALED_SHA="${{ inputs.expected_healed_sha }}" + RUN_SUBJECT_HEAD="${{ github.sha }}" + if [ -n "$EXPECTED_HEALED_SHA" ]; then + ACTUAL_HEAD="$(git rev-parse HEAD)" + if ! [ "$RUN_SUBJECT_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: workflow run subject $RUN_SUBJECT_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + if ! [ "$ACTUAL_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: checkout head $ACTUAL_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + echo "heal revalidation preflight: checkout names expected healed head $EXPECTED_HEALED_SHA" + fi + - name: Download release-bins artifact + uses: actions/download-artifact@v4 + with: + name: release-bins + timeout-minutes: 10 + - name: Unpack + verify release bins (claim_executor --verify-build-artifacts; fail-closed) + id: release_bins + run: | + # 🟡 dissolve-on: ci_release_bins_unpack_verify_script — concat-built foreign-executor (GitHub Actions run:) release-bins unpack + claim_executor --verify-build-artifacts runner; membership of verified paths is derived from gunbc.ci_release_bins, but the unpack/verify transport itself remains hand-shell; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) realizes the unpack/verify runner through orchestration emit or typed host_effect_apply without a medium-as-string concat scaffold + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + mkdir -p "$ROOT/target/release" + tar -xzf "$ROOT/release-bins.tgz" -C "$ROOT/target/release" + rm -f "$ROOT/release-bins.tgz" + "$ROOT/target/release/claim_executor" --verify-build-artifacts "$ROOT/target/release/claim_executor" "$ROOT/target/release/gunbc" "$ROOT/target/release/regen_floor_skip_witness" "$ROOT/target/release/floor_skip_discovery_witness" "$ROOT/target/release/selection_control_skip_witness" "$ROOT/target/release/discover_source_root_ingest" "$ROOT/target/release/claim_batch" "$ROOT/target/release/regen_stage0" "$ROOT/target/release/interp_recorded_fixture_witness" "$ROOT/target/release/v1_src_dag_parse" "$ROOT/target/release/auth_declared_but_unwired_witness" "$ROOT/target/release/bootstrap_witness" "$ROOT/target/release/dag_collect_fingerprint_witness" "$ROOT/target/release/diagnostics_witness" "$ROOT/target/release/effects_rest_transport_witness" "$ROOT/target/release/infer_semantics_witness" "$ROOT/target/release/parse_witness" "$ROOT/target/release/resolution_divergence_census" "$ROOT/target/release/cssl_assemble" "$ROOT/target/release/frontier_probe_survey" "$ROOT/target/release/namespace_structural_root_exposure_generated_witness" + timeout-minutes: 5 + - name: Live fleet reach probe (probe_fleet_reach_wet; outcomes are data) + run: | + # 🟡 dissolve-on: gunbc_ci_fleet_reach_probe_invoke — concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.host_reach_identity_probe probe_fleet_reach_wet plus a cat of the typed receipt file; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the probe step to orchestration emit over the typed gunbc.WitnessBin.Run route + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/host_reach_identity_probe.dag --function probe_fleet_reach_wet + cat "$ROOT/target/fleet-reach-probe-receipt.txt" + timeout-minutes: 5 + install_fleet_converge_timer: + runs-on: [self-hosted, linux, arm64] + needs: [build] + timeout-minutes: 10 + if: github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ inputs.expected_healed_sha || github.sha }} + - name: Refuse a heal revalidation whose run subject or checkout does not name the expected healed SHA + run: | + EXPECTED_HEALED_SHA="${{ inputs.expected_healed_sha }}" + RUN_SUBJECT_HEAD="${{ github.sha }}" + if [ -n "$EXPECTED_HEALED_SHA" ]; then + ACTUAL_HEAD="$(git rev-parse HEAD)" + if ! [ "$RUN_SUBJECT_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: workflow run subject $RUN_SUBJECT_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + if ! [ "$ACTUAL_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: checkout head $ACTUAL_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + echo "heal revalidation preflight: checkout names expected healed head $EXPECTED_HEALED_SHA" + fi + - name: Download release-bins artifact + uses: actions/download-artifact@v4 + with: + name: release-bins + timeout-minutes: 10 + - name: Unpack + verify release bins (claim_executor --verify-build-artifacts; fail-closed) + id: release_bins + run: | + # 🟡 dissolve-on: ci_release_bins_unpack_verify_script — concat-built foreign-executor (GitHub Actions run:) release-bins unpack + claim_executor --verify-build-artifacts runner; membership of verified paths is derived from gunbc.ci_release_bins, but the unpack/verify transport itself remains hand-shell; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) realizes the unpack/verify runner through orchestration emit or typed host_effect_apply without a medium-as-string concat scaffold + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + mkdir -p "$ROOT/target/release" + tar -xzf "$ROOT/release-bins.tgz" -C "$ROOT/target/release" + rm -f "$ROOT/release-bins.tgz" + "$ROOT/target/release/claim_executor" --verify-build-artifacts "$ROOT/target/release/claim_executor" "$ROOT/target/release/gunbc" "$ROOT/target/release/regen_floor_skip_witness" "$ROOT/target/release/floor_skip_discovery_witness" "$ROOT/target/release/selection_control_skip_witness" "$ROOT/target/release/discover_source_root_ingest" "$ROOT/target/release/claim_batch" "$ROOT/target/release/regen_stage0" "$ROOT/target/release/interp_recorded_fixture_witness" "$ROOT/target/release/v1_src_dag_parse" "$ROOT/target/release/auth_declared_but_unwired_witness" "$ROOT/target/release/bootstrap_witness" "$ROOT/target/release/dag_collect_fingerprint_witness" "$ROOT/target/release/diagnostics_witness" "$ROOT/target/release/effects_rest_transport_witness" "$ROOT/target/release/infer_semantics_witness" "$ROOT/target/release/parse_witness" "$ROOT/target/release/resolution_divergence_census" "$ROOT/target/release/cssl_assemble" "$ROOT/target/release/frontier_probe_survey" "$ROOT/target/release/namespace_structural_root_exposure_generated_witness" + timeout-minutes: 5 + - name: Fleet converge timer self-install (fleet_timer_self_install_wet; installs on THIS runner's host) + run: | + # 🟡 dissolve-on: gunbc_ci_fleet_timer_install_invoke — concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.fleet_converge_timer_install fleet_timer_self_install_wet; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the step to orchestration emit over the typed gunbc.WitnessBin.Run route + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/fleet_converge_timer_install.dag --function fleet_timer_self_install_wet + timeout-minutes: 5 + collect_fleet_convergence: + runs-on: [self-hosted, linux, arm64, srv1] + needs: [ci] + timeout-minutes: 10 + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ inputs.expected_healed_sha || github.sha }} + - name: Refuse a heal revalidation whose run subject or checkout does not name the expected healed SHA + run: | + EXPECTED_HEALED_SHA="${{ inputs.expected_healed_sha }}" + RUN_SUBJECT_HEAD="${{ github.sha }}" + if [ -n "$EXPECTED_HEALED_SHA" ]; then + ACTUAL_HEAD="$(git rev-parse HEAD)" + if ! [ "$RUN_SUBJECT_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: workflow run subject $RUN_SUBJECT_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + if ! [ "$ACTUAL_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: checkout head $ACTUAL_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + echo "heal revalidation preflight: checkout names expected healed head $EXPECTED_HEALED_SHA" + fi + - name: Download release-bins artifact + uses: actions/download-artifact@v4 + with: + name: release-bins + timeout-minutes: 10 + - name: Unpack + verify release bins (claim_executor --verify-build-artifacts; fail-closed) + id: release_bins + run: | + # 🟡 dissolve-on: ci_release_bins_unpack_verify_script — concat-built foreign-executor (GitHub Actions run:) release-bins unpack + claim_executor --verify-build-artifacts runner; membership of verified paths is derived from gunbc.ci_release_bins, but the unpack/verify transport itself remains hand-shell; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) realizes the unpack/verify runner through orchestration emit or typed host_effect_apply without a medium-as-string concat scaffold + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + mkdir -p "$ROOT/target/release" + tar -xzf "$ROOT/release-bins.tgz" -C "$ROOT/target/release" + rm -f "$ROOT/release-bins.tgz" + "$ROOT/target/release/claim_executor" --verify-build-artifacts "$ROOT/target/release/claim_executor" "$ROOT/target/release/gunbc" "$ROOT/target/release/regen_floor_skip_witness" "$ROOT/target/release/floor_skip_discovery_witness" "$ROOT/target/release/selection_control_skip_witness" "$ROOT/target/release/discover_source_root_ingest" "$ROOT/target/release/claim_batch" "$ROOT/target/release/regen_stage0" "$ROOT/target/release/interp_recorded_fixture_witness" "$ROOT/target/release/v1_src_dag_parse" "$ROOT/target/release/auth_declared_but_unwired_witness" "$ROOT/target/release/bootstrap_witness" "$ROOT/target/release/dag_collect_fingerprint_witness" "$ROOT/target/release/diagnostics_witness" "$ROOT/target/release/effects_rest_transport_witness" "$ROOT/target/release/infer_semantics_witness" "$ROOT/target/release/parse_witness" "$ROOT/target/release/resolution_divergence_census" "$ROOT/target/release/cssl_assemble" "$ROOT/target/release/frontier_probe_survey" "$ROOT/target/release/namespace_structural_root_exposure_generated_witness" + timeout-minutes: 5 + - name: Collect fleet converge receipts and fold the fleet verdict (collect_fleet_convergence_wet) + run: | + # 🟡 dissolve-on: gunbc_ci_collect_fleet_convergence_invoke — concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.fleet_receipt_collector collect_fleet_convergence_wet plus a cat of the typed collect receipt; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the step to orchestration emit over the typed gunbc.WitnessBin.Run route + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/fleet_receipt_collector.dag --function collect_fleet_convergence_wet + cat "$ROOT/target/fleet-converge-collect-receipt.txt" + timeout-minutes: 5 + advance_fleet_desired: + runs-on: [self-hosted, linux, arm64] + needs: [ci] + timeout-minutes: 10 + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ inputs.expected_healed_sha || github.sha }} + - name: Refuse a heal revalidation whose run subject or checkout does not name the expected healed SHA + run: | + EXPECTED_HEALED_SHA="${{ inputs.expected_healed_sha }}" + RUN_SUBJECT_HEAD="${{ github.sha }}" + if [ -n "$EXPECTED_HEALED_SHA" ]; then + ACTUAL_HEAD="$(git rev-parse HEAD)" + if ! [ "$RUN_SUBJECT_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: workflow run subject $RUN_SUBJECT_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + if ! [ "$ACTUAL_HEAD" = "$EXPECTED_HEALED_SHA" ]; then + echo "::error::heal revalidation refused: checkout head $ACTUAL_HEAD does not equal expected healed head $EXPECTED_HEALED_SHA" + exit 1 + fi + echo "heal revalidation preflight: checkout names expected healed head $EXPECTED_HEALED_SHA" + fi + - name: Advance refs/fleet/desired to the merged revision (CAS, floor-green only) + run: | + # 🟡 dissolve-on: gunbc_ci_advance_fleet_desired_script — concat-built foreign-executor (GitHub Actions run:) CAS ref advance (git ls-remote + push --force-with-lease onto gunbc.fleet_main_revision fleet_desired_ref_name); DISSOLVES WHEN the SCM ref-update realization lands as the typed git.Core CAS service op (membership-diff spine classify_change SCM-ref arm) invoked via host_effect_apply + set -eu + DESIRED_REF=refs/fleet/desired + CURRENT=$(git ls-remote origin "$DESIRED_REF" | cut -f1) + if [ -z "$CURRENT" ]; then + git push origin "$GITHUB_SHA:$DESIRED_REF" + echo "[fleet-desired] created $DESIRED_REF -> $GITHUB_SHA" + else + git push --force-with-lease="$DESIRED_REF:$CURRENT" origin "$GITHUB_SHA:$DESIRED_REF" + echo "[fleet-desired] advanced $DESIRED_REF: $CURRENT -> $GITHUB_SHA (CAS expected-prior held)" + fi + timeout-minutes: 5 diff --git a/dag/extdeps/git/git.dag b/dag/extdeps/git/git.dag index 8ad1ac8a12f..b301ab46c02 100644 --- a/dag/extdeps/git/git.dag +++ b/dag/extdeps/git/git.dag @@ -907,6 +907,28 @@ service git.Core { } } + operation LsRemoteRefInRepo { + input { repo: FilePath, remote: String, ref_name: String } + output { + advertised: String from "stdout" + exit_code: Int from "exit_code" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["git", "-C", "{repo}", "ls-remote", "{remote}", "{ref_name}"] } + } + + operation RevParseInRepo { + input { repo: FilePath, target: GitRef } + output { + revision: String from "stdout" + exit_code: Int from "exit_code" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["git", "-C", "{repo}", "rev-parse", "{target}"] } + } + operation CatFileBlobInRepo { input { repo: FilePath, oid: String } output { diff --git a/dag/extdeps/linux/procfs.dag b/dag/extdeps/linux/procfs.dag index 2db6b0e7dba..9a48564835f 100644 --- a/dag/extdeps/linux/procfs.dag +++ b/dag/extdeps/linux/procfs.dag @@ -23,6 +23,7 @@ type ProcfsPath | ProcSelfStatus | ProcSelfCgroup | ProcSwaps + | ProcSysKernelHostname fn procfs_path_literal(p: ProcfsPath) -> NonEmptyStr { match p { @@ -30,6 +31,7 @@ fn procfs_path_literal(p: ProcfsPath) -> NonEmptyStr { ProcSelfStatus => "/proc/self/status" ProcSelfCgroup => "/proc/self/cgroup" ProcSwaps => "/proc/swaps" + ProcSysKernelHostname => "/proc/sys/kernel/hostname" } } @@ -38,6 +40,7 @@ data procfs_paths_read_by_this_repository: List = [ ProcSelfStatus, ProcSelfCgroup, ProcSwaps, + ProcSysKernelHostname, ] // Provision is a total match over std.os.types.KernelFamily rather than a roster of platforms that happen to have it, so adding a kernel family to that closed enum forces an answer here instead of silently defaulting. LinuxGuestOnWindows answers TRUE deliberately and is the arm most likely to be got wrong by pattern-matching on the name: WSL2 runs a real Linux kernel and synthesizes a real procfs, so a consumer that treated it as a Windows platform would refuse observations the host can actually serve. Darwin and WindowsNt answer false — each has its own mechanism for the same facts (Mach task_info, sysctl, the Windows process API), and modeling those is a separate job this module does not attempt. diff --git a/dag/extdeps/systemd/systemd.dag b/dag/extdeps/systemd/systemd.dag index 8c852db2403..ede980806fe 100644 --- a/dag/extdeps/systemd/systemd.dag +++ b/dag/extdeps/systemd/systemd.dag @@ -1,6 +1,6 @@ module extdeps.systemd -import std.types { NonEmptyStr, String } +import std.types { NonEmptyStr, String, Int } import std.types { List } import std.measure { ByteSize, byte_size, byte_size_count } import extdeps.external_authority { ExternalAuthority } @@ -32,6 +32,9 @@ type SystemdUnitProperty | MemoryHigh | CPUWeight | MemoryCurrent + | TasksMax + | TasksCurrent + | ControlGroup fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr { match property { @@ -40,6 +43,9 @@ fn systemd_unit_property_wire(property: SystemdUnitProperty) -> NonEmptyStr { MemoryHigh => "MemoryHigh" as NonEmptyStr CPUWeight => "CPUWeight" as NonEmptyStr MemoryCurrent => "MemoryCurrent" as NonEmptyStr + TasksMax => "TasksMax" as NonEmptyStr + TasksCurrent => "TasksCurrent" as NonEmptyStr + ControlGroup => "ControlGroup" as NonEmptyStr } } @@ -55,6 +61,10 @@ data systemd_cpu_weight_property: NonEmptyStr = systemd_unit_property_wire(prope data systemd_memory_current_property: NonEmptyStr = systemd_unit_property_wire(property: MemoryCurrent) +data systemd_tasks_max_property: NonEmptyStr = systemd_unit_property_wire(property: TasksMax) + +data systemd_tasks_current_property: NonEmptyStr = systemd_unit_property_wire(property: TasksCurrent) + type SystemdCgroupMemoryLimit = MemoryLimitUnbounded | MemoryLimitBytes { bytes: ByteSize } @@ -77,6 +87,35 @@ fn systemd_memory_limit_wire_label(lim: SystemdCgroupMemoryLimit) -> String { } } +type SystemdCgroupTaskLimit + = TaskLimitUnbounded + | TaskLimitCount { count: Int } + | TaskLimitAbsent + +// parse_systemd_task_limit_show mirrors parse_systemd_memory_limit_show because a systemd +// cgroup limit shown by `systemctl show` is one wire shape — a decimal count or the literal +// "infinity" — differing only in the magnitude it grounds into (ByteSize vs a cardinal task +// count). 🟡 feature:task-count-measure-carrier dissolve-on: a single limit parse +// parameterized by magnitude (the SystemdCgroupLimit consolidation) with the count arm +// grounding into a std.measure TaskCount carrier (the HardwareThreadCount shape) instead of +// bare Int, at which point both hand parses derive from it and the flat scalar retires. +fn parse_systemd_task_limit_show(raw: String) -> SystemdCgroupTaskLimit { + if raw == "" { TaskLimitAbsent } + else { if raw == "infinity" { TaskLimitUnbounded } + else { match parse_int(s: raw) { + Present { value: n } => TaskLimitCount { count: n } + Absent => TaskLimitAbsent + } } } +} + +fn systemd_task_limit_wire_label(lim: SystemdCgroupTaskLimit) -> String { + match lim { + TaskLimitUnbounded => "infinity" + TaskLimitCount { count: n } => to_string(n) + TaskLimitAbsent => "" + } +} + fn parse_systemd_unit_active_state(raw: String) -> SystemdUnitActiveState? { if raw == "active" { Present { value: Active } } else { if raw == "inactive" { Present { value: Inactive } } diff --git a/dag/gunbc/ci_runner_placement.dag b/dag/gunbc/ci_runner_placement.dag index 8f54be0c017..cdfefcb5d14 100644 --- a/dag/gunbc/ci_runner_placement.dag +++ b/dag/gunbc/ci_runner_placement.dag @@ -73,6 +73,11 @@ fn workload_class_cpu_weight(wc: WorkloadClass) -> Int { } } +// 🟡 feature:task-count-measure-carrier dissolve-on: TaskCount measure carrier — per_runner_tasks_max is a pids-controller +// COUNT riding a bare Int while its memory siblings carry ByteSize; same dissolution as +// the ConvergeTarget.PerSlotMemoryCap → PerSlotUnitProperty P3 axis rework (a std.measure +// TaskCount, the HardwareThreadCount shape), which also retires the to_string() re-serialization +// at the host_converge knob seam. type RunnerHostDeployment { identity: HostIdentity runner_count: Int @@ -80,6 +85,7 @@ type RunnerHostDeployment { per_runner_memory_cap: ByteSize per_runner_memory_swap_cap: ByteSize per_runner_memory_high_cap: ByteSize + per_runner_tasks_max: Int build_tokens: HardwareThreadCount runner: RunnerSpec workload_class: WorkloadClass @@ -189,6 +195,7 @@ fn conservation_host_deployment(offer: ComputeOffer, nproc: HardwareThreadCount) per_runner_memory_cap: gunbc_ci_runner_cgroup_memory_cap, per_runner_memory_swap_cap: gunbc_runner_slot_desired().memory_swap_max, per_runner_memory_high_cap: gunbc_runner_slot_desired().memory_high, + per_runner_tasks_max: gunbc_runner_slot_desired().tasks_max, build_tokens: nproc, runner: runner_spec_from_offer(offer: offer), workload_class: BatchLatencyTolerant, diff --git a/dag/gunbc/ci_spec.dag b/dag/gunbc/ci_spec.dag index 59c8c3ad497..418d9e6ff80 100644 --- a/dag/gunbc/ci_spec.dag +++ b/dag/gunbc/ci_spec.dag @@ -46,6 +46,7 @@ import gunbc.heal_push_plan { } import gunbc.ci_failure_class { infra_retry_grep_alternation } import gunbc.ci_deploy_access { DeployAccess, ci_deploy_srv1_access } +import gunbc.fleet_main_revision { fleet_desired_ref_name } import gunbc.ci_deploy_target_host { gunbc_ci_deploy_srv1_runner_spec } import gunbc.merge_admission_produce { ci_repo_root_shell, @@ -503,6 +504,176 @@ fn gunbc_ci_floor_run(spec: CiSpec) -> String { gunbc_ci_floor_only_script(spec: spec) } +// Read-only srv1 tasks pre-apply observation (operator ruling 2026-08-09 on PR 8045; +// manager approval msg_03ec1f78): produces the ONE exact pre-apply receipt (effective +// TasksMax, pids.max, pids.current, ancestor limit, observer slot membership) via the +// typed read seams only. workflow_dispatch-gated at the job, so it never rides PR or +// push events; the printed return value IS the typed receipt. +data gunbc_ci_srv1_tasks_observation_entry: String = "dag/gunbc/srv1_tasks_preapply_observation.dag" +data gunbc_ci_srv1_tasks_observation_function: String = "observe_srv1_tasks_preapply_wet" + +// Main→desired-ref advancement (CONVERGE-0 loop leg, operator directive 2026-08-09): +// on a floor-green main push (needs: [ci] IS the floor-green admission — this job exists +// only downstream of the required check succeeding in the same run), advance +// refs/fleet/desired to the merged revision by CAS: --force-with-lease names the exact +// observed prior (gunbc.fleet_main_revision admit_fleet_desired_advance's from side), so +// a race with a concurrent advance refuses rather than overwriting; ref creation is the +// from=none arm. Hosts follow the ref; no host is contacted here. +data gunbc_ci_fleet_reach_probe_shell_emit_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.ci_spec", + decl_name: "gunbc_ci_fleet_reach_probe_invoke", + field: WholeDeclaration + } +} + +data gunbc_ci_fleet_reach_probe_shell_emit_dissolution_trigger: String = "🟡 dissolve-on: gunbc_ci_fleet_reach_probe_invoke — concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.host_reach_identity_probe probe_fleet_reach_wet plus a cat of the typed receipt file; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the probe step to orchestration emit over the typed gunbc.WitnessBin.Run route" + +fn gunbc_ci_fleet_reach_probe_invoke() -> String { + concat( + concat("# ", concat(gunbc_ci_fleet_reach_probe_shell_emit_dissolution_trigger, "\n")), + concat( + concat( + concat("ROOT=", ci_repo_root_shell(), "\n"), + gunbc_run_shell( + source_roots: witness_layer_roots, + entry: "dag/gunbc/host_reach_identity_probe.dag", + function: "probe_fleet_reach_wet", + rooted: true, + claim_run: false + ) + ), + "\ncat \"$ROOT/target/fleet-reach-probe-receipt.txt\"" + ) + ) +} + +// Fleet timer self-install bootstrap (operator verdict join 2, parent verification +// msg_8d84ae4f): the LIVE invocation wire for fleet_timer_self_install_wet — without +// this job the self-install route had zero callers, a disconnected mechanism. The job +// runs on a fleet runner and installs the timer on THE MACHINE IT RUNS ON (self-install: +// no remote host is contacted); dispatch it once per bootstrap host, thereafter the +// installed timer sustains itself. +data gunbc_ci_fleet_timer_install_shell_emit_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.ci_spec", + decl_name: "gunbc_ci_fleet_timer_install_invoke", + field: WholeDeclaration + } +} + +data gunbc_ci_fleet_timer_install_shell_emit_dissolution_trigger: String = "🟡 dissolve-on: gunbc_ci_fleet_timer_install_invoke — concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.fleet_converge_timer_install fleet_timer_self_install_wet; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the step to orchestration emit over the typed gunbc.WitnessBin.Run route" + +fn gunbc_ci_fleet_timer_install_invoke() -> String { + concat( + concat("# ", concat(gunbc_ci_fleet_timer_install_shell_emit_dissolution_trigger, "\n")), + concat( + concat("ROOT=", ci_repo_root_shell(), "\n"), + gunbc_run_shell( + source_roots: witness_layer_roots, + entry: "dag/gunbc/fleet_converge_timer_install.dag", + function: "fleet_timer_self_install_wet", + rooted: true, + claim_run: false + ) + ) + ) +} + +// Fleet convergence collection (operator verdict join 5, parent verification +// msg_8d84ae4f): the LIVE invocation wire for collect_fleet_convergence_wet — runs on +// main pushes downstream of the required ci check, from srv1 (inside the fleet, over +// the typed SSH probe seam), so the collector->fold composition executes on a real +// cadence and its collect receipt is printed into the job log. +data gunbc_ci_collect_fleet_convergence_shell_emit_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.ci_spec", + decl_name: "gunbc_ci_collect_fleet_convergence_invoke", + field: WholeDeclaration + } +} + +data gunbc_ci_collect_fleet_convergence_shell_emit_dissolution_trigger: String = "🟡 dissolve-on: gunbc_ci_collect_fleet_convergence_invoke — concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.fleet_receipt_collector collect_fleet_convergence_wet plus a cat of the typed collect receipt; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the step to orchestration emit over the typed gunbc.WitnessBin.Run route" + +fn gunbc_ci_collect_fleet_convergence_invoke() -> String { + concat( + concat("# ", concat(gunbc_ci_collect_fleet_convergence_shell_emit_dissolution_trigger, "\n")), + concat( + concat( + concat("ROOT=", ci_repo_root_shell(), "\n"), + gunbc_run_shell( + source_roots: witness_layer_roots, + entry: "dag/gunbc/fleet_receipt_collector.dag", + function: "collect_fleet_convergence_wet", + rooted: true, + claim_run: false + ) + ), + "\ncat \"$ROOT/target/fleet-converge-collect-receipt.txt\"" + ) + ) +} + +data gunbc_ci_advance_fleet_desired_shell_emit_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.ci_spec", + decl_name: "gunbc_ci_advance_fleet_desired_script", + field: WholeDeclaration + } +} + +data gunbc_ci_advance_fleet_desired_shell_emit_dissolution_trigger: String = "🟡 dissolve-on: gunbc_ci_advance_fleet_desired_script — concat-built foreign-executor (GitHub Actions run:) CAS ref advance (git ls-remote + push --force-with-lease onto gunbc.fleet_main_revision fleet_desired_ref_name); DISSOLVES WHEN the SCM ref-update realization lands as the typed git.Core CAS service op (membership-diff spine classify_change SCM-ref arm) invoked via host_effect_apply" + +fn gunbc_ci_advance_fleet_desired_script() -> String { + join([ + "# ", gunbc_ci_advance_fleet_desired_shell_emit_dissolution_trigger, "\n", + "set -eu\n", + "DESIRED_REF=", fleet_desired_ref_name, "\n", + "CURRENT=$(git ls-remote origin \"$DESIRED_REF\" | cut -f1)\n", + "if [ -z \"$CURRENT\" ]; then\n", + "git push origin \"$GITHUB_SHA:$DESIRED_REF\"\n", + "echo \"[fleet-desired] created $DESIRED_REF -> $GITHUB_SHA\"\n", + "else\n", + "git push --force-with-lease=\"$DESIRED_REF:$CURRENT\" origin \"$GITHUB_SHA:$DESIRED_REF\"\n", + "echo \"[fleet-desired] advanced $DESIRED_REF: $CURRENT -> $GITHUB_SHA (CAS expected-prior held)\"\n", + "fi\n", + ], "") +} + +data gunbc_ci_srv1_tasks_observation_shell_emit_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.ci_spec", + decl_name: "gunbc_ci_srv1_tasks_observation_invoke", + field: WholeDeclaration + } +} + +data gunbc_ci_srv1_tasks_observation_shell_emit_dissolution_trigger: String = "🟡 dissolve-on: gunbc_ci_srv1_tasks_observation_invoke — feature:task-count-measure-carrier sibling transport; concat-built foreign-executor (GitHub Actions run:) shell invoking gunbc run on gunbc.srv1_tasks_preapply_observation observe_srv1_tasks_preapply_wet plus a cat of the typed receipt file; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell→intent Phase 2) binds the observation step to orchestration emit over the typed gunbc.WitnessBin.Run route without a medium-as-string concat scaffold" + +fn gunbc_ci_srv1_tasks_observation_invoke() -> String { + concat( + concat("# ", concat(gunbc_ci_srv1_tasks_observation_shell_emit_dissolution_trigger, "\n")), + concat( + concat( + concat("ROOT=", ci_repo_root_shell(), "\n"), + gunbc_run_shell( + source_roots: witness_layer_roots, + entry: gunbc_ci_srv1_tasks_observation_entry, + function: gunbc_ci_srv1_tasks_observation_function, + rooted: true, + claim_run: false + ) + ), + "\ncat \"$ROOT/target/srv1-tasks-preapply-receipt.txt\"" + ) + ) +} + fn gunbc_ci_deploy_invoke(stage: DeployStage) -> String { concat( concat("ROOT=", ci_repo_root_shell(), "\n"), diff --git a/dag/gunbc/ci_workflow.dag b/dag/gunbc/ci_workflow.dag index 392a0b99494..b855e836679 100644 --- a/dag/gunbc/ci_workflow.dag +++ b/dag/gunbc/ci_workflow.dag @@ -20,7 +20,13 @@ import gunbc.ci_spec { ci_heal_author_commit_artifact_includes_hidden_files, gunbc_ci_heal_repo_local_git_config_invoke, ci_fmt_gate_line, + gunbc_ci_srv1_tasks_observation_invoke, + gunbc_ci_advance_fleet_desired_script, + gunbc_ci_fleet_reach_probe_invoke, + gunbc_ci_fleet_timer_install_invoke, + gunbc_ci_collect_fleet_convergence_invoke, } +import gunbc.ci_deploy_target_host { gunbc_ci_deploy_srv1_runner_spec } import gunbc.merge_admission_produce { ci_repo_root_shell } import v2.workflow.ci_retry_emit { ci_cargo_eagain_retry_script } import v2.workflow.ci_v1_compiler_tests_compile_gate_emit { @@ -32,6 +38,7 @@ import v2.workflow.ci_heal_revalidation_preflight_emit { } import gunbc.ci_workflow_expressions { ci_deploy_push_to_main_if, + ci_workflow_dispatch_only_if, ci_floor_runs_unless_cancelled_if, ci_floor_upstream_red_refusal_script } @@ -601,6 +608,201 @@ fn ci_deploy_jobs() -> List { map(gunbc_ci_deploy_stages, s => ci_deploy_job(stage: s)) } +// Read-only srv1 tasks pre-apply observation (operator ruling 2026-08-09 on PR 8045, +// manager approval msg_03ec1f78): workflow_dispatch-ONLY at the job condition, so it +// never rides pull_request or push events; pinned to the srv1 runner so the read runs +// INSIDE the observed universe; needs only the release-bins build (no floor, no deploy +// chain). The step is strictly read-only — the entry it invokes reaches only systemctl +// show and cgroupfs filesystem_read (gunbc.srv1_tasks_preapply_observation), and the +// printed return value is the typed Srv1TasksPreApplyReceipt. +fn ci_srv1_tasks_observation_step() -> Step { + RunStep { + name: Present { value: "Read-only srv1 tasks pre-apply observation (observe_srv1_tasks_preapply_wet)" }, + id: none, + run: gunbc_ci_srv1_tasks_observation_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: none, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +// Main→desired-ref advancement job (CONVERGE-0 loop): push-to-main only, downstream of +// the required ci check (needs: [ci] is the floor-green admission), contents:write so the +// checkout-persisted credential can push the CAS ref advance. No host is contacted; hosts +// follow the ref on their own timers. +// Live fleet reach probe (loop leg e): workflow_dispatch-only, srv1-pinned (the probe +// runs FROM inside the fleet, over the typed SSH probe seam where outcome is data). +fn ci_fleet_reach_probe_step() -> Step { + RunStep { + name: Present { value: "Live fleet reach probe (probe_fleet_reach_wet; outcomes are data)" }, + id: none, + run: gunbc_ci_fleet_reach_probe_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: none, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +fn ci_fleet_reach_probe_job() -> Job { + Job { + id: "probe_fleet_reach", + name: none, + runner: gunbc_ci_deploy_srv1_runner_spec(), + steps: concat( + ci_artifact_consumer_prelude_steps(), + [ci_release_bins_download_step(), ci_release_bins_unpack_verify_step(), ci_fleet_reach_probe_step()] + ), + needs: ["build"], + env: none, + outputs: none, + if_condition: Present { value: ci_workflow_dispatch_only_if }, + timeout_minutes: Present { value: gunbc_ci_deploy_job_backstop_timeout_minutes }, + continue_on_error: none, + concurrency: none, + permissions: none + } +} + +// Fleet timer self-install bootstrap (join 2 invocation wire): workflow_dispatch-only; +// runs on a fleet runner and installs the converge timer on the machine it runs on +// (self-install — no remote host contacted). Dispatch once per bootstrap host. +fn ci_fleet_timer_install_step() -> Step { + RunStep { + name: Present { value: "Fleet converge timer self-install (fleet_timer_self_install_wet; installs on THIS runner's host)" }, + id: none, + run: gunbc_ci_fleet_timer_install_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: none, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +fn ci_fleet_timer_install_job() -> Job { + Job { + id: "install_fleet_converge_timer", + name: none, + runner: gunbc_ci_selected_runner_spec(), + steps: concat( + ci_artifact_consumer_prelude_steps(), + [ci_release_bins_download_step(), ci_release_bins_unpack_verify_step(), ci_fleet_timer_install_step()] + ), + needs: ["build"], + env: none, + outputs: none, + if_condition: Present { value: ci_workflow_dispatch_only_if }, + timeout_minutes: Present { value: gunbc_ci_deploy_job_backstop_timeout_minutes }, + continue_on_error: none, + concurrency: none, + permissions: none + } +} + +// Fleet convergence collection (join 5 invocation wire): push-to-main downstream of the +// required ci check, srv1-pinned (the collector reads published receipts over the typed +// SSH seam FROM inside the fleet) — the collector->fold composition executes per merged +// revision and prints the collect receipt + fold verdict. +fn ci_collect_fleet_convergence_step() -> Step { + RunStep { + name: Present { value: "Collect fleet converge receipts and fold the fleet verdict (collect_fleet_convergence_wet)" }, + id: none, + run: gunbc_ci_collect_fleet_convergence_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: none, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +fn ci_collect_fleet_convergence_job() -> Job { + Job { + id: "collect_fleet_convergence", + name: none, + runner: gunbc_ci_deploy_srv1_runner_spec(), + steps: concat( + ci_artifact_consumer_prelude_steps(), + [ci_release_bins_download_step(), ci_release_bins_unpack_verify_step(), ci_collect_fleet_convergence_step()] + ), + needs: ["ci"], + env: none, + outputs: none, + if_condition: Present { value: ci_deploy_push_to_main_if }, + timeout_minutes: Present { value: gunbc_ci_deploy_job_backstop_timeout_minutes }, + continue_on_error: none, + concurrency: none, + permissions: none + } +} + +fn ci_advance_fleet_desired_step() -> Step { + RunStep { + name: Present { value: "Advance refs/fleet/desired to the merged revision (CAS, floor-green only)" }, + id: none, + run: gunbc_ci_advance_fleet_desired_script(), + shell: none, + env: none, + working_directory: none, + if_condition: none, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +fn ci_advance_fleet_desired_job() -> Job { + Job { + id: "advance_fleet_desired", + name: none, + runner: gunbc_ci_selected_runner_spec(), + steps: concat(ci_artifact_consumer_prelude_steps(), [ci_advance_fleet_desired_step()]), + needs: ["ci"], + env: none, + outputs: none, + if_condition: Present { value: ci_deploy_push_to_main_if }, + timeout_minutes: Present { value: gunbc_ci_deploy_job_backstop_timeout_minutes }, + continue_on_error: none, + concurrency: none, + permissions: Present { + value: WorkflowPermissions { + contents: Present { value: PermWrite }, + pull_requests: none, + issues: none, + actions: none, + id_token: none + } + } + } +} + +fn ci_srv1_tasks_observation_job() -> Job { + Job { + id: "observe_srv1_tasks_preapply", + name: none, + runner: gunbc_ci_deploy_srv1_runner_spec(), + steps: concat( + ci_artifact_consumer_prelude_steps(), + [ci_release_bins_download_step(), ci_release_bins_unpack_verify_step(), ci_srv1_tasks_observation_step()] + ), + needs: ["build"], + env: none, + outputs: none, + if_condition: Present { value: ci_workflow_dispatch_only_if }, + timeout_minutes: Present { value: gunbc_ci_deploy_job_backstop_timeout_minutes }, + continue_on_error: none, + concurrency: none, + permissions: none + } +} + fn gunbc_ci_heal_job_backstop_timeout_minutes() -> Int { gunbc_ci_artifact_transfer_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_aux_step_timeout_minutes + gunbc_ci_artifact_transfer_step_timeout_minutes + gunbc_ci_prelude_allowance_minutes } @@ -830,5 +1032,5 @@ data ci_workflow: Workflow = { id_token: none } }, - jobs: concat([ci_build_job(), ci_regen_job(), ci_job()], concat(ci_deploy_jobs(), [ci_heal_generated_artifacts_job()])) + jobs: concat([ci_build_job(), ci_regen_job(), ci_job()], concat(ci_deploy_jobs(), [ci_heal_generated_artifacts_job(), ci_srv1_tasks_observation_job(), ci_fleet_reach_probe_job(), ci_fleet_timer_install_job(), ci_collect_fleet_convergence_job(), ci_advance_fleet_desired_job()])) } diff --git a/dag/gunbc/ci_workflow_expressions.dag b/dag/gunbc/ci_workflow_expressions.dag index e020c33844a..663aac7e247 100644 --- a/dag/gunbc/ci_workflow_expressions.dag +++ b/dag/gunbc/ci_workflow_expressions.dag @@ -82,6 +82,8 @@ data ci_workflow_expression_templates: List = [ data ci_deploy_push_to_main_if: String = "github.ref == 'refs/heads/main' && github.event_name == 'push'" +data ci_workflow_dispatch_only_if: String = "github.event_name == 'workflow_dispatch'" + data ci_floor_runs_unless_cancelled_if: String = "!cancelled()" data ci_floor_upstream_red_refusal_script: String = "set -e\nBUILD='${{ needs.build.result }}'\nREGEN='${{ needs.regen.result }}'\nHEAL='${{ needs.heal_generated_artifacts.result }}'\nif [ \"$BUILD\" != \"success\" ] || [ \"$REGEN\" != \"success\" ] || [ \"$HEAL\" = \"failure\" ]; then\necho \"::error::FloorUpstreamAlreadyRed build=$BUILD regen=$REGEN heal=$HEAL — an upstream job already established this branch is red, so the floor refuses before spending its budget. The located cause is that job's own diagnostic; this step adds no verdict of its own.\"\nexit 1\nfi\necho \"[floor-preflight] upstream clean: build=$BUILD regen=$REGEN heal=$HEAL\"" diff --git a/dag/gunbc/fleet_converge_cli.dag b/dag/gunbc/fleet_converge_cli.dag index ef68dcca572..91920bf4af6 100644 --- a/dag/gunbc/fleet_converge_cli.dag +++ b/dag/gunbc/fleet_converge_cli.dag @@ -1,5 +1,20 @@ module gunbc.fleet_converge_cli +import extdeps.filesystem.filesystem_io { Filesystem } +import std.algebra { trim } + +import extdeps.linux.procfs { ProcSysKernelHostname, procfs_path_literal } +import gunbc.fleet_converge_apply { converge_apply_for_host } +import gunbc.host_layout { fleet_converge_receipt_dir, srv1_gunbc_repo_root } +import gunbc.fleet_desired_observe { + FleetRevisionStanding, + RevisionConverged, + RevisionDrifted, + RevisionUnobserved, + fleet_revision_standing, +} +import extdeps.git.object_store { git_object_id_wire_hex } + data gunbc_converge_cli_stage0_receiver_disposition: Disposition = Terminal { reason: "stage0's `gunbc converge --host ` argv dispatch (src/v1/stage0/src/cli_run.rs::handle_converge) is the CLI transport handler for this .dag interface (DESIGN §3 transport-is-a-handler): argv parse -> resolve_entry_graph_shared -> run_in_context_with_args(converge_cli_output) -> stdout/exit-code projection, nothing else. All interpretation, the byte-locked receipt grammar, and typed refusal causes are modeled here (gunbc.fleet_converge_cli) and in gunbc.host_converge/gunbc.fleet_converge_apply; the Rust side carries no converge logic and is not scaffold pending dissolution." @@ -8,6 +23,7 @@ data gunbc_converge_cli_stage0_receiver_disposition: Disposition = Terminal { type ConvergeCliRefusalCause = PolicyUnsound { reason: String } | UnknownHost { host: String } + | SparkPreEnrollment { host: String } | ComputeInventoryAbsent { host: String } | KnobFrontier { reason: String } @@ -23,6 +39,7 @@ fn converge_cli_refusal_message(cause: ConvergeCliRefusalCause) -> String { match cause { PolicyUnsound { reason: why } => concat("gunbc converge: fleet policy unsound — ", why) UnknownHost { host: h } => converge_cli_unknown_host_reason(host: h) + SparkPreEnrollment { host: h } => concat(concat("gunbc converge: ", h), " is a Spark pre-enrollment host — no runner slots or host layout exist yet (SPARK-SERVE-0 owns the repair); reach is probed at its modeled endpoint, and no converge route exists until enrollment") ComputeInventoryAbsent { host: h } => converge_cli_compute_inventory_absent_reason(host: h) KnobFrontier { reason: why } => why } @@ -92,6 +109,36 @@ fn converge_cli_routes_to_slice1(host: String) -> Bool { host == (operator_host_srv1 as String) || host == (operator_host_srv3 as String) } +// APPLICABILITY, total over the fleet (operator verdict on PR 8045, join 3): every +// enrolled host identity has exactly one arm — a real converge route, or a TYPED +// frontier refusal naming what unblocks it. "Applicable" = a slice1 circuit exists for +// the host's knobs today. srv2/srv4 are knob-frontier (whole-host typed refusal, the +// pre-existing arm, dissolving as slice1 realizes their knobs); srv5/srv6 are +// PRE-ENROLLMENT (no runner slots, no host layout yet — SPARK-SERVE-0 owns the repair; +// their reach is probed at the modeled endpoint, gunbc.fleet_reach_endpoint) — a state +// DISTINCT from UnknownHost, which now means only an identity outside the fleet. +type FleetConvergeApplicability + = ConvergeSlice1Route + | ConvergeKnobFrontier + | ConvergeSparkPreEnrollment + | ConvergeOutsideFleet + +fn fleet_converge_applicability(host: String) -> FleetConvergeApplicability { + if converge_cli_routes_to_slice1(host: host) { + ConvergeSlice1Route + } else { + if host == (operator_host_srv2 as String) || host == (operator_host_srv4 as String) { + ConvergeKnobFrontier + } else { + if host == (operator_host_srv5 as String) || host == (operator_host_srv6 as String) { + ConvergeSparkPreEnrollment + } else { + ConvergeOutsideFleet + } + } + } +} + fn converge_cli_slice1_srv3_receipt() -> ConvergeCliReceipt { let result = host_converge_slice1_converge_srv3() if host_converge_slice1_host_result_all_valid(result: result) { @@ -152,6 +199,32 @@ fn converge_cli_slice1_srv1_receipt() -> ConvergeCliReceipt { }, } } + // Unreachable today by construction: host_converge_slice1_converge_loop folds the + // tasks outcome into the combined Converged arm (or Refused) and never returns + // TasksConverged at top level. Handled exhaustively anyway — do not wire new callers + // through this arm. + HostConvergeSlice1TasksConverged { applied_knobs: n, receipt: r } => + if host_converge_slice1_tasks_receipt_is_valid(receipt: r) { + ConvergeCliReceipt { + host: operator_host_srv1 as String, + applied: n, + drifted: 0, + verdict: VerdictConverged, + reason: none, + } + } else { + ConvergeCliReceipt { + host: operator_host_srv1 as String, + applied: 0, + drifted: 0, + verdict: Absent, + reason: Present { + value: KnobFrontier { + reason: "host_converge_slice1: tasks receipt validity gate failed (read_back_converged && reapply_noop && verdict=converged)", + }, + }, + } + } HostConvergeSlice1Refused { reason: why } => ConvergeCliReceipt { host: operator_host_srv1 as String, @@ -253,19 +326,139 @@ fn converge_cli_verify_run(host: String) -> ConvergeCliReceipt { } } +// Receipt publication (CONVERGE-0 loop leg): on the SELF route — the per-host timer's +// route, where the deployment guarantees the host layout exists — the receipt line is +// WRITTEN to the declared receipts path before being printed, so a fold-side observation +// job can read the host's last converge standing without a new commit. Publication +// failure on that route FAILS CLOSED (converged=false with a typed reason): the loop's +// product is the published receipt, so an unpublished converge must not report success. +// Non-self routes (operator/CI supervised runs against a named host) are unchanged — +// no write is attempted, because nothing guarantees the receipts directory there. +fn fleet_converge_receipt_path(host: String) -> String { + join([fleet_converge_receipt_dir, "/", host, "-converge-receipt.txt"], "") +} + +fn converge_cli_publish(out: ConvergeCliOutput, resolved: String, revision_line: String) -> ConvergeCliOutput { + let w = Filesystem.Write( + path: fleet_converge_receipt_path(host: resolved), + content: join([out.line, "\n", revision_line, "\n"], ""), + ) + if w.success { + out + } else { + ConvergeCliOutput { + line: out.line, + converged: false, + reason: Present { value: concat("gunbc converge: receipt publication failed — ", w.error) }, + } + } +} + +// Operator verdict join 1 (2026-08-09): the self route is where a host READS the +// desired ref. The knob receipt alone cannot claim convergence while the deployed tree +// is not at the admitted revision — revision drift or an unobserved ref demotes the +// output to not-converged with the typed cause, and the published receipt carries the +// revision cell as its own line so the collector (join 5) sees BOTH cells. Drifted +// names both sides (desired AND local): tree-sync is the catch-up realization, and its +// safe form is a compare-and-swap needing the prior. The repo path is the ONE +// host_layout convention row (byte-identical across enrolled hosts per its note). +fn converge_cli_revision_gate(out: ConvergeCliOutput, standing: FleetRevisionStanding) -> ConvergeCliOutput { + match standing { + RevisionConverged { revision: _ } => out + RevisionDrifted { desired: d, local: l } => + ConvergeCliOutput { + line: out.line, + converged: false, + reason: Present { + value: join([ + "gunbc converge: revision drift — desired ", + git_object_id_wire_hex(oid: d) as String, + ", local tree at ", + git_object_id_wire_hex(oid: l) as String, + "; tree-sync is the catch-up realization", + ], ""), + }, + } + RevisionUnobserved { cause: c } => + ConvergeCliOutput { + line: out.line, + converged: false, + reason: Present { value: concat("gunbc converge: desired revision unobserved — ", c) }, + } + } +} + +fn converge_cli_revision_line(standing: FleetRevisionStanding) -> String { + match standing { + RevisionConverged { revision: r } => + concat("converge-revision verdict=converged revision=", git_object_id_wire_hex(oid: r) as String) + RevisionDrifted { desired: d, local: l } => + join([ + "converge-revision verdict=drifted desired=", + git_object_id_wire_hex(oid: d) as String, + " local=", + git_object_id_wire_hex(oid: l) as String, + ], "") + RevisionUnobserved { cause: c } => concat("converge-revision verdict=unobserved cause=", c) + } +} + fn converge_cli_output(host: String) -> ConvergeCliOutput { - converge_cli_output_with_verify(host: host, verify_only: false) + let out = converge_cli_output_with_verify(host: host, verify_only: false) + if host == "self" { + let standing = fleet_revision_standing(repo: srv1_gunbc_repo_root) + converge_cli_publish( + out: converge_cli_revision_gate(out: out, standing: standing), + resolved: converge_cli_resolve_self(host: host), + revision_line: converge_cli_revision_line(standing: standing), + ) + } else { + out + } } fn converge_cli_verify_output(host: String) -> ConvergeCliOutput { converge_cli_output_with_verify(host: host, verify_only: true) } +// MACHINE-ID HAZARD, checked (msg_1c4e0bce): self identity here derives from the +// kernel hostname — NEVER /etc/machine-id, which both Sparks share from one baked DGX +// image — machine-id is the duplicated non-anchor. Their SSH HOST KEYS ARE DISTINCT +// (bold-eagle measured all three algorithms differing on both hosts, 2026-08-09 — an +// earlier revision of this note claimed identical factory keys, which was FALSE), so +// host-key IS a valid identity anchor for srv5/srv6 today and the fingerprint +// enrollment registry can proceed against the two distinct keys with no repair +// prerequisite; do not plan around host-key regeneration. Hostname is itself only +// sound AFTER P0:host-identity-converge has set it; before that, two machines can +// resolve one name — which is why the fleet fold classifies duplicate receipts for one +// identity as a named ambiguity (never first-wins), so a collision surfaces as +// NotConverged instead of one machine's receipt silently satisfying the other. +// --host self (CONVERGE-0 loop leg): the per-host timer unit converges SELF, so the +// literal "self" resolves to the machine's own kernel-reported short hostname before +// routing — the ONE observation a host can always make about itself, read at the cited +// procfs authority, never an env var or a config file a deploy could skew. Any other +// value routes unchanged (the operator/CI form). An unresolvable read falls through as +// the literal, landing in UnknownHost — a typed refusal, never a fabricated identity. +fn converge_cli_resolve_self(host: String) -> String { + if host == "self" { + let raw = filesystem_read(path: procfs_path_literal(p: ProcSysKernelHostname) as String).content + let line = raw.trim() + if line == "" { + host + } else { + line + } + } else { + host + } +} + fn converge_cli_output_with_verify(host: String, verify_only: Bool) -> ConvergeCliOutput { + let resolved = converge_cli_resolve_self(host: host) let r = if verify_only { - converge_cli_verify_run(host: host) + converge_cli_verify_run(host: resolved) } else { - converge_cli_run(host: host) + converge_cli_run(host: resolved) } ConvergeCliOutput { line: converge_cli_receipt_line(r: r), @@ -288,20 +481,28 @@ fn converge_cli_run(host: String) -> ConvergeCliReceipt { reason: Present { value: PolicyUnsound { reason: why } }, } ConvergePolicyDerived { hosts: hosts } => - if converge_cli_routes_to_slice1(host: host) { - converge_cli_slice1_receipt(host: host) - } else { - match host_converge_for_identity(hosts: hosts, identity: host as HostIdentity) { - Present { value: hc } => converge_cli_receipt_for_host(host_converge: hc) - Absent => + match fleet_converge_applicability(host: host) { + ConvergeSlice1Route => converge_cli_slice1_receipt(host: host) + ConvergeSparkPreEnrollment => ConvergeCliReceipt { host: host, applied: 0, drifted: 0, verdict: Absent, - reason: Present { value: UnknownHost { host: host } }, + reason: Present { value: SparkPreEnrollment { host: host } }, + } + _ => + match host_converge_for_identity(hosts: hosts, identity: host as HostIdentity) { + Present { value: hc } => converge_cli_receipt_for_host(host_converge: hc) + Absent => + ConvergeCliReceipt { + host: host, + applied: 0, + drifted: 0, + verdict: Absent, + reason: Present { value: UnknownHost { host: host } }, + } } - } } } } diff --git a/dag/gunbc/fleet_converge_timer.dag b/dag/gunbc/fleet_converge_timer.dag new file mode 100644 index 00000000000..e310b7d3e37 --- /dev/null +++ b/dag/gunbc/fleet_converge_timer.dag @@ -0,0 +1,96 @@ +module gunbc.fleet_converge_timer + +import std.types { String, NonEmptyStr, Bool, Int } +import std.nat { Nat } +import std.measure { Second, second, second_count } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } + +// Arc item 4 (timer / anti-entropy, operator directive 2026-08-08): the steady-state +// enforcement loop is a PER-HOST systemd timer executing the converge circuit against +// the admitted desired revision (gunbc.fleet_main_revision) — never a central cron, an +// SSH star, or a GHA event as steady state. Every firing emits a receipt; a host that +// was offline across a desired-state change converges on return because the timer's +// next firing reads the admitted ref (catch-up is a property of following the ref, plus +// Persistent= for missed calendar firings). This module is the SINGLE authority for the +// unit names, cadence, and unit CONTENT — the emitted files derive from these rows, so +// a hand-edited unit is drift against this authority, not a second truth. +// +// Timer shape is the design doc's Option A (OnCalendar + Persistent=true + jitter): +// Persistent only functions with OnCalendar (systemd ignores it on monotonic timers), +// and calendar catch-up is exactly the offline-host property the arc requires. +data fleet_converge_service_unit_name: String = "ctrl-fleet-converge.service" +data fleet_converge_timer_unit_name: String = "ctrl-fleet-converge.timer" +data fleet_converge_timer_on_calendar: String = "*:0/15" +data fleet_converge_timer_randomized_delay: Second = second(count: 90) +data fleet_converge_binary_path: String = "/opt/gunbc/bin/gunbc" +data fleet_converge_env_file_path: String = "/etc/gunbc/fleet-converge.env" + +// String-built medium: unit INI text is rendered by join/concat rather than as a Node +// emitted through grammar rows — the medium-as-string shape the shell→intent lane's +// containment exists to stop spreading. Tolerated only while this module is tests-only +// (no production emitter consumes it); the typed row below carries the dissolution. +data fleet_converge_unit_text_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.live_deploy.emit", + decl_name: "live_deploy_emit_shell_dissolution_trigger", + field: WholeDeclaration, + } +} + +// 🟡 dissolve-on: fleet-converge unit-body single render (review 50806) — the unit BODY +// is currently authored TWICE: these content fns (consumed by the srv2–srv4 self-install +// route) and gunbc.live_deploy.emit emit_fleet_converge_{service,timer}_unit_doc (the +// srv1 membership route, which adds After/Wants=network-online.target and +// WorkingDirectory=, omits EnvironmentFile=, and derives only cadence/jitter from this +// authority). A §3 parallel representation across the two consumers — srv1 and srv2–4 +// do NOT receive byte-identical units, and an edit to either producer can drift them. +// Not merged blind here because the emit side's bytes are golden-pinned and deployed on +// srv1 (a silent byte churn of a live unit is the wrong side of the trade mid-arc). +// DISSOLVES WHEN one render authority (these content fns, Doc-wrapped) feeds both +// routes and the emit goldens re-pin from it. +data fleet_converge_unit_body_fork_disposition: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.live_deploy.emit", + decl_name: "emit_fleet_converge_service_unit_doc", + field: WholeDeclaration, + } +} + +fn fleet_converge_service_content(host_arg: String) -> String { + join([ + "[Unit]\n", + "Description=gunbc fleet converge (self)\n", + "\n", + "[Service]\n", + "Type=oneshot\n", + "User=root\n", + "EnvironmentFile=-", fleet_converge_env_file_path, "\n", + "ExecStart=", fleet_converge_binary_path, " converge --host ", host_arg, "\n", + ], "") +} + +fn fleet_converge_timer_content() -> String { + join([ + "[Unit]\n", + "Description=gunbc fleet converge cadence\n", + "\n", + "[Timer]\n", + "OnCalendar=", fleet_converge_timer_on_calendar, "\n", + "Persistent=true\n", + "RandomizedDelaySec=", to_string(second_count(s: fleet_converge_timer_randomized_delay)), "\n", + "Unit=", fleet_converge_service_unit_name, "\n", + "\n", + "[Install]\n", + "WantedBy=timers.target\n", + ], "") +} + +// The service converges SELF only: the unit text never names a remote host, so the +// central-actor-converges-remote-host shape is unwritable from this authority (the +// design doc's invariant: CI never runs converge against a remote N). +fn fleet_converge_service_content_self() -> String { + fleet_converge_service_content(host_arg: "self") +} diff --git a/dag/gunbc/fleet_converge_timer_install.dag b/dag/gunbc/fleet_converge_timer_install.dag new file mode 100644 index 00000000000..e9fbce7b133 --- /dev/null +++ b/dag/gunbc/fleet_converge_timer_install.dag @@ -0,0 +1,113 @@ +module gunbc.fleet_converge_timer_install + +import std.types { String, Bool, List, NonEmptyStr } +import std.process { ProcessExit, ExitSuccess, ExitFailure, exit_failure } +import std.resources { Network } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.exec.command { ArgvCommand, LocalExec } +import gunbc.command_runner { run_shell_commands } +import gunbc.fleet_converge_timer { + fleet_converge_service_unit_name, + fleet_converge_timer_unit_name, + fleet_converge_service_content_self, + fleet_converge_timer_content, +} +import gunbc.fleet_converge_cli { + FleetConvergeApplicability, + ConvergeSlice1Route, + ConvergeKnobFrontier, + ConvergeSparkPreEnrollment, + ConvergeOutsideFleet, + fleet_converge_applicability, +} + +// Operator verdict join 2 (2026-08-09): the timer previously installed through +// deployment_spec_srv1 ONLY — srv2–srv6 had no route, so "per-host timer" was a +// one-host fact wearing a fleet name. This module makes the route TOTAL over the +// applicability join 3 defines: srv1 keeps the live_deploy membership route it already +// has (one authority for that host's owned artifacts — a second installer there would +// fork ownership); every other ENROLLED host (srv2–srv4) gets the SELF-INSTALL route — +// the converge binary on the host writes both unit files from the ONE +// gunbc.fleet_converge_timer authority and enables the timer locally (bootstrap firing +// arrives via that host's CI runner; thereafter the timer sustains itself); srv5/srv6 +// are the typed pre-enrollment arm (known fleet identities awaiting SPARK-SERVE-0 — +// never silently absent); anything else is outside the fleet. No arm is a central +// SSH star: nothing here converges a REMOTE host. +type FleetTimerRoute + = TimerViaLiveDeployMembership { host: String } + | TimerViaSelfInstall { host: String } + | TimerPreEnrollment { host: String } + | TimerOutsideFleet { host: String } + +fn fleet_timer_install_route(host: String) -> FleetTimerRoute { + match fleet_converge_applicability(host: host) { + ConvergeSlice1Route => + if host == "srv1" { + TimerViaLiveDeployMembership { host: host } + } else { + TimerViaSelfInstall { host: host } + } + ConvergeKnobFrontier => TimerViaSelfInstall { host: host } + ConvergeSparkPreEnrollment => TimerPreEnrollment { host: host } + ConvergeOutsideFleet => TimerOutsideFleet { host: host } + } +} + +data fleet_timer_stage_dir: String = "/tmp/gunbc-fleet-timer-stage" + +fn fleet_timer_staged_path(unit_name: String) -> String { + join([fleet_timer_stage_dir, "/", unit_name], "") +} + +fn fleet_timer_installed_path(unit_name: String) -> String { + concat("/etc/systemd/system/", unit_name) +} + +// The install argv plan, derived — unit names come from the single timer authority, so +// a renamed unit cannot leave a stale plan behind. `install -m 0644` then daemon-reload +// then enable --now; the final step is the INDEPENDENT POST-EFFECT READBACK (`systemctl +// is-enabled`), because an apply's success criterion is never its exit code. +fn fleet_timer_install_commands() -> List { + [ + ArgvCommand { argv: ["sudo", "-n", "install", "-m", "0644", fleet_timer_staged_path(unit_name: fleet_converge_service_unit_name), fleet_timer_installed_path(unit_name: fleet_converge_service_unit_name)] }, + ArgvCommand { argv: ["sudo", "-n", "install", "-m", "0644", fleet_timer_staged_path(unit_name: fleet_converge_timer_unit_name), fleet_timer_installed_path(unit_name: fleet_converge_timer_unit_name)] }, + ArgvCommand { argv: ["sudo", "-n", "systemctl", "daemon-reload"] }, + ArgvCommand { argv: ["sudo", "-n", "systemctl", "enable", "--now", fleet_converge_timer_unit_name] }, + ArgvCommand { argv: ["systemctl", "is-enabled", fleet_converge_timer_unit_name] }, + ] +} + +// Wet self-install: stage both unit texts from the authority, then run the derived argv +// plan locally. Fails closed on the first refusing step (run_shell_commands +// short-circuits); a staging write failure refuses before any systemd state changes. +func fleet_timer_self_install_wet() -> ProcessExit + uses net: Network +{ + let mk = run_shell_commands( + commands: [ArgvCommand { argv: ["mkdir", "-p", fleet_timer_stage_dir] }], + transport: LocalExec, + ) + match mk { + ExitFailure { code: _, reason: why } => exit_failure(reason: concat("fleet_timer_self_install: stage dir refused: ", why)) + ExitSuccess => { + let w_service = Filesystem.Write( + path: fleet_timer_staged_path(unit_name: fleet_converge_service_unit_name), + content: fleet_converge_service_content_self(), + ) + let w_timer = Filesystem.Write( + path: fleet_timer_staged_path(unit_name: fleet_converge_timer_unit_name), + content: fleet_converge_timer_content(), + ) + if w_service.success && w_timer.success { + run_shell_commands(commands: fleet_timer_install_commands(), transport: LocalExec) + } else { + exit_failure(reason: join([ + "fleet_timer_self_install: staging write refused: ", + w_service.error, + " ", + w_timer.error, + ], "")) + } + } + } +} diff --git a/dag/gunbc/fleet_convergence_verdict.dag b/dag/gunbc/fleet_convergence_verdict.dag new file mode 100644 index 00000000000..1cfd3c63ad9 --- /dev/null +++ b/dag/gunbc/fleet_convergence_verdict.dag @@ -0,0 +1,161 @@ +module gunbc.fleet_convergence_verdict + +import std.types { String, NonEmptyStr, Bool, Int, List } +import product.placement_supply { HostIdentity } +import std.upsert_decision { + ObservationVerdict, + Converged, + Absent, + Drifted, + Conflict, + Inaccessible, + UnknownRefused, +} +import gunbc.host_phase_status { + HostPhaseCellStatus, + host_phase_cells_for_enrollment, + host_phase_cell_verdict, +} +import gunbc.host_assimilation_program { HostEnrollmentIntent } +import gunbc.fleet_converge_cli { ConvergeCliReceipt, converge_cli_receipt_converged } + +// THE fleet-atomic-convergence-verdict fold (operator directive 2026-08-08): ONE verdict +// over the whole fleet, joining the knob-grain converge receipts (the slice1 circuit's +// ConvergeCliReceipt) with the lifecycle-phase cells (the observe_host_phase matrix). +// The load-bearing law is fail-closed at the CELL: a required cell nobody observed +// PREVENTS fleet convergence — it never defaults green — and the not-converged causes +// keep their remedies distinct (unobserved: build/run the producer; unreachable: fix +// reach; refused/conflict: resolve the contradiction; drifted: run the converge). The +// current defect this fold makes countable rather than prose: the fleet phase matrix has +// exactly ONE real producer cell today (srv3, prefix:os-install-actuated), so every +// other required cell reads unobserved and the fleet verdict is honestly NOT converged +// until producers land receipts — that number going down IS the arc's progress. +type RequiredCellStanding + = CellConverged { host: HostIdentity, phase_label: String } + | CellUnobserved { host: HostIdentity, phase_label: String } + | CellUnreachable { host: HostIdentity, phase_label: String } + | CellDrifted { host: HostIdentity, phase_label: String } + | CellConflict { host: HostIdentity, phase_label: String } + +// Projection keeps the full six-arm ObservationVerdict fidelity: Absent and +// UnknownRefused are both "nobody observed this" (no producer, or a producer with no +// committed observation) and land in CellUnobserved; Inaccessible is a producer that +// ASKED and could not reach — a different remedy, kept distinct; Conflict is its own arm +// (a wrong-machine or contradictory observation is neither drift nor unreachability). +fn required_cell_standing(c: HostPhaseCellStatus) -> RequiredCellStanding { + let host = c.cell.host + let label = c.cell.phase_label + match host_phase_cell_verdict(c: c) { + Converged => CellConverged { host: host, phase_label: label } + Absent => CellUnobserved { host: host, phase_label: label } + UnknownRefused => CellUnobserved { host: host, phase_label: label } + Inaccessible => CellUnreachable { host: host, phase_label: label } + Drifted => CellDrifted { host: host, phase_label: label } + Conflict => CellConflict { host: host, phase_label: label } + } +} + +fn cell_standing_is_converged(s: RequiredCellStanding) -> Bool { + match s { + CellConverged { host: _, phase_label: _ } => true + CellUnobserved { host: _, phase_label: _ } => false + CellUnreachable { host: _, phase_label: _ } => false + CellDrifted { host: _, phase_label: _ } => false + CellConflict { host: _, phase_label: _ } => false + } +} + +// A host's knob-grain standing joins by IDENTITY with its lifecycle cells. A host with +// no converge receipt at all is unobserved at knob grain — the same fail-closed arm as +// an unobserved cell, never a default green. +type HostKnobStanding + = KnobConverged { host: HostIdentity } + | KnobNotConverged { host: HostIdentity, reason: String } + | KnobUnobserved { host: HostIdentity } + +// Duplicate receipts for ONE host identity are a CONTRADICTION to classify, never a +// pick (the machine-id hazard, msg_1c4e0bce: both Sparks share one /etc/machine-id and +// shipped identical host keys, so two machines CAN claim one identity until +// host-identity-converge runs — first-wins would render that collision a verdict). +// 0 receipts -> Unobserved; exactly 1 -> judged on its own validity; >1 -> NotConverged +// with the ambiguity named, whatever the duplicates claim. +fn host_knob_standing(host: HostIdentity, receipts: List) -> HostKnobStanding { + let mine = filter(receipts, r => r.host == (host as String)) + if length(mine) == 0 { + KnobUnobserved { host: host } + } else { + if length(mine) > 1 { + KnobNotConverged { host: host, reason: "ambiguous: multiple converge receipts claim this one host identity — a collision to investigate, never a pick" } + } else { + let r = mine.first() + if converge_cli_receipt_converged(r: r) { + KnobConverged { host: host } + } else { + KnobNotConverged { host: host, reason: "converge receipt present but not converged" } + } + } + } +} + +fn host_knob_standing_is_converged(s: HostKnobStanding) -> Bool { + match s { + KnobConverged { host: _ } => true + KnobNotConverged { host: _, reason: _ } => false + KnobUnobserved { host: _ } => false + } +} + +type HostConvergenceStanding { + host: HostIdentity + knob: HostKnobStanding + cells: List +} + +fn host_convergence_standing( + intent: HostEnrollmentIntent, + receipts: List, +) -> HostConvergenceStanding { + HostConvergenceStanding { + host: intent.host, + knob: host_knob_standing(host: intent.host, receipts: receipts), + cells: map(host_phase_cells_for_enrollment(intent: intent), c => required_cell_standing(c: c)), + } +} + +fn host_standing_is_converged(s: HostConvergenceStanding) -> Bool { + host_knob_standing_is_converged(s: s.knob) + && fold(s.cells, init: true, f: fn(acc, c) { acc && cell_standing_is_converged(s: c) }) +} + +type FleetConvergenceVerdict + = FleetConverged { host_count: Int } + | FleetNotConverged { standings: List } + +// The atomic fold: converged only when EVERY enrolled host converges at BOTH grains. +// The not-converged arm carries every host's full standing rather than a count, so the +// blocking cells are readable without re-deriving anything — the verdict IS the work +// list. An empty enrollment refuses convergence (a fleet of zero hosts is an observation +// gap, not a converged fleet). +fn fleet_convergence_verdict( + enrollments: List, + receipts: List, +) -> FleetConvergenceVerdict { + let standings = map(enrollments, e => host_convergence_standing(intent: e, receipts: receipts)) + if length(standings) > 0 + && fold(standings, init: true, f: fn(acc, s) { acc && host_standing_is_converged(s: s) }) { + FleetConverged { host_count: length(standings) } + } else { + FleetNotConverged { standings: standings } + } +} + +fn fleet_unobserved_cell_count(standings: List) -> Int { + fold(standings, init: 0, f: fn(acc, s) { + acc + fold(s.cells, init: 0, f: fn(a, c) { + match c { + CellUnobserved { host: _, phase_label: _ } => a + 1 + _ => a + } + }) + }) +} diff --git a/dag/gunbc/fleet_desired_observe.dag b/dag/gunbc/fleet_desired_observe.dag new file mode 100644 index 00000000000..00ae931c184 --- /dev/null +++ b/dag/gunbc/fleet_desired_observe.dag @@ -0,0 +1,101 @@ +module gunbc.fleet_desired_observe + +import std.types { String, Bool, FilePath, GitRef } +import std.algebra { trim } +import extdeps.git +import extdeps.git.object_store { + GitObjectId, + git_object_id_from_untagged_hex, + git_object_id_eq, +} +import gunbc.fleet_main_revision { + fleet_desired_ref_name, + FleetDesiredRevision, + FleetDesiredAdmitted, + FleetDesiredUnobserved, + converge_target_revision, +} + +// Operator verdict join 1 (2026-08-09): the LIVE half of gunbc.fleet_main_revision. +// That module models what the desired ref MEANS; until this module landed, nothing on +// any host actually READ refs/fleet/desired — converge_target_revision had no live +// caller, so the anti-entropy property the ref buys (a returning host converges from the +// admitted ref without a new commit) existed only as vocabulary. This module is the +// observation edge: ls-remote the exact ref through the typed git.Core service op +// (runtime-present site — typed service op, never emitted shell, per the shell→intent +// routing), decode the advertised object id through the ONE untagged-hex decoder the +// object-store authority provides, and refuse (typed cause, never a fabricated target) +// on transport refusal, an unadvertised ref, or an undecodable advertisement. +fn observe_fleet_desired_revision(repo: FilePath) -> FleetDesiredRevision { + let read = git.Core.LsRemoteRefInRepo(repo: repo, remote: "origin", ref_name: fleet_desired_ref_name) + if read.exit_code == 0 { + let line = trim(s: read.advertised) + if line == "" { + FleetDesiredUnobserved { + cause: concat("fleet_desired_observe: origin does not advertise ", fleet_desired_ref_name), + } + } else { + match git_object_id_from_untagged_hex(hex: split(s: line, delimiter: "\t").first()) { + Present { value: oid } => FleetDesiredAdmitted { revision: oid } + Absent => + FleetDesiredUnobserved { + cause: concat("fleet_desired_observe: advertisement is not a decodable object id: ", line), + } + } + } + } else { + FleetDesiredUnobserved { + cause: concat("fleet_desired_observe: ls-remote refused: ", read.stderr), + } + } +} + +// The local half of the comparison: what revision THIS host's deployed tree is at. +// Absent (typed, located in the cause the caller renders) when the repo path is not a +// git repository or HEAD does not decode — never a fabricated revision. +fn observe_local_tree_revision(repo: FilePath) -> GitObjectId? { + let read = git.Core.RevParseInRepo(repo: repo, target: "HEAD" as GitRef) + if read.exit_code == 0 { + git_object_id_from_untagged_hex(hex: trim(s: read.revision)) + } else { + none + } +} + +// The revision cell of a host's convergence: three states, none collapsible. Converged +// says the deployed tree IS the admitted revision; Drifted carries BOTH sides (the +// membership-diff `from` ruling — a catch-up realization needs the prior, and the +// receipt must name what it observed, not just that it disagreed); Unobserved says the +// comparison could not be made and why — it is unobserved-GRADE for the fold, never +// satisfied (leg-f ruling shape). +type FleetRevisionStanding + = RevisionConverged { revision: GitObjectId } + | RevisionDrifted { desired: GitObjectId, local: GitObjectId } + | RevisionUnobserved { cause: String } + +fn fleet_revision_standing(repo: FilePath) -> FleetRevisionStanding { + let observed = observe_fleet_desired_revision(repo: repo) + match converge_target_revision(observed: observed) { + Present { value: desired } => + match observe_local_tree_revision(repo: repo) { + Present { value: local } => + if git_object_id_eq(left: desired, right: local) { + RevisionConverged { revision: desired } + } else { + RevisionDrifted { desired: desired, local: local } + } + Absent => + RevisionUnobserved { + cause: "fleet_desired_observe: local tree revision unreadable (not a git repository, or HEAD undecodable)", + } + } + Absent => + match observed { + FleetDesiredUnobserved { cause: c } => RevisionUnobserved { cause: c } + FleetDesiredAdmitted { revision: _ } => + RevisionUnobserved { + cause: "fleet_desired_observe: admitted revision projected no target (unreachable by construction)", + } + } + } +} diff --git a/dag/gunbc/fleet_main_revision.dag b/dag/gunbc/fleet_main_revision.dag new file mode 100644 index 00000000000..43ab4823fad --- /dev/null +++ b/dag/gunbc/fleet_main_revision.dag @@ -0,0 +1,59 @@ +module gunbc.fleet_main_revision + +import std.types { String, NonEmptyStr, Bool } +import extdeps.git.object_store { GitObjectId } + +// fleet-main-revision-authority (arc item 3, operator directive 2026-08-08): the fleet's +// desired revision derives from MERGED, FLOOR-GREEN main — never from a CLI argument, a +// session's checkout, or raw HEAD. Hosts follow ONE admitted ref; "submitted" means +// "survived the floor", not "was pushed". Git-ref polling over service polling is the +// anti-entropy property being bought: a host that cannot reach the control plane keeps +// converging toward last-known-good, and a returning host converges from the admitted +// ref without a new commit or a human step. +data fleet_desired_ref_name: String = "refs/fleet/desired" + +// What a host (or the fold) knows about the admitted desired revision after observing +// the ref. Unobserved is a typed state with a cause — a host that cannot read the ref +// holds last-known-good and says so; it never fabricates a target and never treats +// "could not read" as "nothing desired" (the empty-observation narrow). +type FleetDesiredRevision + = FleetDesiredAdmitted { revision: GitObjectId } + | FleetDesiredUnobserved { cause: String } + +fn fleet_desired_revision_target(d: FleetDesiredRevision) -> GitObjectId? { + match d { + FleetDesiredAdmitted { revision: r } => Present { value: r } + FleetDesiredUnobserved { cause: _ } => none + } +} + +// The admission gate: the ref advances ONLY when the candidate revision's floor is +// green. The advance is a compare-and-swap naming the expected prior (git update-ref +// — the MemberChanged `from` side, per the reconcile spine's CAS +// ruling), so a concurrent admission refuses instead of silently overwriting. A prior +// of none is the first admission (ref creation). +type FleetDesiredAdvance + = AdvanceAdmitted { from: GitObjectId?, to: GitObjectId } + | AdvanceRefused { cause: String } + +fn admit_fleet_desired_advance( + candidate: GitObjectId, + floor_green: Bool, + current: GitObjectId?, +) -> FleetDesiredAdvance { + if floor_green { + AdvanceAdmitted { from: current, to: candidate } + } else { + AdvanceRefused { + cause: "fleet_main_revision: candidate revision's floor is not green; the desired ref never advances onto an unproven revision (raw-HEAD simultaneous fleet-bricking is the prevented class)", + } + } +} + +// Converge target derivation — the seam that retires the CLI argument as the source of +// desired state. The converge circuit asks THIS function what revision it is converging +// toward; an unobserved ref yields none and the caller refuses (typed, located), it +// does not fall back to "whatever tree I was invoked from". +fn converge_target_revision(observed: FleetDesiredRevision) -> GitObjectId? { + fleet_desired_revision_target(d: observed) +} diff --git a/dag/gunbc/fleet_reach_endpoint.dag b/dag/gunbc/fleet_reach_endpoint.dag new file mode 100644 index 00000000000..7b63ea8eb35 --- /dev/null +++ b/dag/gunbc/fleet_reach_endpoint.dag @@ -0,0 +1,46 @@ +module gunbc.fleet_reach_endpoint + +import std.types { String, NonEmptyStr, Bool, List } +import product.placement_supply { HostIdentity } +import gunbc.dgx_spark_procurement { + DgxSparkRouterBinding, + SlotAssigned, + SlotUnassigned, + dgx_spark_router_bindings, +} +import extdeps.network.ipv4 { render_ipv4_address } + +// Probe-endpoint resolution (bold-eagle-107's projection, offered via witty-raven +// msg_1c4e0bce; homed HERE — downstream of both gunbc.dgx_spark_procurement and the +// reach probe — so procurement never imports reach machinery). The srv5/srv6 resolution +// gap is NOT missing DNS: the addresses are already modeled as cited CR1000A DHCP-lease +// rows joined to fleet identity by an OperatorAllocation receipt, and nothing publishes +// 'srv5'/'srv6' as resolvable names. A probe that resolves HOSTNAMES therefore fails for +// every Spark forever regardless of host state; the fix is consuming the modeled +// endpoint. SlotAssigned projects the reservation's address; SlotUnassigned projects NO +// endpoint — never a placeholder (a fabricated address would be the §5 plausible-output +// trap aimed at the network). +fn spark_endpoint_for_binding(binding: DgxSparkRouterBinding) -> String? { + match binding { + SlotAssigned { identity: _, reservation: r, allocation: _ } => + Present { value: render_ipv4_address(addr: r.ipv4_address) as String } + SlotUnassigned { identity: _ } => none + } +} + +// Fleet-wide: a host with a modeled endpoint probes AT that endpoint; every other host +// probes by name (srv1-srv4 resolve via the LAN today). The receipt stays keyed by the +// fleet IDENTITY either way — the endpoint is transport addressing, not identity. +fn fleet_probe_endpoint_for(host: String) -> String { + fold(dgx_spark_router_bindings, init: host, f: fn(acc, b) { + match b { + SlotAssigned { identity: i, reservation: r, allocation: _ } => + if (i as String) == host { + render_ipv4_address(addr: r.ipv4_address) as String + } else { + acc + } + SlotUnassigned { identity: _ } => acc + } + }) +} diff --git a/dag/gunbc/fleet_receipt_collector.dag b/dag/gunbc/fleet_receipt_collector.dag new file mode 100644 index 00000000000..996c0443c10 --- /dev/null +++ b/dag/gunbc/fleet_receipt_collector.dag @@ -0,0 +1,202 @@ +module gunbc.fleet_receipt_collector + +import std.types { String, Bool, List, NonEmptyStr } +import std.algebra { trim } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import extdeps.filesystem.filesystem_io { Filesystem } +import product.placement_supply { HostIdentity } +import gunbc.fleet_converge_cli { + ConvergeCliReceipt, + fleet_converge_receipt_path, +} +import gunbc.host_converge { ConvergeVerdict, VerdictConverged, Drifted, Absent, converge_verdict_label } +import gunbc.typed_argv_exec { + typed_argv_probe_over_ssh, + TypedArgvExecOutcome, + TypedArgvExecConverged, + TypedArgvExecRefused, +} +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } +import gunbc.host_reach_identity_probe { fleet_reach_probe_targets } +import gunbc.fleet_convergence_verdict { + FleetConvergenceVerdict, + FleetConverged, + FleetNotConverged, + fleet_convergence_verdict, +} +import gunbc.host_phase_status { fleet_subsumption_enrollments } + +// Operator verdict join 5 (2026-08-09): the receipts hosts PUBLISH were consumed by +// nothing — converge wrote /opt/gunbc/receipts/-converge-receipt.txt and +// fleet_convergence_verdict folded over whatever list a caller handed it, with no +// collector in between, so the fold's NotConverged came from an empty registry rather +// than from observed state. This module is that collector: read each fleet host's +// published receipt at the modeled endpoint, PARSE it through one typed grammar (the +// byte-locked converge-receipt line read back — the same grammar the CLI renders, +// asked in the other direction), and classify per host with no absorbing arm: a +// collected receipt, an absent one (typed cause — unreachable and not-yet-published +// stay distinct in the cause text carried), or an unparseable one (a receipt that +// exists but does not decode is CONTRADICTION-shaped evidence, never skipped into +// absence). Only Collected rows enter the fold; every other arm keeps the host +// knob-unobserved, which the fold already refuses to call converged. +type CollectedHostReceipt + = ReceiptCollected { host: String, receipt: ConvergeCliReceipt } + | ReceiptAbsent { host: String, cause: String } + | ReceiptUnparseable { host: String, line: String } + +// Forward grammar lives in converge_cli_receipt_line; this is the backward read of the +// SAME line shape: "converge-receipt host=H applied=N drifted=N verdict=V". A missing +// prefix, missing field, or unknown verdict label refuses (Absent), never guesses. +fn parse_converge_receipt_line(line: String) -> ConvergeCliReceipt? { + let tokens = split(s: trim(s: line), delimiter: " ") + if length(tokens) == 5 + && match tokens.first() { Present { value: t } => t == "converge-receipt" Absent => false } { + match receipt_field(tokens: tokens, key: "host=") { + Present { value: host } => + match receipt_field(tokens: tokens, key: "verdict=") { + Present { value: label } => + match parse_converge_verdict_label(label: label) { + Present { value: v } => + match parse_int(s: receipt_field_or_empty(tokens: tokens, key: "applied=")) { + Present { value: applied } => + match parse_int(s: receipt_field_or_empty(tokens: tokens, key: "drifted=")) { + Present { value: drifted } => + Present { + value: ConvergeCliReceipt { + host: host, + applied: applied, + drifted: drifted, + verdict: v, + reason: none, + }, + } + Absent => none + } + Absent => none + } + Absent => none + } + Absent => none + } + Absent => none + } + } else { + none + } +} + +fn receipt_field(tokens: List, key: String) -> String? { + fold(tokens, init: none, f: fn(acc, t) { + if starts_with(s: t, prefix: key) { + Present { value: drop_prefix(s: t, prefix: key) } + } else { + acc + } + }) +} + +fn receipt_field_or_empty(tokens: List, key: String) -> String { + match receipt_field(tokens: tokens, key: key) { + Present { value: v } => v + Absent => "" + } +} + +fn drop_prefix(s: String, prefix: String) -> String { + substring(s: s, start: prefix.length(), end: s.length()) +} + +// Review 50817: decode each label to ITS verdict constructor directly. An earlier +// revision routed drifted/absent through converge_verdict(effective: "", expected: +// label), which returns Absent whenever effective is empty — silently re-classifying a +// published Drifted as Absent on collection, exactly the state-space conflation the +// header forbids (different remedies: run the converge vs build the observation). +fn parse_converge_verdict_label(label: String) -> ConvergeVerdict? { + if label == "converged" { + Present { value: VerdictConverged } + } else { + if label == "drifted" { + Present { value: Drifted } + } else { + if label == "absent" { + Present { value: Absent } + } else { + none + } + } + } +} + +// One host's collection: cat the published receipt over the typed argv seam at the +// modeled endpoint. Wire refusal or transport failure -> Absent with the cause; a first +// line that does not decode -> Unparseable carrying the exact bytes. +fn collect_host_receipt(host: String) -> CollectedHostReceipt { + match typed_argv_probe_over_ssh( + host: fleet_probe_endpoint_for(host: host) as NonEmptyStr, + argv: ["cat", fleet_converge_receipt_path(host: host)], + ) { + TypedArgvExecConverged { result: r } => + if r.success { + match receipt_first_line(raw: r.stdout) { + Present { value: line } => + match parse_converge_receipt_line(line: line) { + Present { value: receipt } => ReceiptCollected { host: host, receipt: receipt } + Absent => ReceiptUnparseable { host: host, line: line } + } + Absent => ReceiptAbsent { host: host, cause: "receipt file empty" } + } + } else { + ReceiptAbsent { host: host, cause: trim(s: r.stderr) } + } + TypedArgvExecRefused { reason: why } => ReceiptAbsent { host: host, cause: why } + } +} + +fn receipt_first_line(raw: String) -> String? { + match split(s: raw, delimiter: "\n").first() { + Present { value: l } => if trim(s: l) == "" { none } else { Present { value: trim(s: l) } } + Absent => none + } +} + +fn collected_receipts_only(collected: List) -> List { + collected.flat_map(c => match c { + ReceiptCollected { host: _, receipt: r } => [r] + ReceiptAbsent { host: _, cause: _ } => [] + ReceiptUnparseable { host: _, line: _ } => [] + }) +} + +fn collected_line(c: CollectedHostReceipt) -> String { + match c { + ReceiptCollected { host: h, receipt: _ } => join(["collect host=", h, " outcome=collected"], "") + ReceiptAbsent { host: h, cause: why } => join(["collect host=", h, " outcome=absent cause=", why], "") + ReceiptUnparseable { host: h, line: l } => join(["collect host=", h, " outcome=unparseable line=", l], "") + } +} + +data fleet_collect_receipt_path: String = "target/fleet-converge-collect-receipt.txt" + +// The wet collector run: collect every fleet target, feed ONLY collected receipts to +// the atomic fold (join 5's composition — executed observation -> registry -> verdict), +// and publish one collect receipt naming each host's collection outcome plus the fold +// verdict. Exit reflects publication only: an honest NotConverged is a successful +// collection, not a process failure. +func collect_fleet_convergence_wet() -> ProcessExit { + let collected = map(fleet_reach_probe_targets, h => collect_host_receipt(host: h)) + let verdict = fleet_convergence_verdict( + enrollments: fleet_subsumption_enrollments, + receipts: collected_receipts_only(collected: collected), + ) + let verdict_line = match verdict { + FleetConverged { host_count: n } => concat("fleet-verdict converged hosts=", to_string(n)) + FleetNotConverged { standings: s } => concat("fleet-verdict not-converged standings=", to_string(length(s))) + } + let body = join([join(map(collected, c => collected_line(c: c)), "\n"), "\n", verdict_line, "\n"], "") + let w = Filesystem.Write(path: fleet_collect_receipt_path, content: body) + if w.success { + ExitSuccess + } else { + exit_failure(reason: concat("fleet_receipt_collector: collect receipt publication failed — ", w.error)) + } +} diff --git a/dag/gunbc/host_axis_caps.dag b/dag/gunbc/host_axis_caps.dag index 05a5165b5e9..540054c894e 100644 --- a/dag/gunbc/host_axis_caps.dag +++ b/dag/gunbc/host_axis_caps.dag @@ -17,6 +17,7 @@ import gunbc.host_converge { gunbc_runner_per_slot_dropin_path, gunbc_runner_per_slot_swap_dropin_path, gunbc_runner_per_slot_high_dropin_path, + gunbc_runner_per_slot_tasks_dropin_path, } data host_axis_caps_note: String = "PerSlotMemoryCap axis, closed vertically on the membership_reconcile spine — the first non-degenerate reconcile in the repo (desired vs a real OBSERVED set, not observed=[]). A CapMember is a per-slot systemd drop-in keyed by its file path (stable IDENTITY, so a content drift is Modified->MemberChanged, never Remove+Add). The interference domain is the drop-in directory itself: ownership is DERIVED from the path (managed set -> Owned; anything else -> a foreign occupant, Ensured), never stored, so a Removed foreign file yields MemberRemovalRefused (R5, no effect arm, never deleted). Idempotence = zero hunks (Unchanged); perturb = Modified->MemberChanged of exactly the drifted row; foreign occupant = ApplyRefused wholesale. Live apply/observe are host_effect_apply typed transports (operator one-liners for the go-live receipts); this module is the pure, hermetically-witnessed fold." @@ -30,6 +31,7 @@ data caps_managed_dropin_paths: List = [ gunbc_runner_per_slot_dropin_path as NonEmptyStr, gunbc_runner_per_slot_swap_dropin_path as NonEmptyStr, gunbc_runner_per_slot_high_dropin_path as NonEmptyStr, + gunbc_runner_per_slot_tasks_dropin_path as NonEmptyStr, ] fn cap_is_managed_path(p: NonEmptyStr) -> Bool { diff --git a/dag/gunbc/host_converge.dag b/dag/gunbc/host_converge.dag index 7378fdc3d99..f9f3c64b2f5 100644 --- a/dag/gunbc/host_converge.dag +++ b/dag/gunbc/host_converge.dag @@ -34,6 +34,7 @@ import extdeps.systemd { systemd_memory_swap_max_property, systemd_memory_high_property, systemd_cpu_weight_property, + systemd_tasks_max_property, } import extdeps.systemd.oomd { oomd_managed_oom_memory_pressure_property, @@ -51,6 +52,7 @@ data gunbc_runner_all_slots_unit_glob: String = "actions-runner@*.service" data gunbc_runner_per_slot_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/20-fleet-width.conf" data gunbc_runner_per_slot_swap_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/30-fleet-swap.conf" data gunbc_runner_per_slot_high_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/40-fleet-high.conf" +data gunbc_runner_per_slot_tasks_dropin_path: String = "/etc/systemd/system/actions-runner@.service.d/50-fleet-tasks.conf" data gunbc_runner_jobserver_env_file: String = "/etc/default/ctrl-jobserver" data gunbc_runner_jobserver_env_key: String = "CTRL_JOBSERVER_TOKENS" data gunbc_runner_jobserver_service: String = "ctrl-jobserver.service" @@ -186,6 +188,15 @@ type GunbcBinaryProvenancePolicy { bootstrap_fallback: GunbcBinaryProvenance } +// PerSlotMemoryCap is the per-slot systemd drop-in/property mechanism (dropin_path x +// instance_glob x property x value), and the per_slot_tasks_max knob (a pids-controller +// count; desired value owned by gunbc.runner_slot_allocation) rides it because the mechanism is property-generic — forking a +// parallel variant per property class would be the §2 duplication the shape exists to +// avoid. The variant name and its bytes_value field over-narrow to the memory axis that +// landed first: a §3 mislabel carried knowingly rather than a ~96-site mechanical rename +// inside this change. dissolve-on: the HostAxisBundle axis framework +// (gunbc.plans.host_convergence_circuit_residue P3), whose landing renames the variant to +// the mechanism it names (PerSlotUnitProperty, value_raw) in one motion. type ConvergeTarget = SliceProperty { unit: String, property: String, apply_value: String, expected_effective: String } | PerSlotMemoryCap { dropin_path: String, instance_glob: String, property: String, bytes_value: String } @@ -476,6 +487,17 @@ fn runner_host_knobs(h: RunnerHostDeployment) -> List { bytes_value: to_string(byte_size_count(b: h.per_runner_memory_swap_cap)), } ), + legacy_converge_knob( + slice: RunnerSlice, + name: "per_slot_tasks_max", + desired: to_string(h.per_runner_tasks_max), + target: PerSlotMemoryCap { + dropin_path: gunbc_runner_per_slot_tasks_dropin_path, + instance_glob: glob, + property: systemd_tasks_max_property, + bytes_value: to_string(h.per_runner_tasks_max), + } + ), legacy_converge_knob( slice: RunnerSlice, name: "per_slot_memory_high_bytes", diff --git a/dag/gunbc/host_converge_slice1.dag b/dag/gunbc/host_converge_slice1.dag index 396d7b10eb6..42def17af62 100644 --- a/dag/gunbc/host_converge_slice1.dag +++ b/dag/gunbc/host_converge_slice1.dag @@ -11,6 +11,14 @@ import extdeps.systemd { MemoryLimitBytes, MemoryLimitUnbounded, MemoryLimitAbsent, + systemd_tasks_max_property, + TasksMax, + TasksCurrent, + SystemdCgroupTaskLimit, + parse_systemd_task_limit_show, + TaskLimitCount, + TaskLimitUnbounded, + TaskLimitAbsent, } import gunbc.systemctl_show_read { systemctl_show_property_read_local, @@ -56,6 +64,30 @@ data host_converge_slice1_probe_unit: NonEmptyStr = "actions-runner@srv1-01.serv data host_converge_slice1_knob_name: String = "per_slot_memory_max_bytes" +data host_converge_slice1_tasks_knob_name: String = "per_slot_tasks_max" + +// TasksMax (gunbc.runner_slot_allocation gunbc_runner_slot_desired, 16384 — the +// 2026-06-21 incident REPAIR value; 512 was the diagnosed EAGAIN outage value, per the +// operator ruling on PR 8045) is the second knob realized on this circuit: same typed +// systemctl show/set-property transports, same fail-closed order (live read -> Drifted +// verdict -> pre-apply TasksCurrent guard -> apply -> independent read-back -> +// reapply-noop), a pids-controller count instead of a byte size. The guard refuses when +// live TasksCurrent meets or exceeds the target cap (>=: equality means zero headroom, +// so the next clone() would EAGAIN) — lowering TasksMax below live usage would not kill +// tasks but would starve every new clone(), the compute_envelope_model failure class — +// typed and retryable, never applied past it. +type HostConvergeSlice1TasksMaxReceipt { + host: HostIdentity + unit: NonEmptyStr + knob: ConvergeKnob + limit: SystemdCgroupTaskLimit + verdict: ConvergeVerdict + probed_at: NonEmptyStr + applied: Bool + read_back_converged: Bool + reapply_noop: Bool +} + type HostConvergeSlice1MemoryCapReceipt { host: HostIdentity unit: NonEmptyStr @@ -70,6 +102,7 @@ type HostConvergeSlice1MemoryCapReceipt { type HostConvergeSlice1ConvergeResult = HostConvergeSlice1Converged { applied_knobs: Int, receipt: HostConvergeSlice1MemoryCapReceipt } + | HostConvergeSlice1TasksConverged { applied_knobs: Int, receipt: HostConvergeSlice1TasksMaxReceipt } | HostConvergeSlice1Refused { reason: String } fn host_converge_identity(h: HostConverge) -> HostIdentity { @@ -319,6 +352,7 @@ fn host_converge_slice1_cross_check_enumeration(host: HostIdentity, units: List< type HostConvergeSlice1UnitOutcome = HostConvergeSlice1UnitConverged { unit: NonEmptyStr, applied: Bool, receipt: HostConvergeSlice1MemoryCapReceipt } + | HostConvergeSlice1UnitTasksConverged { unit: NonEmptyStr, applied: Bool, receipt: HostConvergeSlice1TasksMaxReceipt } | HostConvergeSlice1UnitRefused { unit: NonEmptyStr, reason: String } | HostConvergeSlice1UnitUsageGuardRefused { unit: NonEmptyStr, current_bytes: ByteSize, target_cap_bytes: ByteSize } @@ -414,20 +448,275 @@ fn host_converge_slice1_converge_unit(host: HostIdentity, unit: NonEmptyStr, kno } } +fn host_converge_slice1_tasks_knob(host: HostIdentity) -> ConvergeKnob? { + match host_converge_from_policy(policy: fleet_converge_policy(), host: host) { + Present { value: hc } => + match converge_knob_by_name(host: hc, name: host_converge_slice1_tasks_knob_name) { + Present { value: knob } => + match knob.target { + PerSlotMemoryCap { dropin_path: _, instance_glob: _, property: p, bytes_value: b } => + if p == (systemd_tasks_max_property as String) && b == knob.desired { + Present { value: knob } + } else { + none + } + _ => none + } + _ => none + } + _ => none + } +} + +fn host_converge_slice1_read_tasks_max_raw(unit: NonEmptyStr) -> String? { + match systemctl_show_property_read_local(unit: unit, property: TasksMax) { + SystemdPropertyCaptured { value: raw } => Present { value: raw } + SystemdPropertyCaptureRefused { reason: _ } => none + } +} + +fn host_converge_slice1_read_tasks_current_raw(unit: NonEmptyStr) -> String? { + match systemctl_show_property_read_local(unit: unit, property: TasksCurrent) { + SystemdPropertyCaptured { value: raw } => Present { value: raw } + SystemdPropertyCaptureRefused { reason: _ } => none + } +} + +// Verdict over the live TasksMax read against the knob's desired count. An empty or +// unparseable wire is the Absent verdict (untyped wire, refused upstream), never Drifted: +// Drifted is reserved for a real limit that disagrees, because Drifted is the arm that +// arms the apply path. +fn host_converge_slice1_tasks_verdict(limit: SystemdCgroupTaskLimit, knob: ConvergeKnob) -> ConvergeVerdict { + match limit { + TaskLimitCount { count: n } => + if to_string(n) == knob.desired { VerdictConverged } else { Drifted } + TaskLimitUnbounded => Drifted + TaskLimitAbsent => Absent + } +} + +fn host_converge_slice1_apply_tasks_max(unit: NonEmptyStr, value: String) -> Bool { + systemctl_set_property_local( + unit: unit, + property: systemd_tasks_max_property, + value: value as NonEmptyStr, + ) +} + +type HostConvergeSlice1TasksGuardOutcome + = HostConvergeSlice1TasksSafe { unit: NonEmptyStr, current_count: Int } + | HostConvergeSlice1TasksUnsafe { unit: NonEmptyStr, current_count: Int, target_count: Int } + | HostConvergeSlice1TasksUnknown { unit: NonEmptyStr, reason: String } + +// Pure boundary predicate for the tasks guard, extracted so the >= boundary is +// witnessable: equality means zero headroom (the next clone() EAGAINs), so it refuses. +fn host_converge_slice1_tasks_headroom_refuses(current: Int, target: Int) -> Bool { + current >= target +} + +fn host_converge_slice1_tasks_guard(unit: NonEmptyStr, target_value: String) -> HostConvergeSlice1TasksGuardOutcome { + match parse_int(s: target_value) { + Present { value: target } => + match host_converge_slice1_read_tasks_current_raw(unit: unit) { + Present { value: raw } => + match parse_systemd_task_limit_show(raw: raw) { + TaskLimitCount { count: current } => + if host_converge_slice1_tasks_headroom_refuses(current: current, target: target) { + HostConvergeSlice1TasksUnsafe { unit: unit, current_count: current, target_count: target } + } else { + HostConvergeSlice1TasksSafe { unit: unit, current_count: current } + } + _ => + HostConvergeSlice1TasksUnknown { + unit: unit, + reason: "host_converge_slice1: live TasksCurrent read unbounded or unparseable; cannot bound usage against target cap; fail-closed", + } + } + _ => + HostConvergeSlice1TasksUnknown { + unit: unit, + reason: "host_converge_slice1: live systemctl show TasksCurrent read absent; fail-closed", + } + } + _ => + HostConvergeSlice1TasksUnknown { + unit: unit, + reason: "host_converge_slice1: target TasksMax value did not parse as Int; fail-closed", + } + } +} + +fn host_converge_slice1_tasks_receipt( + host: HostIdentity, + unit: NonEmptyStr, + knob: ConvergeKnob, + limit: SystemdCgroupTaskLimit, + probed_at: NonEmptyStr, + applied: Bool, + read_back_converged: Bool, + reapply_noop: Bool, +) -> HostConvergeSlice1TasksMaxReceipt { + HostConvergeSlice1TasksMaxReceipt { + host: host, + unit: unit, + knob: knob, + limit: limit, + verdict: host_converge_slice1_tasks_verdict(limit: limit, knob: knob), + probed_at: probed_at, + applied: applied, + read_back_converged: read_back_converged, + reapply_noop: reapply_noop, + } +} + +fn host_converge_slice1_tasks_receipt_is_valid(receipt: HostConvergeSlice1TasksMaxReceipt) -> Bool { + receipt.read_back_converged + && receipt.reapply_noop + && receipt.verdict == VerdictConverged +} + +fn host_converge_slice1_tasks_read_limit(unit: NonEmptyStr) -> SystemdCgroupTaskLimit? { + match host_converge_slice1_read_tasks_max_raw(unit: unit) { + Present { value: raw } => Present { value: parse_systemd_task_limit_show(raw: raw) } + _ => none + } +} + +// The reapply-noop second write intentionally bypasses the TasksCurrent guard: it is only +// reachable after an independent read proved the effective TasksMax already equals the +// desired value, and re-applying the effective value cannot lower headroom (TasksCurrent +// is always <= the effective cap at the moment of the read). The read-back and this +// second write are NOT atomic — concurrent task creation can raise TasksCurrent between +// them — but the EAGAIN-starvation class the guard exists for requires setting the cap +// BELOW current, which re-applying the already-effective value cannot cause regardless +// of concurrent spawns (review 50754 wording). Mirrors the memory circuit's reapply-noop. +fn host_converge_slice1_prove_tasks_reapply_noop(unit: NonEmptyStr, knob: ConvergeKnob) -> String? { + if !host_converge_slice1_apply_tasks_max(unit: unit, value: knob.desired) { + Present { + value: "host_converge_slice1: tasks reapply-noop second apply (systemctl set-property TasksMax) failed; fail-closed", + } + } else { + match host_converge_slice1_tasks_read_limit(unit: unit) { + Present { value: limit } => + match host_converge_slice1_tasks_verdict(limit: limit, knob: knob) { + VerdictConverged => none + Drifted => + Present { + value: "host_converge_slice1: tasks reapply-noop failed — still drifted after second apply; fail-closed", + } + Absent => + Present { + value: "host_converge_slice1: tasks reapply-noop read returned Absent verdict after second apply; fail-closed", + } + } + _ => + Present { + value: "host_converge_slice1: tasks reapply-noop independent read after second apply absent; fail-closed", + } + } + } +} + +fn host_converge_slice1_converge_unit_tasks(host: HostIdentity, unit: NonEmptyStr, knob: ConvergeKnob) -> HostConvergeSlice1UnitOutcome { + match host_converge_slice1_probed_at() { + Present { value: probed_at } => + match host_converge_slice1_tasks_read_limit(unit: unit) { + Present { value: limit } => + match host_converge_slice1_tasks_verdict(limit: limit, knob: knob) { + VerdictConverged => + match host_converge_slice1_prove_tasks_reapply_noop(unit: unit, knob: knob) { + Present { value: why } => HostConvergeSlice1UnitRefused { unit: unit, reason: why } + _ => + HostConvergeSlice1UnitTasksConverged { + unit: unit, + applied: false, + receipt: host_converge_slice1_tasks_receipt( + host: host, unit: unit, knob: knob, limit: limit, probed_at: probed_at, + applied: false, read_back_converged: true, reapply_noop: true, + ), + } + } + Drifted => + match host_converge_slice1_tasks_guard(unit: unit, target_value: knob.desired) { + HostConvergeSlice1TasksUnsafe { unit: _, current_count: cc, target_count: tc } => + HostConvergeSlice1UnitRefused { + unit: unit, + reason: concat( + concat(concat("host_converge_slice1: tasks guard refused apply for ", unit as String), concat(" current_tasks=", to_string(cc))), + concat(concat(" exceeds target_tasks_max=", to_string(tc)), "; retryable once the slot is quiet"), + ), + } + HostConvergeSlice1TasksUnknown { unit: _, reason: r } => + HostConvergeSlice1UnitRefused { unit: unit, reason: concat("host_converge_slice1: tasks guard could not prove safety before apply, fail-closed: ", r) } + HostConvergeSlice1TasksSafe { unit: _, current_count: _ } => + if !host_converge_slice1_apply_tasks_max(unit: unit, value: knob.desired) { + HostConvergeSlice1UnitRefused { + unit: unit, + reason: concat( + concat("host_converge_slice1: systemctl set-property TasksMax failed for ", unit as String), + concat(" desired=", knob.desired) + ), + } + } else { + match host_converge_slice1_probed_at() { + Present { value: readback_at } => + match host_converge_slice1_tasks_read_limit(unit: unit) { + Present { value: read_back } => + match host_converge_slice1_tasks_verdict(limit: read_back, knob: knob) { + VerdictConverged => + match host_converge_slice1_prove_tasks_reapply_noop(unit: unit, knob: knob) { + Present { value: why } => HostConvergeSlice1UnitRefused { unit: unit, reason: why } + _ => + HostConvergeSlice1UnitTasksConverged { + unit: unit, + applied: true, + receipt: host_converge_slice1_tasks_receipt( + host: host, unit: unit, knob: knob, limit: read_back, probed_at: readback_at, + applied: true, read_back_converged: true, reapply_noop: true, + ), + } + } + Drifted => + HostConvergeSlice1UnitRefused { unit: unit, reason: "host_converge_slice1: tasks independent read-back after apply still drifted; fail-closed" } + Absent => + HostConvergeSlice1UnitRefused { unit: unit, reason: "host_converge_slice1: tasks read-back after apply returned Absent verdict; fail-closed" } + } + _ => HostConvergeSlice1UnitRefused { unit: unit, reason: "host_converge_slice1: tasks independent read-back after apply returned absent; fail-closed" } + } + _ => HostConvergeSlice1UnitRefused { unit: unit, reason: host_converge_slice1_probed_at_refusal_reason } + } + } + } + Absent => + HostConvergeSlice1UnitRefused { unit: unit, reason: concat("host_converge_slice1: live TasksMax parsed Absent (empty/untyped wire) for ", unit as String) } + } + _ => HostConvergeSlice1UnitRefused { unit: unit, reason: concat("host_converge_slice1: live systemctl show TasksMax read absent for ", unit as String) } + } + _ => HostConvergeSlice1UnitRefused { unit: unit, reason: host_converge_slice1_probed_at_refusal_reason } + } +} + fn host_converge_slice1_converge_host(host: HostIdentity, instance_glob: String) -> HostConvergeSlice1HostResult { match host_converge_slice1_per_slot_memory_max_knob(host: host) { Present { value: knob } => - match host_converge_slice1_enumerate_units(instance_glob: instance_glob) { - Present { value: units } => - match host_converge_slice1_cross_check_enumeration(host: host, units: units) { - Present { value: mismatch } => HostConvergeSlice1HostEnumerationMismatch { mismatch: mismatch } - _ => - HostConvergeSlice1HostEnumerated { - host: host, - outcomes: map(units, u => host_converge_slice1_converge_unit(host: host, unit: u, knob: knob)), + match host_converge_slice1_tasks_knob(host: host) { + Present { value: tasks_knob } => + match host_converge_slice1_enumerate_units(instance_glob: instance_glob) { + Present { value: units } => + match host_converge_slice1_cross_check_enumeration(host: host, units: units) { + Present { value: mismatch } => HostConvergeSlice1HostEnumerationMismatch { mismatch: mismatch } + _ => + HostConvergeSlice1HostEnumerated { + host: host, + outcomes: units.flat_map(u => [ + host_converge_slice1_converge_unit(host: host, unit: u, knob: knob), + host_converge_slice1_converge_unit_tasks(host: host, unit: u, knob: tasks_knob), + ]), + } } + _ => HostConvergeSlice1HostRefused { reason: concat("host_converge_slice1: live unit enumeration absent for glob ", instance_glob) } } - _ => HostConvergeSlice1HostRefused { reason: concat("host_converge_slice1: live unit enumeration absent for glob ", instance_glob) } + _ => HostConvergeSlice1HostRefused { reason: "host_converge_slice1: per_slot_tasks_max knob absent from fleet_converge_policy for host; fail-closed" } } _ => HostConvergeSlice1HostRefused { reason: "host_converge_slice1: per_slot_memory_max_bytes knob absent from fleet_converge_policy for host; fail-closed" } } @@ -486,6 +775,8 @@ fn host_converge_slice1_unit_outcome_to_legacy_result(outcome: HostConvergeSlice match outcome { HostConvergeSlice1UnitConverged { unit: _, applied: a, receipt: r } => HostConvergeSlice1Converged { applied_knobs: if a { 1 } else { 0 }, receipt: r } + HostConvergeSlice1UnitTasksConverged { unit: _, applied: a, receipt: r } => + HostConvergeSlice1TasksConverged { applied_knobs: if a { 1 } else { 0 }, receipt: r } HostConvergeSlice1UnitRefused { unit: _, reason: reason } => HostConvergeSlice1Refused { reason: reason } HostConvergeSlice1UnitUsageGuardRefused { unit: u, current_bytes: cb, target_cap_bytes: tb } => @@ -504,6 +795,7 @@ fn host_converge_slice1_host_result_applied_count(result: HostConvergeSlice1Host fold(os, init: 0, f: fn(acc, o) { match host_converge_slice1_unit_outcome_to_legacy_result(o) { HostConvergeSlice1Converged { applied_knobs: n, receipt: _ } => acc + n + HostConvergeSlice1TasksConverged { applied_knobs: n, receipt: _ } => acc + n _ => acc } }) @@ -519,6 +811,7 @@ fn host_converge_slice1_host_result_all_valid(result: HostConvergeSlice1HostResu acc && match o { HostConvergeSlice1UnitConverged { unit: _, applied: _, receipt: r } => host_converge_slice1_receipt_is_valid(receipt: r) + HostConvergeSlice1UnitTasksConverged { unit: _, applied: _, receipt: r } => host_converge_slice1_tasks_receipt_is_valid(receipt: r) _ => false } }) @@ -550,12 +843,42 @@ fn host_converge_slice1_host_result_refusal_reason(result: HostConvergeSlice1Hos } } +// srv1 probe-unit loop, both knobs: the memory outcome carries the legacy receipt shape; +// the tasks outcome must ALSO be valid-converged for the loop to report converged — an +// invalid or refused tasks outcome refuses the whole loop (never silently dropped), and a +// converged one adds its applied count. fn host_converge_slice1_converge_loop() -> HostConvergeSlice1ConvergeResult { match host_converge_slice1_srv1_per_slot_memory_max_knob() { Present { value: knob } => - host_converge_slice1_unit_outcome_to_legacy_result( - host_converge_slice1_converge_unit(host: operator_host_srv1, unit: host_converge_slice1_probe_unit, knob: knob) - ) + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: tasks_knob } => + match host_converge_slice1_unit_outcome_to_legacy_result( + host_converge_slice1_converge_unit(host: operator_host_srv1, unit: host_converge_slice1_probe_unit, knob: knob) + ) { + HostConvergeSlice1Converged { applied_knobs: n, receipt: r } => + match host_converge_slice1_converge_unit_tasks(host: operator_host_srv1, unit: host_converge_slice1_probe_unit, knob: tasks_knob) { + HostConvergeSlice1UnitTasksConverged { unit: _, applied: a, receipt: tr } => + if host_converge_slice1_tasks_receipt_is_valid(receipt: tr) { + HostConvergeSlice1Converged { applied_knobs: n + (if a { 1 } else { 0 }), receipt: r } + } else { + HostConvergeSlice1Refused { reason: "host_converge_slice1: tasks receipt validity gate failed on srv1 probe unit; fail-closed" } + } + HostConvergeSlice1UnitConverged { unit: _, applied: _, receipt: _ } => + HostConvergeSlice1Refused { reason: "host_converge_slice1: tasks converge returned a memory-cap outcome; fail-closed" } + HostConvergeSlice1UnitRefused { unit: _, reason: why } => + HostConvergeSlice1Refused { reason: why } + HostConvergeSlice1UnitUsageGuardRefused { unit: _, current_bytes: _, target_cap_bytes: _ } => + HostConvergeSlice1Refused { reason: "host_converge_slice1: tasks converge returned a memory usage-guard outcome; fail-closed" } + } + HostConvergeSlice1TasksConverged { applied_knobs: _, receipt: _ } => + HostConvergeSlice1Refused { reason: "host_converge_slice1: memory converge returned a tasks outcome; fail-closed" } + HostConvergeSlice1Refused { reason: why } => HostConvergeSlice1Refused { reason: why } + } + _ => + HostConvergeSlice1Refused { + reason: "host_converge_slice1: per_slot_tasks_max knob absent from fleet_converge_policy for srv1; fail-closed", + } + } _ => HostConvergeSlice1Refused { reason: "host_converge_slice1: per_slot_memory_max_bytes knob absent from fleet_converge_policy for srv1; fail-closed", diff --git a/dag/gunbc/host_layout.dag b/dag/gunbc/host_layout.dag index aafeeab2fb3..0b832b07646 100644 --- a/dag/gunbc/host_layout.dag +++ b/dag/gunbc/host_layout.dag @@ -74,6 +74,14 @@ data belt_spawn_workdir_env_var_note: String = "The ONE authority for the enviro data belt_spawn_workdir_env_var: NonEmptyStr = "GUNBC_BELT_SPAWN_WORKDIR" as NonEmptyStr +// The ONE authority for the fleet converge-receipt directory, homed here for the same +// two-consumers-across-a-boundary reason as the belt env var above: gunbc.fleet_converge_cli +// WRITES per-host receipts under it, gunbc.live_deploy.emit CREATES it at deploy time +// (install -d). Homing it in the CLI put the deploy emitter's closure behind the converge +// CLI, which closed the import cycle fleet_converge_apply -> host_effect_realize -> +// live_deploy.emit -> fleet_converge_cli the moment the CLI imported its own apply seam. +data fleet_converge_receipt_dir: String = "/opt/gunbc/receipts" + data provider_runtime_placement_note: String = "ONE host-parameterized placement for provider package/exec release trees, mutable provider state, and private wire-evidence roots. Parallel to GunbcHostLayout: a second host must be describable without minting a second set of path concepts. Claude/Cursor/Codex lanes consume this record; they do not re-author /opt/... roots. The flat srv1_* path data below are the once-written field values of srv1's row — the same dissolution pattern as gunbc_host_layout_note." type ProviderRuntimePlacement { diff --git a/dag/gunbc/host_phase_status.dag b/dag/gunbc/host_phase_status.dag index 738259549f1..3c808fad0cc 100644 --- a/dag/gunbc/host_phase_status.dag +++ b/dag/gunbc/host_phase_status.dag @@ -19,6 +19,10 @@ import gunbc.host_assimilation_program { inference_serving_enrollment_participation, } import gunbc.host_standup_assimilation_deduction { srv3_post_install_actuation_verdict } +import gunbc.host_reach_identity_probe { + observe_host_identity_converge, + observe_reach_secrets_network, +} import gunbc.membership_reconcile { MembershipPlan, MemberAction, @@ -105,23 +109,48 @@ fn host_phase_cell_has_live_producer(c: HostPhaseCellStatus) -> Bool { data host_phase_observation_frontier_note: String = "The live-observation producer registry, and deliberately a TYPED, COUNTABLE FRONTIER rather than a table of hand-written facts (the self-host frontier shape, DESIGN §7: each cell is either produced or it is a declared absence, never a silent assumption). Exactly one cell carries an observation today — (srv3, prefix:os-install-actuated), which deduces its verdict through gunbc.host_standup_assimilation_deduction; that deduction was ALREADY host-parameterized (deduce_os_install_actuation_verdict takes fleet_slot: HostIdentity), so generalizing it was reading the parameter it already had, not writing new machinery. CORRECTED 2026-08-09, by execution: this sentence previously called that cell a REAL PRODUCER, and it is not one. Its DHCP lease table is srv3_post_install_lease_table_fixture, an authored data constant in gunbc.network_identity_subsumption whose one dynamic row carries the placeholder MAC 00:00:00:00:00:00 — no effect anywhere in the tree produces a DhcpLeaseTableObservation. So host_phase_observation_coverage_count, which this note designates as the number that measures the debt, counts one covered cell and that cell is authored, overstating coverage by its entire population; and host_phase_cell_has_live_producer tests only that the observation is Present, never that anything executed. The rule below was already written and was simply not enforceable — nothing would have reddened had a second host been wired the same way, which is precisely what Spark collector-fact enrollment for srv5/srv6 was about to do. NEXT-RUNG TRIGGER, named rather than silently stalled (DESIGN §4b): observation provenance becomes a carried fact so authored and executed are different states, and coverage counts only the executed one. Until then the honest reading of the coverage number is ZERO. Controls: test.claim.host_phase_status witness_covered_cell_verdict_is_selected_by_authorship_alone and witness_every_covered_cell_today_is_the_authored_one. Every other cell returns Absent, which host_phase_verdict_from renders UnknownRefused. What must NOT happen here is the shape this module exists to kill: hand-entering an observation because someone ssh'd a host once. A probe run by a person is not a modeled producer — encoding its result as a data row would rebuild the parallel ledger the gap-ledger dissolution just removed, and it would go stale in exactly the same way. A cell earns a row here when an effect produces its verdict by execution; until then UnknownRefused is the honest answer and host_phase_observation_coverage_count is the number that measures the debt." -fn host_phase_observation(host: HostIdentity, step: HostStandupStep) -> ObservationVerdict? { - if (step.label as String) == "prefix:os-install-actuated" { +// THE canonical observation seam (operator directive 2026-08-08): every lifecycle-phase +// observation for every host routes through observe_host_phase — never a parallel +// per-program observer beside it (a spark_* sibling would be the reconcile-spine §3 +// fork). The subject is the host's enrollment intent, so a producer can read what the +// host is enrolled FOR without a second roster. Producers are PURE deductions over +// committed typed observations (the srv3 lease-table pattern; gunbc.host_reach_identity_probe +// for the two P0 cells) — the matrix never executes a transport, and observation never +// repairs anything. A phase with no producer row returns none, which the verdict fold +// renders UnknownRefused: absence of a producer is a counted debt, never a silent green. +fn observe_host_phase( + subject: HostEnrollmentIntent, + host: HostIdentity, + phase_label: String, +) -> ObservationVerdict? { + if phase_label == "prefix:os-install-actuated" { if host == operator_host_srv3 { Present { value: srv3_post_install_actuation_verdict(elapsed: millisecond(count: 0)) } } else { none } } else { - none + if phase_label == "P0:host-identity-converge" { + observe_host_identity_converge(host: host) + } else { + if phase_label == "P0:reach-secrets-network" { + observe_reach_secrets_network(host: host) + } else { + none + } + } } } -fn host_phase_cell_status(host: HostIdentity, step: HostStandupStep) -> HostPhaseCellStatus { +fn host_phase_observation(subject: HostEnrollmentIntent, host: HostIdentity, step: HostStandupStep) -> ObservationVerdict? { + observe_host_phase(subject: subject, host: host, phase_label: step.label as String) +} + +fn host_phase_cell_status(subject: HostEnrollmentIntent, host: HostIdentity, step: HostStandupStep) -> HostPhaseCellStatus { HostPhaseCellStatus { cell: HostPhaseCell { host: host, phase_label: step.label as String }, disposition: step.disposition, - observation: host_phase_observation(host: host, step: step), + observation: host_phase_observation(subject: subject, host: host, step: step), } } @@ -130,7 +159,7 @@ data host_phase_program_scoping_note: String = "The cells of a host are the step fn host_phase_cells_for_enrollment(intent: HostEnrollmentIntent) -> List { map( assimilation_program_for(intent: intent).steps, - s => host_phase_cell_status(host: intent.host, step: s), + s => host_phase_cell_status(subject: intent, host: intent.host, step: s), ) } diff --git a/dag/gunbc/host_reach_identity_probe.dag b/dag/gunbc/host_reach_identity_probe.dag new file mode 100644 index 00000000000..5c13588a7e7 --- /dev/null +++ b/dag/gunbc/host_reach_identity_probe.dag @@ -0,0 +1,320 @@ +module gunbc.host_reach_identity_probe + +import extdeps.filesystem.filesystem_io { Filesystem } +import std.algebra { trim } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import gunbc.typed_argv_exec { + typed_argv_probe_over_ssh, + TypedArgvExecOutcome, + TypedArgvExecConverged, + TypedArgvExecRefused, +} +import gunbc.clock_read { clock_now_probed_at } +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } + +import std.types { String, NonEmptyStr, Bool, List, Int } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import product.placement_supply { HostIdentity } +import std.upsert_decision { + ObservationVerdict, + Converged, + Absent, + Drifted, + Conflict, + Inaccessible, + UnknownRefused, +} + +// ONE wet probe, two phase cells (DESIGN §2: one observation, N consumers). An +// authenticated short-hostname read over the host's management transport is +// simultaneously the reach verification (did the transport connect, did the fleet key +// authenticate, did the expected machine answer) and the identity observation (does the +// live short hostname equal the enrolled slot identity). The producers for the +// P0:reach-secrets-network and P0:host-identity-converge phase cells are two PURE +// deductions over the same committed probe receipt — the same shape as the srv3 +// os-install producer deducing from an observed DHCP lease table: the EFFECT lands a +// receipt row; the matrix never executes a transport (hermetic by construction, the +// slice1 landmine class cannot recur here). +// +// The four outcomes are DISTINCT ARMS BY OPERATOR CONSTRAINT and must never collapse: +// transport-unavailable and authentication-refused both project to Inaccessible at +// verdict grain, but their remedies differ (fix reach vs fix the key), so the receipt +// keeps the typed outcome and any consumer needing the distinction reads the outcome, +// never the verdict. Host-key mismatch is a REAL arm, not a formality: srv5/srv6 DO +// carry distinct SSH host keys (bold-eagle-107 measured all three algorithms differing; +// an earlier revision of this note claimed identical factory keys — false, the §3 +// stale-claim class) while SHARING one /etc/machine-id, so identity anchors are +// host-key and NIC MAC, and a host-key check against the wrong expectation is a live +// failure class for exactly the hosts this producer verifies first. +// +// Ordering law (observe, never silently repair): password auth on srv5/srv6 stays +// enabled until a ProbeReachable receipt exists for the fleet key — the fallback is +// never disabled before the primary is proven. That disable is a separate effect with +// its own receipt, never a side effect of observation. +// Two SEPARATE constructors for the two host-key states (operator verdict on PR 8045): +// ProbeHostKeyUnenrolled — the transport flagged verification failure and NO expectation +// is enrolled, so first-contact and changed-key are indistinguishable; remedy = enroll +// the fingerprint registry; carries only the raw transport diagnostic. ProbeHostKeyMismatch +// — an enrolled expectation EXISTS and the observed key contradicts it; remedy = +// investigate wrong-machine vs rotated-key; expected is REQUIRED here by construction +// (the earlier String? field let one constructor carry both remedies). +type ReachProbeOutcome + = ProbeReachable { observed_short_hostname: String } + | ProbeTransportUnavailable { cause: String } + | ProbeAuthenticationRefused { cause: String } + | ProbeHostKeyUnenrolled { transport_diagnostic: String } + | ProbeHostKeyMismatch { expected_fingerprint: String, observed_fingerprint: String } + +type HostReachIdentityProbeReceipt { + host: HostIdentity + probed_at: NonEmptyStr + outcome: ReachProbeOutcome +} + +// Committed probe receipts — the registry the two deductions read. A row lands here only +// when a wet probe EXECUTED and its typed outcome was recorded (the srv3 lease-table +// pattern); a hand-asserted row without an executed probe is the parallel-ledger shape +// host_phase_status exists to kill. +// +// PROVENANCE (operator verdict join 4): the six rows below are the transcription of the +// executed probe_fleet_reach job on GitHub Actions run 31300640122 (branch +// session/wise-moth-430, probed 2026-08-09T07:20:04Z, ExitSuccess, receipt artifact +// FleetReachProbeRun printed by probe_fleet_reach_wet). Every host answered at the SSH +// layer with host-key verification failed and NO enrolled expectation, which the wire +// classifier lands as ProbeHostKeyUnenrolled (first-contact indistinguishable from +// changed-key until a fingerprint registry exists — the named next observation). The +// run also DISCHARGED the endpoint question: srv5/srv6 previously failed a layer +// earlier (name resolution); probed at their modeled reservation addresses they now +// reach SSH — reach is a transport fact, identity stays the receipt key. Both P0 cells +// for all six hosts therefore read Inaccessible (never satisfied, never Conflict — +// leg-f ruling), and the fleet fold stays honestly NOT converged with the remedy named: +// enroll the fingerprint registry. +// Typed, not a prose row (review 50827: run id / job / timestamp / branch are +// machine-consumed facts, the §4c String-commentary shape if left as one string). +type ProbeRunProvenance { + workflow_run_id: Int + job_id: String + probed_at: NonEmptyStr + head_branch: String +} + +data probe_receipts_run_provenance: ProbeRunProvenance = ProbeRunProvenance { + workflow_run_id: 31300640122, + job_id: "probe_fleet_reach", + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + head_branch: "session/wise-moth-430", +} + +// 🟡 dissolve-on (review 50824): the rows below are a HAND TRANSCRIPTION of one executed +// run — provenance-bearing, but still an authored copy of machine output, and the wet +// probe writes only the receipt FILE, never this registry, so a later run cannot move +// fleet phase standing without an author in the loop. Accepted interim (parent seam +// verification msg_8d84ae4f names the transcription acceptable with THIS as the +// next-rung trigger): DISSOLVES WHEN the probe artifact is ingested by machinery — the +// receipt file's typed parse committed into this registry by the commit workflow (the +// same artifact->typed-parse->committed-rows path the operator's join 4 names), at which +// point a stale hand row is unwritable rather than merely provenance-annotated. +data probe_receipt_ledger_transcription_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.host_reach_identity_probe", + decl_name: "host_reach_identity_probe_receipts", + field: WholeDeclaration, + } +} + +data host_reach_identity_probe_receipts: List = [ + HostReachIdentityProbeReceipt { + host: "srv1" as HostIdentity, + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + outcome: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }, + }, + HostReachIdentityProbeReceipt { + host: "srv2" as HostIdentity, + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + outcome: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }, + }, + HostReachIdentityProbeReceipt { + host: "srv3" as HostIdentity, + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + outcome: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }, + }, + HostReachIdentityProbeReceipt { + host: "srv4" as HostIdentity, + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + outcome: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }, + }, + HostReachIdentityProbeReceipt { + host: "srv5" as HostIdentity, + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + outcome: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }, + }, + HostReachIdentityProbeReceipt { + host: "srv6" as HostIdentity, + probed_at: "2026-08-09T07:20:04Z" as NonEmptyStr, + outcome: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }, + }, +] + +fn host_reach_identity_probe_receipt_for(host: HostIdentity) -> HostReachIdentityProbeReceipt? { + fold(host_reach_identity_probe_receipts, init: none, f: fn(acc, r) { + match acc { + Present { value: _ } => acc + _ => if r.host == host { Present { value: r } } else { none } + } + }) +} + +// Wire classification for the wet leg: the transport's terminal facts (exit success, +// stdout, stderr) decide the outcome arm, fail-closed — an unclassifiable failure is +// TransportUnavailable with the raw cause preserved, never a fabricated Reachable. +// The stderr patterns are the OpenSSH client's stable diagnostics for the two +// non-transport failure classes. +fn classify_reach_probe_wire_with_expectation( + success: Bool, + stdout: String, + stderr: String, + expected_fingerprint: String?, +) -> ReachProbeOutcome { + match classify_reach_probe_wire(success: success, stdout: stdout, stderr: stderr) { + ProbeHostKeyUnenrolled { transport_diagnostic: obs } => + match expected_fingerprint { + Present { value: e } => ProbeHostKeyMismatch { expected_fingerprint: e, observed_fingerprint: obs } + _ => ProbeHostKeyUnenrolled { transport_diagnostic: obs } + } + other => other + } +} + +fn classify_reach_probe_wire( + success: Bool, + stdout: String, + stderr: String, +) -> ReachProbeOutcome { + if success { + ProbeReachable { observed_short_hostname: trim(stdout) } + } else { + if string_contains(s: stderr, pattern: "Permission denied") { + ProbeAuthenticationRefused { cause: stderr } + } else { + if string_contains(s: stderr, pattern: "HOST IDENTIFICATION HAS CHANGED") + || string_contains(s: stderr, pattern: "Host key verification failed") { + ProbeHostKeyUnenrolled { transport_diagnostic: stderr } + } else { + ProbeTransportUnavailable { cause: stderr } + } + } + } +} + +// Producer 1 — P0:reach-secrets-network. Projection to the shared verdict vocabulary; +// the outcome stays on the receipt (see the distinctness note above). +// A host-key mismatch is a CONFLICT only when an enrolled expectation exists to be +// contradicted (Present — "we know this host's key and the observed one differs"). +// With Absent expectation the wire's 'verification failed' cannot distinguish first +// contact from a changed key, so the honest verdict is Inaccessible — the transport +// never completed and nothing about the host was verified. Both arms leave the cell +// NOT converged (operator ruling 2026-08-09: a Mismatch/Absent cell counts as +// unobserved-grade, never as satisfied); only the REMEDY differs — enroll the +// fingerprint registry vs investigate a real contradiction. +fn reach_verdict_from_outcome(o: ReachProbeOutcome) -> ObservationVerdict { + match o { + ProbeReachable { observed_short_hostname: _ } => Converged + ProbeTransportUnavailable { cause: _ } => Inaccessible + ProbeAuthenticationRefused { cause: _ } => Inaccessible + ProbeHostKeyUnenrolled { transport_diagnostic: _ } => Inaccessible + ProbeHostKeyMismatch { expected_fingerprint: _, observed_fingerprint: _ } => Conflict + } +} + +fn observe_reach_secrets_network(host: HostIdentity) -> ObservationVerdict? { + match host_reach_identity_probe_receipt_for(host: host) { + Present { value: r } => Present { value: reach_verdict_from_outcome(o: r.outcome) } + _ => none + } +} + +// Producer 2 — P0:host-identity-converge. The enrolled slot identity is the desired +// short hostname (fleet naming convention: the fleet_intent slot label IS the hostname +// the SetHostnameCas converge drives toward). A reachable probe whose hostname differs +// is Drifted — real divergence, the thing the converge corrects — while an unreachable +// or unauthenticated host is Inaccessible (cannot say), and a host-key mismatch is +// Conflict (the machine answering may not be the enrolled host at all; treating its +// hostname as an identity observation would launder a wrong-machine read into drift). +fn host_identity_desired_short_hostname(host: HostIdentity) -> String { + host as String +} + +fn identity_verdict_from_outcome(host: HostIdentity, o: ReachProbeOutcome) -> ObservationVerdict { + match o { + ProbeReachable { observed_short_hostname: observed } => + if observed == host_identity_desired_short_hostname(host: host) { + Converged + } else { + Drifted + } + ProbeTransportUnavailable { cause: _ } => Inaccessible + ProbeAuthenticationRefused { cause: _ } => Inaccessible + ProbeHostKeyUnenrolled { transport_diagnostic: _ } => Inaccessible + ProbeHostKeyMismatch { expected_fingerprint: _, observed_fingerprint: _ } => Conflict + } +} + +fn observe_host_identity_converge(host: HostIdentity) -> ObservationVerdict? { + match host_reach_identity_probe_receipt_for(host: host) { + Present { value: r } => Present { value: identity_verdict_from_outcome(host: host, o: r.outcome) } + _ => none + } +} + +// LIVE probe execution (loop leg e). One wet pass over the fleet reach targets via the +// typed SSH PROBE seam (typed_argv_probe_over_ssh, expect: OutcomeIsData) — a failed leg +// is DATA classified into its typed arm, never a step failure and never an ExpectSuccess +// annotation standing where a check was owed (DEVBOOT retraction class). No enrollment +// registry exists yet, so every mismatch carries expected_fingerprint = Absent honestly. +// The receipt is WRITTEN (typed value) and the entry exits success even when hosts are +// unreachable, because the probe's product is the observation; only a receipt-write +// failure is a run failure. +data fleet_reach_probe_targets: List = ["srv1", "srv2", "srv3", "srv4", "srv5", "srv6"] +data fleet_reach_probe_receipt_path: String = "target/fleet-reach-probe-receipt.txt" + +type FleetReachProbeRun { + probed_at: String + receipts: List +} + +// The probe transports to the host's RESOLVED ENDPOINT (gunbc.fleet_reach_endpoint: +// Sparks probe at their modeled reservation address, everyone else by name) while the +// receipt stays keyed by fleet IDENTITY — addressing is transport, never identity. +fn probe_host_reach_once(host: String, probed_at: String) -> HostReachIdentityProbeReceipt { + let outcome = match typed_argv_probe_over_ssh(host: fleet_probe_endpoint_for(host: host) as NonEmptyStr, argv: ["hostname"]) { + TypedArgvExecConverged { result: r } => + classify_reach_probe_wire_with_expectation( + success: r.success, + stdout: r.stdout, + stderr: r.stderr, + expected_fingerprint: none, + ) + TypedArgvExecRefused { reason: why } => ProbeTransportUnavailable { cause: why } + } + HostReachIdentityProbeReceipt { + host: host as HostIdentity, + probed_at: probed_at as NonEmptyStr, + outcome: outcome, + } +} + +func probe_fleet_reach_wet() -> ProcessExit { + let now = clock_now_probed_at() + let run = FleetReachProbeRun { + probed_at: now as String, + receipts: map(fleet_reach_probe_targets, h => probe_host_reach_once(host: h, probed_at: now as String)), + } + let w = Filesystem.Write(path: fleet_reach_probe_receipt_path, content: to_string(run)) + if w.success { + ExitSuccess + } else { + exit_failure(reason: join(["fleet reach probe receipt write failed: ", w.error], "")) + } +} diff --git a/dag/gunbc/live_deploy/emit.dag b/dag/gunbc/live_deploy/emit.dag index d6a12f42b81..8ca12306bc7 100644 --- a/dag/gunbc/live_deploy/emit.dag +++ b/dag/gunbc/live_deploy/emit.dag @@ -1,5 +1,11 @@ module gunbc.live_deploy.emit +import gunbc.fleet_converge_timer { + fleet_converge_timer_on_calendar, + fleet_converge_timer_randomized_delay, +} +import std.measure { second_count } + import std.types { String, List, Bool, NonEmptyStr, Int, FilePath, CommitSha } import gunbc.managed_directory { ManagedDirectory, @@ -34,10 +40,13 @@ import gunbc.live_deploy.spec { DeploymentStep, ArtifactStep, Dependency, DeploymentDependencyStep, DeploymentArtifactStep, - ServeBinary, GunbcSourceTree, DispatchWorktreeRoot, AttemptStateRoot, SystemdUnit, BeltTimerUnit, TailscaleServeMapping, + ServeBinary, GunbcSourceTree, DispatchWorktreeRoot, AttemptStateRoot, SystemdUnit, BeltTimerUnit, FleetConvergeTimerUnit, TailscaleServeMapping, deployment_belt_service_unit_name, deployment_belt_timer_unit_name, deployment_belt_service_unit_path, + deployment_fleet_converge_service_unit_name, + deployment_fleet_converge_timer_unit_name, + deployment_fleet_converge_service_unit_path, deployment_spec_srv1, deployment_plan_listen_port, deployment_owned_steps_retract_order, @@ -46,7 +55,7 @@ import gunbc.live_deploy.spec { TeardownDisposition, TeardownActs, TeardownRefuses, tree_sync_unit_path, } -import gunbc.host_layout { belt_spawn_workdir_env_var } +import gunbc.host_layout { belt_spawn_workdir_env_var, fleet_converge_receipt_dir } import gunbc.ownership { Ownership } import v2.std.algebra { list_tail, TailFound, TailAbsent } import gunbc.membership_reconcile { @@ -299,6 +308,56 @@ fn apply_belt_service_unit_write_step(spec: DeploymentSpec) -> PipelineStep { ], "")) } +fn emit_fleet_converge_service_unit_doc(spec: DeploymentSpec) -> Doc { + let svc = spec.service + doc_concat(parts: [ + doc_line_of(text: "[Unit]"), + doc_line_of(text: "Description=gunbc fleet converge (self)"), + doc_line_of(text: "After=network-online.target"), + doc_line_of(text: "Wants=network-online.target"), + doc_line_of(text: ""), + doc_line_of(text: "[Service]"), + doc_line_of(text: "Type=oneshot"), + doc_line_of(text: "User=root"), + doc_line_of(text: concat("WorkingDirectory=", svc.repo_root)), + doc_line_of(text: join([ + "ExecStart=", svc.serve_binary, " converge --host self", + ], "")), + ]) +} + +fn emit_fleet_converge_timer_unit_doc(spec: DeploymentSpec) -> Doc { + doc_concat(parts: [ + doc_line_of(text: "[Unit]"), + doc_line_of(text: "Description=gunbc fleet converge cadence"), + doc_line_of(text: ""), + doc_line_of(text: "[Timer]"), + doc_line_of(text: concat("Unit=", deployment_fleet_converge_service_unit_name(names: spec.names) as String)), + doc_line_of(text: concat("OnCalendar=", fleet_converge_timer_on_calendar)), + doc_line_of(text: "Persistent=true"), + doc_line_of(text: concat("RandomizedDelaySec=", to_string(second_count(s: fleet_converge_timer_randomized_delay)))), + doc_line_of(text: ""), + doc_line_of(text: "[Install]"), + doc_line_of(text: "WantedBy=timers.target"), + ]) +} + +fn apply_fleet_converge_service_unit_write_step(spec: DeploymentSpec) -> PipelineStep { + deploy_raw(command: join([ + "cat > \"$_gunbc_stage/", deployment_fleet_converge_service_unit_name(names: spec.names) as String, "\" <<'GUNBC_FLEET_CONVERGE_SERVICE_EOF'\n", + render(doc: emit_fleet_converge_service_unit_doc(spec: spec), proto: live_deploy_protocol), + "GUNBC_FLEET_CONVERGE_SERVICE_EOF", + ], "")) +} + +fn apply_fleet_converge_timer_unit_write_step(spec: DeploymentSpec) -> PipelineStep { + deploy_raw(command: join([ + "cat > \"$_gunbc_stage/", deployment_fleet_converge_timer_unit_name(names: spec.names) as String, "\" <<'GUNBC_FLEET_CONVERGE_TIMER_EOF'\n", + render(doc: emit_fleet_converge_timer_unit_doc(spec: spec), proto: live_deploy_protocol), + "GUNBC_FLEET_CONVERGE_TIMER_EOF", + ], "")) +} + fn apply_belt_timer_unit_write_step(spec: DeploymentSpec) -> PipelineStep { deploy_raw(command: join([ "cat > \"$_gunbc_stage/", deployment_belt_timer_unit_name(names: spec.names) as String, "\" <<'GUNBC_BELT_TIMER_EOF'\n", @@ -506,6 +565,22 @@ fn emit_artifact_upsert(art: DeploymentArtifactStep, spec: DeploymentSpec, revis deploy_raw(command: systemctl_enable_command(unit: deployment_belt_timer_unit_name(names: spec.names), privileged: true)), deploy_raw(command: systemctl_restart_command(unit: deployment_belt_timer_unit_name(names: spec.names), privileged: true)), ] + FleetConvergeTimerUnit => [ + deploy_raw(command: join(["sudo -n install -d -m 0755 ", fleet_converge_receipt_dir], "")), + apply_fleet_converge_service_unit_write_step(spec: spec), + apply_fleet_converge_timer_unit_write_step(spec: spec), + deploy_raw(command: join([ + "sudo -n install -m 0644 \"$_gunbc_stage/", deployment_fleet_converge_service_unit_name(names: spec.names) as String, "\" ", + deployment_fleet_converge_service_unit_path(names: spec.names) as String, + ], "")), + deploy_raw(command: join([ + "sudo -n install -m 0644 \"$_gunbc_stage/", deployment_fleet_converge_timer_unit_name(names: spec.names) as String, "\" ", + art.path as String, + ], "")), + deploy_raw(command: systemctl_daemon_reload_command(privileged: true)), + deploy_raw(command: systemctl_enable_command(unit: deployment_fleet_converge_timer_unit_name(names: spec.names), privileged: true)), + deploy_raw(command: systemctl_restart_command(unit: deployment_fleet_converge_timer_unit_name(names: spec.names), privileged: true)), + ] TailscaleServeMapping => [ tailscale_serve_apply_step(spec: spec), ] @@ -535,6 +610,15 @@ fn emit_artifact_teardown(art: DeploymentArtifactStep, spec: DeploymentSpec) -> deploy_raw(command: systemctl_daemon_reload_command(privileged: true)), ] ) + FleetConvergeTimerUnit => concat( + systemctl_disable_now_if_present_steps(unit: deployment_fleet_converge_timer_unit_name(names: spec.names), privileged: true), + [ + deploy_raw(command: join([ + "sudo -n rm -f ", art.path as String, " ", deployment_fleet_converge_service_unit_path(names: spec.names) as String, + ], "")), + deploy_raw(command: systemctl_daemon_reload_command(privileged: true)), + ] + ) DispatchWorktreeRoot => [ deploy_raw(command: concat("sudo -n rm -d ", art.path as String)), ] diff --git a/dag/gunbc/live_deploy/spec.dag b/dag/gunbc/live_deploy/spec.dag index 6346c0bf629..bc2d9b506f5 100644 --- a/dag/gunbc/live_deploy/spec.dag +++ b/dag/gunbc/live_deploy/spec.dag @@ -56,6 +56,7 @@ type OwnedArtifactKind = | AttemptStateRoot | SystemdUnit | BeltTimerUnit + | FleetConvergeTimerUnit | TailscaleServeMapping type EnsuredDependencyKind = @@ -100,6 +101,7 @@ fn owned_artifact_teardown_disposition(kind: OwnedArtifactKind) -> TeardownDispo AttemptStateRoot => TeardownActs SystemdUnit => TeardownActs BeltTimerUnit => TeardownActs + FleetConvergeTimerUnit => TeardownActs TailscaleServeMapping => TeardownActs } } @@ -175,6 +177,33 @@ fn deployment_belt_timer_unit_path(names: DeploymentNames) -> NonEmptyStr { join(["/etc/systemd/system/", deployment_belt_timer_unit_name(names: names) as String], "") as NonEmptyStr } +// CONVERGE-0 loop member (operator directive 2026-08-09: 'timer in deployment membership'). +// The fleet-converge timer rides the belt-timer precedent EXACTLY — per-deployment names +// derived from the instance stem (unconditional membership, so the twin mirror holds and +// two instances cannot mint one path), one member owning .timer + .service, teardown +// removes both, retirement stops the runtime. The converge pass itself is idempotent +// observe→verdict→apply-on-drift, so a twin's duplicate cadence against the same host is +// duplicated observation, never duplicated mutation. CONTENT cadence/catch-up facts +// (OnCalendar, Persistent=true, jitter) derive from the gunbc.fleet_converge_timer +// authority rows; only the NAMES derive here. That module's fixed ctrl-fleet-converge +// unit-name rows remain the standalone/host-standup authority and are not consumed by +// this member. +fn deployment_fleet_converge_service_unit_name(names: DeploymentNames) -> NonEmptyStr { + join([deployment_belt_unit_stem(names: names), "-fleet-converge.service"], "") as NonEmptyStr +} + +fn deployment_fleet_converge_timer_unit_name(names: DeploymentNames) -> NonEmptyStr { + join([deployment_belt_unit_stem(names: names), "-fleet-converge.timer"], "") as NonEmptyStr +} + +fn deployment_fleet_converge_service_unit_path(names: DeploymentNames) -> NonEmptyStr { + join(["/etc/systemd/system/", deployment_fleet_converge_service_unit_name(names: names) as String], "") as NonEmptyStr +} + +fn deployment_fleet_converge_timer_unit_path(names: DeploymentNames) -> NonEmptyStr { + join(["/etc/systemd/system/", deployment_fleet_converge_timer_unit_name(names: names) as String], "") as NonEmptyStr +} + data tailscale_default_https_port: Port = 443 data derived_mount_is_the_instance_namespace_note: String = "THE MOUNT PATH IS NAMESPACED BY instance_id, which is the same axis that already namespaces the instance root, the repo root, the serve binary, the listen port, the worktree root, the unit name and the tmux session. The twin's endpoint is /srv1-lab because the twin IS srv1-lab, so two instances cannot produce one endpoint without sharing an identity the instance registry already forbids — the collision is unwritable rather than merely witnessed (operator: use literal namespacing in the repo, 2026-07-30).\\n\\nHand-picking a mount would have re-created the defect this replaces. tailscale_serve_path was hand-picked too, and being distinct is precisely what it was: distinct, unconsumed, and unrelated to the thing tailscale keys on. A second hand-picked string differs only in being the right KIND of string, and the next instance would re-litigate it.\\n\\nDISSOLVES ON the containment tree becoming the naming authority (namespace-only resolution): instance_id is itself a flat identifier, so this derivation is namespacing by string prefix where namespacing by position is the end state. The mount then projects from the instance's position rather than from its name, and this function stops spelling a separator." @@ -265,6 +294,7 @@ fn deployment_owned_steps( DeploymentArtifactStep { kind: AttemptStateRoot, path: dashboard_instance_dispatch_attempt_state_root(instance: instance) as String as NonEmptyStr }, DeploymentArtifactStep { kind: SystemdUnit, path: deployment_systemd_unit_path(names: names) }, DeploymentArtifactStep { kind: BeltTimerUnit, path: deployment_belt_timer_unit_path(names: names) }, + DeploymentArtifactStep { kind: FleetConvergeTimerUnit, path: deployment_fleet_converge_timer_unit_path(names: names) }, DeploymentArtifactStep { kind: TailscaleServeMapping, path: tailscale_serve_endpoint_key(endpoint: names.tailscale_serve_endpoint) }, ] } @@ -389,6 +419,7 @@ fn owned_artifact_retirement_disposition(kind: OwnedArtifactKind) -> RetirementD AttemptStateRoot => RetirementPreserves SystemdUnit => RetiresRuntime BeltTimerUnit => RetiresRuntime + FleetConvergeTimerUnit => RetiresRuntime TailscaleServeMapping => RetirementPreserves } } diff --git a/dag/gunbc/runner_capacity_operator_access.dag b/dag/gunbc/runner_capacity_operator_access.dag index 00cff48c299..5b93f29bc86 100644 --- a/dag/gunbc/runner_capacity_operator_access.dag +++ b/dag/gunbc/runner_capacity_operator_access.dag @@ -168,7 +168,7 @@ fn in_band_privilege_observation_for_host(host: HostIdentity) -> InBandPrivilege } } -data srv1_already_converged_caps_control_note: String = "srv1 acceptance row 6 control (operator 2026-08-07): five slots hand-converged to modeled 14GiB caps (MemoryMax=15032385536 MemoryHigh=13958643712 MemorySwapMax=34359738368 TasksMax=512 CPUWeight=100), verified by read-back. A second identical apply against srv1 must emit ZERO effects — discriminates directory-counting or write-blind realizers." +data srv1_already_converged_caps_control_note: String = "srv1 acceptance row 6 control (operator 2026-08-07): five slots hand-converged to modeled 14GiB caps (MemoryMax=15032385536 MemoryHigh=13958643712 MemorySwapMax=34359738368 CPUWeight=100) plus (per the original note) TasksMax=512 — but the 2026-08-08 operator ruling on PR 8045 made 16384 the desired TasksMax (512 is the diagnosed EAGAIN outage value), and the 512 claim was then DECIDED FALSE BY OBSERVATION (executed pre-apply receipt, run 31288091024, 2026-08-09, operator-accepted: all five slots at effective TasksMax=16384 with matching cgroup pids.max). The zero-effects premise therefore holds on every converged axis including tasks, and the no-op control is exact. Still discriminates directory-counting or write-blind realizers." type RunnerCapacityHostAdmissionVerdict = InBandAdmissionRequirementDocumented { requirement: RunnerCapacityHostInBandAdmissionRequirement } diff --git a/dag/gunbc/runner_lifecycle.dag b/dag/gunbc/runner_lifecycle.dag index 4cce808eba8..80d1c4e1900 100644 --- a/dag/gunbc/runner_lifecycle.dag +++ b/dag/gunbc/runner_lifecycle.dag @@ -235,7 +235,7 @@ data runner_lifecycle_jit_wrapper_path: String = join([actions_runner_base_dir, data runner_lifecycle_systemd_template_unit: String = gunbc_runner_unit_template -data runner_lifecycle_memory_cap_apply_note: String = "Per-slot caps are applied IMPERATIVELY on hosts (operator probe 2026-08-07): systemctl set-property writes /etc/systemd/system.control/.d/50-*.conf; read back with systemctl show -p MemoryMax/MemoryHigh/MemorySwapMax — not a hand-authored unit drop-in file. host_converge_slice1 is the precedent for EnsureRunnerMemoryCaps realization. srv1's five slots were hand-converged to the 14GiB model (MemoryMax=15032385536 MemoryHigh=13958643712 MemorySwapMax=34359738368 TasksMax=512 CPUWeight=100) and serve as the acceptance row 6 no-op control; srv2/srv3/srv4 remain at 16GiB until the workflow applies." +data runner_lifecycle_memory_cap_apply_note: String = "Per-slot caps are applied IMPERATIVELY on hosts (operator probe 2026-08-07): systemctl set-property writes /etc/systemd/system.control/.d/50-*.conf; read back with systemctl show -p MemoryMax/MemoryHigh/MemorySwapMax — not a hand-authored unit drop-in file. host_converge_slice1 is the precedent for EnsureRunnerMemoryCaps realization. srv1's five slots were hand-converged 2026-08-07 to the 14GiB memory model (MemoryMax=15032385536 MemoryHigh=13958643712 MemorySwapMax=34359738368 CPUWeight=100) AND (per that note's own text) to TasksMax=512 — which the 2026-08-08 operator ruling on PR 8045 identified as the diagnosed EAGAIN outage value, not the repair; desired is 16384 (gunbc.runner_slot_allocation). The 512 claim was DECIDED FALSE BY OBSERVATION (executed pre-apply receipt, workflow_dispatch run 31288091024, 2026-08-09, operator-accepted): all five srv1 slots read effective TasksMax=16384 with matching cgroup pids.max=16384, so the tasks axis is Converged and srv1 serves as the acceptance row 6 no-op control on ALL converged axes. srv2/srv3/srv4 remain at 16GiB until the workflow applies; their tasks state is unobserved." data runner_lifecycle_memory_cap_dropin: String = gunbc_runner_per_slot_dropin_path diff --git a/dag/gunbc/runner_slot_allocation.dag b/dag/gunbc/runner_slot_allocation.dag index a7b81b8a9ad..6829d42ae97 100644 --- a/dag/gunbc/runner_slot_allocation.dag +++ b/dag/gunbc/runner_slot_allocation.dag @@ -32,19 +32,29 @@ type HostRunnerSlotAllocation { } data gunbc_runner_slot_allocation_authority_disposition: Disposition = Terminal { - reason: "Single authority for fleet runner-slot desired allocation (operator rulings 2026-07-10 superseding the 2026-07-05 brief; MemoryHigh-emit + swap-provisioning 2026-07-11; srv3 host-standup converge 2026-07-21; PER-HOST WIDTH + 14GiB slot 2026-08-05; srv4 disk row-pin retired + apply-time preflight 2026-08-06). Fleet-uniform per-slot row: 8 cores · 14GiB MemoryMax · 13GiB MemoryHigh (the authored max−1GiB relation, its own emitted knob per the 2026-07-11 ruling — a hit at 14GiB spills to swap after throttling at 13) · 32GiB MemorySwapMax · TasksMax 512 · cpu_weight 100. PER-HOST width: MEMORY admits 5/5/7/7 for srv1/srv2/srv3/srv4; COMMITTED width is the minimum over allocation-time axes — srv1/srv2/srv3 commit 5/5/7; srv4 commits 7 at memory with disk DiskWidthUnconstrained at allocation (apply-time gate gunbc.runner_lifecycle runner_width_disk_preflight; width_is_a_minimum_over_axes_note) — fleet total 24. GUARANTEED mode is now STRUCTURAL rather than checked — runner_slice_cap is DERIVED as slots × MemoryMax instead of being a second literal, so Σ per-slot MemoryMax ≤ slice_cap holds by construction and an oversubscribed slice has no representation here (DESIGN 5 construction-over-validation; the prior 80GiB literal was kept coherent with 5 × 16GiB by hand, which the old note admitted when it called them 'coherent by construction at these values' — a coincidence maintained by an author, not a derivation). The real remaining constraint is CONSERVATION against host RAM, which is not this module's to check: gunbc.ci_runner_placement holds it, now against OS-usable RAM with the per-host session slice and per-host fixed overhead charged. Per-axis targets are operator-set; never reconcile this row toward live readback (live is not automatically truth — the 32-vs-0 swap fork existed precisely because the apply was assumed rather than readback-proven). Downstream: gunbc.ci_floor_measurement (declared vs live cap fork), gunbc.ci_runner_placement, gunbc.host_converge, gunbc.host_standup (prefix:host-converge + P1:runner-deploy-slot), gunbc.runner_slot_enforcement, gunbc.fleet_show_effective_read. ctrl applies host-side; gunbc emits converge — never apply from this module." + reason: "Single authority for fleet runner-slot desired allocation (operator rulings 2026-07-10 superseding the 2026-07-05 brief; MemoryHigh-emit + swap-provisioning 2026-07-11; srv3 host-standup converge 2026-07-21; PER-HOST WIDTH + 14GiB slot 2026-08-05; srv4 disk row-pin retired + apply-time preflight 2026-08-06). Fleet-uniform per-slot row: 8 cores · 14GiB MemoryMax · 13GiB MemoryHigh (the authored max−1GiB relation, its own emitted knob per the 2026-07-11 ruling — a hit at 14GiB spills to swap after throttling at 13) · 32GiB MemorySwapMax · TasksMax 16384 (the 2026-06-21 incident REPAIR value restored by operator ruling 2026-08-08 on PR 8045: TasksMax=512 WAS the diagnosed EAGAIN/process-creation outage value, and the earlier 512 row here inverted that diagnosis; any lower derived envelope value must be separately accepted and never below live task demand) · cpu_weight 100. PER-HOST width: MEMORY admits 5/5/7/7 for srv1/srv2/srv3/srv4; COMMITTED width is the minimum over allocation-time axes — srv1/srv2/srv3 commit 5/5/7; srv4 commits 7 at memory with disk DiskWidthUnconstrained at allocation (apply-time gate gunbc.runner_lifecycle runner_width_disk_preflight; width_is_a_minimum_over_axes_note) — fleet total 24. GUARANTEED mode is now STRUCTURAL rather than checked — runner_slice_cap is DERIVED as slots × MemoryMax instead of being a second literal, so Σ per-slot MemoryMax ≤ slice_cap holds by construction and an oversubscribed slice has no representation here (DESIGN 5 construction-over-validation; the prior 80GiB literal was kept coherent with 5 × 16GiB by hand, which the old note admitted when it called them 'coherent by construction at these values' — a coincidence maintained by an author, not a derivation). The real remaining constraint is CONSERVATION against host RAM, which is not this module's to check: gunbc.ci_runner_placement holds it, now against OS-usable RAM with the per-host session slice and per-host fixed overhead charged. Per-axis targets are operator-set; never reconcile this row toward live readback (live is not automatically truth — the 32-vs-0 swap fork existed precisely because the apply was assumed rather than readback-proven). Downstream: gunbc.ci_floor_measurement (declared vs live cap fork), gunbc.ci_runner_placement, gunbc.host_converge, gunbc.host_standup (prefix:host-converge + P1:runner-deploy-slot), gunbc.runner_slot_enforcement, gunbc.fleet_show_effective_read. ctrl applies host-side; gunbc emits converge — never apply from this module." } data gunbc_runner_slot_memory_max_ruling_note: String = "WHY 14GiB AND NOT 16, measured rather than preferred (operator ruling 2026-08-05). 186 floor-job cgroup peaks from 2026-08-05 plus five sampled days back to 2026-07-16, read from the [calibration] floor_peak_post receipts the ci job emits: successful runs peak at p50 10.32GiB and max 10.96GiB, and NOT ONE run in the sample reached the 16GiB MemoryMax. Four of 186 reached 15.0GiB, which is the 15GiB MemoryHigh exactly — those are throttle pins, censored lower bounds rather than demand, and they are why the cap does not fall further.\n\nTHE FLOOR UNDER IT IS ALREADY MEASURED AND IT IS 12GiB: gunbc_floor_minimum_viable_armed_budget in this module, enforced by claim_executor as an arm-time refusal. MemoryHigh is what the run-time governor arms against, so MemoryHigh is the axis that number constrains — 13GiB leaves exactly 1GiB of margin over it, and clears the worst p90 in the three-week sample (12.94GiB on 2026-07-30). Going to 13/12 to buy an eighth slot on the managed hosts would put the throttle line ON the measured refusal floor with zero margin, and the 2026-07-30 distribution says that day would have spent most of its runs in swap.\n\nWHAT MOVING 16 TO 14 IS NOT: it is not oversubscription and does not touch the guaranteed-mode property. Every slot still owns its ceiling outright. The ceilings are simply sized closer to what the workload was measured to want, which is what freed the width." data gunbc_runner_slot_width_ruling_note: String = "WHY srv3/srv4 GET 7 AND srv1/srv2 STAY AT 5 — the width difference is DERIVED from two per-host facts, not granted by preference. First, sessions: gunbc.session_residency host_session_residency places interactive session containers on srv1 and srv2 and on neither managed host, so srv1/srv2 owe a sessions reservation out of the same host RAM and srv3/srv4 do not. Second, fixed overhead: gunbc.ci_floor_measurement carries a measured upper bound for the managed hosts and an operator allowance for the legacy pair, which differ by 10.4GiB per host.\n\nRun the conservation arithmetic with both charged and the MEMORY admissions fall out — srv3/srv4 admit 7 slots at 14GiB and refuse 8 (an eighth would need 112GiB against ~111.6GiB available, over by less than a gibibyte), srv1/srv2 admit 5 and refuse 6. Neither number is authored here as a target; both are the largest width gunbc.ci_runner_placement's wall will pass, and if either per-host fact is corrected the admissible width moves with it.\n\nMEMORY ADMISSION IS NOT THE FULL STORY FOR DISK ON srv4. This note explains the memory axis; gunbc_runner_slots_per_host derives committed width as the minimum over allocation-time axes. srv4's disk row-pin is retired: host_disk_admitted_width returns DiskWidthUnconstrained and srv4 commits 7 like srv3 at allocation; insufficient or unobservable disk at apply is refused by gunbc.runner_lifecycle runner_width_disk_preflight rather than lowering allocation to 5.\n\nTHE SESSIONS FACT IS RESIDENCY, AND SAYING SO HERE IS LOAD-BEARING. An earlier revision of this note derived the same widths from gunbc.build_cache_instance instead — whether a host carried an InteractiveSessionCompiles cache instance — which merely CORRELATED with sessions living there. That proxy is the defect this change removes, and gunbc.session_residency why_this_is_not_derived_from_the_build_cache_roster_note records how close it came: the session cache on srv1 and srv2 was found provably dead on 2026-08-06 and deleted, so dropping its now-meaningless roster rows would have stopped charging both hosts their reservation and admitted 7 slots each on a wall returning true. A note still reasoning from the roster would invite exactly that edit, which is why it is corrected rather than left as harmless prose.\n\nTHE FLEET-UNIFORM ROW IS WHY THIS COULD NOT BE EXPRESSED BEFORE. gunbc_committed_host_runner_slot_allocation took a host and then read three globals, so 'this host is different' had nowhere to live and the only way to raise srv3/srv4 was to raise all four — which on srv1/srv2 would have overcommitted by 17GiB against a wall that could not see the sessions it was overcommitting against." +// 🟡 REGISTERED FOLLOW-UP — task-budget envelope model (operator ruling on PR 8045, 2026-08-09). +// tasks_max=16384 below is APPROVED as an EMERGENCY PARITY ROLLBACK ONLY: it is not accepted as +// a derived value, and RunnerSlotDesired.tasks_max is not accepted as the model home. The +// follow-up model must distinguish THREE task budgets, each derived from one measured envelope +// (concurrent-build admission × compiler/thread fan-out): (1) runner-slot task budget; (2) shared +// compile-pool task budget — sccache cache-miss compiles run as children of the SERVER'S cgroup, +// not the requester's (gunbc.build_cache_instance placement axis), so the pool owns their task +// count exactly as compile_pool owns their memory; (3) host-parent reserve — with the conservation +// law sum(slots) + pool + reserve <= parent ancestor limit. Dissolve-on: that model landing, at +// which point this row's tasks_max derives rather than being authored. Do not build it in PR 8045. fn gunbc_runner_slot_desired() -> RunnerSlotDesired { RunnerSlotDesired { memory_max: byte_size(15032385536), memory_swap_max: byte_size(34359738368), memory_high: byte_size(13958643712), - tasks_max: 512, + tasks_max: 16384, cpu_weight: 100, labels: RunnerSlotLabels { github_labels: ["self-hosted", "linux", "arm64", "gunbc-heavy"] } } diff --git a/dag/gunbc/srv1_tasks_preapply_observation.dag b/dag/gunbc/srv1_tasks_preapply_observation.dag new file mode 100644 index 00000000000..ce2ee0ebe52 --- /dev/null +++ b/dag/gunbc/srv1_tasks_preapply_observation.dag @@ -0,0 +1,144 @@ +module gunbc.srv1_tasks_preapply_observation + +import extdeps.filesystem.filesystem_io { Filesystem } +import std.types { String, NonEmptyStr, Bool, List } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import gunbc.host_converge { runner_instance_glob } +import gunbc.host_converge_slice1 { + host_converge_slice1_enumerate_units, + host_converge_slice1_read_tasks_max_raw, + host_converge_slice1_read_tasks_current_raw, +} +import gunbc.systemctl_show_read { + systemctl_show_property_read_local, + SystemdPropertyCapture, + SystemdPropertyCaptured, + SystemdPropertyCaptureRefused, +} +import extdeps.systemd { ControlGroup } +import extdeps.linux.procfs { ProcSelfCgroup, procfs_path_literal } +import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } + +// STRICTLY READ-ONLY pre-apply observation for the PR 8045 supervised TasksMax apply +// (operator ruling 2026-08-09: before the apply, re-observe and reduce to ONE exact +// receipt recording effective TasksMax, pids.max, pids.current, and the relevant +// ancestor limit; success criterion for the later apply is the independent post-effect +// readback, never the exit code). Every transport this module reaches is a read: +// systemctl show (TasksMax / TasksCurrent / ControlGroup) through the typed +// gunbc.systemctl_show_read seam, and filesystem_read on the cgroupfs pids files at +// the path SYSTEMD reports (ControlGroup), never a hand-escaped slice path — the +// system-actions-runner.slice escaping decoy is exactly why the path is observed, not +// derived. No set-property, no unit operation, no write of any kind exists in this +// module's reach. The observation runs AS a job on the observed host (LocalShell, +// the slice1 precedent), so the receipt records the observer's own cgroup membership +// (/proc/self/cgroup, raw) — a reader can see the read ran INSIDE the observed +// universe and which slot performed it. +data srv1_tasks_preapply_cgroupfs_root: String = "/sys/fs/cgroup" + +type Srv1TasksAncestorRead { + cgroup_dir: String + pids_max_raw: String +} + +type Srv1TasksUnitObservation + = Srv1TasksUnitObserved { + unit: NonEmptyStr + tasks_max_effective: String + tasks_current: String + control_group: String + pids_max_raw: String + pids_current_raw: String + ancestor: Srv1TasksAncestorRead + } + | Srv1TasksUnitObservationRefused { unit: NonEmptyStr, cause: String } + +type Srv1TasksPreApplyReceipt + = Srv1TasksPreApplyObserved { + host: String + desired_tasks_max: String + observer_cgroup_membership: String + units: List + } + | Srv1TasksPreApplyRefused { cause: String } + +fn cgroup_parent_dir(control_group: String) -> String { + let segs = split(s: control_group, delimiter: "/") + if length(segs) < 3 { + control_group + } else { + join(segs.take(n: length(segs) - 1), "/") + } +} + +fn read_pids_file(control_group: String, leaf: String) -> String { + filesystem_read(path: join([srv1_tasks_preapply_cgroupfs_root, control_group, "/", leaf], "")).content +} + +fn observe_unit_tasks(unit: NonEmptyStr) -> Srv1TasksUnitObservation { + match host_converge_slice1_read_tasks_max_raw(unit: unit) { + Present { value: tmax } => + match host_converge_slice1_read_tasks_current_raw(unit: unit) { + Present { value: tcur } => + match systemctl_show_property_read_local(unit: unit, property: ControlGroup) { + SystemdPropertyCaptured { value: cg } => + if cg == "" { + Srv1TasksUnitObservationRefused { unit: unit, cause: "ControlGroup read returned empty — cannot locate cgroupfs pids files" } + } else { + Srv1TasksUnitObserved { + unit: unit, + tasks_max_effective: tmax, + tasks_current: tcur, + control_group: cg, + pids_max_raw: read_pids_file(control_group: cg, leaf: "pids.max"), + pids_current_raw: read_pids_file(control_group: cg, leaf: "pids.current"), + ancestor: Srv1TasksAncestorRead { + cgroup_dir: cgroup_parent_dir(control_group: cg), + pids_max_raw: read_pids_file(control_group: cgroup_parent_dir(control_group: cg), leaf: "pids.max"), + }, + } + } + SystemdPropertyCaptureRefused { reason: why } => + Srv1TasksUnitObservationRefused { unit: unit, cause: join(["ControlGroup read refused: ", why], "") } + } + _ => Srv1TasksUnitObservationRefused { unit: unit, cause: "TasksCurrent read refused" } + } + _ => Srv1TasksUnitObservationRefused { unit: unit, cause: "TasksMax read refused" } + } +} + +fn observe_srv1_tasks_preapply() -> Srv1TasksPreApplyReceipt { + match host_converge_slice1_enumerate_units(instance_glob: runner_instance_glob(host: "srv1")) { + Present { value: units } => + if length(units) == 0 { + Srv1TasksPreApplyRefused { cause: "unit enumeration returned zero actions-runner@srv1-* units — refusing an empty observation rather than reporting a vacuous receipt" } + } else { + Srv1TasksPreApplyObserved { + host: "srv1", + desired_tasks_max: to_string(gunbc_runner_slot_desired().tasks_max), + observer_cgroup_membership: filesystem_read(path: procfs_path_literal(p: ProcSelfCgroup) as String).content, + units: map(units, u => observe_unit_tasks(unit: u)), + } + } + _ => Srv1TasksPreApplyRefused { cause: "unit enumeration refused (systemctl list-units transport)" } + } +} + +// Receipt landing: gunbc run admits only ProcessExit from a --function wet entry (the +// first dispatch printed the receipt through the NotProcessExit refusal arm and exited +// 2 — receipt captured, job red). The typed receipt therefore lands as a FILE the job +// uploads/cats, and the exit derives from the receipt's own arm — never from transport +// exit codes (the ruling's success criterion applies to the later apply the same way). +data srv1_tasks_preapply_receipt_path: String = "target/srv1-tasks-preapply-receipt.txt" + +func observe_srv1_tasks_preapply_wet() -> ProcessExit { + let receipt = observe_srv1_tasks_preapply() + let w = Filesystem.Write(path: srv1_tasks_preapply_receipt_path, content: to_string(receipt)) + if !w.success { + exit_failure(reason: join(["receipt write failed: ", w.error], "")) + } else { + match receipt { + Srv1TasksPreApplyObserved { host: _, desired_tasks_max: _, observer_cgroup_membership: _, units: _ } => ExitSuccess + Srv1TasksPreApplyRefused { cause: why } => exit_failure(reason: why) + } + } +} diff --git a/dag/test/claim/fleet_converge_cli_witness_test.dag b/dag/test/claim/fleet_converge_cli_witness_test.dag index be8f41d2c5a..a3b592ec98a 100644 --- a/dag/test/claim/fleet_converge_cli_witness_test.dag +++ b/dag/test/claim/fleet_converge_cli_witness_test.dag @@ -28,6 +28,7 @@ test fn witness_converge_cli_run_srv2_fails_closed_on_knob_frontier() -> Bool { KnobFrontier { reason: why } => string_contains(s: why, pattern: "host_converge_realize") PolicyUnsound { reason: _ } => false UnknownHost { host: _ } => false + SparkPreEnrollment { host: _ } => false ComputeInventoryAbsent { host: _ } => false } _ => false @@ -47,6 +48,7 @@ test fn witness_converge_cli_run_unknown_host_refuses() -> Bool { UnknownHost { host: h } => h == "no-such-fleet-host" PolicyUnsound { reason: _ } => false KnobFrontier { reason: _ } => false + SparkPreEnrollment { host: _ } => false ComputeInventoryAbsent { host: _ } => false } _ => false @@ -63,10 +65,12 @@ test fn witness_converge_cli_unknown_host_and_compute_inventory_absent_are_disti UnknownHost { host: _ } => false PolicyUnsound { reason: _ } => false KnobFrontier { reason: _ } => false + SparkPreEnrollment { host: _ } => false } ComputeInventoryAbsent { host: _ } => false PolicyUnsound { reason: _ } => false KnobFrontier { reason: _ } => false + SparkPreEnrollment { host: _ } => false } && converge_cli_refusal_message(cause: unknown) != converge_cli_refusal_message(cause: inventory_absent) && string_contains(s: converge_cli_refusal_message(cause: unknown), pattern: "unknown fleet host identity") @@ -89,10 +93,12 @@ test fn witness_converge_cli_refusal_causes_discriminate() -> Bool { UnknownHost { host: _ } => true PolicyUnsound { reason: _ } => false KnobFrontier { reason: _ } => false + SparkPreEnrollment { host: _ } => false ComputeInventoryAbsent { host: _ } => false } PolicyUnsound { reason: _ } => false UnknownHost { host: _ } => false + SparkPreEnrollment { host: _ } => false ComputeInventoryAbsent { host: _ } => false } _ => false @@ -124,3 +130,68 @@ test fn witness_converge_cli_output_projects_receipt_and_exit_signal() -> Bool { } } + +// Join 1 (operator verdict): the revision cell gates the self route. Converged passes +// the knob output through untouched; Drifted demotes to not-converged carrying BOTH +// revisions (the catch-up realization needs the prior); Unobserved demotes with the +// typed cause and is never satisfied. The rendered receipt line carries the cell for +// the collector, one verdict token per arm. +test fn witness_revision_gate_demotes_drift_and_unobserved_never_converged_passthrough() -> Bool { + let a = revision_fixture_oid(hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + let b = revision_fixture_oid(hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + let out = ConvergeCliOutput { line: "converge-receipt host=srv1 applied=1 drifted=0 verdict=converged", converged: true, reason: none } + let gated_ok = converge_cli_revision_gate(out: out, standing: RevisionConverged { revision: a }) + let gated_drift = converge_cli_revision_gate(out: out, standing: RevisionDrifted { desired: a, local: b }) + let gated_unobs = converge_cli_revision_gate(out: out, standing: RevisionUnobserved { cause: "ls-remote refused: no route" }) + gated_ok.converged + && !gated_drift.converged + && match gated_drift.reason { + Present { value: why } => + string_contains(s: why, pattern: "revision drift") + && string_contains(s: why, pattern: "aaaaaaaa") + && string_contains(s: why, pattern: "bbbbbbbb") + _ => false + } + && !gated_unobs.converged + && match gated_unobs.reason { + Present { value: why } => string_contains(s: why, pattern: "unobserved") + _ => false + } + && string_contains(s: converge_cli_revision_line(standing: RevisionConverged { revision: a }), pattern: "verdict=converged") + && string_contains(s: converge_cli_revision_line(standing: RevisionDrifted { desired: a, local: b }), pattern: "verdict=drifted") + && string_contains(s: converge_cli_revision_line(standing: RevisionUnobserved { cause: "x" }), pattern: "verdict=unobserved") +} + +fn revision_fixture_oid(hex: String) -> GitObjectId { + match git_sha1_object_id(hex: hex) { + Present { value: oid } => oid + Absent => + GitSha1ObjectId { + digest: Sha1Digest { hex: hex as Sha1DigestHex }, + } + } +} + +// Join 3 (operator verdict): applicability is TOTAL over the fleet — srv5/srv6 are a +// typed pre-enrollment refusal DISTINCT from UnknownHost (they are known fleet +// identities awaiting SPARK-SERVE-0), and every enrolled host lands one arm. +test fn witness_spark_pre_enrollment_is_typed_and_distinct_from_unknown() -> Bool { + let r = converge_cli_run(host: "srv5") + !converge_cli_receipt_converged(r: r) + && match r.reason { + Present { value: cause } => + match cause { + SparkPreEnrollment { host: h } => h == "srv5" + _ => false + } + _ => false + } + && match converge_cli_run(host: "no-such-fleet-host").reason { + Present { value: c2 } => + match c2 { + UnknownHost { host: _ } => true + _ => false + } + _ => false + } +} diff --git a/dag/test/claim/fleet_converge_timer_install_witness_test.dag b/dag/test/claim/fleet_converge_timer_install_witness_test.dag new file mode 100644 index 00000000000..ab7976234ed --- /dev/null +++ b/dag/test/claim/fleet_converge_timer_install_witness_test.dag @@ -0,0 +1,62 @@ +module test.claim.fleet_converge_timer_install_witness + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// Join 2 (operator verdict): the timer route is TOTAL over the fleet and each arm is +// typed — srv1 stays on live_deploy membership (its owned-artifact authority), every +// other enrolled host self-installs, Sparks are pre-enrollment (never silently absent), +// and a non-fleet name is outside. No two hosts collapse into an untyped default. +test fn witness_timer_route_is_total_and_discriminates() -> Bool { + match fleet_timer_install_route(host: "srv1") { + TimerViaLiveDeployMembership { host: h } => h == "srv1" + _ => false + } + && match fleet_timer_install_route(host: "srv2") { + TimerViaSelfInstall { host: h } => h == "srv2" + _ => false + } + && match fleet_timer_install_route(host: "srv3") { + TimerViaSelfInstall { host: h } => h == "srv3" + _ => false + } + && match fleet_timer_install_route(host: "srv4") { + TimerViaSelfInstall { host: h } => h == "srv4" + _ => false + } + && match fleet_timer_install_route(host: "srv5") { + TimerPreEnrollment { host: h } => h == "srv5" + _ => false + } + && match fleet_timer_install_route(host: "no-such-host") { + TimerOutsideFleet { host: _ } => true + _ => false + } +} + +// The install plan derives from the ONE timer authority: staged and installed paths +// carry the authority's unit names, the enable step names the timer unit, and the LAST +// step is the independent post-effect readback (is-enabled), not an exit-code trust. +test fn witness_timer_install_plan_derives_from_authority_and_ends_in_readback() -> Bool { + let cmds = fleet_timer_install_commands() + cmds.length() == 5 + && match cmds.first() { + Present { value: head } => + match head { + ArgvCommand { argv: argv } => + string_contains(s: join(argv, " "), pattern: fleet_converge_service_unit_name) + && string_contains(s: join(argv, " "), pattern: "/etc/systemd/system/") + } + Absent => false + } + && match last_command(cmds: cmds) { + Present { value: readback } => + match readback { + ArgvCommand { argv: argv } => join(argv, " ") == join(["systemctl is-enabled ", fleet_converge_timer_unit_name], "") + } + Absent => false + } +} + +fn last_command(cmds: List) -> ArgvCommand? { + fold(cmds, init: none, f: (acc, c) => Present { value: c }) +} diff --git a/dag/test/claim/fleet_converge_timer_witness_test.dag b/dag/test/claim/fleet_converge_timer_witness_test.dag new file mode 100644 index 00000000000..4117d963760 --- /dev/null +++ b/dag/test/claim/fleet_converge_timer_witness_test.dag @@ -0,0 +1,49 @@ +module test.claim.fleet_converge_timer_witness + +import std.types { Bool, String } +import gunbc.fleet_converge_timer { + fleet_converge_service_unit_name, + fleet_converge_timer_unit_name, + fleet_converge_timer_on_calendar, + fleet_converge_service_content, + fleet_converge_service_content_self, + fleet_converge_timer_content, +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// Option A shape: calendar cadence WITH Persistent=true (catch-up after downtime — the +// offline-host property) and jitter. Persistent on a monotonic timer would be silently +// ignored by systemd, so the content deriving both from one authority is the wall. +test fn timer_content_is_calendar_persistent_with_jitter() -> Bool { + let t = fleet_converge_timer_content() + string_contains(s: t, pattern: "OnCalendar=*:0/15") + && string_contains(s: t, pattern: "Persistent=true") + && string_contains(s: t, pattern: "RandomizedDelaySec=90") + && string_contains(s: t, pattern: "Unit=ctrl-fleet-converge.service") + && !string_contains(s: t, pattern: "OnUnitActiveSec") +} + +// The service converges SELF only — no remote host name is writable into the emitted +// unit from this authority, and the oneshot runs the pinned binary path. +test fn service_content_converges_self_only_via_pinned_binary() -> Bool { + let s = fleet_converge_service_content_self() + string_contains(s: s, pattern: "ExecStart=/opt/gunbc/bin/gunbc converge --host self") + && string_contains(s: s, pattern: "Type=oneshot") + && !string_contains(s: s, pattern: "--host srv") +} + +// RED-shaped control: a content builder invoked with a remote host WOULD name it — the +// self-only law lives in the self projection, so the discriminating fact is that the +// generic builder and the self projection differ exactly on the host argument. +test fn generic_builder_with_remote_host_is_not_the_self_projection() -> Bool { + fleet_converge_service_content(host_arg: "srv2") != fleet_converge_service_content_self() + && string_contains(s: fleet_converge_service_content(host_arg: "srv2"), pattern: "--host srv2") +} + +test fn unit_names_are_the_single_authority() -> Bool { + fleet_converge_service_unit_name == "ctrl-fleet-converge.service" + && fleet_converge_timer_unit_name == "ctrl-fleet-converge.timer" + && fleet_converge_timer_on_calendar == "*:0/15" +} diff --git a/dag/test/claim/fleet_convergence_verdict_witness_test.dag b/dag/test/claim/fleet_convergence_verdict_witness_test.dag new file mode 100644 index 00000000000..856a4719930 --- /dev/null +++ b/dag/test/claim/fleet_convergence_verdict_witness_test.dag @@ -0,0 +1,124 @@ +module test.claim.fleet_convergence_verdict_witness + +import std.types { Bool, String, NonEmptyStr, List, Int } +import product.placement_supply { HostIdentity } +import gunbc.fleet_intent_network { operator_host_srv1 } +import gunbc.host_phase_status { fleet_subsumption_enrollments } +import gunbc.host_assimilation_program { HostEnrollmentIntent } +import gunbc.fleet_converge_cli { ConvergeCliReceipt, ConvergeCliRefusalCause, KnobFrontier } +import gunbc.host_converge { VerdictConverged, Absent, Drifted } +import gunbc.fleet_convergence_verdict { + fleet_convergence_verdict, + FleetConvergenceVerdict, + FleetConverged, + FleetNotConverged, + fleet_unobserved_cell_count, + host_knob_standing, + HostKnobStanding, + KnobConverged, + KnobNotConverged, + KnobUnobserved, + RequiredCellStanding, + CellConverged, + CellUnobserved, + CellUnreachable, + CellDrifted, + CellConflict, + cell_standing_is_converged, +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +fn no_receipts() -> List { + [] +} + +fn converged_receipt(host: String) -> ConvergeCliReceipt { + ConvergeCliReceipt { host: host, applied: 0, drifted: 0, verdict: VerdictConverged, reason: none } +} + +// THE LAW: the live fleet, with its one real producer cell and zero knob receipts, is +// NOT converged — required unobserved cells prevent convergence rather than defaulting +// green, and the unobserved count is a positive number (the arc's honest progress +// denominator). This witness is the named current defect made executable. +test fn live_fleet_is_not_converged_and_unobserved_cells_are_counted() -> Bool { + match fleet_convergence_verdict(enrollments: fleet_subsumption_enrollments, receipts: no_receipts()) { + FleetNotConverged { standings: ss } => fleet_unobserved_cell_count(standings: ss) > 0 + FleetConverged { host_count: _ } => false + } +} + +// RED control (the forbidden absorbing arm): even if every enrolled host carried a +// CONVERGED knob receipt, the unobserved lifecycle cells still prevent fleet +// convergence — knob green never outvotes an unobserved required cell. +test fn knob_receipts_alone_cannot_converge_the_fleet() -> Bool { + let receipts = map(fleet_subsumption_enrollments, e => converged_receipt(host: e.host as String)) + match fleet_convergence_verdict(enrollments: fleet_subsumption_enrollments, receipts: receipts) { + FleetNotConverged { standings: _ } => true + FleetConverged { host_count: _ } => false + } +} + +// An empty enrollment is an observation gap, never a vacuously converged fleet. +test fn empty_fleet_refuses_convergence() -> Bool { + let empty: List = [] + match fleet_convergence_verdict(enrollments: empty, receipts: no_receipts()) { + FleetNotConverged { standings: ss } => length(ss) == 0 + FleetConverged { host_count: _ } => false + } +} + +// Knob standing discriminates its three arms: no receipt is Unobserved (never green), a +// non-converged receipt is NotConverged, a converged receipt is Converged. +test fn knob_standing_three_arms_discriminate() -> Bool { + match host_knob_standing(host: operator_host_srv1, receipts: no_receipts()) { + KnobUnobserved { host: _ } => true + _ => false + } + && match host_knob_standing(host: operator_host_srv1, receipts: [converged_receipt(host: operator_host_srv1 as String)]) { + KnobConverged { host: _ } => true + _ => false + } + && match host_knob_standing( + host: operator_host_srv1, + receipts: [ConvergeCliReceipt { + host: operator_host_srv1 as String, + applied: 0, + drifted: 1, + verdict: Absent, + reason: Present { value: KnobFrontier { reason: "witness fixture" } }, + }], + ) { + KnobNotConverged { host: _, reason: _ } => true + _ => false + } +} + +// The five cell arms stay distinct — unobserved, unreachable, drifted, and conflict are +// different remedies and none of them is converged. +test fn cell_standings_discriminate_and_only_converged_counts() -> Bool { + let h = operator_host_srv1 + cell_standing_is_converged(s: CellConverged { host: h, phase_label: "p" }) + && !cell_standing_is_converged(s: CellUnobserved { host: h, phase_label: "p" }) + && !cell_standing_is_converged(s: CellUnreachable { host: h, phase_label: "p" }) + && !cell_standing_is_converged(s: CellDrifted { host: h, phase_label: "p" }) + && !cell_standing_is_converged(s: CellConflict { host: h, phase_label: "p" }) +} + +// Machine-id hazard control (msg_1c4e0bce): two receipts claiming ONE host identity — +// possible while both Sparks share a machine-id and factory host keys — classify as a +// named ambiguity, never first-wins. Even two CONVERGED duplicates cannot converge the +// knob, because which machine produced which receipt is exactly what is unestablished. +test fn duplicate_receipts_for_one_host_are_ambiguity_never_a_pick() -> Bool { + let a = converged_receipt(host: "srv5") + let b = converged_receipt(host: "srv5") + match host_knob_standing(host: "srv5" as HostIdentity, receipts: [a, b]) { + KnobNotConverged { host: _, reason: r } => string_contains(s: r, pattern: "ambiguous") + _ => false + } + && match host_knob_standing(host: "srv5" as HostIdentity, receipts: [a]) { + KnobConverged { host: _ } => true + _ => false + } +} diff --git a/dag/test/claim/fleet_main_revision_witness_test.dag b/dag/test/claim/fleet_main_revision_witness_test.dag new file mode 100644 index 00000000000..ca045417096 --- /dev/null +++ b/dag/test/claim/fleet_main_revision_witness_test.dag @@ -0,0 +1,95 @@ +module test.claim.fleet_main_revision_witness + +import std.types { Bool, String } +import extdeps.git.object_store { GitObjectId, git_sha1_object_id } +import gunbc.fleet_main_revision { + fleet_desired_ref_name, + FleetDesiredRevision, + FleetDesiredAdmitted, + FleetDesiredUnobserved, + fleet_desired_revision_target, + FleetDesiredAdvance, + AdvanceAdmitted, + AdvanceRefused, + admit_fleet_desired_advance, + converge_target_revision, +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +fn fixture_rev(hex_char: String) -> GitObjectId? { + git_sha1_object_id(hex: join([hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char,hex_char], "")) +} + +// RED — the load-bearing admission law: a revision whose floor is not green NEVER +// advances the desired ref, whatever the current ref state. +test fn advance_refused_without_floor_green() -> Bool { + match fixture_rev(hex_char: "a") { + Present { value: cand } => + match admit_fleet_desired_advance(candidate: cand, floor_green: false, current: none) { + AdvanceRefused { cause: _ } => true + _ => false + } + && match fixture_rev(hex_char: "b") { + Present { value: cur } => + match admit_fleet_desired_advance(candidate: cand, floor_green: false, current: Present { value: cur }) { + AdvanceRefused { cause: _ } => true + _ => false + } + _ => false + } + _ => false + } +} + +// The admitted advance is a CAS carrying the expected prior — first admission carries +// from=none (ref creation), a subsequent one carries the exact prior revision. +test fn admitted_advance_is_cas_carrying_expected_prior() -> Bool { + match fixture_rev(hex_char: "a") { + Present { value: cand } => + match admit_fleet_desired_advance(candidate: cand, floor_green: true, current: none) { + AdvanceAdmitted { from: f, to: _ } => + match f { + Present { value: _ } => false + _ => true + } + _ => false + } + && match fixture_rev(hex_char: "b") { + Present { value: cur } => + match admit_fleet_desired_advance(candidate: cand, floor_green: true, current: Present { value: cur }) { + AdvanceAdmitted { from: f2, to: _ } => + match f2 { + Present { value: _ } => true + _ => false + } + _ => false + } + _ => false + } + _ => false + } +} + +// An unobserved desired ref yields NO converge target — never a fabricated one, never +// the invoking tree. Admitted yields exactly the admitted revision. +test fn converge_target_derives_only_from_admitted_ref() -> Bool { + match converge_target_revision(observed: FleetDesiredUnobserved { cause: "ref unreadable" }) { + Present { value: _ } => false + _ => true + } + && match fixture_rev(hex_char: "c") { + Present { value: rev } => + match converge_target_revision(observed: FleetDesiredAdmitted { revision: rev }) { + Present { value: _ } => true + _ => false + } + _ => false + } +} + +// The ref name is the single authority and is a fleet-scoped ref, not a branch. +test fn desired_ref_name_is_the_fleet_scoped_authority() -> Bool { + fleet_desired_ref_name == "refs/fleet/desired" +} diff --git a/dag/test/claim/fleet_receipt_collector_witness_test.dag b/dag/test/claim/fleet_receipt_collector_witness_test.dag new file mode 100644 index 00000000000..97868f97aae --- /dev/null +++ b/dag/test/claim/fleet_receipt_collector_witness_test.dag @@ -0,0 +1,94 @@ +module test.claim.fleet_receipt_collector_witness + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// Join 5 (operator verdict): the collector's parse is the BACKWARD read of the same +// byte-locked line the CLI renders forward — a rendered receipt round-trips to an equal +// receipt, and the three refusal shapes (wrong prefix, missing field, unknown verdict +// label) all refuse rather than guessing a verdict. +test fn witness_receipt_line_round_trips_and_refuses_malformed() -> Bool { + let r = ConvergeCliReceipt { host: "srv2", applied: 3, drifted: 1, verdict: VerdictConverged, reason: none } + match parse_converge_receipt_line(line: converge_cli_receipt_line(r: r)) { + Present { value: back } => + back.host == "srv2" + && back.applied == 3 + && back.drifted == 1 + && converge_cli_receipt_converged(r: back) + Absent => false + } + && match parse_converge_receipt_line(line: "not-a-receipt host=srv2 applied=0 drifted=0 verdict=converged") { + Present { value: _ } => false + Absent => true + } + && match parse_converge_receipt_line(line: "converge-receipt host=srv2 applied=0 drifted=0 verdict=plausible") { + Present { value: _ } => false + Absent => true + } + && match parse_converge_receipt_line(line: "converge-receipt host=srv2 verdict=converged") { + Present { value: _ } => false + Absent => true + } +} + +// A parsed non-converged label stays non-converged (drifted/absent never launder into +// converged through the round trip). +test fn witness_parse_preserves_non_converged_verdicts() -> Bool { + match parse_converge_receipt_line(line: "converge-receipt host=srv1 applied=0 drifted=2 verdict=drifted") { + Present { value: r } => + !converge_cli_receipt_converged(r: r) + && r.drifted == 2 + && converge_verdict_label(v: r.verdict) == "drifted" + Absent => false + } + && match parse_converge_receipt_line(line: "converge-receipt host=srv1 applied=0 drifted=0 verdict=absent") { + Present { value: r } => + !converge_cli_receipt_converged(r: r) && converge_verdict_label(v: r.verdict) == "absent" + Absent => false + } + && match parse_converge_receipt_line(line: "converge-receipt host=srv1 applied=x drifted=0 verdict=converged") { + Present { value: _ } => false + Absent => true + } +} + +// Only Collected rows reach the fold: Absent and Unparseable are typed evidence that +// keep the host knob-unobserved, never silently dropped INTO the fold as data and never +// widened into a fabricated receipt. +test fn witness_only_collected_rows_enter_the_fold_registry() -> Bool { + let collected = [ + ReceiptCollected { + host: "srv1", + receipt: ConvergeCliReceipt { host: "srv1", applied: 1, drifted: 0, verdict: VerdictConverged, reason: none }, + }, + ReceiptAbsent { host: "srv2", cause: "No such file or directory" }, + ReceiptUnparseable { host: "srv3", line: "converge-receipt host=srv3 verdict=???" }, + ] + let rows = collected_receipts_only(collected: collected) + length(rows) == 1 + && match rows.first() { + Present { value: r } => r.host == "srv1" + Absent => false + } + && string_contains(s: collected_line(c: ReceiptAbsent { host: "srv2", cause: "x" }), pattern: "outcome=absent") + && string_contains( + s: collected_line(c: ReceiptUnparseable { host: "srv3", line: "y" }), + pattern: "outcome=unparseable" + ) +} + +// The composed fold over a collector-shaped registry: one collected-converged receipt +// alone still yields FleetNotConverged (unobserved cells and knob-unobserved hosts +// block), pinning that the collector cannot manufacture fleet convergence from partial +// collection — the leg-f hermetic RED at the collector seam. +test fn witness_partial_collection_never_yields_fleet_converged() -> Bool { + let rows = collected_receipts_only(collected: [ + ReceiptCollected { + host: "srv1", + receipt: ConvergeCliReceipt { host: "srv1", applied: 1, drifted: 0, verdict: VerdictConverged, reason: none }, + }, + ]) + match fleet_convergence_verdict(enrollments: fleet_subsumption_enrollments, receipts: rows) { + FleetConverged { host_count: _ } => false + FleetNotConverged { standings: s } => length(s) > 0 + } +} diff --git a/dag/test/claim/host_converge_delta_witness_test.dag b/dag/test/claim/host_converge_delta_witness_test.dag index d5e564c342b..1dd6c4b831f 100644 --- a/dag/test/claim/host_converge_delta_witness_test.dag +++ b/dag/test/claim/host_converge_delta_witness_test.dag @@ -10,6 +10,7 @@ fn fixture_srv2_host() -> RunnerHostDeployment { per_runner_memory_cap: byte_size(30000000000), per_runner_memory_swap_cap: byte_size(5000000000), per_runner_memory_high_cap: byte_size(4000000000), + per_runner_tasks_max: 16384, build_tokens: hardware_thread_count(count: 12), runner: SelfHosted { labels: ["self-hosted", "linux", "arm64"] }, workload_class: BatchLatencyTolerant, diff --git a/dag/test/claim/host_converge_slice1_witness_test.dag b/dag/test/claim/host_converge_slice1_witness_test.dag index a5ff0b692e3..4f6e9bdab94 100644 --- a/dag/test/claim/host_converge_slice1_witness_test.dag +++ b/dag/test/claim/host_converge_slice1_witness_test.dag @@ -2,8 +2,8 @@ module test.claim.host_converge_slice1_witness import std.types { Bool, NonEmptyStr, List, String } import gunbc.runner_host_deploy { runner_index_seq } -import extdeps.systemd { systemd_memory_max_property, MemoryMax } -import gunbc.host_converge { Drifted, PerSlotMemoryCap } +import extdeps.systemd { systemd_memory_max_property, MemoryMax, systemd_tasks_max_property, parse_systemd_task_limit_show } +import gunbc.host_converge { Drifted, PerSlotMemoryCap, VerdictConverged } import gunbc.fleet_intent_network { operator_host_srv1 } import gunbc.runner_unit_live_read { gunbc_srv1_runner_unit_live_read_fixture, @@ -41,6 +41,14 @@ import gunbc.host_converge_slice1 { HostConvergeSlice1UnitUsageGuardRefused, HostConvergeSlice1Refused, host_converge_slice1_unit_outcome_to_legacy_result, + host_converge_slice1_tasks_knob_name, + host_converge_slice1_tasks_knob, + host_converge_slice1_tasks_verdict, + host_converge_slice1_tasks_receipt, + host_converge_slice1_tasks_receipt_is_valid, + host_converge_slice1_tasks_headroom_refuses, + HostConvergeSlice1UnitTasksConverged, + HostConvergeSlice1TasksConverged, } import gunbc.fleet_intent_network { operator_host_srv3 } import gunbc.fleet_host_budget { declared_runner_count_for } @@ -189,3 +197,110 @@ test fn witness_usage_guard_refusal_routes_to_legacy_refused_with_retryable_reas } } + +// --- TasksMax knob (second knob on the slice1 circuit; desired = 16384, the incident +// repair value — 512 is the diagnosed EAGAIN outage value and must read Drifted) --- + +test fn witness_tasks_knob_is_per_slot_tasks_max_on_srv1_and_srv3() -> Bool { + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: knob } => + knob.name == host_converge_slice1_tasks_knob_name + && knob.desired == "16384" + && match knob.target { + PerSlotMemoryCap { dropin_path: _, instance_glob: _, property: p, bytes_value: b } => + p == (systemd_tasks_max_property as String) && b == knob.desired + _ => false + } + && match host_converge_slice1_tasks_knob(host: operator_host_srv3) { + Present { value: k3 } => k3.desired == "16384" + Absent => false + } + Absent => false + } +} + +// TasksMax=512 is the diagnosed EAGAIN/process-creation outage value (operator ruling on +// PR 8045, restoring the 2026-06-21 repair 16384); the discriminating claim is that the +// outage value reads Drifted against the repaired desired, never Converged — a converge +// circuit that would re-apply 512 is the outage made reproducible by construction. +test fn witness_tasks_verdict_outage_512_drifts() -> Bool { + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: knob } => + host_converge_slice1_tasks_verdict(limit: parse_systemd_task_limit_show(raw: "512"), knob: knob) == Drifted + && host_converge_slice1_tasks_verdict(limit: parse_systemd_task_limit_show(raw: "infinity"), knob: knob) == Drifted + Absent => false + } +} + +test fn witness_tasks_verdict_desired_count_converges_and_offbyone_drifts() -> Bool { + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: knob } => + host_converge_slice1_tasks_verdict(limit: parse_systemd_task_limit_show(raw: "16384"), knob: knob) == VerdictConverged + && host_converge_slice1_tasks_verdict(limit: parse_systemd_task_limit_show(raw: "16383"), knob: knob) == Drifted + Absent => false + } +} + +test fn witness_tasks_parse_empty_or_junk_is_absent_verdict() -> Bool { + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: knob } => + host_converge_slice1_tasks_verdict(limit: parse_systemd_task_limit_show(raw: ""), knob: knob) == Absent + && host_converge_slice1_tasks_verdict(limit: parse_systemd_task_limit_show(raw: "not-a-count"), knob: knob) == Absent + Absent => false + } +} + +test fn witness_tasks_outcome_routes_to_tasks_legacy_arm_preserving_applied() -> Bool { + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: knob } => { + let receipt = host_converge_slice1_tasks_receipt( + host: operator_host_srv1, + unit: host_converge_slice1_probe_unit, + knob: knob, + limit: parse_systemd_task_limit_show(raw: "16384"), + probed_at: "witness-fixture" as NonEmptyStr, + applied: true, + read_back_converged: true, + reapply_noop: true, + ) + host_converge_slice1_tasks_receipt_is_valid(receipt: receipt) + && match host_converge_slice1_unit_outcome_to_legacy_result( + HostConvergeSlice1UnitTasksConverged { unit: host_converge_slice1_probe_unit, applied: true, receipt: receipt } + ) { + HostConvergeSlice1TasksConverged { applied_knobs: n, receipt: r } => + n == 1 && r.verdict == VerdictConverged + _ => false + } + } + Absent => false + } +} + +// RED control: a receipt whose live read still shows the outage value must fail the +// validity gate even when the boolean flags claim success — the verdict is derived from +// the read, never from the flags. +test fn witness_tasks_receipt_invalid_when_read_still_drifted() -> Bool { + match host_converge_slice1_tasks_knob(host: operator_host_srv1) { + Present { value: knob } => { + let receipt = host_converge_slice1_tasks_receipt( + host: operator_host_srv1, + unit: host_converge_slice1_probe_unit, + knob: knob, + limit: parse_systemd_task_limit_show(raw: "512"), + probed_at: "witness-fixture" as NonEmptyStr, + applied: true, + read_back_converged: true, + reapply_noop: true, + ) + !host_converge_slice1_tasks_receipt_is_valid(receipt: receipt) + } + Absent => false + } +} + +// Guard boundary: equality is zero headroom and refuses; one below the cap admits. +test fn witness_tasks_guard_boundary_equality_refuses() -> Bool { + host_converge_slice1_tasks_headroom_refuses(current: 16384, target: 16384) + && host_converge_slice1_tasks_headroom_refuses(current: 16385, target: 16384) + && !host_converge_slice1_tasks_headroom_refuses(current: 16383, target: 16384) +} diff --git a/dag/test/claim/host_phase_status_witness_test.dag b/dag/test/claim/host_phase_status_witness_test.dag index d1e23a3ff56..e9eaba8a2cd 100644 --- a/dag/test/claim/host_phase_status_witness_test.dag +++ b/dag/test/claim/host_phase_status_witness_test.dag @@ -159,8 +159,12 @@ test fn witness_matrix_is_exact_join_over_participation_programs() -> Bool { matched.length() == fleet_subsumption_enrollments.length() } +// Re-derived DELIBERATELY on the join-4 flip (run 31300640122): the executed fleet reach +// probe landed a receipt row for all six targets, and each row feeds BOTH P0 deduction +// cells — so the frontier is the authored srv3 cell plus 2 cells per probed host, +// DERIVED from the probe target roster rather than pinned as a tree-copied literal. test fn witness_live_observation_frontier_is_one_cell() -> Bool { - host_phase_observation_coverage_count(enrollments: fleet_subsumption_enrollments) == 1 + host_phase_observation_coverage_count(enrollments: fleet_subsumption_enrollments) == 1 + 2 * fleet_reach_probe_targets.length() } // Cross-check between two independent producers over the same cells rather than a rectangle: the @@ -358,16 +362,31 @@ test fn witness_enrolled_sparks_carry_exactly_the_shared_obligations() -> Bool { // The unobserved half of the same claim, kept separate so a future producer landing on a Spark // cell reds this witness instead of silently upgrading the sentence above. test fn witness_enrolled_spark_cells_are_all_unobserved() -> Bool { + // Re-derived on the join-4 flip: the executed probe observed BOTH P0 cells on each + // Spark at the modeled endpoint (ProbeHostKeyUnenrolled -> Inaccessible — observed, + // NOT satisfied), and every non-P0 Spark cell stays unobserved. The witness now pins + // that exact split so a producer landing on any further Spark cell still reds here. let matrix = host_fleet_phase_matrix(enrollments: fleet_subsumption_enrollments) - let observed = filter( + let sparks = filter( matrix, - c => (c.cell.host == operator_host_srv5 || c.cell.host == operator_host_srv6) + c => c.cell.host == operator_host_srv5 || c.cell.host == operator_host_srv6, + ) + let observed = filter( + sparks, + c => match c.observation { + Present { value: _ } => true + Absent => false + }, + ) + let observed_p0 = filter( + observed, + c => (c.cell.phase_label == "P0:reach-secrets-network" || c.cell.phase_label == "P0:host-identity-converge") && match c.observation { - Present { value: _ } => true + Present { value: v } => v == Inaccessible Absent => false }, ) - observed.length() == 0 + observed.length() == 4 && observed_p0.length() == 4 } // Non-vacuity guard for the gap join above: if the spine carried no DeclaredGap step at all, that @@ -431,6 +450,12 @@ test fn witness_every_covered_cell_today_is_the_authored_one() -> Bool { covered, c => c.cell.host == operator_host_srv3 && c.cell.phase_label == "prefix:os-install-actuated", ) + // Join-4 flip: the rest of the covered population must be EXACTLY the probe-deduced + // P0 cells — identity grain, not a widened count. + let probe_deduced = filter( + covered, + c => c.cell.phase_label == "P0:reach-secrets-network" || c.cell.phase_label == "P0:host-identity-converge", + ) // The PRODUCTION metric itself, not a rebuilt filter beside it. host_phase_observation_coverage_count // is the subject of this finding, so a control that never calls it proves nothing about the number // anyone actually reads. @@ -439,6 +464,7 @@ test fn witness_every_covered_cell_today_is_the_authored_one() -> Bool { // population is this single authored cell, so the assertion must fail the moment that stops holding // -- including when a genuinely executed producer is wired, which is the outcome this lane wants. metric == covered.length() - && covered.length() == authored.length() + && covered.length() == authored.length() + probe_deduced.length() && authored.length() == 1 + && probe_deduced.length() == 2 * fleet_reach_probe_targets.length() } diff --git a/dag/test/claim/host_reach_identity_probe_witness_test.dag b/dag/test/claim/host_reach_identity_probe_witness_test.dag new file mode 100644 index 00000000000..7baa56bd28c --- /dev/null +++ b/dag/test/claim/host_reach_identity_probe_witness_test.dag @@ -0,0 +1,191 @@ +module test.claim.host_reach_identity_probe_witness + +import std.types { Bool, String, NonEmptyStr, List } +import std.upsert_decision { ObservationVerdict, Converged, Drifted, Conflict, Inaccessible, UnknownRefused } +import product.placement_supply { HostIdentity } +import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv3 } +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } +import gunbc.host_reach_identity_probe { + ReachProbeOutcome, + ProbeReachable, + ProbeTransportUnavailable, + ProbeAuthenticationRefused, + ProbeHostKeyMismatch, + ProbeHostKeyUnenrolled, + HostReachIdentityProbeReceipt, + host_reach_identity_probe_receipt_for, + classify_reach_probe_wire, + classify_reach_probe_wire_with_expectation, + reach_verdict_from_outcome, + identity_verdict_from_outcome, + observe_reach_secrets_network, + observe_host_identity_converge, + host_identity_desired_short_hostname, +} +import gunbc.host_phase_status { + observe_host_phase, + fleet_subsumption_enrollments, + host_phase_observation_coverage_count, +} +import gunbc.host_assimilation_program { HostEnrollmentIntent } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +fn fixture_subject() -> HostEnrollmentIntent { + fleet_subsumption_enrollments.first() +} + +// The four wire classes land in four distinct outcome arms, fail-closed: an +// unclassifiable failure is TransportUnavailable with the cause preserved, never a +// fabricated Reachable. +test fn probe_wire_classifies_four_distinct_outcomes() -> Bool { + match classify_reach_probe_wire(success: true, stdout: "srv1\n", stderr: "") { + ProbeReachable { observed_short_hostname: h } => h == "srv1" + _ => false + } + && match classify_reach_probe_wire(success: false, stdout: "", stderr: "briansrls@srv5: Permission denied (publickey).") { + ProbeAuthenticationRefused { cause: _ } => true + _ => false + } + && match classify_reach_probe_wire(success: false, stdout: "", stderr: "Host key verification failed.") { + ProbeHostKeyUnenrolled { transport_diagnostic: _ } => true + _ => false + } + && match classify_reach_probe_wire(success: false, stdout: "", stderr: "ssh: connect to host srv5 port 22: Connection timed out") { + ProbeTransportUnavailable { cause: _ } => true + _ => false + } +} + +// RED-shaped fail-closed control: empty stderr on failure is still TransportUnavailable +// (unknown never widens to Reachable), and a success wire never lands in a failure arm. +test fn probe_wire_unknown_failure_is_transport_unavailable_never_reachable() -> Bool { + match classify_reach_probe_wire(success: false, stdout: "srv1\n", stderr: "") { + ProbeTransportUnavailable { cause: _ } => true + _ => false + } +} + +// Reach verdict projection: transport-unavailable and authentication-refused stay +// DISTINCT outcome arms even though both project Inaccessible — the distinction lives on +// the receipt, and this witness pins that both arms exist separately AND project the +// same fail-closed verdict (neither is ever Converged or Absent). +test fn reach_projection_covers_all_arms() -> Bool { + reach_verdict_from_outcome(o: ProbeReachable { observed_short_hostname: "srv1" }) == Converged + && reach_verdict_from_outcome(o: ProbeTransportUnavailable { cause: "timeout" }) == Inaccessible + && reach_verdict_from_outcome(o: ProbeAuthenticationRefused { cause: "denied" }) == Inaccessible + && reach_verdict_from_outcome(o: ProbeHostKeyMismatch { expected_fingerprint: "a", observed_fingerprint: "b" }) == Conflict + && reach_verdict_from_outcome(o: ProbeHostKeyUnenrolled { transport_diagnostic: "Host key verification failed." }) == Inaccessible +} + +// Identity deduction: reachable-and-matching converges; reachable-but-different is +// Drifted (the divergence SetHostnameCas corrects); unreachable/unauthenticated cannot +// say (Inaccessible); a host-key mismatch is Conflict — the machine answering may not be +// the enrolled host, so its hostname must never launder into an identity observation. +test fn identity_projection_discriminates_match_drift_and_wrong_machine() -> Bool { + let desired = host_identity_desired_short_hostname(host: operator_host_srv1) + identity_verdict_from_outcome(host: operator_host_srv1, o: ProbeReachable { observed_short_hostname: desired }) == Converged + && identity_verdict_from_outcome(host: operator_host_srv1, o: ProbeReachable { observed_short_hostname: "altrad8ud-1l2t-1" }) == Drifted + && identity_verdict_from_outcome(host: operator_host_srv1, o: ProbeTransportUnavailable { cause: "x" }) == Inaccessible + && identity_verdict_from_outcome(host: operator_host_srv1, o: ProbeAuthenticationRefused { cause: "x" }) == Inaccessible + && identity_verdict_from_outcome(host: operator_host_srv1, o: ProbeHostKeyMismatch { expected_fingerprint: "a", observed_fingerprint: "b" }) == Conflict + && identity_verdict_from_outcome(host: operator_host_srv1, o: ProbeHostKeyUnenrolled { transport_diagnostic: "x" }) == Inaccessible +} + +// Join 4 (operator verdict): the executed probe run's receipts ARE the registry now — +// the producers answer Present for a probed host, the observed verdict is Inaccessible +// (unenrolled host key: never satisfied, never Conflict), and coverage GREW from the +// executed run, not from wiring. A host outside the registry still answers none. +test fn committed_probe_receipts_feed_producers_and_grow_coverage() -> Bool { + match host_reach_identity_probe_receipt_for(host: operator_host_srv1) { + Present { value: r } => + match r.outcome { + ProbeHostKeyUnenrolled { transport_diagnostic: d } => string_contains(s: d, pattern: "Host key verification failed") + _ => false + } + _ => false + } + && match observe_reach_secrets_network(host: operator_host_srv1) { + Present { value: v } => v == Inaccessible + _ => false + } + && match observe_host_phase(subject: fixture_subject(), host: operator_host_srv1, phase_label: "P0:reach-secrets-network") { + Present { value: v } => v == Inaccessible + _ => false + } + && match host_reach_identity_probe_receipt_for(host: "no-such-host" as HostIdentity) { + Present { value: _ } => false + _ => true + } + && host_phase_observation_coverage_count(enrollments: fleet_subsumption_enrollments) > 1 +} + +// The canonical seam routes each label to its producer and answers none for a label with +// no producer row; the srv3 lease producer is preserved and still host-scoped. +test fn observe_host_phase_routes_by_label_and_stays_fail_closed() -> Bool { + match observe_host_phase(subject: fixture_subject(), host: operator_host_srv3, phase_label: "prefix:os-install-actuated") { + Present { value: _ } => true + _ => false + } + && match observe_host_phase(subject: fixture_subject(), host: operator_host_srv1, phase_label: "prefix:os-install-actuated") { + Present { value: _ } => false + _ => true + } + && match observe_host_phase(subject: fixture_subject(), host: operator_host_srv1, phase_label: "P1:runner-deploy-slot") { + Present { value: _ } => false + _ => true + } + && match observe_host_phase(subject: fixture_subject(), host: operator_host_srv1, phase_label: "P0:host-identity-converge") { + Present { value: v } => v == Inaccessible + _ => false + } +} + +// With a synthetic receipt, both deductions produce verdicts from the SAME probe — one +// observation, two cells (§2) — and the two Inaccessible-projecting arms remain +// distinguishable at the receipt's outcome grain. +test fn one_receipt_feeds_both_cells() -> Bool { + let r = HostReachIdentityProbeReceipt { + host: operator_host_srv1, + probed_at: "witness-fixture" as NonEmptyStr, + outcome: ProbeReachable { observed_short_hostname: host_identity_desired_short_hostname(host: operator_host_srv1) }, + } + reach_verdict_from_outcome(o: r.outcome) == Converged + && identity_verdict_from_outcome(host: r.host, o: r.outcome) == Converged +} + +// Operator verdict join 7 (supersedes the review 50695 Optional shape): "no enrollment to +// compare against" and "enrolled expectation contradicted" are SEPARATE CONSTRUCTORS with +// different remedies — Unenrolled{transport_diagnostic} says enroll a fingerprint; +// Mismatch{expected, observed} says the machine answering may not be the enrolled host. +// The wire classifier alone can only produce Unenrolled; the expectation-threading +// wrapper PROMOTES it to Mismatch exactly when an enrolled fingerprint is Present, and +// never fabricates an expected value. +test fn mismatch_expectation_threads_without_fabrication() -> Bool { + match classify_reach_probe_wire(success: false, stdout: "", stderr: "Host key verification failed.") { + ProbeHostKeyUnenrolled { transport_diagnostic: d } => string_contains(s: d, pattern: "Host key verification failed") + _ => false + } + && match classify_reach_probe_wire_with_expectation(success: false, stdout: "", stderr: "Host key verification failed.", expected_fingerprint: Present { value: "SHA256:abc" }) { + ProbeHostKeyMismatch { expected_fingerprint: e2, observed_fingerprint: _ } => e2 == "SHA256:abc" + _ => false + } + && match classify_reach_probe_wire_with_expectation(success: false, stdout: "", stderr: "Host key verification failed.", expected_fingerprint: none) { + ProbeHostKeyUnenrolled { transport_diagnostic: _ } => true + _ => false + } + && match classify_reach_probe_wire_with_expectation(success: true, stdout: "srv5\n", stderr: "", expected_fingerprint: Present { value: "SHA256:abc" }) { + ProbeReachable { observed_short_hostname: h } => h == "srv5" + _ => false + } +} + +// Endpoint resolution (msg_1c4e0bce): the Sparks' probe target is the MODELED +// reservation address — 'srv5'/'srv6' are not published names anywhere — while +// name-resolvable hosts pass through unchanged. Identity stays the receipt key. +test fn spark_probe_targets_are_modeled_addresses_not_names() -> Bool { + fleet_probe_endpoint_for(host: "srv5") == "192.168.1.222" + && fleet_probe_endpoint_for(host: "srv6") == "192.168.1.223" + && fleet_probe_endpoint_for(host: "srv1") == "srv1" +} diff --git a/dag/test/claim/live_deploy/emit_test.dag b/dag/test/claim/live_deploy/emit_test.dag index e94a1f89953..cd6e58a8ecf 100644 --- a/dag/test/claim/live_deploy/emit_test.dag +++ b/dag/test/claim/live_deploy/emit_test.dag @@ -687,3 +687,25 @@ test fn retract_is_not_emitted_without_an_observed_scoped_removal() -> Bool { && !string_contains(s: measured, pattern: "__GUNBC_DEPLOY_REFUSED__") && string_contains(s: measured, pattern: "sudo -n tailscale serve --https=443 off") } + +// CONVERGE-0 loop member: the deployment installs a per-instance fleet-converge timer +// whose service converges SELF via the deployed binary, with calendar catch-up +// (Persistent=true — the offline-host property) and the receipts dir the publication +// leg writes into. Cadence facts derive from the gunbc.fleet_converge_timer authority, +// so a drifted cadence here reds against that one row, not a second copy. +test fn fleet_converge_member_installs_self_converging_timer_with_catchup() -> Bool { + let live = live_deploy_apply_script_for(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }) + string_contains(s: live, pattern: " converge --host self") + && string_contains(s: live, pattern: "OnCalendar=*:0/15") + && string_contains(s: live, pattern: "Persistent=true") + && string_contains(s: live, pattern: "-fleet-converge.timer") + && string_contains(s: live, pattern: "install -d -m 0755 /opt/gunbc/receipts") +} + +// Teardown owns both files: the retract removes the timer AND the service beside it +// (the belt-member one-member-two-files shape), never production's fixed-name units. +test fn fleet_converge_member_retract_removes_both_unit_files() -> Bool { + let retract = live_deploy_retract_script_for(spec: deployment_spec_srv1()) + string_contains(s: retract, pattern: "-fleet-converge.timer") + && string_contains(s: retract, pattern: "-fleet-converge.service") +} diff --git a/dag/test/claim/srv1_tasks_preapply_observation_witness_test.dag b/dag/test/claim/srv1_tasks_preapply_observation_witness_test.dag new file mode 100644 index 00000000000..e00cbc8ec44 --- /dev/null +++ b/dag/test/claim/srv1_tasks_preapply_observation_witness_test.dag @@ -0,0 +1,21 @@ +module test.claim.srv1_tasks_preapply_observation_witness + +import std.types { Bool, String } +import gunbc.srv1_tasks_preapply_observation { cgroup_parent_dir } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// The ancestor read targets the PARENT of the systemd-reported ControlGroup — the +// slice directory — never a hand-escaped reconstruction (the decoy-slice-name class). +test fn ancestor_dir_is_parent_of_reported_control_group() -> Bool { + let parent = cgroup_parent_dir(control_group: "/system.slice/system-actions.slice/system-actions-runner.slice/actions-runner@srv1-01.service") + parent == "/system.slice/system-actions.slice/system-actions-runner.slice" +} + +// RED-shaped boundary: a degenerate one-segment path has no parent to read — the fn +// returns it unchanged rather than fabricating an empty path that would read cgroupfs root. +test fn degenerate_control_group_does_not_fabricate_a_parent() -> Bool { + cgroup_parent_dir(control_group: "degenerate") == "degenerate" + && cgroup_parent_dir(control_group: "/leaf") == "/leaf" +}