Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions dag/extdeps/systemd/systemctl.dag
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,29 @@ fn systemctl_stop_argv(unit: NonEmptyStr) -> List<String> {
["systemctl", "stop", unit as String]
}

// MASK IS NOT A STRONGER DISABLE, IT IS A DIFFERENT OPERATION, and systemctl(1) separates them on
// two axes this repository has a caller for. FIRST, REACH: `disable` removes the enablement
// symlinks, which is why it does nothing at all to a STATIC unit -- one with no [Install] section,
// which has no such symlinks to remove -- and why it never prevents `systemctl start` or activation
// by another unit. `mask` links the unit name to /dev/null, so activation is refused whatever
// requests it: a timer, a dependency, or a hand at a terminal. SECOND, AND IT IS THE REASON A
// TEMPORARY QUIESCENCE WANTS MASK RATHER THAN DISABLE: `disable` DESTROYS the enablement state,
// so restoring it means re-deciding what that state was, and a restore-by-enable would enable a
// unit that had been deliberately disabled before the quiescence began. `mask` LAYERS over the
// existing state and `unmask` removes the layer, leaving prior enablement exactly as it was found.
// So the pair is state-preserving where enable/disable is state-asserting.
data systemctl_mask_verb: NonEmptyStr = "mask" as NonEmptyStr

data systemctl_unmask_verb: NonEmptyStr = "unmask" as NonEmptyStr

fn systemctl_mask_now_argv(unit: NonEmptyStr) -> List<String> {
["systemctl", systemctl_mask_verb as String, "--now", unit as String]
}

fn systemctl_unmask_argv(unit: NonEmptyStr) -> List<String> {
["systemctl", systemctl_unmask_verb as String, unit as String]
}

fn systemctl_set_property_argv(unit: NonEmptyStr, property: NonEmptyStr, value: NonEmptyStr) -> List<String> {
[
"systemctl",
Expand Down
1,119 changes: 1,119 additions & 0 deletions dag/gunbc/deploy_transition.dag

Large diffs are not rendered by default.

522 changes: 522 additions & 0 deletions dag/gunbc/deploy_transition_bootstrap.dag

Large diffs are not rendered by default.

10 changes: 7 additions & 3 deletions dag/gunbc/fleet/fleet_workflow_steps.dag
Original file line number Diff line number Diff line change
Expand Up @@ -490,9 +490,13 @@ data gunbc_ci_live_deploy_step_timeout_minutes: Duration = 30
// serialization is not ORDERING -- a slow build of an older revision can acquire the lease after a
// fast build of a newer one and install backwards -- and it is not COMPLETE INGRESS, because
// `gunbc.apply` DashboardSrv1 reaches live_deploy_apply_srv1_wet directly and never traverses a
// GitHub concurrency group at all. Both are open; the closing construction for the second is a
// host-local lease the host effect owns, since GitHub owns only one transport into the critical
// section. Recorded here rather than left implicit because a concurrency group is exactly the kind
// GitHub concurrency group at all. The first is open. The second's closing construction NOW EXISTS:
// `gunbc.deploy_transition` `acquire_transition_lease` is the host-local lease, an O_EXCL create
// under the deployment's own instance root, so both ingresses contend for one file and the loser
// takes a typed refusal naming the holder. GitHub owns only one transport into the critical
// section, which is why the wall could never live in a concurrency group. WHAT IS NOT CLAIMED: the
// lease exists and is witnessed, and this workflow does not yet acquire it -- binding it into the
// mutating ingresses is a separate change, so the bypass is closable rather than closed. Recorded here rather than left implicit because a concurrency group is exactly the kind
// of artifact a later reader cites as the serialization authority.
data srv1_dashboard_deploy_concurrency_group: String = "srv1-dashboard-deploy"

Expand Down
27 changes: 27 additions & 0 deletions dag/gunbc/live_deploy/operations.dag
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import extdeps.systemd.systemctl {
systemctl_enable_argv,
systemctl_restart_argv,
systemctl_disable_now_argv,
systemctl_mask_now_argv,
systemctl_unmask_argv,
systemctl_stop_argv,
}
import extdeps.tools.gnu_coreutils {
coreutils_install_directory_argv,
Expand Down Expand Up @@ -94,6 +97,30 @@ fn systemctl_disable_now_command(unit: NonEmptyStr, privileged: Bool) -> String
}
}

fn systemctl_mask_now_command(unit: NonEmptyStr, privileged: Bool) -> String {
if privileged {
shell_privileged_command_text_of_argv(argv: systemctl_mask_now_argv(unit: unit))
} else {
shell_command_text_of_argv(argv: systemctl_mask_now_argv(unit: unit))
}
}

fn systemctl_stop_command(unit: NonEmptyStr, privileged: Bool) -> String {
if privileged {
shell_privileged_command_text_of_argv(argv: systemctl_stop_argv(unit: unit))
} else {
shell_command_text_of_argv(argv: systemctl_stop_argv(unit: unit))
}
}

fn systemctl_unmask_command(unit: NonEmptyStr, privileged: Bool) -> String {
if privileged {
shell_privileged_command_text_of_argv(argv: systemctl_unmask_argv(unit: unit))
} else {
shell_command_text_of_argv(argv: systemctl_unmask_argv(unit: unit))
}
}

// THE DEPLOY STAGING DIRECTORY IS ONE FACT. Apply and retract both create a temp directory and
// hold it in this shell variable; every staged file is named relative to it. Spelled at each use
// site it was a literal inside seven different quotations, each of which decided independently
Expand Down
90 changes: 86 additions & 4 deletions dag/gunbc/roadmap/roadmap_belt_actuate.dag
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ import gunbc.worker_lifecycle {
}
import gunbc.dispatch_selection { default_provider_selection_request }
import gunbc.host_layout { srv1_gunbc_repo_root, belt_spawn_workdir_env_var, instance_receipts_dir }
import gunbc.deploy_transition {
ActuationAdmission,
ActuationAdmitted,
ActuationRefused,
actuation_admission,
transition_actuation_standing,
}
import gunbc.roadmap_dashboard_instance {
HostDashboardInstance,
DashboardObserveOnly,
Expand Down Expand Up @@ -2200,10 +2207,38 @@ fn belt_tick_classify_sessions(live: List<DispatchLiveSession>) -> BeltTickSessi
}
}

data belt_publish_transition_inhibited_reason: String = "publication deferred: a deployment transition is in force or unresolved on this host, so the attempt evidence this publication would read is being rewritten or was left half-rewritten --"

data belt_tick_transition_inhibited_reason: String = "belt tick refused: a deployment transition is in force or unresolved on this host --"

data belt_tick_revision_drift_reason: String = "belt tick refused: deployed tree is not the admitted fleet revision; spawn and reap withheld"
data belt_tick_revision_unobserved_reason: String = "belt tick refused: deployed revision could not be observed; spawn and reap withheld"

fn belt_tick_revision_refused(
// A TRANSITION REFUSAL WITHHOLDS PUBLICATION AND VERIFICATION AS WELL AS SPAWN AND REAP, and that is
// the one place it departs from the revision refusal below. The revision gate's ruling is that a
// refusal blocks spawn and teardown ONLY, because verification and publication read independently
// bound receipts and stopping them would widen a spawn-admission refusal into a work stoppage. That
// argument does not survive here: a transition means the tree those receipts are read FROM is being
// rewritten underneath, or was left half-rewritten by something that died, so every one of the four
// passes is consuming a release that is not coherent. Publishing a pull request off a receipt read
// from a half-converged tree is not a smaller mistake than spawning into it -- it is the one that
// leaves the house.
fn belt_tick_transition_refused(
state: BeltRunState,
reason: String,
) -> BeltTickResult {
BeltTickResult {
observe_refused: true,
observe_refusal_reason: reason,
reconcile: belt_tick_empty_reconcile(state: state),
spawn_outcomes: [],
teardown_outcomes: [],
verify_outcomes: [],
publish_outcomes: [],
}
}

fn belt_tick_actuation_refused(
instance: HostDashboardInstance,
state: BeltRunState,
reason: String,
Expand All @@ -2226,6 +2261,13 @@ fn belt_tick_for_instance(
cap: BeltCapacity,
spawn_workdir: String,
) -> BeltTickResult {
match belt_transition_admission_for_instance(instance: instance) {
ActuationRefused { reason: tr } =>
belt_tick_transition_refused(
state: state,
reason: join([belt_tick_transition_inhibited_reason, " ", tr], ""),
)
ActuationAdmitted =>
match belt_sessions_observe_for_instance(instance: instance) {
ObserveRefused { reason: r } =>
BeltTickResult {
Expand All @@ -2242,13 +2284,13 @@ fn belt_tick_for_instance(
standing: fleet_revision_standing(repo: instance.repo_root),
) {
BeltRevisionRefusedDrifted { desired: d, local: l } =>
belt_tick_revision_refused(
belt_tick_actuation_refused(
instance: instance,
state: state,
reason: join([belt_tick_revision_drift_reason, " desired=", d, " local=", l], ""),
)
BeltRevisionRefusedUnobserved { cause: c } =>
belt_tick_revision_refused(
belt_tick_actuation_refused(
instance: instance,
state: state,
reason: join([belt_tick_revision_unobserved_reason, " cause=", c], ""),
Expand Down Expand Up @@ -2286,6 +2328,7 @@ fn belt_tick_for_instance(
}
}
}
}
}

fn belt_tick(
Expand Down Expand Up @@ -4738,6 +4781,7 @@ type BeltDispatchResult
| DispatchSessionLivenessUnobserved { node_id: String, session_name: String, reason: String }
| DispatchRevisionDrifted { node_id: String, desired: String, local: String }
| DispatchRevisionUnobserved { node_id: String, cause: String }
| DispatchTransitionInhibited { node_id: String, reason: String }
| DispatchSessionMultiplicityConflict {
node_id: String,
session_names: String,
Expand All @@ -4746,6 +4790,22 @@ type BeltDispatchResult

data belt_dispatch_revision_gate_note: String = "DISPATCH READS BACK THE DEPLOYED REVISION BEFORE IT ACTS, and the reason is that every fact this path consumes comes from the deployed tree — the node roster, the signoff, the contract, the command it is about to run. `gunbc.fleet_desired_observe` `fleet_revision_standing` already decided that cell three ways, and until now its ONLY consumer was `gunbc.fleet_converge_cli`: convergence knew the host was drifted while the served page dispatched against the drifted tree anyway, saying nothing. That is not a stale display, it is a worker spawned from code the fleet never admitted, reported as an ordinary spawn.\\n\\nSo the standing is consulted FIRST — before node lookup, before session observation, before the worktree — because a roster read from an unadmitted tree cannot be trusted to name the right node, and refusing later would mean refusing after acting on it.\\n\\nBOTH non-converged arms refuse, and neither is an absorbing widen: `RevisionDrifted` carries BOTH revisions (the membership-diff `from` ruling — a catch-up needs the prior, and a receipt must name what it observed), `RevisionUnobserved` carries its cause and is unobserved-GRADE, never satisfied. A `RevisionUnobserved` that dispatched anyway would be exactly the empty-observation narrow: could-not-compare rendered as nothing-is-wrong.\\n\\nThe repo path is `instance.repo_root`, not the `srv1_gunbc_repo_root` constant `fleet_converge_cli` uses, because the gate is a fact about the instance being dispatched — hardcoding srv1 here would mint a second path authority and answer for the wrong host on any other instance."

// THE TRANSITION GATE DOMINATES THE REVISION GATE, and the order is the safety property rather than
// a preference. A repository convergence advances the base ref BEFORE it transitions the worktree,
// so a converge that died in between leaves the ref at the admitted revision over a tree that never
// moved -- and `fleet_revision_standing` reads that ref. The revision gate would therefore answer
// CONVERGED and admit a dispatch into exactly the half-transitioned tree the belt must stay out of.
// Asking the inhibition first is what stops a gate from being satisfied by the artifact of the
// failure it is meant to notice.
//
// UNREADABLE REFUSES. `actuation_admission` admits only `ActuationPermitted`; a standing that could
// not be read is a refusal carrying its cause, never a proceed.
fn belt_transition_admission_for_instance(instance: HostDashboardInstance) -> ActuationAdmission {
actuation_admission(
standing: transition_actuation_standing(instance_root: instance.instance_root as String),
)
}

fn belt_dispatch_node_for_instance(
instance: HostDashboardInstance,
node_id: String,
Expand All @@ -4758,6 +4818,10 @@ fn belt_dispatch_node_for_instance(
reason: "dashboard instance is observe-only; dispatch refused before observation, provider preflight, or worktree creation",
}
DashboardActuating =>
match belt_transition_admission_for_instance(instance: instance) {
ActuationRefused { reason: tr } =>
DispatchTransitionInhibited { node_id: node_id, reason: tr }
ActuationAdmitted =>
match belt_revision_admission(
standing: fleet_revision_standing(repo: instance.repo_root),
) {
Expand Down Expand Up @@ -4829,6 +4893,7 @@ fn belt_dispatch_node_for_instance(
}
}
}
}
}
}

Expand Down Expand Up @@ -4859,6 +4924,7 @@ fn belt_dispatch_status_label(r: BeltDispatchResult) -> String {
DispatchSessionLivenessUnobserved { node_id: _, session_name: _, reason: _ } => "session_liveness_unobserved"
DispatchRevisionDrifted { node_id: _, desired: _, local: _ } => "deployed_revision_drifted"
DispatchRevisionUnobserved { node_id: _, cause: _ } => "deployed_revision_unobserved"
DispatchTransitionInhibited { node_id: _, reason: _ } => "deployment_transition_inhibited"
DispatchSessionMultiplicityConflict { node_id: _, session_names: _, session_count: _ } => "session_multiplicity_conflict"
}
}
Expand Down Expand Up @@ -4895,6 +4961,7 @@ fn belt_dispatch_result_band(r: BeltDispatchResult) -> DispatchBand {
DispatchSessionLivenessUnobserved { node_id: _, session_name: _, reason: _ } => BandLoud
DispatchRevisionDrifted { node_id: _, desired: _, local: _ } => BandLoud
DispatchRevisionUnobserved { node_id: _, cause: _ } => BandLoud
DispatchTransitionInhibited { node_id: _, reason: _ } => BandLoud
DispatchSessionMultiplicityConflict { node_id: _, session_names: _, session_count: _ } => BandLoud
}
}
Expand Down Expand Up @@ -4962,6 +5029,7 @@ fn belt_dispatch_result_exemplars() -> List<BeltDispatchResult> {
DispatchSessionLivenessUnobserved { node_id: "x", session_name: "s", reason: "r" },
DispatchRevisionDrifted { node_id: "x", desired: "a", local: "b" },
DispatchRevisionUnobserved { node_id: "x", cause: "c" },
DispatchTransitionInhibited { node_id: "x", reason: "r" },
DispatchSessionMultiplicityConflict { node_id: "x", session_names: "s1, s2", session_count: 2 },
]
}
Expand Down Expand Up @@ -5074,6 +5142,12 @@ fn belt_dispatch_result_json_value(r: BeltDispatchResult) -> JsonValue {
json_kv(key: "status", value: json_string(s: belt_dispatch_status_label(r: r))),
json_kv(key: "cause", value: json_string(s: c2)),
])
DispatchTransitionInhibited { node_id: nid, reason: tr } =>
json_object(members: [
json_kv(key: "node_id", value: json_string(s: nid)),
json_kv(key: "status", value: json_string(s: belt_dispatch_status_label(r: r))),
json_kv(key: "reason", value: json_string(s: tr)),
])
DispatchSessionMultiplicityConflict { node_id: nid, session_names: sn, session_count: n } =>
json_object(members: [
json_kv(key: "node_id", value: json_string(s: nid)),
Expand Down Expand Up @@ -5494,7 +5568,14 @@ fn belt_publish_node_for_instance(
" is observe-only, so publication was neither observed nor recorded",
], ""),
}
DashboardActuating => {
DashboardActuating =>
match belt_transition_admission_for_instance(instance: instance) {
ActuationRefused { reason: tr } =>
PublishDeferred {
node_id: node_id,
reason: join([belt_publish_transition_inhibited_reason, " ", tr], ""),
}
ActuationAdmitted => {
let key_read = Filesystem.Read(
path: dispatch_current_attempt_key_path_for_instance(
instance: instance,
Expand All @@ -5515,6 +5596,7 @@ fn belt_publish_node_for_instance(
)
}
}
}
}
}

Expand Down
2 changes: 2 additions & 0 deletions dag/gunbc/roadmap/roadmap_serve.dag
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import gunbc.roadmap_belt_actuate {
DispatchDisabled, DispatchNodeAlreadyDone, DispatchObserveRefused, DispatchAuthorityRefused, DispatchSpawnFailed,
DispatchStaleSession, DispatchSessionLivenessUnobserved,
DispatchRevisionDrifted, DispatchRevisionUnobserved,
DispatchTransitionInhibited,
DispatchSessionMultiplicityConflict,
belt_dispatch_node, belt_dispatch_node_for_instance,
belt_dispatch_result_json_value,
Expand Down Expand Up @@ -270,6 +271,7 @@ fn belt_dispatch_status(r: BeltDispatchResult) -> HttpStatus {
DispatchStaleSession { node_id: _, session_name: _, detail: _ } => 409
DispatchRevisionDrifted { node_id: _, desired: _, local: _ } => 409
DispatchRevisionUnobserved { node_id: _, cause: _ } => 503
DispatchTransitionInhibited { node_id: _, reason: _ } => 409
DispatchSessionMultiplicityConflict { node_id: _, session_names: _, session_count: _ } => 409
DispatchSessionLivenessUnobserved { node_id: _, session_name: _, reason: _ } => 503
}
Expand Down
9 changes: 8 additions & 1 deletion dag/test/claim/component_dispatch_button_witness_test.dag
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,17 @@ test fn witness_scale_well_formed() -> Bool {
scale_well_formed()
}

// THE PIN MOVES 13 -> 14 WITH `DispatchTransitionInhibited`, and it is maintenance rather than a
// loosening: the roster is one exemplar per BeltDispatchResult variant over a CLOSED coproduct, so
// the literal pins a hand-enumerated fixture's size and not a measured population. The two clauses
// above it are the ones with the failure mode -- both directions of totality, derived on each side --
// and they held through this change because the component's states are `map`ped from the label
// authority rather than listed beside it. What this literal catches is a variant added to the wire
// without its exemplar, which is the half derivation cannot see.
test fn witness_dispatch_button_states_total_over_wire() -> Bool {
dispatch_button_wire_total()
&& dispatch_button_terminal_count() == belt_dispatch_all_status_labels().length()
&& belt_dispatch_all_status_labels().length() == 13
&& belt_dispatch_all_status_labels().length() == 14
}

test fn witness_dropped_state_is_located_red() -> Bool {
Expand Down
Loading
Loading