Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
4149839
WIP: dag roadmap work
briansrls Jul 20, 2026
e6ea4dd
WIP: dag roadmap work
briansrls Jul 20, 2026
ba268d3
WIP: dag roadmap work
briansrls Jul 20, 2026
80a7b26
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 20, 2026
c839222
Belt A review #6942: single-authority dispatch route prefix
briansrls Jul 20, 2026
2658585
WIP: dag roadmap work
briansrls Jul 20, 2026
52af1e3
WIP: dag roadmap work
briansrls Jul 20, 2026
1119da2
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 20, 2026
e119687
WIP: dag roadmap work
briansrls Jul 20, 2026
fbca528
Belt A #6942: address cursor REQUEST_CHANGES (4 findings)
briansrls Jul 20, 2026
f28da88
WIP: dag roadmap work
briansrls Jul 20, 2026
ebe4488
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 20, 2026
d8cb1c2
WIP: dag roadmap work
briansrls Jul 20, 2026
23dd3eb
WIP: dag roadmap work
briansrls Jul 20, 2026
b273e82
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 20, 2026
dedd824
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
b35ad7e
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
7210589
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
95a392e
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
963dce4
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
8cf037a
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
404d1ae
temp: srv1 runner diagnostic workflow (read-only; removed before PR)
briansrls Jul 21, 2026
1c00a86
temp: extend srv1 diag with perms chain + grant reality-check
briansrls Jul 21, 2026
f49ea61
WIP: dag roadmap work
briansrls Jul 21, 2026
dd06f61
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
4b1d2dc
WIP: dag roadmap work
briansrls Jul 21, 2026
2273b9e
live-deploy: unprivileged ci-runner principal — tree sync as service …
briansrls Jul 21, 2026
c55a17d
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
ac42e8c
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 21, 2026
bedffa3
WIP: dag roadmap work
briansrls Jul 21, 2026
58145b8
WIP: dag roadmap work
briansrls Jul 21, 2026
72a10e5
WIP: dag roadmap work
briansrls Jul 21, 2026
bc8972f
WIP: dag roadmap work
briansrls Jul 22, 2026
b8f8828
WIP: dag roadmap work
briansrls Jul 22, 2026
52d41a9
UnitDiagnosis becomes a real coproduct; collapse the duplicate reason…
briansrls Jul 22, 2026
d708f14
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 22, 2026
c9c4978
Merge origin/main into session/royal-carp-451 (pick up ci_deploy_sudo…
briansrls Jul 22, 2026
1e36d8b
Merge remote-tracking branch 'origin/main' into session/royal-carp-451
briansrls Jul 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions dag/gunbc/live_deploy/intent.dag
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import gunbc.live_deploy.service_ready {
live_deploy_service_ready_poll_bound,
live_deploy_service_ready_unit_authority,
live_deploy_service_ready_endpoint_authority,
UnitDiagnosisUnavailable,
}
import gunbc.roadmap_static_site { roadmap_site_health_path }
import gunbc.live_deploy.spec { DeploymentSpec, deployment_plan_listen_port }
Expand Down Expand Up @@ -47,6 +48,7 @@ fn live_deploy_service_ready_gate_for_spec(spec: DeploymentSpec) -> ServiceReady
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: "health probe never succeeded",
unit_diagnosis: UnitDiagnosisUnavailable,
},
}
}
Expand All @@ -58,6 +60,17 @@ fn live_deploy_readback_for_spec(spec: DeploymentSpec) -> LiveDeployReadback {
}
}

data live_deploy_unit_diagnosis_capture_exit_note: String = "The capture ends in 'exit 0' DELIBERATELY, and this is not the escape hatch DESIGN 5 forbids -- read what the exit code answers. Here it answers 'did the capture run', NOT 'is the service healthy'. The health verdict was already decided, fail-closed, by the poll bound this capture runs after; the captured value is a String that is printed and never matched, compared, or branched on, so no arm here can turn a refusal into a pass. Conflating capture-failure with unit-failure is the state-space conflation to avoid: the unit's real state is in the captured TEXT, which says 'Active: failed' in its own words, and a capture that fails yields a descriptive string rather than a pass.

WHAT THE exit 0 IS AND IS NOT, stated precisely, because an overstated rationale is the same debt this module's poll-bound reason was just corrected for. systemctl status exits 3 for a dead-or-failed unit, which is exactly the case we capture in. But on the path this actually runs -- interpreter runtime through host_effect_apply_gated, plain 'bash -s', no pipefail -- the trailing pipe to tail ALREADY yields status 0, because a pipeline takes its LAST command's status and tail succeeds. Measured both ways, 2026-07-21: without pipefail 'exit 3 | tail' reports 0; with 'set -o pipefail' the same pipeline reports 3. So systemctl's nonzero is masked by tail before the exit 0 is ever reached, and the exit 0 is DEFENSIVE, not load-bearing today. It earns its place by covering two live futures rather than a present need: it still holds if the tail is later dropped, and it becomes load-bearing the moment this capture routes through a pipefail shell -- which is not hypothetical, since several live_deploy and gunbc scripts set 'set -euo pipefail', the emitted apply among them. Without either guard the realizer would see a nonzero exit, return NotConverged, and DISCARD stdout, so the diagnostic would go dark exactly when it is needed and report only that it failed to report."

fn live_deploy_unit_diagnosis_command(unit: NonEmptyStr) -> String {
concat(
concat("systemctl status --no-pager --full ", unit as String),
" 2>&1 | tail -n 40; exit 0"
)
}

fn live_deploy_health_probe_curl_command(health_url: NonEmptyStr) -> String {
concat(
concat(
Expand Down
126 changes: 103 additions & 23 deletions dag/gunbc/live_deploy/readiness.dag
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,20 @@ import std.realization_reconcile {
import gunbc.live_deploy.service_ready {
ServiceNotReady,
service_not_ready_reason,
service_not_ready_with_captured_diagnosis,
live_deploy_service_ready_poll_bound,
live_deploy_service_ready_poll_cadence,
live_deploy_service_ready_poll_max_attempts,
live_deploy_service_ready_unit_authority,
live_deploy_service_ready_endpoint_authority,
UnitDiagnosis,
UnitDiagnosisUnavailable,
UnitDiagnosisCaptured,
}
import gunbc.live_deploy.intent {
live_deploy_health_probe_curl_command,
live_deploy_health_url_for_port,
live_deploy_unit_diagnosis_command,
}
import gunbc.roadmap_site_healthz_validate { roadmap_site_healthz_json_string_parses }
import gunbc.live_deploy.spec {
Expand Down Expand Up @@ -200,57 +205,119 @@ fn live_deploy_service_ready_once() -> Reconciliation<ServiceReadyApplyEffect, S
reconcile_service_ready_read(read: show_healthz_read_fixture_healthy())
}

fn live_deploy_poll_service_ready_after_attempts(
fn unit_diagnosis_from_shell_apply(
r: Reconciliation<HostEffectIntent, HostEffectEvidence>
) -> String {
match r {
Converged { evidence: e, applied: _ } =>
match e {
ShellEffectApplied { stdout: out, exit_code: _ } => out
ConvergePlanApplied { stdout: _, exit_code: _ } =>
"unit diagnosis capture returned ConvergePlanApplied; typed mismatch"
}
NotConverged { reason: why, applied: _ } =>
concat("unit diagnosis capture failed: ", why)
}
}

fn capture_unit_diagnosis(
spec: DeploymentSpec,
transport: HostEffectTransport,
access: DeployAccess,
) -> String {
unit_diagnosis_from_shell_apply(
r: host_effect_apply_gated(
access: access,
target: HostOs { node: spec.target.host },
effect: ShellCommand {
script: live_deploy_unit_diagnosis_command(
unit: live_deploy_service_ready_unit_authority
),
},
evidence: host_effect_read_evidence,
transport: transport,
fixture_actor: Absent,
)
)
}

type ServiceReadyPollOutcome =
PollSettled { result: Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence> }
| PollRefusedAtBound { refusal: ServiceNotReady }

fn poll_bound_refusal(last_probe: String) -> ServiceNotReady {
ServiceNotReady {
unit: live_deploy_service_ready_unit_authority,
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: last_probe,
unit_diagnosis: UnitDiagnosisUnavailable,
}
}

fn refusal_to_reconciliation(refusal: ServiceNotReady) -> Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence> {
NotConverged {
reason: service_not_ready_reason(refusal: refusal),
applied: service_ready_apply_effect(),
}
}

fn live_deploy_poll_service_ready_outcome_after_attempts(
spec: DeploymentSpec,
transport: HostEffectTransport,
access: DeployAccess,
attempt: Int,
last_probe: String,
) -> Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence> {
) -> ServiceReadyPollOutcome {
let max_attempts = live_deploy_service_ready_poll_max_attempts()
if attempt >= max_attempts {
NotConverged {
reason: service_not_ready_reason(refusal: ServiceNotReady {
unit: live_deploy_service_ready_unit_authority,
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: last_probe,
}),
applied: service_ready_apply_effect(),
}
PollRefusedAtBound { refusal: poll_bound_refusal(last_probe: last_probe) }
} else {
let probe = live_deploy_service_ready_once_for_deploy(
spec: spec,
transport: transport,
access: access,
)
if reconciliation_converged(r: probe) {
probe
PollSettled { result: probe }
} else {
let delay = sleep_delay_seconds_second_carrier_projection(duration: live_deploy_service_ready_poll_cadence)
if delay {
live_deploy_poll_service_ready_after_attempts(
live_deploy_poll_service_ready_outcome_after_attempts(
spec: spec,
transport: transport,
access: access,
attempt: attempt + 1,
last_probe: last_probe_from_attempt(attempt: probe)
)
} else {
NotConverged {
reason: service_not_ready_reason(refusal: ServiceNotReady {
unit: live_deploy_service_ready_unit_authority,
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: "sleep between health probes failed",
}),
applied: service_ready_apply_effect(),
PollRefusedAtBound {
refusal: poll_bound_refusal(last_probe: "sleep between health probes failed")
}
}
}
}
}

fn live_deploy_poll_service_ready_after_attempts(
spec: DeploymentSpec,
transport: HostEffectTransport,
access: DeployAccess,
attempt: Int,
last_probe: String,
) -> Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence> {
match live_deploy_poll_service_ready_outcome_after_attempts(
spec: spec,
transport: transport,
access: access,
attempt: attempt,
last_probe: last_probe,
) {
PollSettled { result: r } => r
PollRefusedAtBound { refusal: refusal } => refusal_to_reconciliation(refusal: refusal)
}
}

fn last_probe_from_attempt(
attempt: Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence>
) -> String {
Expand All @@ -265,13 +332,26 @@ fn live_deploy_poll_until_service_ready_for_deploy(
transport: HostEffectTransport,
access: DeployAccess,
) -> Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence> {
live_deploy_poll_service_ready_after_attempts(
match live_deploy_poll_service_ready_outcome_after_attempts(
spec: spec,
transport: transport,
access: access,
attempt: 0,
last_probe: "no probe yet",
)
) {
PollSettled { result: r } => r
PollRefusedAtBound { refusal: refusal } =>
refusal_to_reconciliation(
refusal: service_not_ready_with_captured_diagnosis(
refusal: refusal,
text: capture_unit_diagnosis(
spec: spec,
transport: transport,
access: access,
)
)
)
}
}

fn live_deploy_poll_until_service_ready() -> Reconciliation<ServiceReadyApplyEffect, ServiceReadyEvidence> {
Expand Down
33 changes: 30 additions & 3 deletions dag/gunbc/live_deploy/service_ready.dag
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,25 @@ data live_deploy_service_ready_scaffold_dissolution_trigger: Disposition = Scaff
}
}

type UnitDiagnosis =
UnitDiagnosisUnavailable
| UnitDiagnosisCaptured { text: String }

data live_deploy_unit_diagnosis_coproduct_note: String = "UnitDiagnosis is a coproduct, not a String with a reserved 'unavailable' value, and the distinction is the whole point of the field. A sentinel string makes 'nothing was captured' and 'the unit literally emitted the sentinel text' the SAME representation -- an illegal state that is representable, and one an equality test would silently misclassify the day a captured stdout happened to match. That is precisely the state-space conflation this module's diagnosis note invokes against folding unit_diagnosis into last_probe, so encoding absence as a magic string here would have contradicted the field's own reason for existing (caught in review of PR 7038, review 41182). Absence is a variant; rendering matches on it; no comparison against a magic value exists anywhere."

type ServiceNotReady {
unit: NonEmptyStr
endpoint: NonEmptyStr
waited: Second
last_probe: String
unit_diagnosis: UnitDiagnosis
}

data live_deploy_service_ready_unit_diagnosis_note: String = "unit_diagnosis carries the UNIT's own account of itself (systemctl status: Active state, main-PID exit code, recent journal lines), captured once when the poll bound is exhausted. It is a SEPARATE fact from last_probe and must not be folded into it: last_probe is what the health endpoint said (a curl-level observation from outside), unit_diagnosis is what the service said (a process-level observation from inside). Conflating them is the state-space conflation DESIGN 3 names -- 'the endpoint refused connection' and 'the process exited 1 while compiling' are different states whose remedies differ, and only the second locates the defect. Motivating incident: the srv1 deploy refused with ServiceNotReady on every run from belt B's merge (PR 6940) onward, and because the refusal carried only the curl exit the cause stayed unlocated for the whole window -- the line stopped, typed, but nobody could see WHY. gunbc serve has four distinct exit paths (source load, blocking diagnostics, no graph, bind failure) and the probe alone cannot tell them apart."

data live_deploy_roadmap_unit_expected_startup: Second = second(count: 30)

data live_deploy_service_ready_poll_bound_reason: String = "Conservative default for gunbc-roadmap.service cold-start after systemctl restart (node HTTP bind + localhost healthz); revisit if false ServiceNotReady observed in production."
data live_deploy_service_ready_poll_bound_reason: String = "Cold-start bound for gunbc-roadmap.service after systemctl restart. The startup SHAPE changed at belt B (PR 6940) and this rationale was corrected with it: the unit is no longer the former Node HTTP server, which bound its socket almost immediately. gunbc serve loads its source roots and COMPILES the .dag graph to a resolved form, and binds the port only after that compile succeeds, so readiness is gated on compile time rather than on process spawn -- and a compile that fails exits before any bind, which presents to the probe as connection-refused for the entire bound, indistinguishable from slow. Measured on the roadmap_serve entry (green-by-execution control, 2026-07-21): 92 sources resolved, analyses-done at 3.3s, then listening and healthz OK. The bound stays 30s, which that measurement supports with wide margin. If a REAL srv1 cold start is ever observed above it, raise it as its own decision carrying its own receipt -- never to make a failing probe pass, which would be the absorbing fallback DESIGN 5 forbids."

data live_deploy_service_ready_poll_bound: Second = live_deploy_roadmap_unit_expected_startup

Expand All @@ -57,8 +66,10 @@ data live_deploy_service_ready_unit_authority: NonEmptyStr = gunbc_roadmap_unit_

data live_deploy_service_ready_endpoint_authority: NonEmptyStr = roadmap_site_health_path

data live_deploy_unit_diagnosis_capture_placement_note: String = "The capture is performed by the DEPLOY ENTRY (readiness.live_deploy_poll_until_service_ready_for_deploy), never by the recursive poller, and that placement is load-bearing rather than stylistic. The poller is driven directly by claim witnesses; the deploy access it is handed carries an SshShell transport pointed at srv1, so performing the capture inside the poller's exhaustion branch made a hermetic refusal-shape witness attempt a LIVE cross-host SSH from whatever runner executed the corpus. That is non-hermetic by construction and it duly behaved differently per environment -- green locally, and on srv2-03 it returned a value that blew a downstream match (PR 7038, first CI run). So the recursive poller stays free of the capture effect and builds its refusal with UnitDiagnosisUnavailable, which is the honest statement at that point: nothing has been captured yet. The entry, which only the real deploy reaches, performs the capture once on the refusal and rebuilds it through service_not_ready_with_captured_diagnosis. Rendering matches the variant and emits the segment only for UnitDiagnosisCaptured, so exactly one unit_diagnosis segment ever appears, it is never a placeholder masquerading as an observation, and there is ONE rendering authority rather than a second copy of the format string at the entry."

fn service_not_ready_reason(refusal: ServiceNotReady) -> String {
concat(
let base = concat(
concat(
concat(
concat("ServiceNotReady unit=", refusal.unit as String),
Expand All @@ -68,6 +79,20 @@ fn service_not_ready_reason(refusal: ServiceNotReady) -> String {
),
concat(" last_probe=", refusal.last_probe)
)
match refusal.unit_diagnosis {
UnitDiagnosisUnavailable => base
UnitDiagnosisCaptured { text: t } => concat(base, concat(" unit_diagnosis=", t))
}
}

fn service_not_ready_with_captured_diagnosis(refusal: ServiceNotReady, text: String) -> ServiceNotReady {
ServiceNotReady {
unit: refusal.unit,
endpoint: refusal.endpoint,
waited: refusal.waited,
last_probe: refusal.last_probe,
unit_diagnosis: UnitDiagnosisCaptured { text: text },
}
}

fn live_deploy_readback_ready_set(graph_node: Node) -> List<Node> {
Expand All @@ -90,11 +115,13 @@ fn live_deploy_service_ready_poll_cadence_seconds() -> Second {
live_deploy_service_ready_poll_cadence
}

fn service_not_ready_refusal_at_poll_bound(last_probe: String) -> ServiceNotReady {
fn service_not_ready_refusal_at_poll_bound(last_probe: String, unit_diagnosis: UnitDiagnosis) -> ServiceNotReady {
ServiceNotReady {
unit: live_deploy_service_ready_unit_authority,
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: last_probe,
unit_diagnosis: unit_diagnosis,
}
}

2 changes: 2 additions & 0 deletions dag/test/claim/live_deploy/apply_witness_test.dag
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import gunbc.live_deploy.service_ready {
live_deploy_service_ready_poll_bound,
live_deploy_service_ready_unit_authority,
live_deploy_service_ready_endpoint_authority,
UnitDiagnosisUnavailable,
}
import gunbc.host_effect {
ShellCommand,
Expand Down Expand Up @@ -65,6 +66,7 @@ fn fixture_service_not_ready_readiness() -> Reconciliation<ServiceReadyApplyEffe
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: "health probe never succeeded",
unit_diagnosis: UnitDiagnosisUnavailable,
}),
applied: service_ready_apply_effect(),
}
Expand Down
3 changes: 3 additions & 0 deletions dag/test/claim/live_deploy/readiness_witness_test.dag
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import gunbc.live_deploy.service_ready {
ServiceNotReady,
live_deploy_service_ready_unit_authority,
live_deploy_service_ready_endpoint_authority,
UnitDiagnosisCaptured,
}
import gunbc.live_deploy.spec { deployment_spec_srv1, deployment_plan_listen_port }
import gunbc.ci_deploy_access { ci_deploy_srv1_access }
Expand Down Expand Up @@ -201,9 +202,11 @@ fn witness_service_not_ready_refusal_shape_matches_contract() -> Bool {
endpoint: live_deploy_service_ready_endpoint_authority,
waited: live_deploy_service_ready_poll_bound,
last_probe: "connection refused",
unit_diagnosis: UnitDiagnosisCaptured { text: "Active: failed (Result: exit-code)" },
})
string_contains(s: reason, pattern: "ServiceNotReady")
&& string_contains(s: reason, pattern: "gunbc-roadmap.service")
&& string_contains(s: reason, pattern: "Active: failed (Result: exit-code)")
}

test fn live_deploy_readiness_witnesses() -> Bool {
Expand Down
Loading
Loading