Spark remote legs carry a liveness deadline; a leg that never began is typed dark-or-down undecided

dag/extdeps/ssh/client_options.dag

@@ -1,6 +1,6 @@
 module extdeps.ssh.client_options
 
-import std.types { NonEmptyStr, String, Bool, List }
+import std.types { NonEmptyStr, String, Bool, Int, List }
 import std.measure { Second, second, second_count }
 import std.decl_ref { DeclarationRef, WholeDeclaration }
 import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef }
@@ -99,7 +99,16 @@ type UpdateHostKeysValue
   | UpdateHostKeysAsk
 
 // One variant per keyword, each carrying the value domain THAT keyword admits, so a nonsensical
-// pairing has no constructor. ConnectTimeout takes an Int because ssh_config(5) specifies seconds;
+// pairing has no constructor. ServerAliveInterval and ServerAliveCountMax are the LIVENESS half of a
+// deadline, distinct from ConnectTimeout, which bounds only the TCP connect: ssh_config(5) --
+// ServerAliveInterval "Sets a timeout interval in seconds after which if no data has been received
+// from the server, ssh(1) will send a message through the encrypted channel to request a response",
+// default 0 (never); ServerAliveCountMax "Sets the number of server alive messages ... which may be
+// sent without ssh(1) receiving any messages back from the server. If this threshold is reached while
+// server alive messages are being sent, ssh will disconnect from the server, terminating the session",
+// default 3. With the interval at its default of 0 an established session whose path goes dark waits
+// on TCP alone, which is how fleet-converge run 37156689444 sat ~1h inside one leg.
+// ConnectTimeout takes an Int because ssh_config(5) specifies seconds;
 // BatchMode takes a Bool because its domain is yes|no and Bool is that domain already grounded
 // (section 2: no net new concepts by re-invention).
 type SshClientOption
@@ -112,6 +121,8 @@ type SshClientOption
   | BatchMode { enabled: Bool }
   | ExitOnForwardFailure { enabled: Bool }
   | ConnectTimeout { seconds: Second }
+  | ServerAliveInterval { seconds: Second }
+  | ServerAliveCountMax { count: Int }
   | ConfigFile { path: NonEmptyStr }
   | IdentityFile { path: NonEmptyStr }
   | IdentitiesOnly { enabled: Bool }
@@ -134,6 +145,8 @@ fn ssh_client_option_keyword(option: SshClientOption) -> String {
     BatchMode { enabled: _ } => "BatchMode"
     ExitOnForwardFailure { enabled: _ } => "ExitOnForwardFailure"
     ConnectTimeout { seconds: _ } => "ConnectTimeout"
+    ServerAliveInterval { seconds: _ } => "ServerAliveInterval"
+    ServerAliveCountMax { count: _ } => "ServerAliveCountMax"
     ConfigFile { path: _ } => "ConfigFile"
     IdentityFile { path: _ } => "IdentityFile"
     IdentitiesOnly { enabled: _ } => "IdentitiesOnly"
@@ -182,6 +195,8 @@ fn ssh_client_option_value(option: SshClientOption) -> String {
     BatchMode { enabled: e } => if e { "yes" } else { "no" }
     ExitOnForwardFailure { enabled: e } => if e { "yes" } else { "no" }
     ConnectTimeout { seconds: s } => to_string(second_count(s: s))
+    ServerAliveInterval { seconds: s } => to_string(second_count(s: s))
+    ServerAliveCountMax { count: n } => to_string(n)
     ConfigFile { path: p } => p as String
     IdentityFile { path: p } => p as String
     IdentitiesOnly { enabled: e } => if e { "yes" } else { "no" }
@@ -338,6 +353,8 @@ fn serialize_ssh_config_file_flag(option: SshClientOption) -> List<String> {
     BatchMode { enabled: _ } => serialize_ssh_client_option(option: option)
     ExitOnForwardFailure { enabled: _ } => serialize_ssh_client_option(option: option)
     ConnectTimeout { seconds: _ } => serialize_ssh_client_option(option: option)
+    ServerAliveInterval { seconds: _ } => serialize_ssh_client_option(option: option)
+    ServerAliveCountMax { count: _ } => serialize_ssh_client_option(option: option)
     IdentityFile { path: _ } => serialize_ssh_client_option(option: option)
     IdentitiesOnly { enabled: _ } => serialize_ssh_client_option(option: option)
     IdentityAgent { path: _ } => serialize_ssh_client_option(option: option)

dag/gunbc/compute/host_occupancy.dag

@@ -19,7 +19,7 @@ import extdeps.systemd {
   SystemdUnitActiveState, Active, Inactive, Activating, Deactivating, Reloading, Failed,
   parse_systemd_unit_active_state, systemd_unit_active_state_wire_label,
 }
-import gunbc.spark.host_effect_quiescence { ArgvRun, ArgvRan, ArgvLegDidNotRun }
+import gunbc.spark.host_effect_quiescence { ArgvRun, ArgvRan, ArgvLegDidNotRun, ArgvLegMayHaveRun }
 
 // ── WHO IS ALREADY USING THIS HOST: A READING, TAKEN ON THE HOST, BEFORE AN EFFECT IS ADMITTED ─
 //
@@ -85,13 +85,15 @@ type HostOperationLeg
 type HostOperationRan
   = HostOperationExited { exit_code: Int, stdout: String, stderr: String }
   | HostOperationNotRun { cause: String }
+  | HostOperationResultUnreceived { cause: String }
 
 fn host_operation_ran(run: fn(List<String>) -> ArgvRun, operation: HostOperation) -> HostOperationRan {
   match host_operation_materialize_argv(operation: operation) {
     ArgvMaterializationRefused { at: _, cause: _ } => HostOperationNotRun { cause: join([host_operation_label(operation: operation), " could not be materialized as an argv"], "") }
     ArgvMaterialized { argv: argv } =>
       match run(argv) {
         ArgvLegDidNotRun { cause: c } => HostOperationNotRun { cause: join([host_operation_label(operation: operation), " did not run: ", c], "") }
+        ArgvLegMayHaveRun { cause: c } => HostOperationResultUnreceived { cause: join([host_operation_label(operation: operation), " may have run, no result received: ", c], "") }
         ArgvRan { exit_code: code, stdout: out, stderr: err } => HostOperationExited { exit_code: code, stdout: out, stderr: err }
       }
   }
@@ -104,6 +106,7 @@ fn host_operation_exit_unread(operation: HostOperation, code: Int, err: String)
 fn host_operation_leg(run: fn(List<String>) -> ArgvRun, operation: HostOperation, accepted_exits: List<Int>) -> HostOperationLeg {
   match host_operation_ran(run: run, operation: operation) {
     HostOperationNotRun { cause: c } => HostOperationLegUnread { cause: c }
+    HostOperationResultUnreceived { cause: c } => HostOperationLegUnread { cause: c }
     HostOperationExited { exit_code: code, stdout: out, stderr: err } =>
       if any(accepted_exits, e => e == code) { HostOperationLegRead { stdout: out } }
       else { HostOperationLegUnread { cause: host_operation_exit_unread(operation: operation, code: code, err: err) } }
@@ -125,6 +128,7 @@ fn read_unit_activity(run: fn(List<String>) -> ArgvRun, unit: NonEmptyStr) -> Un
   let operation = SystemctlIsActive { unit: unit }
   match host_operation_ran(run: run, operation: operation) {
     HostOperationNotRun { cause: c } => UnitUnread { cause: c }
+    HostOperationResultUnreceived { cause: c } => UnitUnread { cause: c }
     HostOperationExited { exit_code: code, stdout: out, stderr: err } =>
       match systemctl_is_active_exit(code: code) {
         Absent => UnitUnread { cause: host_operation_exit_unread(operation: operation, code: code, err: err) }

dag/gunbc/fleet/fleet_known_hosts_anchor.dag

@@ -1,7 +1,7 @@
 module gunbc.fleet_known_hosts_anchor
 
 import std.types { String, Bool, List, NonEmptyStr, PathSegment, path_segment_is_safe, brand }
-import std.measure { second }
+import std.measure { Second, second }
 import std.algebra { trim }
 import std.content_hash {
   ContentHash,
@@ -28,6 +28,8 @@ import extdeps.ssh.client_options {
   User,
   BatchMode,
   ConnectTimeout,
+  ServerAliveInterval,
+  ServerAliveCountMax,
 }
 import gunbc.fleet_host_key_enrollment { KnownHostKeyRow, fleet_host_key_enrollments, known_hosts_line, host_key_provenance_label }
 import gunbc.spark.fabric_reach { fabric_rail_known_host_rows }
@@ -667,12 +669,43 @@ fn fleet_ssh_password_client_options(
   trust: List<SshClientOption>,
   principal: NonEmptyStr,
 ) -> List<SshClientOption> {
-  concat(trust, [
+  concat(trust, concat([
     PreferredAuthentications { methods: "password,keyboard-interactive" as NonEmptyStr },
     User { name: principal },
     BatchMode { enabled: false },
-    ConnectTimeout { seconds: second(count: 10) },
-  ])
+  ], fleet_ssh_leg_deadline_options(deadline: fleet_ssh_leg_deadline)))
+}
+
+// EVERY FLEET LEG CARRIES A DEADLINE, AND IT IS ONE FACT. A leg has two ways to stall and ssh bounds
+// them with different keywords: the TCP connect (ConnectTimeout) and an established session whose
+// path then goes dark (ServerAliveInterval x ServerAliveCountMax, extdeps.ssh.client_options). The
+// shapers below carried the first, inline, and never the second -- so on 2026-10-03 when the Spark
+// management LAN (wlP9s9, every Spark's only route to the executor) stopped forwarding mid-session,
+// fleet-converge run 37156689444 sat ~1h inside one ssh leg instead of refusing.
+//
+// UNCONSTRUCTIBLE WITHOUT IT, at the grain this module owns: neither shaper takes an options
+// parameter, so no caller can shape a fleet leg that omits these words, and both derive them from
+// this ONE row rather than each spelling its own number. A dark path now ends the leg with ssh's own
+// 255 within interval x count_max (60 s) of the last reply; the sshd answers keepalives itself, so
+// a remote command that is merely silent is NOT cut off -- only a path that returns nothing is.
+type FleetSshLegDeadline {
+  connect: Second
+  alive_interval: Second
+  alive_count_max: Int
+}
+
+data fleet_ssh_leg_deadline: FleetSshLegDeadline = FleetSshLegDeadline {
+  connect: second(count: 10),
+  alive_interval: second(count: 15),
+  alive_count_max: 4,
+}
+
+fn fleet_ssh_leg_deadline_options(deadline: FleetSshLegDeadline) -> List<SshClientOption> {
+  [
+    ConnectTimeout { seconds: deadline.connect },
+    ServerAliveInterval { seconds: deadline.alive_interval },
+    ServerAliveCountMax { count: deadline.alive_count_max },
+  ]
 }
 
 // The password session needs only the trust anchor, so a caller holding no fleet key (the first
@@ -714,11 +747,13 @@ fn shape_fleet_ssh_exec(
     fleet_openssh_trust_options(policy: context.trust),
     concat(
       fleet_ssh_credential_options(binding: context.credential),
-      [
-        User { name: target.principal },
-        BatchMode { enabled: true },
-        ConnectTimeout { seconds: second(count: 10) },
-      ],
+      concat(
+        [
+          User { name: target.principal },
+          BatchMode { enabled: true },
+        ],
+        fleet_ssh_leg_deadline_options(deadline: fleet_ssh_leg_deadline),
+      ),
     ),
   )
   concat(

dag/gunbc/fleet/fleet_wireless_link.dag

@@ -29,7 +29,7 @@ import extdeps.networkmanager.nmcli {
 }
 import extdeps.systemd.journalctl { journalctl_kernel_current_boot_argv, journalctl_unit_current_boot_argv }
 import gunbc.spark.dgx_procurement { dgx_spark_procurement_intent }
-import gunbc.spark.host_effect_quiescence { ArgvRun, ArgvRan, ArgvLegDidNotRun }
+import gunbc.spark.host_effect_quiescence { ArgvRun, ArgvRan, ArgvLegDidNotRun, ArgvLegMayHaveRun }
 import gunbc.fleet_host_identity {
   operator_host_srv5, operator_host_srv6, operator_host_srv7, operator_host_srv8,
   operator_host_srv9, operator_host_srv10, operator_host_srv11, operator_host_srv12,
@@ -195,6 +195,7 @@ fn networkmanager_no_secrets_line_count(journal: String) -> Int {
 fn argv_run_failure(run: ArgvRun) -> String? {
   match run {
     ArgvLegDidNotRun { cause: c } => Present { value: join(["leg did not run: ", c], "") }
+    ArgvLegMayHaveRun { cause: c } => Present { value: join(["leg may have run, no result received (read back before retrying): ", c], "") }
     ArgvRan { exit_code: code, stdout: _, stderr: err } =>
       if code != 0 { Present { value: join(["exit=", to_string(code), " stderr=", trim(s: err)], "") } } else { none }
   }
@@ -203,6 +204,7 @@ fn argv_run_failure(run: ArgvRun) -> String? {
 fn argv_run_stdout(run: ArgvRun) -> String {
   match run {
     ArgvLegDidNotRun { cause: _ } => ""
+    ArgvLegMayHaveRun { cause: _ } => ""
     ArgvRan { exit_code: _, stdout: o, stderr: _ } => o
   }
 }

dag/gunbc/recurring_failure_mode/a_dark_management_lan_leg_hangs_and_reads_as_a_down_host.dag

@@ -0,0 +1,22 @@
+module gunbc.recurring_failure_mode.a_dark_management_lan_leg_hangs_and_reads_as_a_down_host
+
+import std.types { NonEmptyStr }
+import gunbc.recurring_failure_mode { RecurringFailureMode }
+
+data a_dark_management_lan_leg_hangs_and_reads_as_a_down_host: RecurringFailureMode = RecurringFailureMode {
+  identity: "a_dark_management_lan_leg_hangs_and_reads_as_a_down_host" as NonEmptyStr,
+
+  receipts: [
+    "**a dark management-LAN leg hangs, then reads as a down host** (a fleet ssh leg bounded only at connect waits indefinitely when an established session's path goes dark, and when it does end, the run reports the host as down although the evidence in hand cannot tell a dark LAN leg from a host that is off).",
+
+    "INVALID STATE: gunbc.fleet_known_hosts_anchor shape_fleet_ssh_exec and fleet_ssh_password_client_options carried ConnectTimeout=10 and no ServerAliveInterval (ssh_config(5) default 0, never), so an established session had no liveness bound; and gunbc.spark.model_snapshot_materialize spark_remote_run folded ssh's own 255 into RemoteRan, so a leg whose session died read as a command that exited 255.",
+
+    "HARM: 2026-10-03, Spark Group A bring-up. Every Spark reaches the executor only over wlP9s9 (MediaTek mt7925e). Router band steering roamed the hosts; a failed 4-way handshake left NetworkManager DISCONNECTED ('no secrets: No agents were available') on srv8 and Group B spark-3336, and srv7 stayed associated but forwarded nothing 05:33-06:53 EDT (read on the hosts by valiant-crab-775). Fleet-converge run 37156689444 sat ~1h inside one ssh leg. No host rebooted or wedged; the runs called them down.",
+
+    "DISTINGUISHING FACTS: liveness is a different bound from connect (ssh_config(5) ServerAliveInterval x ServerAliveCountMax); sshd answers keepalives itself, so a silent remote command is not cut off, only a path that returns nothing. Whether the leg's began-line ARRIVED is read in band (gunbc.spark.remote_leg_began remote_leg_began_word on stdout, the one protocol gunbc.spark.pair_serving_apply also reads), not from stderr -- and its absence does not establish that nothing ran, because the marker and the command share one unacknowledged stream: so the arm is LegBeginUnobserved, projected to ArgvLegMayHaveRun, never to ArgvLegDidNotRun (review of gunbc#13204). 'Host down' needs an observation this boundary does not hold, so the type has no HostDown arm: LegBeginUnobserved is dark-or-down UNDECIDED and names the readbacks that close it: first a probe of the host's fabric-rail address (192.168.110.x) from a same-group peer (rail answers -> LAN dark, host up), then uptime and the wlP9s9 journal. Two further triggers, separate from the deadline one: a two-phase leg protocol (the client receives an acknowledgement before it sends the effect command), SUFFICIENT FOR a NeverBegan arm a consumer may retry without a readback; and an automatic rail-jump classifier with its own consumer, SUFFICIENT FOR LegBeginUnobserved to be split into LAN-dark-host-up vs rail-also-dark without a human.",
+
+    "RUNG: found at 1 (a human read the hang and the hosts). Now 2: gunbc.fleet_known_hosts_anchor fleet_ssh_leg_deadline is the one row both fleet shapers derive their deadline words from and neither shaper takes an options parameter, and dag/test/claim/spark/spark_remote_leg_deadline_witness_test.dag pins the argv, the route through the password session, a zeroed-keepalive red control and the never-began reading. CEILING 3: the deadline as a required field of the operation every fleet ssh transport declares, so no ssh leg in the corpus (including extdeps.bmc.openbmc_password_ssh_transport and extdeps.ssh.password_session CopyFile, which still carry ConnectTimeout only) is expressible without it. NEXT-RUNG TRIGGER: those transport literals moved onto extdeps.ssh.client_options with the deadline derived, SUFFICIENT FOR no ssh argv in the corpus lacking a liveness bound.",
+  ],
+
+  evidence: [],
+}

dag/gunbc/recurring_failure_mode/an_unreached_effect_leg_reads_as_one_that_ran_and_refused.dag

@@ -17,7 +17,7 @@ data an_unreached_effect_leg_reads_as_one_that_ran_and_refused: RecurringFailure
 
     "RESIDUE, STATED: a connection lost after the shell started but before its began-line crossed also reads Unreached. gunbc.spark.native_serving_apply therefore retries only idempotent stages: preflight and the front door read, preserve/commit/rollback are no-ops where this transaction's desired state already holds, and apply's unconditional restart is the module's declared epoch semantics.",
 
-    "RUNG: found at 1 (mitigatable -- a human could read the log and delete the files). Now 2 for the native serving arm: witnesses in dag/test/claim/spark/native_serving_resumable_apply_witness_test.dag pin the reading, the bounded retry (spark_native_unreached_leg_attempt_budget) and the leftover decision (spark_native_leftover_decision) with discriminating reds. CEILING 3: an outcome type every remote-effect leg must produce, so no consumer can match a leg result without facing the Unreached arm. NEXT-RUNG TRIGGER: the other SshSessionExecResult consumers that fold 255 into a refusal (gunbc.spark.model_snapshot_materialize spark_remote_run, gunbc.host_command_binding host_command_outcome_of's Unconfirmed arm) consuming one shared in-band leg reading, SUFFICIENT FOR no remote-effect consumer in the corpus to construct a ran-and-refused value from a leg that never began.",
+    "RUNG: found at 1 (mitigatable -- a human could read the log and delete the files). Now 2 for the native serving arm: witnesses in dag/test/claim/spark/native_serving_resumable_apply_witness_test.dag pin the reading, the bounded retry (spark_native_unreached_leg_attempt_budget) and the leftover decision (spark_native_leftover_decision) with discriminating reds. CEILING 3: an outcome type every remote-effect leg must produce, so no consumer can match a leg result without facing the Unreached arm. NEXT-RUNG TRIGGER: the other SshSessionExecResult consumers that fold 255 into a refusal (gunbc.spark.model_snapshot_materialize spark_remote_run, gunbc.host_command_binding host_command_outcome_of's Unconfirmed arm) consuming one shared in-band leg reading, SUFFICIENT FOR no remote-effect consumer in the corpus to construct a ran-and-refused value from a leg that never began. PROGRESS (2026-10-04): gunbc.spark.model_snapshot_materialize spark_remote_run now consumes the ONE shared in-band reading, gunbc.spark.remote_leg_began (lifted out of gunbc.spark.pair_serving_apply, which consumes it too; review of gunbc#13204), and returns RemoteUnreachable with reach LegBeginUnobserved for ssh's 255 with no began-line received, projected to ArgvLegMayHaveRun, never ArgvLegDidNotRun: the marker and the command share one unacknowledged stream, so a missing marker does not establish that nothing ran (see a_dark_management_lan_leg_hangs_and_reads_as_a_down_host). The same residue applies to this row's own Unreached reading. gunbc.host_command_binding host_command_outcome_of remains.",
   ],
 
   evidence: [],