acpmux: a working lane whose model went quiet with no tool call running says it has stalled (lane.stalledSince)

cmux-tui/crates/acpmux/README.md

@@ -199,6 +199,11 @@ responding model produces, such as Claude's stream and thinking or Codex's moder
 notice; the agent's own usage and turn notices do not count) for that long after its
 prompt (a stalled model) logs `turn_silent` {turnId, silentMs} once, so a client can say
 the model has not answered. The model's first sign after it logs `turn_heard` {turnId}.
+`stalledTurnMs` (default 600000; 0 turns it off): a running turn with no tool call in
+flight whose model has shown no sign for that long has stalled: `lane.stalledSince` (and
+`turn.stalledSince`) say since when, and the lane's state stays `working`. A long build in
+one tool call is in flight, so it never stalls. It is checked when a summary is read, with
+no timer.
 Files dropped into a chat (`_acpmux/file_put`, under `<home>/uploads/<session>/`) are
 deleted with the chat (`_acpmux/kill` with purge) and once older than `uploadsMaxAgeDays`
 (default 30; 0 keeps them), checked at the daemon's start and daily.

cmux-tui/crates/acpmux/docs/acpmux-schema.json

@@ -119,7 +119,7 @@
     "permission_chat_allowance": {"active":"boolean"},
     "SessionDefaults": {"model": "string?", "effort": "string?", "policy": "string?", "prefer": "string[]? (profiles to use for a family, in order; first installed wins)", "env": "{key: value}? (${cwd}, ${home}, ${model}, ~/ expand)", "note": "keyed by family or profile"},
     "HarnessProfile": {"kind": "acp|claude-stdio", "argv": "string[] (${model} = the requested model at spawn)", "env": "{key: value}", "family": "string? (derived from argv when absent)", "models": "(string | {id, name})[] declared catalog, shown ahead of the reported one", "model": "string?", "effort": "string?", "policy": "string?", "fallback": "string? (profile to move to on limit or auth errors)", "description": "string?"},
-    "SessionSummary": {"sessionId": "string", "name": "string", "harness": "string", "family": "string|null", "cwd": "string", "status": "idle|ready|running|waiting|disconnected|closed", "peer": "string? (host name for remote sessions)", "stateSeq": "number (bumps on status, permission and turn changes)", "unread": "boolean (a turn ended while no attached client showed the chat; cleared when one shows it)", "shown": "boolean (some attached client shows the chat: _acpmux/attach visible, _acpmux/view)", "indicator": "input|working|failed|lost|done|null (the status dot every client draws; first match wins: input = a permission or question waits, working = a turn runs, failed = the last turn failed, lost = the agent process is gone (disconnected or unreachable) and the last turn did not answer, done = the last turn answered (seen is client-side), null = nothing to show)", "lastTurn": "{turnId, promptId, status, stopReason?, errorText?, errorSource?, detail?, endedAt} | null", "lane": "{state: working|input|idle|failed|lost, since: number|null, error: string|null, step: string|null, title: string|null} | null (a coordinator's view of the session; null when closed: working since its turn started, input since it began waiting, idle since its last turn ended, failed with that turn's errorText, lost when the agent process or peer is gone; step is the running turn's prompt, else the agent's last reply line, one line of at most 240 characters; title is the session's title, else its latest prompt, cut the same way; error, step and title have secrets masked, e.g. [GitHub Personal Access Token], and error is cut to 3072 characters; the summary's other text (title, lastPrompt, preview, lastMessage.text, the turn's and queued prompts, lastTurn.errorText) is raw for every client except a proven peer daemon (peer token), which reads it masked and cut to 16384 characters and may not rewrite a queued prompt that holds a secret)", "inputSince": "number|null (when the session began waiting on the person)", "attached": "number (clients attached now)", "pendingPermissions": "number", "queued": "number", "turnCount": "number", "lastSeq": "number", "lastPrompt": "string|null", "preview": "string|null", "model": "string|null", "currentModeId": "string|null", "policy": "string|null", "rules": "boolean", "tools": "ToolRow[] (the chat's tools: cmux's own, then what the harness reported)", "toolsMutable": "boolean (_acpmux/set_tools can turn its MCP servers off: Claude Code only)", "tags": "{key: value}", "createdAt": "number", "updatedAt": "number", "queue": "[{promptId, turnId, client, prompt, queuedAt, held, images: [{id, mimeType, name: string|null}]}] prompts waiting for the running turn (images: what each sends, in order; _acpmux/attachment reads each by id before it runs)", "turn": "{startedAt, client, prompt, turnId, promptId, turnSeq} | null (the running turn)", "enforcement": "Enforcement", "gitBranch": "string|null (the branch the session's cwd has checked out now; null on a detached HEAD or outside a repository. Live while a client holds the chat (a git record on each change); otherwise the last branch this daemon read, else the one the session started on)"},
+    "SessionSummary": {"sessionId": "string", "name": "string", "harness": "string", "family": "string|null", "cwd": "string", "status": "idle|ready|running|waiting|disconnected|closed", "peer": "string? (host name for remote sessions)", "stateSeq": "number (bumps on status, permission and turn changes)", "unread": "boolean (a turn ended while no attached client showed the chat; cleared when one shows it)", "shown": "boolean (some attached client shows the chat: _acpmux/attach visible, _acpmux/view)", "indicator": "input|working|failed|lost|done|null (the status dot every client draws; first match wins: input = a permission or question waits, working = a turn runs, failed = the last turn failed, lost = the agent process is gone (disconnected or unreachable) and the last turn did not answer, done = the last turn answered (seen is client-side), null = nothing to show)", "lastTurn": "{turnId, promptId, status, stopReason?, errorText?, errorSource?, detail?, endedAt} | null", "lane": "{state: working|input|idle|failed|lost, since: number|null, error: string|null, step: string|null, title: string|null, stalledSince: number|null} | null (a coordinator's view of the session; null when closed: working since its turn started (stalledSince: since when a working turn has had no tool call in flight and no sign from its model for stalledTurnMs, default 10 minutes, 0 off), input since it began waiting, idle since its last turn ended, failed with that turn's errorText, lost when the agent process or peer is gone; step is the running turn's prompt, else the agent's last reply line, one line of at most 240 characters; title is the session's title, else its latest prompt, cut the same way; error, step and title have secrets masked, e.g. [GitHub Personal Access Token], and error is cut to 3072 characters; the summary's other text (title, lastPrompt, preview, lastMessage.text, the turn's and queued prompts, lastTurn.errorText) is raw for every client except a proven peer daemon (peer token), which reads it masked and cut to 16384 characters and may not rewrite a queued prompt that holds a secret)", "inputSince": "number|null (when the session began waiting on the person)", "attached": "number (clients attached now)", "pendingPermissions": "number", "queued": "number", "turnCount": "number", "lastSeq": "number", "lastPrompt": "string|null", "preview": "string|null", "model": "string|null", "currentModeId": "string|null", "policy": "string|null", "rules": "boolean", "tools": "ToolRow[] (the chat's tools: cmux's own, then what the harness reported)", "toolsMutable": "boolean (_acpmux/set_tools can turn its MCP servers off: Claude Code only)", "tags": "{key: value}", "createdAt": "number", "updatedAt": "number", "queue": "[{promptId, turnId, client, prompt, queuedAt, held, images: [{id, mimeType, name: string|null}]}] prompts waiting for the running turn (images: what each sends, in order; _acpmux/attachment reads each by id before it runs)", "turn": "{startedAt, client, prompt, turnId, promptId, turnSeq, stalledSince} | null (the running turn; stalledSince as lane.stalledSince)", "enforcement": "Enforcement", "gitBranch": "string|null (the branch the session's cwd has checked out now; null on a detached HEAD or outside a repository. Live while a client holds the chat (a git record on each change); otherwise the last branch this daemon read, else the one the session started on)"},
     "SessionDetail": "SessionSummary plus modes, configOptions, models, agentInfo, agentCapabilities, usage, pending: [{permissionId, request, groupId, turnId}] (groupId and turnId are string|null), rulesJson, gitHead: string|null (the cwd's full HEAD commit when the session started or forked; null outside a repository; gitBranch is SessionSummary's, the branch checked out now), totals: TurnUsage|null (the sum of this session's own turns; null until a harness reports any; a fork starts its own)",
     "TurnUsage": {"costUsd": "number? (USD, the harness's estimate)", "inputTokens": "number?", "outputTokens": "number?", "cacheReadTokens": "number?", "cacheWriteTokens": "number?", "note": "Only what the harness reported: Claude Code reports all of them (its tokens cover the main loop, its cost includes subagents); other harnesses report none, so the fields are absent, never 0. A turn's costUsd is the rise in Claude Code's running total, and is absent when that cannot be told: a failed turn, a total that went down, the first turn of a fork through an earlier turn or of a session older than this field."},
     "Process": {"id": "string (<sessionId>/<harness task id>: Claude's task id, Codex's processId)", "sessionId": "string", "subagentSessionId": "string|null (the subagent whose call started it)", "toolCallId": "string (Codex: the commandExecution item id)", "taskId": "string (Codex: the processId)", "harness": "string (the session's harness family: claude, codex)", "command": "string", "cwd": "string", "startedAt": "number (unix ms)", "endedAt": "number|null", "state": "running|exited|failed|killed|lost (lost: it ended with its harness, unseen)", "exitCode": "number|null (Codex reports it when the process exits on its own; Claude Code does not)", "signal": "null", "reason": "person|agent|harness_exit|null (why it stopped)", "detail": "string|null (the harness's own words on its end)", "origin": "background|auto_backgrounded|user_backgrounded (user_backgrounded: moved there by _acpmux/process/background, Ctrl-B)", "outputBytes": "number|null (its output file's length when listed; null while the harness named no file, and for Codex)", "outputFile": "string? (the file the harness writes its output to; read it with _acpmux/process/tail)"},

cmux-tui/crates/acpmux/spec/acpmux-protocol.json

@@ -1711,6 +1711,16 @@
             "null"
           ]
         },
+        "stalledSince": {
+          "default": null,
+          "description": "A `working` lane that has stalled: since when its turn has had no tool\ncall in flight and no sign of its model (`turn.stalledSince`); null\notherwise. The state stays `working`, so a client that reads only it\nis unchanged.",
+          "format": "uint64",
+          "minimum": 0,
+          "type": [
+            "integer",
+            "null"
+          ]
+        },
         "state": {
           "$ref": "#/types/LaneState"
         },
@@ -1732,6 +1742,7 @@
       "required": [
         "error",
         "since",
+        "stalledSince",
         "state",
         "step",
         "title"
@@ -3595,6 +3606,15 @@
         "promptId": {
           "type": "string"
         },
+        "stalledSince": {
+          "description": "Since when it has stalled: no tool call in flight and no sign of its\nmodel since then, for `stalledTurnMs` or more (`hub/activity.rs`);\nnull while it has not.",
+          "format": "uint64",
+          "minimum": 0,
+          "type": [
+            "integer",
+            "null"
+          ]
+        },
         "startedAt": {
           "format": "uint64",
           "minimum": 0,
@@ -3615,6 +3635,7 @@
         "planStep",
         "prompt",
         "promptId",
+        "stalledSince",
         "startedAt",
         "turnId",
         "turnSeq"

cmux-tui/crates/acpmux/src/cli/task.rs

@@ -130,11 +130,16 @@ fn report(state: &str, r: ResultArgs) -> (&'static str, Value) {
     (method::MUX_REPORT, params)
 }
 
-/// One task: id, state, its lane and the lane's state now, the title, and the result's summary.
+/// One task: id, state, its lane and the lane's state now (and how long it has stalled), the
+/// title, and the result's summary.
 fn line(row: &Value) -> String {
     let s = |v: &Value| v.as_str().unwrap_or("?").to_owned();
-    let lane =
+    let mut lane =
         row["lane"]["state"].as_str().map_or_else(|| s(&row["sessionStatus"]), str::to_owned);
+    if let Some(since) = row["lane"]["stalledSince"].as_u64() {
+        let minutes = crate::store::now_ms().saturating_sub(since) / 60_000;
+        lane.push_str(&format!(", stalled {minutes}m"));
+    }
     let mut text = format!(
         "{:<11} {:<8} {} ({lane})  {}",
         s(&row["taskId"]),

cmux-tui/crates/acpmux/src/config.rs

@@ -812,6 +812,12 @@ pub struct Config {
     /// Default 90000; 0 turns it off.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub silent_turn_ms: Option<u64>,
+    /// `stalledTurnMs`: how long a running turn may go with no tool call in
+    /// flight and no sign of its model at work before its lane says it has
+    /// stalled (`lane.stalledSince`, `hub/views.rs`). A long build in one tool
+    /// call never stalls. Default 600000; 0 turns it off.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub stalled_turn_ms: Option<u64>,
     /// `autoResumeDelaysMs`: the wait before each automatic continuation of a turn a
     /// connection error cut off (`hub/auto_resume.rs`), one entry per attempt. Default
     /// `[10000, 60000, 300000, 1200000]`; `[]` turns it off. Checked on load: at most 4
@@ -936,6 +942,14 @@ impl Config {
         }
     }
 
+    /// `stalledTurnMs`, or its 10 minute default; None when it is 0 (off).
+    pub fn stalled_turn(&self) -> Option<u64> {
+        match self.stalled_turn_ms.unwrap_or(600_000) {
+            0 => None,
+            ms => Some(ms),
+        }
+    }
+
     /// Family of a configured profile.
     pub fn family(&self, profile: &str) -> Option<String> {
         self.harnesses.get(profile).map(|p| derive_family(profile, p))