From 8f6527a27faaa6167f6d6bb658862e159f45a478 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 9 Jun 2026 21:03:10 -0700 Subject: [PATCH 1/9] Add failing reconnect-policy tests for halt-on-unreachable (#5734) The SSH remote auto-reconnect loop retries forever while the host is unreachable. Introduce the WorkspaceRemoteReconnectPolicy seam encoding today's never-suspend behavior, plus tests asserting the desired policy: suspend the loop after consecutive failed reachability probes so the user controls when reconnection happens. The suspend assertions fail (red) until the policy change lands in the follow-up commit. Co-Authored-By: Claude Fable 5 --- Sources/WorkspaceRemoteReconnectPolicy.swift | 60 ++++++++++ cmux.xcodeproj/project.pbxproj | 8 ++ .../WorkspaceRemoteReconnectPolicyTests.swift | 105 ++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 Sources/WorkspaceRemoteReconnectPolicy.swift create mode 100644 cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift diff --git a/Sources/WorkspaceRemoteReconnectPolicy.swift b/Sources/WorkspaceRemoteReconnectPolicy.swift new file mode 100644 index 000000000000..3a300a3b6674 --- /dev/null +++ b/Sources/WorkspaceRemoteReconnectPolicy.swift @@ -0,0 +1,60 @@ +import Foundation + +/// Outcome of a quick reachability probe against the SSH endpoint of a remote +/// workspace, used by the auto-reconnect loop to decide whether retrying makes +/// sense at all. +enum WorkspaceRemoteHostProbeOutcome: Equatable, Sendable { + /// A TCP connection to the resolved SSH endpoint succeeded. + case reachable + /// The endpoint could not be reached (DNS failure, connection refused, + /// timeout, or no route). The associated reason is a short human-readable + /// detail for logs and connection-state messages. + case unreachable(reason: String) + /// Reachability could not be determined (for example ProxyCommand-based + /// transports that cannot be probed with a direct TCP connection). The + /// policy must never suspend the reconnect loop based on this outcome. + case indeterminate +} + +/// Pure decision logic for the SSH remote workspace auto-reconnect loop +/// (https://github.com/manaflow-ai/cmux/issues/5734). +/// +/// While the host stays reachable the loop keeps its existing exponential +/// backoff behavior. Once consecutive reachability probes keep failing, the +/// loop should suspend instead of retrying indefinitely, leaving the user in +/// control of when reconnection happens. +enum WorkspaceRemoteReconnectPolicy { + enum Decision: Equatable, Sendable { + /// Keep the scheduled backoff retry armed. + case scheduleRetry + /// Halt the automatic reconnect loop and wait for a manual reconnect. + case suspend + } + + struct Evaluation: Equatable, Sendable { + /// Consecutive failed probes after accounting for the latest outcome. + let consecutiveUnreachableProbes: Int + let decision: Decision + } + + /// Number of consecutive unreachable probes after which the automatic + /// reconnect loop suspends. Sized to absorb short network transitions + /// (sleep/wake, wifi handoff) without retrying indefinitely against a + /// host that cannot be reached. + static let maxConsecutiveUnreachableProbes = 3 + + static func evaluate( + outcome: WorkspaceRemoteHostProbeOutcome, + previousConsecutiveUnreachableProbes: Int + ) -> Evaluation { + switch outcome { + case .reachable, .indeterminate: + return Evaluation(consecutiveUnreachableProbes: 0, decision: .scheduleRetry) + case .unreachable: + return Evaluation( + consecutiveUnreachableProbes: previousConsecutiveUnreachableProbes + 1, + decision: .scheduleRetry + ) + } + } +} diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index c178fe048bec..4ddca0743511 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -700,6 +700,8 @@ DCC935C5F55C1DCB33E25521 /* WorkspacePullRequestSidebarTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */; }; E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */; }; F6100000A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */; }; + E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */; }; + F6357300A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */; }; E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */; }; 619D509BC9B1EA946CBD6A8A /* WorkspaceRuntimeSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9960CC3992F3C44489E7627C /* WorkspaceRuntimeSettings.swift */; }; F0F0A001A1B2C3D4E5F60718 /* WorkspaceSidebarScrollUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F0F0A002A1B2C3D4E5F60718 /* WorkspaceSidebarScrollUITests.swift */; }; @@ -1426,6 +1428,8 @@ 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspacePullRequestSidebarTests.swift; sourceTree = ""; }; E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConfiguration.swift; sourceTree = ""; }; F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConnectionTests.swift; sourceTree = ""; }; + E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicy.swift; sourceTree = ""; }; + F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicyTests.swift; sourceTree = ""; }; E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteSSHBatchCommandBuilder.swift; sourceTree = ""; }; 9960CC3992F3C44489E7627C /* WorkspaceRuntimeSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/WorkspaceRuntimeSettings.swift; sourceTree = ""; }; F0F0A002A1B2C3D4E5F60718 /* WorkspaceSidebarScrollUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceSidebarScrollUITests.swift; sourceTree = ""; }; @@ -1782,6 +1786,7 @@ D7AB3605C10DEF0000000004 /* WorkspaceCloseTabsBatching.swift */, C3744E020000000000000001 /* Workspace+PanelLifecycle.swift */, E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */, + E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */, D0C0D0C0D0C0D0C0D0C00002 /* RemoteLoopbackProxyAlias.swift */, D0C0D0C0D0C0D0C0D0C00004 /* RemoteLoopbackRuntimeBridge.swift */, E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */, @@ -2126,6 +2131,7 @@ E3309A0A /* AppDelegateEqualizeSplitsShortcutTests.swift */, F6001001A1B2C3D4E5F60718 /* ShortcutUnbindingTests.swift */, F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */, + F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */, F6355601A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift */, F6120001A1B2C3D4E5F60718 /* WorkspaceSSHFishShellTests.swift */, F7000001A1B2C3D4E5F60718 /* WorkspaceContentViewVisibilityTests.swift */, @@ -3056,6 +3062,7 @@ D0B10004A1B2C3D4E5F60001 /* WorkspacePortalPaneDrop.swift in Sources */, 1A1B2C3D4E5F607180000001 /* WorkspacePromptSubmit.swift in Sources */, E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */, + E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */, E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */, 619D509BC9B1EA946CBD6A8A /* WorkspaceRuntimeSettings.swift in Sources */, D7AB34400000000000000001 /* WorkspaceSurfaceConfig.swift in Sources */, @@ -3325,6 +3332,7 @@ 1A1B2C3D4E5F607180000003 /* WorkspacePromptSubmitTests.swift in Sources */, DCC935C5F55C1DCB33E25521 /* WorkspacePullRequestSidebarTests.swift in Sources */, F6100000A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift in Sources */, + F6357300A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift in Sources */, 6B524A0CB34FD46A771335AB /* WorkspaceSplitStartupCommandTests.swift in Sources */, F6120000A1B2C3D4E5F60718 /* WorkspaceSSHFishShellTests.swift in Sources */, FA000000A1B2C3D4E5F60718 /* WorkspaceStressProfileTests.swift in Sources */, diff --git a/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift new file mode 100644 index 000000000000..a38cd3efa67a --- /dev/null +++ b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift @@ -0,0 +1,105 @@ +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +// Regression coverage for https://github.com/manaflow-ai/cmux/issues/5734: +// the SSH remote auto-reconnect loop must stop retrying once the host stays +// unreachable, instead of retrying indefinitely, so the user controls when +// reconnection happens. +@Suite("Workspace remote reconnect policy") +struct WorkspaceRemoteReconnectPolicyTests { + private func evaluate( + _ outcome: WorkspaceRemoteHostProbeOutcome, + previous: Int + ) -> WorkspaceRemoteReconnectPolicy.Evaluation { + WorkspaceRemoteReconnectPolicy.evaluate( + outcome: outcome, + previousConsecutiveUnreachableProbes: previous + ) + } + + @Test("Reachable host keeps the existing backoff retry loop") + func reachableHostKeepsRetrying() { + for previous in [0, 1, WorkspaceRemoteReconnectPolicy.maxConsecutiveUnreachableProbes] { + let evaluation = evaluate(.reachable, previous: previous) + #expect(evaluation.decision == .scheduleRetry) + #expect(evaluation.consecutiveUnreachableProbes == 0) + } + } + + @Test("Indeterminate probes keep retrying and reset the unreachable streak") + func indeterminateProbeKeepsRetrying() { + for previous in [0, 1, WorkspaceRemoteReconnectPolicy.maxConsecutiveUnreachableProbes] { + let evaluation = evaluate(.indeterminate, previous: previous) + #expect(evaluation.decision == .scheduleRetry) + #expect(evaluation.consecutiveUnreachableProbes == 0) + } + } + + @Test("Unreachable probes below the threshold keep retrying") + func unreachableBelowThresholdKeepsRetrying() { + for previous in 0..<(WorkspaceRemoteReconnectPolicy.maxConsecutiveUnreachableProbes - 1) { + let evaluation = evaluate(.unreachable(reason: "connection refused"), previous: previous) + #expect(evaluation.decision == .scheduleRetry) + #expect(evaluation.consecutiveUnreachableProbes == previous + 1) + } + } + + @Test("Reconnect loop suspends once the host stays unreachable") + func suspendsAtUnreachableThreshold() { + var streak = 0 + var decisions: [WorkspaceRemoteReconnectPolicy.Decision] = [] + for _ in 0.. Date: Tue, 9 Jun 2026 23:29:02 -0700 Subject: [PATCH 2/9] Stop SSH auto-reconnect when host is unreachable; add manual Reconnect control Fixes #5734. Every scheduled reconnect retry now kicks a quick reachability probe (ssh -G endpoint resolution + short-timeout TCP connect; first ProxyJump hop when present, indeterminate for ProxyCommand). Three consecutive unreachable probes suspend the loop instead of retrying indefinitely: the controller cancels the pending retry and publishes the new .suspended connection state with a localized detail. Manual control shares the existing Workspace.reconnectRemoteConnection path: an inline sidebar Reconnect button on the (still visible) SSH row, the workspace context menu, and new "cmux workspace reconnect" / "cmux workspace disconnect" CLI verbs over the existing workspace.remote.reconnect/disconnect socket methods. Suspended workspaces are exempt from remote demotion, surface a status entry, sidebar log line, and notification, and reattach to persistent PTY sessions on manual reconnect where the relay layer supports it. New strings localized in en/ja/ko/uk, matching their sibling families; previously uncataloged context-menu reconnect/disconnect labels added in the same locales. Co-Authored-By: Claude Fable 5 --- CLI/cmux.swift | 61 +++- Resources/Localizable.xcstrings | 319 ++++++++++++++++++ Sources/ContentView.swift | 46 ++- ...idebarWorkspaceSnapshotRefreshPolicy.swift | 1 + Sources/Workspace.swift | 136 +++++++- ...WorkspaceRemoteHostReachabilityProbe.swift | 252 ++++++++++++++ Sources/WorkspaceRemoteReconnectPolicy.swift | 5 +- cmux.xcodeproj/project.pbxproj | 4 + ...rWorkspaceSnapshotRefreshPolicyTests.swift | 1 + .../WorkspaceRemoteReconnectPolicyTests.swift | 162 +++++++++ docs/cli-contract.md | 1 + 11 files changed, 980 insertions(+), 8 deletions(-) create mode 100644 Sources/WorkspaceRemoteHostReachabilityProbe.swift diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 389940bd7cbb..9d16839cee43 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -7114,7 +7114,7 @@ struct CMUXCLI { windowOverride: String? ) throws { guard let sub = commandArgs.first?.lowercased() else { - throw CLIError(message: "workspace requires a subcommand. Try: list, create, close, rename, select, group") + throw CLIError(message: "workspace requires a subcommand. Try: list, create, close, rename, select, reconnect, disconnect, group") } let rest = Array(commandArgs.dropFirst()) switch sub { @@ -7174,9 +7174,66 @@ struct CMUXCLI { windowOverride: windowOverride, requireWorkspaceFlag: false ) + case "reconnect": + try runWorkspaceRemoteConnectionCommand( + commandName: "workspace reconnect", + method: "workspace.remote.reconnect", + commandArgs: rest, + client: client, + jsonOutput: jsonOutput, + idFormat: idFormat, + windowOverride: windowOverride + ) + case "disconnect": + try runWorkspaceRemoteConnectionCommand( + commandName: "workspace disconnect", + method: "workspace.remote.disconnect", + commandArgs: rest, + client: client, + jsonOutput: jsonOutput, + idFormat: idFormat, + windowOverride: windowOverride + ) default: - throw CLIError(message: "Unknown workspace subcommand: \(sub). Try: list, create, close, rename, select, group") + throw CLIError(message: "Unknown workspace subcommand: \(sub). Try: list, create, close, rename, select, reconnect, disconnect, group") + } + } + + /// `cmux workspace reconnect|disconnect` — manual control over a remote + /// (SSH) workspace's connection. Targets the positional/`--workspace` + /// handle, then the caller's workspace, then the selected workspace. + private func runWorkspaceRemoteConnectionCommand( + commandName: String, + method: String, + commandArgs: [String], + client: SocketClient, + jsonOutput: Bool, + idFormat: CLIIDFormat, + windowOverride: String? + ) throws { + let (workspaceArg, rem0) = parseOption(commandArgs, name: "--workspace") + let (_, rem1) = parseOption(rem0, name: "--window") + let positional = rem1.first(where: { !$0.hasPrefix("--") }) + let target = workspaceArg + ?? positional + ?? ProcessInfo.processInfo.environment["CMUX_WORKSPACE_ID"] + + var params: [String: Any] = [:] + let winId = try normalizeWindowHandle( + windowFromArgsOrOverride(commandArgs, windowOverride: windowOverride), + client: client + ) + if let winId { params["window_id"] = winId } + if let wsId = try normalizeWorkspaceHandle(target, client: client, windowHandle: winId) { + params["workspace_id"] = wsId } + let payload = try client.sendV2(method: method, params: params) + printV2Payload( + payload, + jsonOutput: jsonOutput, + idFormat: idFormat, + fallbackText: v2OKSummary(payload, idFormat: idFormat, kinds: ["workspace", "window"]) + ) } /// Emit a `cmux workspace-group` mutation response: JSON when --json, diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index a1c39e5d9dc2..6d48ff64cd71 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -188946,6 +188946,325 @@ } } } + }, + "remote.status.suspended": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Unreachable" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "到達不能" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "연결 불가" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Недоступний" + } + } + } + }, + "sidebar.remote.help.suspended": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH host %@ is unreachable. Automatic reconnect is paused — use Reconnect to retry." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "SSH ホスト %@ に到達できません。自動再接続は一時停止中です。「再接続」で再試行してください。" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "SSH 호스트 %@에 연결할 수 없습니다. 자동 재연결이 일시 중지되었습니다. 재연결로 다시 시도하세요." + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "SSH-хост %@ недоступний. Автоматичне перепідключення призупинено — скористайтеся «Перепідключити», щоб повторити спробу." + } + } + } + }, + "sidebar.remote.reconnect.button": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "재연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключити" + } + } + } + }, + "sidebar.remote.reconnect.help": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect to %@" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "%@ に再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "%@에 다시 연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключитися до %@" + } + } + } + }, + "remote.state.suspended.detail": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Can't reach %@ — automatic reconnect is paused. Use Reconnect when your network is back." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "%@ に到達できないため、自動再接続を一時停止しました。ネットワーク復帰後に「再接続」を実行してください。" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "%@에 연결할 수 없어 자동 재연결을 일시 중지했습니다. 네트워크가 복구되면 재연결을 사용하세요." + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Не вдається з'єднатися з %@ — автоматичне перепідключення призупинено. Скористайтеся «Перепідключити», коли мережа відновиться." + } + } + } + }, + "remote.statusEntry.suspended": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH reconnect paused (%@): %@" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "SSH 再接続を一時停止 (%@): %@" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "SSH 재연결 일시 중지됨 (%@): %@" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключення SSH призупинено (%@): %@" + } + } + } + }, + "remote.notification.suspendedTitle": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH Reconnect Paused" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "SSH 再接続を一時停止" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "SSH 재연결 일시 중지됨" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключення SSH призупинено" + } + } + } + }, + "contextMenu.reconnectWorkspace": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect Workspace" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 재연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключити робочий простір" + } + } + } + }, + "contextMenu.reconnectWorkspaces": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect Workspaces" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 재연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключити робочі простори" + } + } + } + }, + "contextMenu.disconnectWorkspace": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Disconnect Workspace" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを切断" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 연결 해제" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Відключити робочий простір" + } + } + } + }, + "contextMenu.disconnectWorkspaces": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Disconnect Workspaces" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを切断" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 연결 해제" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Відключити робочі простори" + } + } + } } } } diff --git a/Sources/ContentView.swift b/Sources/ContentView.swift index a6abb00c08c7..785428a3b6af 100644 --- a/Sources/ContentView.swift +++ b/Sources/ContentView.swift @@ -15064,6 +15064,7 @@ struct SidebarWorkspaceSnapshotBuilder { let remoteWorkspaceSidebarText: String? let remoteConnectionStatusText: String let remoteStateHelpText: String + let showsRemoteReconnectAffordance: Bool let copyableSidebarSSHError: String? let latestConversationMessage: String? let metadataEntries: [SidebarStatusEntry] @@ -15342,7 +15343,12 @@ struct TabItemView: View, Equatable { } private var remoteWorkspaceSidebarText: String? { - guard tab.hasActiveRemoteTerminalSessions else { return nil } + // Keep the SSH row visible while auto-reconnect is suspended even if + // every remote terminal session already died — it hosts the manual + // Reconnect affordance. + guard tab.hasActiveRemoteTerminalSessions || tab.remoteConnectionState == .suspended else { + return nil + } let trimmedTarget = tab.remoteDisplayTarget?.trimmingCharacters(in: .whitespacesAndNewlines) if let trimmedTarget, !trimmedTarget.isEmpty { return trimmedTarget @@ -15356,7 +15362,8 @@ struct TabItemView: View, Equatable { defaultValue: "remote host" ) let trimmedDetail = tab.remoteConnectionDetail?.trimmingCharacters(in: .whitespacesAndNewlines) - if tab.remoteConnectionState == .error, let trimmedDetail, !trimmedDetail.isEmpty { + if tab.remoteConnectionState == .error || tab.remoteConnectionState == .suspended, + let trimmedDetail, !trimmedDetail.isEmpty { let entry = SidebarRemoteErrorCopyEntry( workspaceTitle: tab.title, target: fallbackTarget, @@ -15389,6 +15396,8 @@ struct TabItemView: View, Equatable { return String(localized: "remote.status.error", defaultValue: "Error") case .disconnected: return String(localized: "remote.status.disconnected", defaultValue: "Disconnected") + case .suspended: + return String(localized: "remote.status.suspended", defaultValue: "Unreachable") } } @@ -15422,6 +15431,29 @@ struct TabItemView: View, Equatable { .font(.system(size: scaledFontSize(9), weight: .medium)) .foregroundColor(activeSecondaryColor(0.58)) .lineLimit(1) + + if workspaceSnapshot.showsRemoteReconnectAffordance { + Button { + tab.reconnectRemoteConnection() + } label: { + Label( + String(localized: "sidebar.remote.reconnect.button", defaultValue: "Reconnect"), + systemImage: "arrow.clockwise" + ) + .labelStyle(.titleAndIcon) + .font(.system(size: scaledFontSize(9), weight: .semibold)) + } + .buttonStyle(.borderless) + .foregroundColor(activeSecondaryColor(0.9)) + .safeHelp(String( + format: String( + localized: "sidebar.remote.reconnect.help", + defaultValue: "Reconnect to %@" + ), + locale: .current, + remoteWorkspaceSidebarText + )) + } } } .padding(.top, latestNotificationText == nil ? 1 : 2) @@ -16474,6 +16506,15 @@ struct TabItemView: View, Equatable { locale: .current, target ) + case .suspended: + return String( + format: String( + localized: "sidebar.remote.help.suspended", + defaultValue: "SSH host %@ is unreachable. Automatic reconnect is paused — use Reconnect to retry." + ), + locale: .current, + target + ) } } @@ -16526,6 +16567,7 @@ struct TabItemView: View, Equatable { remoteWorkspaceSidebarText: remoteWorkspaceSidebarText, remoteConnectionStatusText: remoteConnectionStatusText, remoteStateHelpText: remoteStateHelpText, + showsRemoteReconnectAffordance: tab.remoteConnectionState == .suspended, copyableSidebarSSHError: copyableSidebarSSHError, latestConversationMessage: tab.latestConversationMessage, metadataEntries: detailVisibility.showsMetadata ? tab.sidebarStatusEntriesInDisplayOrder() : [], diff --git a/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift b/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift index 07f4c8eda3ff..85ca4e2bf93b 100644 --- a/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift +++ b/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift @@ -26,6 +26,7 @@ extension SidebarWorkspaceSnapshotBuilder.Snapshot { remoteWorkspaceSidebarText: remoteWorkspaceSidebarText, remoteConnectionStatusText: remoteConnectionStatusText, remoteStateHelpText: remoteStateHelpText, + showsRemoteReconnectAffordance: showsRemoteReconnectAffordance, copyableSidebarSSHError: copyableSidebarSSHError, latestConversationMessage: latestConversationMessage, metadataEntries: metadataEntries, diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index e2319edf2223..4c004d072554 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -6437,6 +6437,9 @@ final class WorkspaceRemoteSessionController { private var reverseRelayStderrBuffer = "" private var reconnectRetryCount = 0 private var reconnectWorkItem: DispatchWorkItem? + private var consecutiveUnreachableProbeCount = 0 + private var reconnectSuspended = false + private var reachabilityProbeGeneration: UInt64 = 0 private var heartbeatCount: Int = 0 private var connectionAttemptStartedAt: Date? private var pendingPTYBridgeStarts: [UUID: PendingPTYBridgeStart] = [:] @@ -6824,6 +6827,9 @@ final class WorkspaceRemoteSessionController { reconnectWorkItem?.cancel() reconnectWorkItem = nil reconnectRetryCount = 0 + consecutiveUnreachableProbeCount = 0 + reconnectSuspended = false + reachabilityProbeGeneration &+= 1 reverseRelayRestartWorkItem?.cancel() reverseRelayRestartWorkItem = nil remotePortScanCoalesceWorkItem?.cancel() @@ -7206,6 +7212,8 @@ final class WorkspaceRemoteSessionController { reconnectWorkItem?.cancel() reconnectWorkItem = nil reconnectRetryCount = 0 + consecutiveUnreachableProbeCount = 0 + reachabilityProbeGeneration &+= 1 guard proxyEndpoint != endpoint else { recordHeartbeatActivityLocked() fulfillPendingPTYBridgeStartsLocked() @@ -7260,7 +7268,9 @@ final class WorkspaceRemoteSessionController { private func scheduleReconnectLocked(baseDelay: TimeInterval) -> RetrySchedule { let retryNumber = reconnectRetryCount + 1 let retryDelay = Self.retryDelay(baseDelay: baseDelay, retry: retryNumber) - guard !isStopping else { return RetrySchedule(retry: retryNumber, delay: retryDelay) } + guard !isStopping, !reconnectSuspended else { + return RetrySchedule(retry: retryNumber, delay: retryDelay) + } reconnectWorkItem?.cancel() reconnectRetryCount = retryNumber let workItem = DispatchWorkItem { [weak self] in @@ -7272,9 +7282,89 @@ final class WorkspaceRemoteSessionController { } reconnectWorkItem = workItem queue.asyncAfter(deadline: .now() + retryDelay, execute: workItem) + evaluateReconnectPolicyLocked() return RetrySchedule(retry: retryNumber, delay: retryDelay) } + /// Probe whether the SSH endpoint is reachable at all after a failed + /// connection attempt. While the host stays unreachable the retry loop is + /// allowed a short streak of attempts (absorbing sleep/wake and network + /// handoffs) and then suspends instead of retrying indefinitely + /// (https://github.com/manaflow-ai/cmux/issues/5734). + private func evaluateReconnectPolicyLocked() { + guard configuration.transport == .ssh else { return } + reachabilityProbeGeneration &+= 1 + let generation = reachabilityProbeGeneration + WorkspaceRemoteHostReachabilityProbe.probe( + destination: configuration.destination, + port: configuration.port, + identityFile: configuration.identityFile, + sshOptions: configuration.sshOptions + ) { [weak self] outcome in + guard let self else { return } + self.queue.async { + self.handleReachabilityProbeOutcomeLocked(outcome, generation: generation) + } + } + } + + private func handleReachabilityProbeOutcomeLocked( + _ outcome: WorkspaceRemoteHostProbeOutcome, + generation: UInt64 + ) { + guard generation == reachabilityProbeGeneration else { return } + guard !isStopping, !reconnectSuspended else { return } + // The probe only judges a still-pending retry; if the retry resolved + // while the probe ran, the connected/stopped paths own the state. + guard reconnectWorkItem != nil else { return } + let evaluation = WorkspaceRemoteReconnectPolicy.evaluate( + outcome: outcome, + previousConsecutiveUnreachableProbes: consecutiveUnreachableProbeCount + ) + consecutiveUnreachableProbeCount = evaluation.consecutiveUnreachableProbes + debugLog( + "remote.session.reachability outcome=\(Self.debugDescription(for: outcome)) " + + "streak=\(evaluation.consecutiveUnreachableProbes) " + + "decision=\(evaluation.decision == .suspend ? "suspend" : "retry") \(debugConfigSummary())" + ) + if evaluation.decision == .suspend { + suspendAutoReconnectLocked() + } + } + + /// Halt the automatic reconnect loop and surface a suspended state with a + /// manual Reconnect affordance. `Workspace.reconnectRemoteConnection()` + /// (sidebar button, workspace context menu, `cmux workspace reconnect`, + /// and the `workspace.remote.reconnect` socket command) replaces this + /// controller, which resets the policy state. + private func suspendAutoReconnectLocked() { + reconnectWorkItem?.cancel() + reconnectWorkItem = nil + reconnectSuspended = true + debugLog( + "remote.session.reconnect.suspended afterUnreachableProbes=\(consecutiveUnreachableProbeCount) " + + debugConfigSummary() + ) + let detailFormat = String( + localized: "remote.state.suspended.detail", + defaultValue: "Can't reach %@ — automatic reconnect is paused. Use Reconnect when your network is back." + ) + let detail = String(format: detailFormat, configuration.displayTarget) + publishDaemonStatus(.unavailable, detail: detail) + publishState(.suspended, detail: detail) + } + + private static func debugDescription(for outcome: WorkspaceRemoteHostProbeOutcome) -> String { + switch outcome { + case .reachable: + return "reachable" + case .unreachable(let reason): + return "unreachable(\(debugLogSnippet(reason, limit: 80)))" + case .indeterminate: + return "indeterminate" + } + } + private func publishState(_ state: WorkspaceRemoteConnectionState, detail: String?) { let controllerID = self.controllerID DispatchQueue.main.async { [weak workspace] in @@ -9810,6 +9900,9 @@ enum WorkspaceRemoteConnectionState: String { case reconnecting case connected case error + /// Automatic reconnect halted because the host stayed unreachable; the + /// user reconnects manually (sidebar Reconnect, context menu, CLI). + case suspended } enum WorkspaceRemoteDaemonState: String { @@ -14008,7 +14101,8 @@ final class Workspace: Identifiable, ObservableObject { if remoteConnectionState == .error || remoteDaemonStatus.state == .error || remoteConnectionState == .connecting || - remoteConnectionState == .reconnecting { + remoteConnectionState == .reconnecting || + remoteConnectionState == .suspended { return } disconnectRemoteConnection(clearConfiguration: true) @@ -14233,6 +14327,44 @@ final class Workspace: Identifiable, ObservableObject { remoteConnectionDetail = detail applyBrowserRemoteWorkspaceStatusToPanels() + if state == .suspended { + let entryDetail = trimmedDetail ?? "" + let entryValue = String( + format: String( + localized: "remote.statusEntry.suspended", + defaultValue: "SSH reconnect paused (%@): %@" + ), + locale: .current, + target, + entryDetail + ) + statusEntries[Self.remoteErrorStatusKey] = SidebarStatusEntry( + key: Self.remoteErrorStatusKey, + value: entryValue, + icon: "pause.circle", + color: nil, + timestamp: Date() + ) + let fingerprint = "suspended:\(entryDetail)" + if remoteLastErrorFingerprint != fingerprint { + remoteLastErrorFingerprint = fingerprint + appendSidebarLog(message: entryValue, level: .warning, source: "remote") + AppDelegate.shared?.notificationStore?.addNotification( + tabId: id, + surfaceId: nil, + title: String( + localized: "remote.notification.suspendedTitle", + defaultValue: "SSH Reconnect Paused" + ), + subtitle: target, + body: entryDetail, + cooldownKey: remoteNotificationCooldownKey(target: target), + cooldownInterval: Self.remoteNotificationCooldown + ) + } + return + } + if let trimmedDetail, !trimmedDetail.isEmpty, (state == .error || proxyOnlyError) { let statusPrefix = proxyOnlyError ? "Remote proxy unavailable" : "SSH error" let statusIcon = proxyOnlyError ? "exclamationmark.triangle.fill" : "network.slash" diff --git a/Sources/WorkspaceRemoteHostReachabilityProbe.swift b/Sources/WorkspaceRemoteHostReachabilityProbe.swift new file mode 100644 index 000000000000..89008c61e8cc --- /dev/null +++ b/Sources/WorkspaceRemoteHostReachabilityProbe.swift @@ -0,0 +1,252 @@ +import Foundation +import Network + +/// Quick reachability probe for the SSH endpoint behind a remote workspace. +/// +/// The auto-reconnect loop uses this to distinguish "the host is temporarily +/// failing" (keep retrying with backoff) from "the host cannot be reached at +/// all" (suspend the loop and wait for a manual reconnect, see +/// https://github.com/manaflow-ai/cmux/issues/5734). +/// +/// The probe resolves the effective endpoint with `ssh -G` so ~/.ssh/config +/// aliases, `HostName` overrides, and `ProxyJump` hops are honored. Transports +/// that cannot be probed with a direct TCP connection (`ProxyCommand`) report +/// `.indeterminate`, which the policy treats like a reachable host so the +/// loop never suspends on a guess. +enum WorkspaceRemoteHostReachabilityProbe { + struct Endpoint: Equatable { + let host: String + let port: Int + } + + static let defaultTimeout: TimeInterval = 2.5 + private static let sshResolveTimeout: TimeInterval = 3.0 + private static let probeQueue = DispatchQueue( + label: "com.cmux.remote-host-reachability", + qos: .utility + ) + + /// Probe the SSH endpoint for `destination`. The completion runs on an + /// arbitrary queue; callers hop back to their own queue. + static func probe( + destination: String, + port: Int?, + identityFile: String?, + sshOptions: [String], + timeout: TimeInterval = defaultTimeout, + completion: @escaping (WorkspaceRemoteHostProbeOutcome) -> Void + ) { + probeQueue.async { + guard let endpoint = resolveEndpoint( + destination: destination, + port: port, + identityFile: identityFile, + sshOptions: sshOptions + ) else { + completion(.indeterminate) + return + } + probeTCP( + host: endpoint.host, + port: endpoint.port, + timeout: timeout, + completion: completion + ) + } + } + + /// Resolve the effective `(host, port)` for an SSH destination using + /// `ssh -G`. Returns nil when the endpoint cannot be probed directly + /// (ProxyCommand transports, unparsable output, or resolution failure). + static func resolveEndpoint( + destination: String, + port: Int?, + identityFile: String?, + sshOptions: [String] + ) -> Endpoint? { + let trimmedDestination = destination.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedDestination.isEmpty, !trimmedDestination.hasPrefix("-") else { return nil } + + var arguments = ["-G"] + if let port, port > 0 { + arguments += ["-p", String(port)] + } + if let identityFile, + !identityFile.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + arguments += ["-i", identityFile] + } + for option in sshOptions { + let trimmed = option.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { continue } + arguments += ["-o", trimmed] + } + arguments.append(trimmedDestination) + + guard let output = runSSHConfigResolution(arguments: arguments) else { return nil } + let resolved = parseSSHConfigOutput(output) + + if let proxyJump = resolved.proxyJump { + // Reachability of the first hop is the right signal for "can this + // network reach the SSH entry point at all". + guard let jump = parseJumpSpec(proxyJump) else { return nil } + guard let jumpOutput = runSSHConfigResolution(arguments: ["-G", jump.destination]) else { + return nil + } + let jumpResolved = parseSSHConfigOutput(jumpOutput) + guard jumpResolved.proxyCommand == nil, jumpResolved.proxyJump == nil else { return nil } + guard let host = jumpResolved.hostName ?? jump.host else { return nil } + let jumpPort = jump.port ?? jumpResolved.port ?? 22 + return Endpoint(host: host, port: jumpPort) + } + + guard resolved.proxyCommand == nil else { return nil } + guard let host = resolved.hostName else { return nil } + return Endpoint(host: host, port: resolved.port ?? port ?? 22) + } + + struct ResolvedSSHConfig: Equatable { + var hostName: String? + var port: Int? + var proxyJump: String? + var proxyCommand: String? + } + + /// Parse the key/value lines printed by `ssh -G`. + static func parseSSHConfigOutput(_ output: String) -> ResolvedSSHConfig { + var resolved = ResolvedSSHConfig() + for line in output.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard let spaceIndex = trimmed.firstIndex(of: " ") else { continue } + let key = trimmed[.. JumpSpec? { + guard let firstHop = spec.split(separator: ",").first else { return nil } + var hop = firstHop.trimmingCharacters(in: .whitespaces) + guard !hop.isEmpty else { return nil } + + var user: String? + if let atIndex = hop.lastIndex(of: "@") { + user = String(hop[.. String? { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") + process.arguments = arguments + let stdout = Pipe() + process.standardOutput = stdout + process.standardError = Pipe() + do { + try process.run() + } catch { + return nil + } + let deadline = Date().addingTimeInterval(sshResolveTimeout) + while process.isRunning, Date() < deadline { + usleep(20_000) + } + if process.isRunning { + process.terminate() + return nil + } + guard process.terminationStatus == 0 else { return nil } + let data = stdout.fileHandleForReading.readDataToEndOfFile() + return String(data: data, encoding: .utf8) + } + + /// Attempt a TCP connection to `host:port`, reporting `.reachable` on a + /// successful handshake and `.unreachable` on refusal, timeout, DNS + /// failure, or no route. + static func probeTCP( + host: String, + port: Int, + timeout: TimeInterval = defaultTimeout, + completion: @escaping (WorkspaceRemoteHostProbeOutcome) -> Void + ) { + guard port > 0, port <= 65_535, let nwPort = NWEndpoint.Port(rawValue: UInt16(port)) else { + completion(.indeterminate) + return + } + let connection = NWConnection(host: NWEndpoint.Host(host), port: nwPort, using: .tcp) + let lock = NSLock() + var finished = false + func finish(_ outcome: WorkspaceRemoteHostProbeOutcome) { + lock.lock() + let alreadyFinished = finished + finished = true + lock.unlock() + guard !alreadyFinished else { return } + connection.cancel() + completion(outcome) + } + connection.stateUpdateHandler = { state in + switch state { + case .ready: + finish(.reachable) + case .failed(let error): + finish(.unreachable(reason: error.localizedDescription)) + case .waiting(let error): + // NWConnection parks refused/no-route attempts in `.waiting` + // hoping for a better path; for a quick probe that means the + // endpoint is not reachable right now. + finish(.unreachable(reason: error.localizedDescription)) + default: + break + } + } + connection.start(queue: probeQueue) + probeQueue.asyncAfter(deadline: .now() + timeout) { + finish(.unreachable(reason: "probe timed out after \(Int(timeout.rounded()))s")) + } + } +} diff --git a/Sources/WorkspaceRemoteReconnectPolicy.swift b/Sources/WorkspaceRemoteReconnectPolicy.swift index 3a300a3b6674..c39a088ddc40 100644 --- a/Sources/WorkspaceRemoteReconnectPolicy.swift +++ b/Sources/WorkspaceRemoteReconnectPolicy.swift @@ -51,9 +51,10 @@ enum WorkspaceRemoteReconnectPolicy { case .reachable, .indeterminate: return Evaluation(consecutiveUnreachableProbes: 0, decision: .scheduleRetry) case .unreachable: + let streak = previousConsecutiveUnreachableProbes + 1 return Evaluation( - consecutiveUnreachableProbes: previousConsecutiveUnreachableProbes + 1, - decision: .scheduleRetry + consecutiveUnreachableProbes: streak, + decision: streak >= Self.maxConsecutiveUnreachableProbes ? .suspend : .scheduleRetry ) } } diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 4ddca0743511..74db61a51d7f 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -700,6 +700,7 @@ DCC935C5F55C1DCB33E25521 /* WorkspacePullRequestSidebarTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */; }; E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */; }; F6100000A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */; }; + E307A0000000000000000002 /* WorkspaceRemoteHostReachabilityProbe.swift in Sources */ = {isa = PBXBuildFile; fileRef = E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */; }; E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */; }; F6357300A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */; }; E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */; }; @@ -1428,6 +1429,7 @@ 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspacePullRequestSidebarTests.swift; sourceTree = ""; }; E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConfiguration.swift; sourceTree = ""; }; F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConnectionTests.swift; sourceTree = ""; }; + E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteHostReachabilityProbe.swift; sourceTree = ""; }; E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicy.swift; sourceTree = ""; }; F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicyTests.swift; sourceTree = ""; }; E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteSSHBatchCommandBuilder.swift; sourceTree = ""; }; @@ -1787,6 +1789,7 @@ C3744E020000000000000001 /* Workspace+PanelLifecycle.swift */, E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */, E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */, + E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */, D0C0D0C0D0C0D0C0D0C00002 /* RemoteLoopbackProxyAlias.swift */, D0C0D0C0D0C0D0C0D0C00004 /* RemoteLoopbackRuntimeBridge.swift */, E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */, @@ -3062,6 +3065,7 @@ D0B10004A1B2C3D4E5F60001 /* WorkspacePortalPaneDrop.swift in Sources */, 1A1B2C3D4E5F607180000001 /* WorkspacePromptSubmit.swift in Sources */, E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */, + E307A0000000000000000002 /* WorkspaceRemoteHostReachabilityProbe.swift in Sources */, E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */, E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */, 619D509BC9B1EA946CBD6A8A /* WorkspaceRuntimeSettings.swift in Sources */, diff --git a/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift b/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift index 174330600b14..f663e6616c1e 100644 --- a/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift +++ b/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift @@ -106,6 +106,7 @@ final class SidebarWorkspaceSnapshotRefreshPolicyTests: XCTestCase { remoteWorkspaceSidebarText: nil, remoteConnectionStatusText: remoteConnectionStatusText, remoteStateHelpText: "", + showsRemoteReconnectAffordance: false, copyableSidebarSSHError: nil, latestConversationMessage: latestConversationMessage, metadataEntries: [], diff --git a/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift index a38cd3efa67a..bc806d3f11ca 100644 --- a/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift +++ b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift @@ -103,3 +103,165 @@ struct WorkspaceRemoteReconnectPolicyTests { ) } } + +@Suite("Workspace remote host reachability probe") +struct WorkspaceRemoteHostReachabilityProbeTests { + @Test("Parses hostname, port, and proxy fields from ssh -G output") + func parsesSSHConfigOutput() { + let output = """ + user nobody + hostname devbox.internal + port 2222 + proxyjump bastion@jump.example.com:2200 + addressfamily any + """ + let resolved = WorkspaceRemoteHostReachabilityProbe.parseSSHConfigOutput(output) + #expect(resolved.hostName == "devbox.internal") + #expect(resolved.port == 2222) + #expect(resolved.proxyJump == "bastion@jump.example.com:2200") + #expect(resolved.proxyCommand == nil) + } + + @Test("Treats `proxycommand none` as no proxy") + func ignoresProxyCommandNone() { + let resolved = WorkspaceRemoteHostReachabilityProbe.parseSSHConfigOutput( + "hostname example.com\nport 22\nproxycommand none\n" + ) + #expect(resolved.proxyCommand == nil) + #expect(resolved.hostName == "example.com") + } + + @Test("Parses ProxyJump hop specs") + func parsesJumpSpecs() { + let plain = WorkspaceRemoteHostReachabilityProbe.parseJumpSpec("jump.example.com") + #expect(plain == WorkspaceRemoteHostReachabilityProbe.JumpSpec( + destination: "jump.example.com", host: "jump.example.com", port: nil + )) + + let userAndPort = WorkspaceRemoteHostReachabilityProbe.parseJumpSpec("ops@jump.example.com:2200") + #expect(userAndPort == WorkspaceRemoteHostReachabilityProbe.JumpSpec( + destination: "ops@jump.example.com", host: "jump.example.com", port: 2200 + )) + + let chained = WorkspaceRemoteHostReachabilityProbe.parseJumpSpec("first.example.com:22,second.example.com") + #expect(chained?.host == "first.example.com") + #expect(chained?.port == 22) + + let bracketedV6 = WorkspaceRemoteHostReachabilityProbe.parseJumpSpec("[2001:db8::1]:2200") + #expect(bracketedV6 == WorkspaceRemoteHostReachabilityProbe.JumpSpec( + destination: "2001:db8::1", host: "2001:db8::1", port: 2200 + )) + + let bareV6 = WorkspaceRemoteHostReachabilityProbe.parseJumpSpec("2001:db8::1") + #expect(bareV6?.host == "2001:db8::1") + #expect(bareV6?.port == nil) + } + + @Test("ProxyCommand destinations cannot be probed directly") + func proxyCommandResolvesToNil() { + let endpoint = WorkspaceRemoteHostReachabilityProbe.resolveEndpoint( + destination: "nobody@127.0.0.1", + port: 22, + identityFile: nil, + sshOptions: ["ProxyCommand=/usr/bin/nc %h %p"] + ) + #expect(endpoint == nil) + } + + @Test("Resolves a direct destination's endpoint via ssh -G") + func resolvesDirectEndpoint() throws { + let endpoint = WorkspaceRemoteHostReachabilityProbe.resolveEndpoint( + destination: "nobody@127.0.0.1", + port: 2222, + identityFile: nil, + sshOptions: [] + ) + let resolved = try #require(endpoint) + #expect(resolved.host == "127.0.0.1") + #expect(resolved.port == 2222) + } + + @Test("TCP probe reports a listening endpoint as reachable") + func tcpProbeReachable() async throws { + let listener = try BlockingTCPListener() + defer { listener.close() } + let outcome = await probeOutcome(host: "127.0.0.1", port: listener.port) + #expect(outcome == .reachable) + } + + @Test("TCP probe reports a refused connection as unreachable") + func tcpProbeRefused() async throws { + // Bind then close a listener so the port is known-free; the + // subsequent probe gets an immediate connection refusal. + let listener = try BlockingTCPListener() + let refusedPort = listener.port + listener.close() + let outcome = await probeOutcome(host: "127.0.0.1", port: refusedPort) + guard case .unreachable = outcome else { + Issue.record("Expected .unreachable for a refused port, got \(outcome)") + return + } + } + + private func probeOutcome(host: String, port: Int) async -> WorkspaceRemoteHostProbeOutcome { + await withCheckedContinuation { continuation in + WorkspaceRemoteHostReachabilityProbe.probeTCP( + host: host, + port: port, + timeout: 3.0 + ) { outcome in + continuation.resume(returning: outcome) + } + } + } +} + +/// Minimal loopback TCP listener for probe tests. +private final class BlockingTCPListener { + let port: Int + private var fd: Int32 + + init() throws { + let socketFD = socket(AF_INET, SOCK_STREAM, 0) + guard socketFD >= 0 else { throw POSIXError(.EMFILE) } + var reuse: Int32 = 1 + setsockopt(socketFD, SOL_SOCKET, SO_REUSEADDR, &reuse, socklen_t(MemoryLayout.size)) + var addr = sockaddr_in() + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = 0 + addr.sin_addr.s_addr = inet_addr("127.0.0.1") + let bindResult = withUnsafePointer(to: &addr) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + bind(socketFD, sockaddrPointer, socklen_t(MemoryLayout.size)) + } + } + guard bindResult == 0, listen(socketFD, 4) == 0 else { + Darwin.close(socketFD) + throw POSIXError(.EADDRINUSE) + } + var boundAddr = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let nameResult = withUnsafeMutablePointer(to: &boundAddr) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + getsockname(socketFD, sockaddrPointer, &length) + } + } + guard nameResult == 0 else { + Darwin.close(socketFD) + throw POSIXError(.EADDRNOTAVAIL) + } + fd = socketFD + port = Int(UInt16(bigEndian: boundAddr.sin_port)) + } + + func close() { + if fd >= 0 { + Darwin.close(fd) + fd = -1 + } + } + + deinit { + close() + } +} diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 45677e3ff091..c291a9cc77a5 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -95,6 +95,7 @@ Environment: | `reorder-workspace` | Reorder a workspace inside a window. | | `reorder-workspaces` | Atomically reorder workspaces inside pinned and unpinned groups. | | `workspace-action` | Run workspace context-menu actions from the CLI. | +| `workspace` | Namespace for workspace verbs: `list`, `create`, `close`, `rename`, `select`, `reconnect`, `disconnect`, `group`. `workspace reconnect` manually reconnects a remote (SSH) workspace — including one whose automatic reconnect suspended because the host was unreachable — and `workspace disconnect` stops its remote connection. Both accept a positional workspace handle or `--workspace `, defaulting to the caller's workspace, then the selected one. | | `move-tab-to-new-workspace` | Move a tab or surface into a newly created workspace. | | `list-workspaces` | List workspaces. | | `new-workspace` | Create a workspace, optionally with cwd, command, description, and layout. | From 87a73e62131623f87c9e17eb2784d0ab1c95baa2 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 10:51:32 -0700 Subject: [PATCH 3/9] Address autoreview: window-scoped CLI targeting + hermetic resolver tests - "cmux workspace reconnect/disconnect" no longer falls back to the caller's CMUX_WORKSPACE_ID when an explicit --window is supplied, so the server resolves that window's selected workspace instead of a workspace from a different window (matches the rename command's convention). - WorkspaceRemoteHostReachabilityProbe.resolveEndpoint gains an sshConfigFile test seam (ssh -F) so resolver tests pin /dev/null and stay hermetic against the ambient ~/.ssh/config; production callers keep honoring the user's real config. Co-Authored-By: Claude Fable 5 --- CLI/cmux.swift | 11 ++++++----- .../WorkspaceRemoteHostReachabilityProbe.swift | 18 ++++++++++++++++-- .../WorkspaceRemoteReconnectPolicyTests.swift | 8 ++++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 3ddc8e3d9fd4..6753da663805 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -7220,15 +7220,16 @@ struct CMUXCLI { let (workspaceArg, rem0) = parseOption(commandArgs, name: "--workspace") let (_, rem1) = parseOption(rem0, name: "--window") let positional = rem1.first(where: { !$0.hasPrefix("--") }) + let windowRaw = windowFromArgsOrOverride(commandArgs, windowOverride: windowOverride) + // With an explicit --window and no workspace argument, target that + // window's selected workspace on the server instead of the caller's + // CMUX_WORKSPACE_ID, which may live in a different window. let target = workspaceArg ?? positional - ?? ProcessInfo.processInfo.environment["CMUX_WORKSPACE_ID"] + ?? (windowRaw == nil ? ProcessInfo.processInfo.environment["CMUX_WORKSPACE_ID"] : nil) var params: [String: Any] = [:] - let winId = try normalizeWindowHandle( - windowFromArgsOrOverride(commandArgs, windowOverride: windowOverride), - client: client - ) + let winId = try normalizeWindowHandle(windowRaw, client: client) if let winId { params["window_id"] = winId } if let wsId = try normalizeWorkspaceHandle(target, client: client, windowHandle: winId) { params["workspace_id"] = wsId diff --git a/Sources/WorkspaceRemoteHostReachabilityProbe.swift b/Sources/WorkspaceRemoteHostReachabilityProbe.swift index 89008c61e8cc..30c65167c7fa 100644 --- a/Sources/WorkspaceRemoteHostReachabilityProbe.swift +++ b/Sources/WorkspaceRemoteHostReachabilityProbe.swift @@ -58,16 +58,25 @@ enum WorkspaceRemoteHostReachabilityProbe { /// Resolve the effective `(host, port)` for an SSH destination using /// `ssh -G`. Returns nil when the endpoint cannot be probed directly /// (ProxyCommand transports, unparsable output, or resolution failure). + /// + /// `sshConfigFile` is a test seam: passing a path (such as `/dev/null`) + /// pins resolution to that config via `ssh -F`, keeping tests hermetic + /// against the ambient `~/.ssh/config`. Production callers leave it nil + /// so the user's real config (aliases, HostName, ProxyJump) is honored. static func resolveEndpoint( destination: String, port: Int?, identityFile: String?, - sshOptions: [String] + sshOptions: [String], + sshConfigFile: String? = nil ) -> Endpoint? { let trimmedDestination = destination.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmedDestination.isEmpty, !trimmedDestination.hasPrefix("-") else { return nil } var arguments = ["-G"] + if let sshConfigFile, !sshConfigFile.isEmpty { + arguments += ["-F", sshConfigFile] + } if let port, port > 0 { arguments += ["-p", String(port)] } @@ -89,7 +98,12 @@ enum WorkspaceRemoteHostReachabilityProbe { // Reachability of the first hop is the right signal for "can this // network reach the SSH entry point at all". guard let jump = parseJumpSpec(proxyJump) else { return nil } - guard let jumpOutput = runSSHConfigResolution(arguments: ["-G", jump.destination]) else { + var jumpArguments = ["-G"] + if let sshConfigFile, !sshConfigFile.isEmpty { + jumpArguments += ["-F", sshConfigFile] + } + jumpArguments.append(jump.destination) + guard let jumpOutput = runSSHConfigResolution(arguments: jumpArguments) else { return nil } let jumpResolved = parseSSHConfigOutput(jumpOutput) diff --git a/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift index bc806d3f11ca..abe888733451 100644 --- a/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift +++ b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift @@ -159,11 +159,14 @@ struct WorkspaceRemoteHostReachabilityProbeTests { @Test("ProxyCommand destinations cannot be probed directly") func proxyCommandResolvesToNil() { + // sshConfigFile pins resolution to an empty config so the test stays + // hermetic against the developer/CI user's ~/.ssh/config. let endpoint = WorkspaceRemoteHostReachabilityProbe.resolveEndpoint( destination: "nobody@127.0.0.1", port: 22, identityFile: nil, - sshOptions: ["ProxyCommand=/usr/bin/nc %h %p"] + sshOptions: ["ProxyCommand=/usr/bin/nc %h %p"], + sshConfigFile: "/dev/null" ) #expect(endpoint == nil) } @@ -174,7 +177,8 @@ struct WorkspaceRemoteHostReachabilityProbeTests { destination: "nobody@127.0.0.1", port: 2222, identityFile: nil, - sshOptions: [] + sshOptions: [], + sshConfigFile: "/dev/null" ) let resolved = try #require(endpoint) #expect(resolved.host == "127.0.0.1") From f2cd32950280f691c10ecb07c7ba92982bfe509d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 10:53:13 -0700 Subject: [PATCH 4/9] Refresh CLI/cmux.swift length budget after window-targeting fix Co-Authored-By: Claude Fable 5 --- .github/swift-file-length-budget.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 4c73ccbdad3f..9aa6d6861800 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -1,7 +1,7 @@ # cmux-owned Swift file length budget. # Format: max_linesrelative path # Reduce counts as files shrink. CI fails if tracked files exceed this budget. -32619 CLI/cmux.swift +32620 CLI/cmux.swift 21996 Sources/TerminalController.swift 19952 Sources/Workspace.swift 19203 Sources/ContentView.swift From 6d34ec20509dae785193329d083406a46134dc0c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 11:03:47 -0700 Subject: [PATCH 5/9] Address autoreview: probe handler retain cycle + workspace help text - probeTCP clears the NWConnection stateUpdateHandler before cancel so the handler/connection cycle can't leak one connection per backoff retry while the host stays reachable but bootstrap keeps failing. - "cmux workspace --help" now lists the reconnect/disconnect verbs with their targeting rules and examples, keeping CLI discovery coherent with docs/cli-contract.md. - Refresh the CLI/cmux.swift length budget for the help-text growth. Co-Authored-By: Claude Fable 5 --- .github/swift-file-length-budget.tsv | 2 +- CLI/cmux.swift | 10 ++++++++++ Sources/WorkspaceRemoteHostReachabilityProbe.swift | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 9aa6d6861800..2c4f97815b77 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -1,7 +1,7 @@ # cmux-owned Swift file length budget. # Format: max_linesrelative path # Reduce counts as files shrink. CI fails if tracked files exceed this budget. -32620 CLI/cmux.swift +32630 CLI/cmux.swift 21996 Sources/TerminalController.swift 19952 Sources/Workspace.swift 19203 Sources/ContentView.swift diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 6753da663805..3c707bad2966 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -13572,12 +13572,22 @@ struct CMUXCLI { close Close a workspace rename --title select Make a workspace active + reconnect [workspace] Reconnect a remote (SSH) workspace, including one + whose automatic reconnect paused because the host + was unreachable + disconnect [workspace] Stop a remote (SSH) workspace's connection group Workspace group operations (see cmux workspace-group --help) + reconnect/disconnect accept a positional handle or --workspace + , defaulting to the caller's workspace, then the + selected one (of --window's window when given). + Examples: cmux workspace list --json cmux workspace create --name Build --cwd ~/projects/myapp cmux workspace close workspace:3 + cmux workspace reconnect + cmux workspace disconnect --workspace workspace:3 """ case "workspace-group": return """ diff --git a/Sources/WorkspaceRemoteHostReachabilityProbe.swift b/Sources/WorkspaceRemoteHostReachabilityProbe.swift index 30c65167c7fa..0de4f42c8fbc 100644 --- a/Sources/WorkspaceRemoteHostReachabilityProbe.swift +++ b/Sources/WorkspaceRemoteHostReachabilityProbe.swift @@ -240,6 +240,10 @@ enum WorkspaceRemoteHostReachabilityProbe { finished = true lock.unlock() guard !alreadyFinished else { return } + // NWConnection retains its handler and the handler's context + // captures the connection; clear it before canceling so each + // backoff-retry probe doesn't leak a connection. + connection.stateUpdateHandler = nil connection.cancel() completion(outcome) } From a1cc2029025e5998405eec92df2d945f7d142e19 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 11:27:22 -0700 Subject: [PATCH 6/9] Restore ghostty pointer; split probe outcome type; drop redundant probe lock - Restore the ghostty submodule pointer to origin/main's commit (34cbf180d): the main-merge conflict resolution accidentally staged a stale local submodule checkout via git add -A. No ghostty change belongs to this PR. - Move WorkspaceRemoteHostProbeOutcome into its own file so the policy file carries one top-level type (Aziz file-organization policy). - probeTCP's first-finisher latch needs no NSLock: both finish paths (NWConnection state updates and the timeout) already run on the serial probeQueue, so plain queue confinement replaces the lock (Aziz concurrency policy). Co-Authored-By: Claude Fable 5 --- Sources/WorkspaceRemoteHostProbeOutcome.swift | 17 +++++++++++++++++ .../WorkspaceRemoteHostReachabilityProbe.swift | 9 ++++----- Sources/WorkspaceRemoteReconnectPolicy.swift | 16 ---------------- cmux.xcodeproj/project.pbxproj | 4 ++++ ghostty | 2 +- 5 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 Sources/WorkspaceRemoteHostProbeOutcome.swift diff --git a/Sources/WorkspaceRemoteHostProbeOutcome.swift b/Sources/WorkspaceRemoteHostProbeOutcome.swift new file mode 100644 index 000000000000..a62772cfcaf2 --- /dev/null +++ b/Sources/WorkspaceRemoteHostProbeOutcome.swift @@ -0,0 +1,17 @@ +import Foundation + +/// Outcome of a quick reachability probe against the SSH endpoint of a remote +/// workspace, used by the auto-reconnect loop to decide whether retrying makes +/// sense at all. +enum WorkspaceRemoteHostProbeOutcome: Equatable, Sendable { + /// A TCP connection to the resolved SSH endpoint succeeded. + case reachable + /// The endpoint could not be reached (DNS failure, connection refused, + /// timeout, or no route). The associated reason is a short human-readable + /// detail for logs and connection-state messages. + case unreachable(reason: String) + /// Reachability could not be determined (for example ProxyCommand-based + /// transports that cannot be probed with a direct TCP connection). The + /// policy must never suspend the reconnect loop based on this outcome. + case indeterminate +} diff --git a/Sources/WorkspaceRemoteHostReachabilityProbe.swift b/Sources/WorkspaceRemoteHostReachabilityProbe.swift index 0de4f42c8fbc..f8677d7cc266 100644 --- a/Sources/WorkspaceRemoteHostReachabilityProbe.swift +++ b/Sources/WorkspaceRemoteHostReachabilityProbe.swift @@ -232,14 +232,13 @@ enum WorkspaceRemoteHostReachabilityProbe { return } let connection = NWConnection(host: NWEndpoint.Host(host), port: nwPort, using: .tcp) - let lock = NSLock() + // Both finish paths (state updates via `connection.start(queue:)` and + // the timeout below) run on the serial `probeQueue`, so the + // first-finisher latch needs no separate synchronization. var finished = false func finish(_ outcome: WorkspaceRemoteHostProbeOutcome) { - lock.lock() - let alreadyFinished = finished + guard !finished else { return } finished = true - lock.unlock() - guard !alreadyFinished else { return } // NWConnection retains its handler and the handler's context // captures the connection; clear it before canceling so each // backoff-retry probe doesn't leak a connection. diff --git a/Sources/WorkspaceRemoteReconnectPolicy.swift b/Sources/WorkspaceRemoteReconnectPolicy.swift index c39a088ddc40..7afb1dc19957 100644 --- a/Sources/WorkspaceRemoteReconnectPolicy.swift +++ b/Sources/WorkspaceRemoteReconnectPolicy.swift @@ -1,21 +1,5 @@ import Foundation -/// Outcome of a quick reachability probe against the SSH endpoint of a remote -/// workspace, used by the auto-reconnect loop to decide whether retrying makes -/// sense at all. -enum WorkspaceRemoteHostProbeOutcome: Equatable, Sendable { - /// A TCP connection to the resolved SSH endpoint succeeded. - case reachable - /// The endpoint could not be reached (DNS failure, connection refused, - /// timeout, or no route). The associated reason is a short human-readable - /// detail for logs and connection-state messages. - case unreachable(reason: String) - /// Reachability could not be determined (for example ProxyCommand-based - /// transports that cannot be probed with a direct TCP connection). The - /// policy must never suspend the reconnect loop based on this outcome. - case indeterminate -} - /// Pure decision logic for the SSH remote workspace auto-reconnect loop /// (https://github.com/manaflow-ai/cmux/issues/5734). /// diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index f73ea594af6a..d91b6d59f560 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -707,6 +707,7 @@ DCC935C5F55C1DCB33E25521 /* WorkspacePullRequestSidebarTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */; }; E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */; }; F6100000A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */; }; + E307B0000000000000000002 /* WorkspaceRemoteHostProbeOutcome.swift in Sources */ = {isa = PBXBuildFile; fileRef = E307B0000000000000000001 /* WorkspaceRemoteHostProbeOutcome.swift */; }; E307A0000000000000000002 /* WorkspaceRemoteHostReachabilityProbe.swift in Sources */ = {isa = PBXBuildFile; fileRef = E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */; }; E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */; }; F6357300A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */; }; @@ -1445,6 +1446,7 @@ 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspacePullRequestSidebarTests.swift; sourceTree = ""; }; E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConfiguration.swift; sourceTree = ""; }; F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConnectionTests.swift; sourceTree = ""; }; + E307B0000000000000000001 /* WorkspaceRemoteHostProbeOutcome.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteHostProbeOutcome.swift; sourceTree = ""; }; E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteHostReachabilityProbe.swift; sourceTree = ""; }; E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicy.swift; sourceTree = ""; }; F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicyTests.swift; sourceTree = ""; }; @@ -1812,6 +1814,7 @@ E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */, E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */, E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */, + E307B0000000000000000001 /* WorkspaceRemoteHostProbeOutcome.swift */, D0C0D0C0D0C0D0C0D0C00002 /* RemoteLoopbackProxyAlias.swift */, D0C0D0C0D0C0D0C0D0C00004 /* RemoteLoopbackRuntimeBridge.swift */, E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */, @@ -3095,6 +3098,7 @@ D0B10004A1B2C3D4E5F60001 /* WorkspacePortalPaneDrop.swift in Sources */, 1A1B2C3D4E5F607180000001 /* WorkspacePromptSubmit.swift in Sources */, E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */, + E307B0000000000000000002 /* WorkspaceRemoteHostProbeOutcome.swift in Sources */, E307A0000000000000000002 /* WorkspaceRemoteHostReachabilityProbe.swift in Sources */, E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */, E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */, diff --git a/ghostty b/ghostty index f78189ac17ff..34cbf180d891 160000 --- a/ghostty +++ b/ghostty @@ -1 +1 @@ -Subproject commit f78189ac17ff21d577c8b3c55422576fd689e241 +Subproject commit 34cbf180d8917b802d61d9929cfb493594f2ab52 From 84892da0d3f046fa8bd8e646b8884ed18fe74f37 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 11:47:06 -0700 Subject: [PATCH 7/9] CI: re-trigger after dropped pull_request synchronize event Co-Authored-By: Claude Fable 5 From 2fe454e611a9edd94d9af1b38c701a879d9c9d35 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 14:14:59 -0700 Subject: [PATCH 8/9] Address review: concurrent probe resolution + clear suspension on ready - Run ssh -G endpoint resolution on the global utility pool instead of the serial probeQueue so simultaneous probes from multiple suspended workspaces don't serialize behind its bounded blocking wait, and NWConnection callbacks stay unblocked (Greptile P1). - Clear reconnectSuspended in the proxy .ready handler alongside the other policy resets so a connection established while suspended can't leave a later failure unable to reschedule retries (Greptile P1). - Refresh the Workspace.swift length budget. Co-Authored-By: Claude Fable 5 --- .github/swift-file-length-budget.tsv | 2 +- Sources/Workspace.swift | 3 +++ Sources/WorkspaceRemoteHostReachabilityProbe.swift | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 3121d176f1b1..3cb7b9928fd7 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -3,7 +3,7 @@ # Reduce counts as files shrink. CI fails if tracked files exceed this budget. 32723 CLI/cmux.swift 22073 Sources/TerminalController.swift -19952 Sources/Workspace.swift +19955 Sources/Workspace.swift 19251 Sources/ContentView.swift 18044 Sources/AppDelegate.swift 16537 Sources/GhosttyTerminalView.swift diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index ec5d2f46b530..053193bcd803 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -7213,6 +7213,9 @@ final class WorkspaceRemoteSessionController { reconnectWorkItem = nil reconnectRetryCount = 0 consecutiveUnreachableProbeCount = 0 + // A live connection ends any suspension; without this a future + // failure would hit the suspended guard and never reschedule. + reconnectSuspended = false reachabilityProbeGeneration &+= 1 guard proxyEndpoint != endpoint else { recordHeartbeatActivityLocked() diff --git a/Sources/WorkspaceRemoteHostReachabilityProbe.swift b/Sources/WorkspaceRemoteHostReachabilityProbe.swift index f8677d7cc266..2644f89ae490 100644 --- a/Sources/WorkspaceRemoteHostReachabilityProbe.swift +++ b/Sources/WorkspaceRemoteHostReachabilityProbe.swift @@ -36,7 +36,11 @@ enum WorkspaceRemoteHostReachabilityProbe { timeout: TimeInterval = defaultTimeout, completion: @escaping (WorkspaceRemoteHostProbeOutcome) -> Void ) { - probeQueue.async { + // Endpoint resolution shells out to `ssh -G` and can block for up to + // its timeout; run it on the concurrent utility pool so simultaneous + // probes from multiple workspaces don't serialize behind it (the + // serial probeQueue is reserved for NWConnection callbacks). + DispatchQueue.global(qos: .utility).async { guard let endpoint = resolveEndpoint( destination: destination, port: port, From e462f1a68305b28e36c79ad99eaea49d2a11dc1a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 10 Jun 2026 14:34:13 -0700 Subject: [PATCH 9/9] Localize workspace CLI help and subcommand errors The CLI target already localizes usage blocks (cli.claude-teams.usage et al) and error strings (cli.error.*), so the new workspace reconnect/disconnect help and the touched subcommand error messages follow the same pattern: cli.workspace.usage, cli.error.workspaceSubcommandRequired, and cli.error.workspaceSubcommandUnknown, each with en/ja/ko/uk entries in Localizable.xcstrings. This also corrects the PR's earlier audit note that claimed CLI text was conventionally unlocalized. Co-Authored-By: Claude Fable 5 --- .github/swift-file-length-budget.tsv | 2 +- CLI/cmux.swift | 18 ++++-- Resources/Localizable.xcstrings | 87 ++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 3cb7b9928fd7..3dad0ccb7258 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -1,7 +1,7 @@ # cmux-owned Swift file length budget. # Format: max_linesrelative path # Reduce counts as files shrink. CI fails if tracked files exceed this budget. -32723 CLI/cmux.swift +32733 CLI/cmux.swift 22073 Sources/TerminalController.swift 19955 Sources/Workspace.swift 19251 Sources/ContentView.swift diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 092401e0bcf2..3af8f1c25cee 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -7213,7 +7213,10 @@ struct CMUXCLI { windowOverride: String? ) throws { guard let sub = commandArgs.first?.lowercased() else { - throw CLIError(message: "workspace requires a subcommand. Try: list, create, close, rename, select, reconnect, disconnect, group") + throw CLIError(message: String( + localized: "cli.error.workspaceSubcommandRequired", + defaultValue: "workspace requires a subcommand. Try: list, create, close, rename, select, reconnect, disconnect, group" + )) } let rest = Array(commandArgs.dropFirst()) switch sub { @@ -7294,7 +7297,14 @@ struct CMUXCLI { windowOverride: windowOverride ) default: - throw CLIError(message: "Unknown workspace subcommand: \(sub). Try: list, create, close, rename, select, reconnect, disconnect, group") + throw CLIError(message: String( + format: String( + localized: "cli.error.workspaceSubcommandUnknown", + defaultValue: "Unknown workspace subcommand: %@. Try: list, create, close, rename, select, reconnect, disconnect, group" + ), + locale: .current, + sub + )) } } @@ -13651,7 +13661,7 @@ struct CMUXCLI { cmux list-workspaces """ case "workspace": - return """ + return String(localized: "cli.workspace.usage", defaultValue: """ Usage: cmux workspace [flags] Canonical noun for workspace operations. Legacy verbs @@ -13681,7 +13691,7 @@ struct CMUXCLI { cmux workspace close workspace:3 cmux workspace reconnect cmux workspace disconnect --workspace workspace:3 - """ + """) case "workspace-group": return """ Usage: cmux workspace-group [flags] diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index 1483858beac3..57b62c23b9c9 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -191248,6 +191248,93 @@ } } } + }, + "cli.workspace.usage": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Usage: cmux workspace [flags]\n\nCanonical noun for workspace operations. Legacy verbs\n(new-workspace, list-workspaces, close-workspace,\nrename-workspace, select-workspace) keep working and print a\none-time deprecation hint pointing here.\n\nSubcommands:\n list List workspaces in a window\n create [flags] Create a workspace (same flags as new-workspace)\n close Close a workspace\n rename --title \n select Make a workspace active\n reconnect [workspace] Reconnect a remote (SSH) workspace, including one\n whose automatic reconnect paused because the host\n was unreachable\n disconnect [workspace] Stop a remote (SSH) workspace's connection\n group Workspace group operations (see cmux workspace-group --help)\n\nreconnect/disconnect accept a positional handle or --workspace\n, defaulting to the caller's workspace, then the\nselected one (of --window's window when given).\n\nExamples:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "使い方: cmux workspace <サブコマンド> [フラグ]\n\nワークスペース操作の正規の名前空間です。従来の動詞コマンド\n(new-workspace、list-workspaces、close-workspace、\nrename-workspace、select-workspace) は引き続き動作し、ここを指す\n非推奨ヒントを一度だけ表示します。\n\nサブコマンド:\n list ウィンドウ内のワークスペースを一覧表示\n create [flags] ワークスペースを作成 (new-workspace と同じフラグ)\n close ワークスペースを閉じる\n rename --title \n select ワークスペースをアクティブにする\n reconnect [workspace] リモート (SSH) ワークスペースを再接続。ホストに\n 到達できず自動再接続が一時停止したものも含む\n disconnect [workspace] リモート (SSH) ワークスペースの接続を停止\n group ワークスペースグループ操作 (cmux workspace-group --help を参照)\n\nreconnect/disconnect は位置引数のハンドルまたは --workspace\n を受け付け、省略時は呼び出し元のワークスペース、\n次に選択中のワークスペース (--window 指定時はそのウィンドウ) を\n対象にします。\n\n例:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "사용법: cmux workspace <하위 명령> [플래그]\n\n워크스페이스 작업의 표준 네임스페이스입니다. 기존 동사형 명령\n(new-workspace, list-workspaces, close-workspace,\nrename-workspace, select-workspace)은 계속 작동하며 이곳을\n가리키는 사용 중단 안내를 한 번 표시합니다.\n\n하위 명령:\n list 창의 워크스페이스 목록 표시\n create [flags] 워크스페이스 생성 (new-workspace와 동일한 플래그)\n close 워크스페이스 닫기\n rename --title \n select 워크스페이스를 활성화\n reconnect [workspace] 원격 (SSH) 워크스페이스 재연결. 호스트에 연결할\n 수 없어 자동 재연결이 일시 중지된 경우 포함\n disconnect [workspace] 원격 (SSH) 워크스페이스 연결 중지\n group 워크스페이스 그룹 작업 (cmux workspace-group --help 참고)\n\nreconnect/disconnect는 위치 핸들 또는 --workspace\n를 받으며, 기본값은 호출자의 워크스페이스, 그다음\n선택된 워크스페이스입니다 (--window 지정 시 해당 창 기준).\n\n예시:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Використання: cmux workspace <підкоманда> [прапорці]\n\nКанонічний простір імен для операцій з робочими просторами.\nЗастарілі дієслівні команди (new-workspace, list-workspaces,\nclose-workspace, rename-workspace, select-workspace) продовжують\nпрацювати й одноразово показують підказку про застарілість, що\nвказує сюди.\n\nПідкоманди:\n list Список робочих просторів у вікні\n create [flags] Створити робочий простір (ті самі прапорці, що й new-workspace)\n close Закрити робочий простір\n rename --title \n select Зробити робочий простір активним\n reconnect [workspace] Перепідключити віддалений (SSH) робочий простір,\n зокрема той, де автоматичне перепідключення\n призупинено через недосяжний хост\n disconnect [workspace] Зупинити з'єднання віддаленого (SSH) робочого простору\n group Операції з групами робочих просторів (див. cmux workspace-group --help)\n\nreconnect/disconnect приймають позиційний дескриптор або --workspace\n; типово використовується робочий простір викликача,\nдалі — вибраний (у вікні з --window, якщо задано).\n\nПриклади:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + } + } + }, + "cli.error.workspaceSubcommandRequired": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "workspace requires a subcommand. Try: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "workspace にはサブコマンドが必要です。利用可能: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "workspace에는 하위 명령이 필요합니다. 사용 가능: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "workspace потребує підкоманди. Доступні: list, create, close, rename, select, reconnect, disconnect, group" + } + } + } + }, + "cli.error.workspaceSubcommandUnknown": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Unknown workspace subcommand: %@. Try: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "不明な workspace サブコマンド: %@。利用可能: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "알 수 없는 workspace 하위 명령: %@. 사용 가능: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Невідома підкоманда workspace: %@. Доступні: list, create, close, rename, select, reconnect, disconnect, group" + } + } + } } } }