diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 322d888134b9..fde8837d8e48 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -1,10 +1,10 @@ # cmux-owned Swift file length budget. # Format: max_linesrelative path # Reduce counts as files shrink. CI fails if tracked files exceed this budget. -32655 CLI/cmux.swift +32733 CLI/cmux.swift 22098 Sources/TerminalController.swift -19820 Sources/Workspace.swift -19209 Sources/ContentView.swift +19955 Sources/Workspace.swift +19251 Sources/ContentView.swift 18044 Sources/AppDelegate.swift 16537 Sources/GhosttyTerminalView.swift 13589 Sources/Panels/BrowserPanel.swift diff --git a/CLI/cmux.swift b/CLI/cmux.swift index f1b213d6f0e1..3af8f1c25cee 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -7213,7 +7213,10 @@ struct CMUXCLI { windowOverride: String? ) throws { guard let sub = commandArgs.first?.lowercased() else { - throw CLIError(message: "workspace requires a subcommand. Try: list, create, close, rename, select, group") + throw CLIError(message: String( + localized: "cli.error.workspaceSubcommandRequired", + defaultValue: "workspace requires a subcommand. Try: list, create, close, rename, select, reconnect, disconnect, group" + )) } let rest = Array(commandArgs.dropFirst()) switch sub { @@ -7273,11 +7276,76 @@ struct CMUXCLI { windowOverride: windowOverride, requireWorkspaceFlag: false ) + case "reconnect": + try runWorkspaceRemoteConnectionCommand( + commandName: "workspace reconnect", + method: "workspace.remote.reconnect", + commandArgs: rest, + client: client, + jsonOutput: jsonOutput, + idFormat: idFormat, + windowOverride: windowOverride + ) + case "disconnect": + try runWorkspaceRemoteConnectionCommand( + commandName: "workspace disconnect", + method: "workspace.remote.disconnect", + commandArgs: rest, + client: client, + jsonOutput: jsonOutput, + idFormat: idFormat, + windowOverride: windowOverride + ) default: - throw CLIError(message: "Unknown workspace subcommand: \(sub). Try: list, create, close, rename, select, group") + throw CLIError(message: String( + format: String( + localized: "cli.error.workspaceSubcommandUnknown", + defaultValue: "Unknown workspace subcommand: %@. Try: list, create, close, rename, select, reconnect, disconnect, group" + ), + locale: .current, + sub + )) } } + /// `cmux workspace reconnect|disconnect` — manual control over a remote + /// (SSH) workspace's connection. Targets the positional/`--workspace` + /// handle, then the caller's workspace, then the selected workspace. + private func runWorkspaceRemoteConnectionCommand( + commandName: String, + method: String, + commandArgs: [String], + client: SocketClient, + jsonOutput: Bool, + idFormat: CLIIDFormat, + windowOverride: String? + ) throws { + let (workspaceArg, rem0) = parseOption(commandArgs, name: "--workspace") + let (_, rem1) = parseOption(rem0, name: "--window") + let positional = rem1.first(where: { !$0.hasPrefix("--") }) + let windowRaw = windowFromArgsOrOverride(commandArgs, windowOverride: windowOverride) + // With an explicit --window and no workspace argument, target that + // window's selected workspace on the server instead of the caller's + // CMUX_WORKSPACE_ID, which may live in a different window. + let target = workspaceArg + ?? positional + ?? (windowRaw == nil ? ProcessInfo.processInfo.environment["CMUX_WORKSPACE_ID"] : nil) + + var params: [String: Any] = [:] + let winId = try normalizeWindowHandle(windowRaw, client: client) + if let winId { params["window_id"] = winId } + if let wsId = try normalizeWorkspaceHandle(target, client: client, windowHandle: winId) { + params["workspace_id"] = wsId + } + let payload = try client.sendV2(method: method, params: params) + printV2Payload( + payload, + jsonOutput: jsonOutput, + idFormat: idFormat, + fallbackText: v2OKSummary(payload, idFormat: idFormat, kinds: ["workspace", "window"]) + ) + } + /// Emit a `cmux workspace-group` mutation response: JSON when --json, /// otherwise a compact `OK`. Centralized so every mutating subcommand /// honors --json the same way the list/create paths do. @@ -13593,7 +13661,7 @@ struct CMUXCLI { cmux list-workspaces """ case "workspace": - return """ + return String(localized: "cli.workspace.usage", defaultValue: """ Usage: cmux workspace [flags] Canonical noun for workspace operations. Legacy verbs @@ -13607,13 +13675,23 @@ struct CMUXCLI { close Close a workspace rename --title select Make a workspace active + reconnect [workspace] Reconnect a remote (SSH) workspace, including one + whose automatic reconnect paused because the host + was unreachable + disconnect [workspace] Stop a remote (SSH) workspace's connection group Workspace group operations (see cmux workspace-group --help) + reconnect/disconnect accept a positional handle or --workspace + , defaulting to the caller's workspace, then the + selected one (of --window's window when given). + Examples: cmux workspace list --json cmux workspace create --name Build --cwd ~/projects/myapp cmux workspace close workspace:3 - """ + cmux workspace reconnect + cmux workspace disconnect --workspace workspace:3 + """) case "workspace-group": return """ Usage: cmux workspace-group [flags] diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index da09336b5d88..623d91e36e0b 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -191235,6 +191235,412 @@ } } } + }, + "remote.status.suspended": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Unreachable" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "到達不能" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "연결 불가" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Недоступний" + } + } + } + }, + "sidebar.remote.help.suspended": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH host %@ is unreachable. Automatic reconnect is paused — use Reconnect to retry." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "SSH ホスト %@ に到達できません。自動再接続は一時停止中です。「再接続」で再試行してください。" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "SSH 호스트 %@에 연결할 수 없습니다. 자동 재연결이 일시 중지되었습니다. 재연결로 다시 시도하세요." + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "SSH-хост %@ недоступний. Автоматичне перепідключення призупинено — скористайтеся «Перепідключити», щоб повторити спробу." + } + } + } + }, + "sidebar.remote.reconnect.button": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "재연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключити" + } + } + } + }, + "sidebar.remote.reconnect.help": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect to %@" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "%@ に再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "%@에 다시 연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключитися до %@" + } + } + } + }, + "remote.state.suspended.detail": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Can't reach %@ — automatic reconnect is paused. Use Reconnect when your network is back." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "%@ に到達できないため、自動再接続を一時停止しました。ネットワーク復帰後に「再接続」を実行してください。" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "%@에 연결할 수 없어 자동 재연결을 일시 중지했습니다. 네트워크가 복구되면 재연결을 사용하세요." + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Не вдається з'єднатися з %@ — автоматичне перепідключення призупинено. Скористайтеся «Перепідключити», коли мережа відновиться." + } + } + } + }, + "remote.statusEntry.suspended": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH reconnect paused (%@): %@" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "SSH 再接続を一時停止 (%@): %@" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "SSH 재연결 일시 중지됨 (%@): %@" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключення SSH призупинено (%@): %@" + } + } + } + }, + "remote.notification.suspendedTitle": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH Reconnect Paused" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "SSH 再接続を一時停止" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "SSH 재연결 일시 중지됨" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключення SSH призупинено" + } + } + } + }, + "contextMenu.reconnectWorkspace": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect Workspace" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 재연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключити робочий простір" + } + } + } + }, + "contextMenu.reconnectWorkspaces": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Reconnect Workspaces" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを再接続" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 재연결" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Перепідключити робочі простори" + } + } + } + }, + "contextMenu.disconnectWorkspace": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Disconnect Workspace" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを切断" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 연결 해제" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Відключити робочий простір" + } + } + } + }, + "contextMenu.disconnectWorkspaces": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Disconnect Workspaces" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "ワークスペースを切断" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "워크스페이스 연결 해제" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Відключити робочі простори" + } + } + } + }, + "cli.workspace.usage": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Usage: cmux workspace [flags]\n\nCanonical noun for workspace operations. Legacy verbs\n(new-workspace, list-workspaces, close-workspace,\nrename-workspace, select-workspace) keep working and print a\none-time deprecation hint pointing here.\n\nSubcommands:\n list List workspaces in a window\n create [flags] Create a workspace (same flags as new-workspace)\n close Close a workspace\n rename --title \n select Make a workspace active\n reconnect [workspace] Reconnect a remote (SSH) workspace, including one\n whose automatic reconnect paused because the host\n was unreachable\n disconnect [workspace] Stop a remote (SSH) workspace's connection\n group Workspace group operations (see cmux workspace-group --help)\n\nreconnect/disconnect accept a positional handle or --workspace\n, defaulting to the caller's workspace, then the\nselected one (of --window's window when given).\n\nExamples:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "使い方: cmux workspace <サブコマンド> [フラグ]\n\nワークスペース操作の正規の名前空間です。従来の動詞コマンド\n(new-workspace、list-workspaces、close-workspace、\nrename-workspace、select-workspace) は引き続き動作し、ここを指す\n非推奨ヒントを一度だけ表示します。\n\nサブコマンド:\n list ウィンドウ内のワークスペースを一覧表示\n create [flags] ワークスペースを作成 (new-workspace と同じフラグ)\n close ワークスペースを閉じる\n rename --title \n select ワークスペースをアクティブにする\n reconnect [workspace] リモート (SSH) ワークスペースを再接続。ホストに\n 到達できず自動再接続が一時停止したものも含む\n disconnect [workspace] リモート (SSH) ワークスペースの接続を停止\n group ワークスペースグループ操作 (cmux workspace-group --help を参照)\n\nreconnect/disconnect は位置引数のハンドルまたは --workspace\n を受け付け、省略時は呼び出し元のワークスペース、\n次に選択中のワークスペース (--window 指定時はそのウィンドウ) を\n対象にします。\n\n例:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "사용법: cmux workspace <하위 명령> [플래그]\n\n워크스페이스 작업의 표준 네임스페이스입니다. 기존 동사형 명령\n(new-workspace, list-workspaces, close-workspace,\nrename-workspace, select-workspace)은 계속 작동하며 이곳을\n가리키는 사용 중단 안내를 한 번 표시합니다.\n\n하위 명령:\n list 창의 워크스페이스 목록 표시\n create [flags] 워크스페이스 생성 (new-workspace와 동일한 플래그)\n close 워크스페이스 닫기\n rename --title \n select 워크스페이스를 활성화\n reconnect [workspace] 원격 (SSH) 워크스페이스 재연결. 호스트에 연결할\n 수 없어 자동 재연결이 일시 중지된 경우 포함\n disconnect [workspace] 원격 (SSH) 워크스페이스 연결 중지\n group 워크스페이스 그룹 작업 (cmux workspace-group --help 참고)\n\nreconnect/disconnect는 위치 핸들 또는 --workspace\n를 받으며, 기본값은 호출자의 워크스페이스, 그다음\n선택된 워크스페이스입니다 (--window 지정 시 해당 창 기준).\n\n예시:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Використання: cmux workspace <підкоманда> [прапорці]\n\nКанонічний простір імен для операцій з робочими просторами.\nЗастарілі дієслівні команди (new-workspace, list-workspaces,\nclose-workspace, rename-workspace, select-workspace) продовжують\nпрацювати й одноразово показують підказку про застарілість, що\nвказує сюди.\n\nПідкоманди:\n list Список робочих просторів у вікні\n create [flags] Створити робочий простір (ті самі прапорці, що й new-workspace)\n close Закрити робочий простір\n rename --title \n select Зробити робочий простір активним\n reconnect [workspace] Перепідключити віддалений (SSH) робочий простір,\n зокрема той, де автоматичне перепідключення\n призупинено через недосяжний хост\n disconnect [workspace] Зупинити з'єднання віддаленого (SSH) робочого простору\n group Операції з групами робочих просторів (див. cmux workspace-group --help)\n\nreconnect/disconnect приймають позиційний дескриптор або --workspace\n; типово використовується робочий простір викликача,\nдалі — вибраний (у вікні з --window, якщо задано).\n\nПриклади:\n cmux workspace list --json\n cmux workspace create --name Build --cwd ~/projects/myapp\n cmux workspace close workspace:3\n cmux workspace reconnect\n cmux workspace disconnect --workspace workspace:3" + } + } + } + }, + "cli.error.workspaceSubcommandRequired": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "workspace requires a subcommand. Try: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "workspace にはサブコマンドが必要です。利用可能: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "workspace에는 하위 명령이 필요합니다. 사용 가능: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "workspace потребує підкоманди. Доступні: list, create, close, rename, select, reconnect, disconnect, group" + } + } + } + }, + "cli.error.workspaceSubcommandUnknown": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Unknown workspace subcommand: %@. Try: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "不明な workspace サブコマンド: %@。利用可能: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "알 수 없는 workspace 하위 명령: %@. 사용 가능: list, create, close, rename, select, reconnect, disconnect, group" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Невідома підкоманда workspace: %@. Доступні: list, create, close, rename, select, reconnect, disconnect, group" + } + } + } } } } diff --git a/Sources/ContentView.swift b/Sources/ContentView.swift index 02357b3c2994..8188a75cd6e2 100644 --- a/Sources/ContentView.swift +++ b/Sources/ContentView.swift @@ -15211,6 +15211,7 @@ struct SidebarWorkspaceSnapshotBuilder { let remoteWorkspaceSidebarText: String? let remoteConnectionStatusText: String let remoteStateHelpText: String + let showsRemoteReconnectAffordance: Bool let copyableSidebarSSHError: String? let latestConversationMessage: String? let metadataEntries: [SidebarStatusEntry] @@ -15489,7 +15490,12 @@ struct TabItemView: View, Equatable { } private var remoteWorkspaceSidebarText: String? { - guard tab.hasActiveRemoteTerminalSessions else { return nil } + // Keep the SSH row visible while auto-reconnect is suspended even if + // every remote terminal session already died — it hosts the manual + // Reconnect affordance. + guard tab.hasActiveRemoteTerminalSessions || tab.remoteConnectionState == .suspended else { + return nil + } let trimmedTarget = tab.remoteDisplayTarget?.trimmingCharacters(in: .whitespacesAndNewlines) if let trimmedTarget, !trimmedTarget.isEmpty { return trimmedTarget @@ -15503,7 +15509,8 @@ struct TabItemView: View, Equatable { defaultValue: "remote host" ) let trimmedDetail = tab.remoteConnectionDetail?.trimmingCharacters(in: .whitespacesAndNewlines) - if tab.remoteConnectionState == .error, let trimmedDetail, !trimmedDetail.isEmpty { + if tab.remoteConnectionState == .error || tab.remoteConnectionState == .suspended, + let trimmedDetail, !trimmedDetail.isEmpty { let entry = SidebarRemoteErrorCopyEntry( workspaceTitle: tab.title, target: fallbackTarget, @@ -15536,6 +15543,8 @@ struct TabItemView: View, Equatable { return String(localized: "remote.status.error", defaultValue: "Error") case .disconnected: return String(localized: "remote.status.disconnected", defaultValue: "Disconnected") + case .suspended: + return String(localized: "remote.status.suspended", defaultValue: "Unreachable") } } @@ -15569,6 +15578,29 @@ struct TabItemView: View, Equatable { .font(.system(size: scaledFontSize(9), weight: .medium)) .foregroundColor(activeSecondaryColor(0.58)) .lineLimit(1) + + if workspaceSnapshot.showsRemoteReconnectAffordance { + Button { + tab.reconnectRemoteConnection() + } label: { + Label( + String(localized: "sidebar.remote.reconnect.button", defaultValue: "Reconnect"), + systemImage: "arrow.clockwise" + ) + .labelStyle(.titleAndIcon) + .font(.system(size: scaledFontSize(9), weight: .semibold)) + } + .buttonStyle(.borderless) + .foregroundColor(activeSecondaryColor(0.9)) + .safeHelp(String( + format: String( + localized: "sidebar.remote.reconnect.help", + defaultValue: "Reconnect to %@" + ), + locale: .current, + remoteWorkspaceSidebarText + )) + } } } .padding(.top, latestNotificationText == nil ? 1 : 2) @@ -16621,6 +16653,15 @@ struct TabItemView: View, Equatable { locale: .current, target ) + case .suspended: + return String( + format: String( + localized: "sidebar.remote.help.suspended", + defaultValue: "SSH host %@ is unreachable. Automatic reconnect is paused — use Reconnect to retry." + ), + locale: .current, + target + ) } } @@ -16673,6 +16714,7 @@ struct TabItemView: View, Equatable { remoteWorkspaceSidebarText: remoteWorkspaceSidebarText, remoteConnectionStatusText: remoteConnectionStatusText, remoteStateHelpText: remoteStateHelpText, + showsRemoteReconnectAffordance: tab.remoteConnectionState == .suspended, copyableSidebarSSHError: copyableSidebarSSHError, latestConversationMessage: tab.latestConversationMessage, metadataEntries: detailVisibility.showsMetadata ? tab.sidebarStatusEntriesInDisplayOrder() : [], diff --git a/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift b/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift index 07f4c8eda3ff..85ca4e2bf93b 100644 --- a/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift +++ b/Sources/Sidebar/SidebarWorkspaceSnapshotRefreshPolicy.swift @@ -26,6 +26,7 @@ extension SidebarWorkspaceSnapshotBuilder.Snapshot { remoteWorkspaceSidebarText: remoteWorkspaceSidebarText, remoteConnectionStatusText: remoteConnectionStatusText, remoteStateHelpText: remoteStateHelpText, + showsRemoteReconnectAffordance: showsRemoteReconnectAffordance, copyableSidebarSSHError: copyableSidebarSSHError, latestConversationMessage: latestConversationMessage, metadataEntries: metadataEntries, diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index a698f7785cf0..053193bcd803 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -6437,6 +6437,9 @@ final class WorkspaceRemoteSessionController { private var reverseRelayStderrBuffer = "" private var reconnectRetryCount = 0 private var reconnectWorkItem: DispatchWorkItem? + private var consecutiveUnreachableProbeCount = 0 + private var reconnectSuspended = false + private var reachabilityProbeGeneration: UInt64 = 0 private var heartbeatCount: Int = 0 private var connectionAttemptStartedAt: Date? private var pendingPTYBridgeStarts: [UUID: PendingPTYBridgeStart] = [:] @@ -6824,6 +6827,9 @@ final class WorkspaceRemoteSessionController { reconnectWorkItem?.cancel() reconnectWorkItem = nil reconnectRetryCount = 0 + consecutiveUnreachableProbeCount = 0 + reconnectSuspended = false + reachabilityProbeGeneration &+= 1 reverseRelayRestartWorkItem?.cancel() reverseRelayRestartWorkItem = nil remotePortScanCoalesceWorkItem?.cancel() @@ -7206,6 +7212,11 @@ final class WorkspaceRemoteSessionController { reconnectWorkItem?.cancel() reconnectWorkItem = nil reconnectRetryCount = 0 + consecutiveUnreachableProbeCount = 0 + // A live connection ends any suspension; without this a future + // failure would hit the suspended guard and never reschedule. + reconnectSuspended = false + reachabilityProbeGeneration &+= 1 guard proxyEndpoint != endpoint else { recordHeartbeatActivityLocked() fulfillPendingPTYBridgeStartsLocked() @@ -7260,7 +7271,9 @@ final class WorkspaceRemoteSessionController { private func scheduleReconnectLocked(baseDelay: TimeInterval) -> RetrySchedule { let retryNumber = reconnectRetryCount + 1 let retryDelay = Self.retryDelay(baseDelay: baseDelay, retry: retryNumber) - guard !isStopping else { return RetrySchedule(retry: retryNumber, delay: retryDelay) } + guard !isStopping, !reconnectSuspended else { + return RetrySchedule(retry: retryNumber, delay: retryDelay) + } reconnectWorkItem?.cancel() reconnectRetryCount = retryNumber let workItem = DispatchWorkItem { [weak self] in @@ -7272,9 +7285,89 @@ final class WorkspaceRemoteSessionController { } reconnectWorkItem = workItem queue.asyncAfter(deadline: .now() + retryDelay, execute: workItem) + evaluateReconnectPolicyLocked() return RetrySchedule(retry: retryNumber, delay: retryDelay) } + /// Probe whether the SSH endpoint is reachable at all after a failed + /// connection attempt. While the host stays unreachable the retry loop is + /// allowed a short streak of attempts (absorbing sleep/wake and network + /// handoffs) and then suspends instead of retrying indefinitely + /// (https://github.com/manaflow-ai/cmux/issues/5734). + private func evaluateReconnectPolicyLocked() { + guard configuration.transport == .ssh else { return } + reachabilityProbeGeneration &+= 1 + let generation = reachabilityProbeGeneration + WorkspaceRemoteHostReachabilityProbe.probe( + destination: configuration.destination, + port: configuration.port, + identityFile: configuration.identityFile, + sshOptions: configuration.sshOptions + ) { [weak self] outcome in + guard let self else { return } + self.queue.async { + self.handleReachabilityProbeOutcomeLocked(outcome, generation: generation) + } + } + } + + private func handleReachabilityProbeOutcomeLocked( + _ outcome: WorkspaceRemoteHostProbeOutcome, + generation: UInt64 + ) { + guard generation == reachabilityProbeGeneration else { return } + guard !isStopping, !reconnectSuspended else { return } + // The probe only judges a still-pending retry; if the retry resolved + // while the probe ran, the connected/stopped paths own the state. + guard reconnectWorkItem != nil else { return } + let evaluation = WorkspaceRemoteReconnectPolicy.evaluate( + outcome: outcome, + previousConsecutiveUnreachableProbes: consecutiveUnreachableProbeCount + ) + consecutiveUnreachableProbeCount = evaluation.consecutiveUnreachableProbes + debugLog( + "remote.session.reachability outcome=\(Self.debugDescription(for: outcome)) " + + "streak=\(evaluation.consecutiveUnreachableProbes) " + + "decision=\(evaluation.decision == .suspend ? "suspend" : "retry") \(debugConfigSummary())" + ) + if evaluation.decision == .suspend { + suspendAutoReconnectLocked() + } + } + + /// Halt the automatic reconnect loop and surface a suspended state with a + /// manual Reconnect affordance. `Workspace.reconnectRemoteConnection()` + /// (sidebar button, workspace context menu, `cmux workspace reconnect`, + /// and the `workspace.remote.reconnect` socket command) replaces this + /// controller, which resets the policy state. + private func suspendAutoReconnectLocked() { + reconnectWorkItem?.cancel() + reconnectWorkItem = nil + reconnectSuspended = true + debugLog( + "remote.session.reconnect.suspended afterUnreachableProbes=\(consecutiveUnreachableProbeCount) " + + debugConfigSummary() + ) + let detailFormat = String( + localized: "remote.state.suspended.detail", + defaultValue: "Can't reach %@ — automatic reconnect is paused. Use Reconnect when your network is back." + ) + let detail = String(format: detailFormat, configuration.displayTarget) + publishDaemonStatus(.unavailable, detail: detail) + publishState(.suspended, detail: detail) + } + + private static func debugDescription(for outcome: WorkspaceRemoteHostProbeOutcome) -> String { + switch outcome { + case .reachable: + return "reachable" + case .unreachable(let reason): + return "unreachable(\(debugLogSnippet(reason, limit: 80)))" + case .indeterminate: + return "indeterminate" + } + } + private func publishState(_ state: WorkspaceRemoteConnectionState, detail: String?) { let controllerID = self.controllerID DispatchQueue.main.async { [weak workspace] in @@ -9802,6 +9895,9 @@ enum WorkspaceRemoteConnectionState: String, Equatable { case reconnecting case connected case error + /// Automatic reconnect halted because the host stayed unreachable; the + /// user reconnects manually (sidebar Reconnect, context menu, CLI). + case suspended } enum WorkspaceRemoteDaemonState: String { @@ -13953,7 +14049,8 @@ final class Workspace: Identifiable, ObservableObject { if remoteConnectionState == .error || remoteDaemonStatus.state == .error || remoteConnectionState == .connecting || - remoteConnectionState == .reconnecting { + remoteConnectionState == .reconnecting || + remoteConnectionState == .suspended { return } disconnectRemoteConnection(clearConfiguration: true) @@ -14178,6 +14275,44 @@ final class Workspace: Identifiable, ObservableObject { remoteConnectionDetail = detail applyBrowserRemoteWorkspaceStatusToPanels() + if state == .suspended { + let entryDetail = trimmedDetail ?? "" + let entryValue = String( + format: String( + localized: "remote.statusEntry.suspended", + defaultValue: "SSH reconnect paused (%@): %@" + ), + locale: .current, + target, + entryDetail + ) + statusEntries[Self.remoteErrorStatusKey] = SidebarStatusEntry( + key: Self.remoteErrorStatusKey, + value: entryValue, + icon: "pause.circle", + color: nil, + timestamp: Date() + ) + let fingerprint = "suspended:\(entryDetail)" + if remoteLastErrorFingerprint != fingerprint { + remoteLastErrorFingerprint = fingerprint + appendSidebarLog(message: entryValue, level: .warning, source: "remote") + AppDelegate.shared?.notificationStore?.addNotification( + tabId: id, + surfaceId: nil, + title: String( + localized: "remote.notification.suspendedTitle", + defaultValue: "SSH Reconnect Paused" + ), + subtitle: target, + body: entryDetail, + cooldownKey: remoteNotificationCooldownKey(target: target), + cooldownInterval: Self.remoteNotificationCooldown + ) + } + return + } + if let trimmedDetail, !trimmedDetail.isEmpty, (state == .error || proxyOnlyError) { let statusPrefix = proxyOnlyError ? "Remote proxy unavailable" : "SSH error" let statusIcon = proxyOnlyError ? "exclamationmark.triangle.fill" : "network.slash" diff --git a/Sources/WorkspaceRemoteHostProbeOutcome.swift b/Sources/WorkspaceRemoteHostProbeOutcome.swift new file mode 100644 index 000000000000..a62772cfcaf2 --- /dev/null +++ b/Sources/WorkspaceRemoteHostProbeOutcome.swift @@ -0,0 +1,17 @@ +import Foundation + +/// Outcome of a quick reachability probe against the SSH endpoint of a remote +/// workspace, used by the auto-reconnect loop to decide whether retrying makes +/// sense at all. +enum WorkspaceRemoteHostProbeOutcome: Equatable, Sendable { + /// A TCP connection to the resolved SSH endpoint succeeded. + case reachable + /// The endpoint could not be reached (DNS failure, connection refused, + /// timeout, or no route). The associated reason is a short human-readable + /// detail for logs and connection-state messages. + case unreachable(reason: String) + /// Reachability could not be determined (for example ProxyCommand-based + /// transports that cannot be probed with a direct TCP connection). The + /// policy must never suspend the reconnect loop based on this outcome. + case indeterminate +} diff --git a/Sources/WorkspaceRemoteHostReachabilityProbe.swift b/Sources/WorkspaceRemoteHostReachabilityProbe.swift new file mode 100644 index 000000000000..2644f89ae490 --- /dev/null +++ b/Sources/WorkspaceRemoteHostReachabilityProbe.swift @@ -0,0 +1,273 @@ +import Foundation +import Network + +/// Quick reachability probe for the SSH endpoint behind a remote workspace. +/// +/// The auto-reconnect loop uses this to distinguish "the host is temporarily +/// failing" (keep retrying with backoff) from "the host cannot be reached at +/// all" (suspend the loop and wait for a manual reconnect, see +/// https://github.com/manaflow-ai/cmux/issues/5734). +/// +/// The probe resolves the effective endpoint with `ssh -G` so ~/.ssh/config +/// aliases, `HostName` overrides, and `ProxyJump` hops are honored. Transports +/// that cannot be probed with a direct TCP connection (`ProxyCommand`) report +/// `.indeterminate`, which the policy treats like a reachable host so the +/// loop never suspends on a guess. +enum WorkspaceRemoteHostReachabilityProbe { + struct Endpoint: Equatable { + let host: String + let port: Int + } + + static let defaultTimeout: TimeInterval = 2.5 + private static let sshResolveTimeout: TimeInterval = 3.0 + private static let probeQueue = DispatchQueue( + label: "com.cmux.remote-host-reachability", + qos: .utility + ) + + /// Probe the SSH endpoint for `destination`. The completion runs on an + /// arbitrary queue; callers hop back to their own queue. + static func probe( + destination: String, + port: Int?, + identityFile: String?, + sshOptions: [String], + timeout: TimeInterval = defaultTimeout, + completion: @escaping (WorkspaceRemoteHostProbeOutcome) -> Void + ) { + // Endpoint resolution shells out to `ssh -G` and can block for up to + // its timeout; run it on the concurrent utility pool so simultaneous + // probes from multiple workspaces don't serialize behind it (the + // serial probeQueue is reserved for NWConnection callbacks). + DispatchQueue.global(qos: .utility).async { + guard let endpoint = resolveEndpoint( + destination: destination, + port: port, + identityFile: identityFile, + sshOptions: sshOptions + ) else { + completion(.indeterminate) + return + } + probeTCP( + host: endpoint.host, + port: endpoint.port, + timeout: timeout, + completion: completion + ) + } + } + + /// Resolve the effective `(host, port)` for an SSH destination using + /// `ssh -G`. Returns nil when the endpoint cannot be probed directly + /// (ProxyCommand transports, unparsable output, or resolution failure). + /// + /// `sshConfigFile` is a test seam: passing a path (such as `/dev/null`) + /// pins resolution to that config via `ssh -F`, keeping tests hermetic + /// against the ambient `~/.ssh/config`. Production callers leave it nil + /// so the user's real config (aliases, HostName, ProxyJump) is honored. + static func resolveEndpoint( + destination: String, + port: Int?, + identityFile: String?, + sshOptions: [String], + sshConfigFile: String? = nil + ) -> Endpoint? { + let trimmedDestination = destination.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedDestination.isEmpty, !trimmedDestination.hasPrefix("-") else { return nil } + + var arguments = ["-G"] + if let sshConfigFile, !sshConfigFile.isEmpty { + arguments += ["-F", sshConfigFile] + } + if let port, port > 0 { + arguments += ["-p", String(port)] + } + if let identityFile, + !identityFile.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + arguments += ["-i", identityFile] + } + for option in sshOptions { + let trimmed = option.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { continue } + arguments += ["-o", trimmed] + } + arguments.append(trimmedDestination) + + guard let output = runSSHConfigResolution(arguments: arguments) else { return nil } + let resolved = parseSSHConfigOutput(output) + + if let proxyJump = resolved.proxyJump { + // Reachability of the first hop is the right signal for "can this + // network reach the SSH entry point at all". + guard let jump = parseJumpSpec(proxyJump) else { return nil } + var jumpArguments = ["-G"] + if let sshConfigFile, !sshConfigFile.isEmpty { + jumpArguments += ["-F", sshConfigFile] + } + jumpArguments.append(jump.destination) + guard let jumpOutput = runSSHConfigResolution(arguments: jumpArguments) else { + return nil + } + let jumpResolved = parseSSHConfigOutput(jumpOutput) + guard jumpResolved.proxyCommand == nil, jumpResolved.proxyJump == nil else { return nil } + guard let host = jumpResolved.hostName ?? jump.host else { return nil } + let jumpPort = jump.port ?? jumpResolved.port ?? 22 + return Endpoint(host: host, port: jumpPort) + } + + guard resolved.proxyCommand == nil else { return nil } + guard let host = resolved.hostName else { return nil } + return Endpoint(host: host, port: resolved.port ?? port ?? 22) + } + + struct ResolvedSSHConfig: Equatable { + var hostName: String? + var port: Int? + var proxyJump: String? + var proxyCommand: String? + } + + /// Parse the key/value lines printed by `ssh -G`. + static func parseSSHConfigOutput(_ output: String) -> ResolvedSSHConfig { + var resolved = ResolvedSSHConfig() + for line in output.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard let spaceIndex = trimmed.firstIndex(of: " ") else { continue } + let key = trimmed[.. JumpSpec? { + guard let firstHop = spec.split(separator: ",").first else { return nil } + var hop = firstHop.trimmingCharacters(in: .whitespaces) + guard !hop.isEmpty else { return nil } + + var user: String? + if let atIndex = hop.lastIndex(of: "@") { + user = String(hop[.. String? { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") + process.arguments = arguments + let stdout = Pipe() + process.standardOutput = stdout + process.standardError = Pipe() + do { + try process.run() + } catch { + return nil + } + let deadline = Date().addingTimeInterval(sshResolveTimeout) + while process.isRunning, Date() < deadline { + usleep(20_000) + } + if process.isRunning { + process.terminate() + return nil + } + guard process.terminationStatus == 0 else { return nil } + let data = stdout.fileHandleForReading.readDataToEndOfFile() + return String(data: data, encoding: .utf8) + } + + /// Attempt a TCP connection to `host:port`, reporting `.reachable` on a + /// successful handshake and `.unreachable` on refusal, timeout, DNS + /// failure, or no route. + static func probeTCP( + host: String, + port: Int, + timeout: TimeInterval = defaultTimeout, + completion: @escaping (WorkspaceRemoteHostProbeOutcome) -> Void + ) { + guard port > 0, port <= 65_535, let nwPort = NWEndpoint.Port(rawValue: UInt16(port)) else { + completion(.indeterminate) + return + } + let connection = NWConnection(host: NWEndpoint.Host(host), port: nwPort, using: .tcp) + // Both finish paths (state updates via `connection.start(queue:)` and + // the timeout below) run on the serial `probeQueue`, so the + // first-finisher latch needs no separate synchronization. + var finished = false + func finish(_ outcome: WorkspaceRemoteHostProbeOutcome) { + guard !finished else { return } + finished = true + // NWConnection retains its handler and the handler's context + // captures the connection; clear it before canceling so each + // backoff-retry probe doesn't leak a connection. + connection.stateUpdateHandler = nil + connection.cancel() + completion(outcome) + } + connection.stateUpdateHandler = { state in + switch state { + case .ready: + finish(.reachable) + case .failed(let error): + finish(.unreachable(reason: error.localizedDescription)) + case .waiting(let error): + // NWConnection parks refused/no-route attempts in `.waiting` + // hoping for a better path; for a quick probe that means the + // endpoint is not reachable right now. + finish(.unreachable(reason: error.localizedDescription)) + default: + break + } + } + connection.start(queue: probeQueue) + probeQueue.asyncAfter(deadline: .now() + timeout) { + finish(.unreachable(reason: "probe timed out after \(Int(timeout.rounded()))s")) + } + } +} diff --git a/Sources/WorkspaceRemoteReconnectPolicy.swift b/Sources/WorkspaceRemoteReconnectPolicy.swift new file mode 100644 index 000000000000..7afb1dc19957 --- /dev/null +++ b/Sources/WorkspaceRemoteReconnectPolicy.swift @@ -0,0 +1,45 @@ +import Foundation + +/// Pure decision logic for the SSH remote workspace auto-reconnect loop +/// (https://github.com/manaflow-ai/cmux/issues/5734). +/// +/// While the host stays reachable the loop keeps its existing exponential +/// backoff behavior. Once consecutive reachability probes keep failing, the +/// loop should suspend instead of retrying indefinitely, leaving the user in +/// control of when reconnection happens. +enum WorkspaceRemoteReconnectPolicy { + enum Decision: Equatable, Sendable { + /// Keep the scheduled backoff retry armed. + case scheduleRetry + /// Halt the automatic reconnect loop and wait for a manual reconnect. + case suspend + } + + struct Evaluation: Equatable, Sendable { + /// Consecutive failed probes after accounting for the latest outcome. + let consecutiveUnreachableProbes: Int + let decision: Decision + } + + /// Number of consecutive unreachable probes after which the automatic + /// reconnect loop suspends. Sized to absorb short network transitions + /// (sleep/wake, wifi handoff) without retrying indefinitely against a + /// host that cannot be reached. + static let maxConsecutiveUnreachableProbes = 3 + + static func evaluate( + outcome: WorkspaceRemoteHostProbeOutcome, + previousConsecutiveUnreachableProbes: Int + ) -> Evaluation { + switch outcome { + case .reachable, .indeterminate: + return Evaluation(consecutiveUnreachableProbes: 0, decision: .scheduleRetry) + case .unreachable: + let streak = previousConsecutiveUnreachableProbes + 1 + return Evaluation( + consecutiveUnreachableProbes: streak, + decision: streak >= Self.maxConsecutiveUnreachableProbes ? .suspend : .scheduleRetry + ) + } + } +} diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 757734c5a88b..47d6f2c3342e 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -709,6 +709,10 @@ DCC935C5F55C1DCB33E25521 /* WorkspacePullRequestSidebarTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */; }; E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */; }; F6100000A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */; }; + E307B0000000000000000002 /* WorkspaceRemoteHostProbeOutcome.swift in Sources */ = {isa = PBXBuildFile; fileRef = E307B0000000000000000001 /* WorkspaceRemoteHostProbeOutcome.swift */; }; + E307A0000000000000000002 /* WorkspaceRemoteHostReachabilityProbe.swift in Sources */ = {isa = PBXBuildFile; fileRef = E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */; }; + E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */; }; + F6357300A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */; }; E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */; }; 619D509BC9B1EA946CBD6A8A /* WorkspaceRuntimeSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9960CC3992F3C44489E7627C /* WorkspaceRuntimeSettings.swift */; }; 5659A0015659A0015659A001 /* WorkspaceSidebarObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5659A0025659A0025659A002 /* WorkspaceSidebarObservation.swift */; }; @@ -1446,6 +1450,10 @@ 14A7DC53B9CA33BE2A421711 /* WorkspacePullRequestSidebarTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspacePullRequestSidebarTests.swift; sourceTree = ""; }; E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConfiguration.swift; sourceTree = ""; }; F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteConnectionTests.swift; sourceTree = ""; }; + E307B0000000000000000001 /* WorkspaceRemoteHostProbeOutcome.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteHostProbeOutcome.swift; sourceTree = ""; }; + E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteHostReachabilityProbe.swift; sourceTree = ""; }; + E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicy.swift; sourceTree = ""; }; + F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteReconnectPolicyTests.swift; sourceTree = ""; }; E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceRemoteSSHBatchCommandBuilder.swift; sourceTree = ""; }; 9960CC3992F3C44489E7627C /* WorkspaceRuntimeSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/WorkspaceRuntimeSettings.swift; sourceTree = ""; }; 5659A0025659A0025659A002 /* WorkspaceSidebarObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceSidebarObservation.swift; sourceTree = ""; }; @@ -1809,6 +1817,9 @@ D7AB3605C10DEF0000000004 /* WorkspaceCloseTabsBatching.swift */, C3744E020000000000000001 /* Workspace+PanelLifecycle.swift */, E30780000000000000000001 /* WorkspaceRemoteConfiguration.swift */, + E30790000000000000000001 /* WorkspaceRemoteReconnectPolicy.swift */, + E307A0000000000000000001 /* WorkspaceRemoteHostReachabilityProbe.swift */, + E307B0000000000000000001 /* WorkspaceRemoteHostProbeOutcome.swift */, D0C0D0C0D0C0D0C0D0C00002 /* RemoteLoopbackProxyAlias.swift */, D0C0D0C0D0C0D0C0D0C00004 /* RemoteLoopbackRuntimeBridge.swift */, E30770000000000000000001 /* WorkspaceRemoteSSHBatchCommandBuilder.swift */, @@ -2155,6 +2166,7 @@ E3309A0A /* AppDelegateEqualizeSplitsShortcutTests.swift */, F6001001A1B2C3D4E5F60718 /* ShortcutUnbindingTests.swift */, F6100001A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift */, + F6357301A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift */, F6355601A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift */, F6120001A1B2C3D4E5F60718 /* WorkspaceSSHFishShellTests.swift */, F7000001A1B2C3D4E5F60718 /* WorkspaceContentViewVisibilityTests.swift */, @@ -3092,6 +3104,9 @@ D0B10004A1B2C3D4E5F60001 /* WorkspacePortalPaneDrop.swift in Sources */, 1A1B2C3D4E5F607180000001 /* WorkspacePromptSubmit.swift in Sources */, E30780000000000000000002 /* WorkspaceRemoteConfiguration.swift in Sources */, + E307B0000000000000000002 /* WorkspaceRemoteHostProbeOutcome.swift in Sources */, + E307A0000000000000000002 /* WorkspaceRemoteHostReachabilityProbe.swift in Sources */, + E30790000000000000000002 /* WorkspaceRemoteReconnectPolicy.swift in Sources */, E30770000000000000000002 /* WorkspaceRemoteSSHBatchCommandBuilder.swift in Sources */, 619D509BC9B1EA946CBD6A8A /* WorkspaceRuntimeSettings.swift in Sources */, 5659A0015659A0015659A001 /* WorkspaceSidebarObservation.swift in Sources */, @@ -3368,6 +3383,7 @@ 1A1B2C3D4E5F607180000003 /* WorkspacePromptSubmitTests.swift in Sources */, DCC935C5F55C1DCB33E25521 /* WorkspacePullRequestSidebarTests.swift in Sources */, F6100000A1B2C3D4E5F60718 /* WorkspaceRemoteConnectionTests.swift in Sources */, + F6357300A1B2C3D4E5F60718 /* WorkspaceRemoteReconnectPolicyTests.swift in Sources */, 5659A0035659A0035659A003 /* WorkspaceSidebarObservationTests.swift in Sources */, 6B524A0CB34FD46A771335AB /* WorkspaceSplitStartupCommandTests.swift in Sources */, F6120000A1B2C3D4E5F60718 /* WorkspaceSSHFishShellTests.swift in Sources */, diff --git a/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift b/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift index 174330600b14..f663e6616c1e 100644 --- a/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift +++ b/cmuxTests/SidebarWorkspaceSnapshotRefreshPolicyTests.swift @@ -106,6 +106,7 @@ final class SidebarWorkspaceSnapshotRefreshPolicyTests: XCTestCase { remoteWorkspaceSidebarText: nil, remoteConnectionStatusText: remoteConnectionStatusText, remoteStateHelpText: "", + showsRemoteReconnectAffordance: false, copyableSidebarSSHError: nil, latestConversationMessage: latestConversationMessage, metadataEntries: [], diff --git a/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift new file mode 100644 index 000000000000..abe888733451 --- /dev/null +++ b/cmuxTests/WorkspaceRemoteReconnectPolicyTests.swift @@ -0,0 +1,271 @@ +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +// Regression coverage for https://github.com/manaflow-ai/cmux/issues/5734: +// the SSH remote auto-reconnect loop must stop retrying once the host stays +// unreachable, instead of retrying indefinitely, so the user controls when +// reconnection happens. +@Suite("Workspace remote reconnect policy") +struct WorkspaceRemoteReconnectPolicyTests { + private func evaluate( + _ outcome: WorkspaceRemoteHostProbeOutcome, + previous: Int + ) -> WorkspaceRemoteReconnectPolicy.Evaluation { + WorkspaceRemoteReconnectPolicy.evaluate( + outcome: outcome, + previousConsecutiveUnreachableProbes: previous + ) + } + + @Test("Reachable host keeps the existing backoff retry loop") + func reachableHostKeepsRetrying() { + for previous in [0, 1, WorkspaceRemoteReconnectPolicy.maxConsecutiveUnreachableProbes] { + let evaluation = evaluate(.reachable, previous: previous) + #expect(evaluation.decision == .scheduleRetry) + #expect(evaluation.consecutiveUnreachableProbes == 0) + } + } + + @Test("Indeterminate probes keep retrying and reset the unreachable streak") + func indeterminateProbeKeepsRetrying() { + for previous in [0, 1, WorkspaceRemoteReconnectPolicy.maxConsecutiveUnreachableProbes] { + let evaluation = evaluate(.indeterminate, previous: previous) + #expect(evaluation.decision == .scheduleRetry) + #expect(evaluation.consecutiveUnreachableProbes == 0) + } + } + + @Test("Unreachable probes below the threshold keep retrying") + func unreachableBelowThresholdKeepsRetrying() { + for previous in 0..<(WorkspaceRemoteReconnectPolicy.maxConsecutiveUnreachableProbes - 1) { + let evaluation = evaluate(.unreachable(reason: "connection refused"), previous: previous) + #expect(evaluation.decision == .scheduleRetry) + #expect(evaluation.consecutiveUnreachableProbes == previous + 1) + } + } + + @Test("Reconnect loop suspends once the host stays unreachable") + func suspendsAtUnreachableThreshold() { + var streak = 0 + var decisions: [WorkspaceRemoteReconnectPolicy.Decision] = [] + for _ in 0.. WorkspaceRemoteHostProbeOutcome { + await withCheckedContinuation { continuation in + WorkspaceRemoteHostReachabilityProbe.probeTCP( + host: host, + port: port, + timeout: 3.0 + ) { outcome in + continuation.resume(returning: outcome) + } + } + } +} + +/// Minimal loopback TCP listener for probe tests. +private final class BlockingTCPListener { + let port: Int + private var fd: Int32 + + init() throws { + let socketFD = socket(AF_INET, SOCK_STREAM, 0) + guard socketFD >= 0 else { throw POSIXError(.EMFILE) } + var reuse: Int32 = 1 + setsockopt(socketFD, SOL_SOCKET, SO_REUSEADDR, &reuse, socklen_t(MemoryLayout.size)) + var addr = sockaddr_in() + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = 0 + addr.sin_addr.s_addr = inet_addr("127.0.0.1") + let bindResult = withUnsafePointer(to: &addr) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + bind(socketFD, sockaddrPointer, socklen_t(MemoryLayout.size)) + } + } + guard bindResult == 0, listen(socketFD, 4) == 0 else { + Darwin.close(socketFD) + throw POSIXError(.EADDRINUSE) + } + var boundAddr = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let nameResult = withUnsafeMutablePointer(to: &boundAddr) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + getsockname(socketFD, sockaddrPointer, &length) + } + } + guard nameResult == 0 else { + Darwin.close(socketFD) + throw POSIXError(.EADDRNOTAVAIL) + } + fd = socketFD + port = Int(UInt16(bigEndian: boundAddr.sin_port)) + } + + func close() { + if fd >= 0 { + Darwin.close(fd) + fd = -1 + } + } + + deinit { + close() + } +} diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 4945c57cfcbd..b64990a648f2 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -97,6 +97,7 @@ Environment: | `reorder-workspace` | Reorder a workspace inside a window. | | `reorder-workspaces` | Atomically reorder workspaces inside pinned and unpinned groups. | | `workspace-action` | Run workspace context-menu actions from the CLI. | +| `workspace` | Namespace for workspace verbs: `list`, `create`, `close`, `rename`, `select`, `reconnect`, `disconnect`, `group`. `workspace reconnect` manually reconnects a remote (SSH) workspace — including one whose automatic reconnect suspended because the host was unreachable — and `workspace disconnect` stops its remote connection. Both accept a positional workspace handle or `--workspace `, defaulting to the caller's workspace, then the selected one. | | `move-tab-to-new-workspace` | Move a tab or surface into a newly created workspace. | | `list-workspaces` | List workspaces. | | `new-workspace` | Create a workspace, optionally with cwd, command, description, and layout. |