diff --git a/Sources/RemoteTmuxController.swift b/Sources/RemoteTmuxController.swift index d7140d2987f1..14d2d63849c4 100644 --- a/Sources/RemoteTmuxController.swift +++ b/Sources/RemoteTmuxController.swift @@ -247,7 +247,7 @@ final class RemoteTmuxController { return nil } catch let error as RemoteTmuxError { if case .commandFailed(_, let stderr) = error, - RemoteTmuxSSHTransport.indicatesAuthRequired(stderr) { + RemoteTmuxSSHTransport.indicatesInteractiveRetryWillHelp(stderr) { return host.interactiveAuthInvocation() } throw error @@ -259,7 +259,7 @@ final class RemoteTmuxController { result: RemoteTmuxCommandResult ) -> [String]? { guard !result.succeeded, - RemoteTmuxSSHTransport.indicatesAuthRequired(result.stderr) else { + RemoteTmuxSSHTransport.indicatesInteractiveRetryWillHelp(result.stderr) else { return nil } return host.interactiveAuthInvocation() @@ -339,20 +339,20 @@ final class RemoteTmuxController { // prompt). A key/agent host — or one with an already-live master — succeeds // here and mirrors directly, with no interactive step, so it also works from // non-tty callers (scripts). A host that needs interactive auth fails here - // (BatchMode can't prompt); classify that and hand back the interactive - // `ssh` argv so the `cmux ssh-tmux` CLI authenticates in the user's terminal - // and retries — the retry then rides the now-open master. `transport.run()` - // creates the control-socket dir, so the returned auth `ssh` can open the - // master. No window has been created yet — nothing to tear down here. Both - // discovery calls (including the create-then-relist for an empty server) are - // inside the catch so an auth failure on any preflight/discovery command is - // classified uniformly. + // (BatchMode can't prompt); classify recoverable stderr via + // ``RemoteTmuxSSHTransport/indicatesInteractiveRetryWillHelp`` and hand back + // the interactive `ssh` argv so the `cmux ssh-tmux` CLI authenticates in the + // user's terminal and retries on the now-open master. `transport.run()` creates + // the control-socket dir, so the returned auth `ssh` can open the master. No + // window has been created yet — nothing to tear down here. Both discovery calls + // (including the create-then-relist for an empty server) are inside the catch so + // a recoverable failure on any preflight/discovery command is classified uniformly. let sessions: [RemoteTmuxSession] do { sessions = try await transport(for: host).discoverMirrorSessions(createIfEmpty: true) } catch let error as RemoteTmuxError { if case .commandFailed(_, let stderr) = error, - RemoteTmuxSSHTransport.indicatesAuthRequired(stderr) { + RemoteTmuxSSHTransport.indicatesInteractiveRetryWillHelp(stderr) { return .authRequired(sshArgv: host.interactiveAuthInvocation()) } throw error diff --git a/Sources/RemoteTmuxError.swift b/Sources/RemoteTmuxError.swift index 04f8e559080a..f2d1bed4173a 100644 --- a/Sources/RemoteTmuxError.swift +++ b/Sources/RemoteTmuxError.swift @@ -32,7 +32,8 @@ extension RemoteTmuxError { /// so a noisy or hostile remote can't inject control bytes or unbounded output into /// our error bodies. Only the rendered `message` is sanitized — the stored /// associated `stderr`/`detail` are left untouched for the stderr-classification - /// paths that pattern-match them (`indicatesNoServer`, `indicatesAuthRequired`). + /// paths that pattern-match them (`indicatesNoServer`, `indicatesAuthRequired`, + /// `indicatesProxyCommandTransportClosed`). var message: String { switch self { case let .commandFailed(exitCode, stderr): diff --git a/Sources/RemoteTmuxSSHTransport+InteractiveRetry.swift b/Sources/RemoteTmuxSSHTransport+InteractiveRetry.swift new file mode 100644 index 000000000000..2e6fe04377be --- /dev/null +++ b/Sources/RemoteTmuxSSHTransport+InteractiveRetry.swift @@ -0,0 +1,56 @@ +extension RemoteTmuxSSHTransport { + /// Whether a failed `BatchMode=yes` connect failed because the local + /// `ProxyCommand` closed the transport *silently* before SSH could surface + /// an explicit auth error string. + /// + /// A `ProxyCommand` with its own pre-handshake authentication or 2FA leg + /// can silently abort under BatchMode because it has no tty to prompt on. + /// An interactive retry lets that prompt surface. The match is anchored to + /// OpenSSH's pipe-transport placeholders (`to UNKNOWN port 65535`, + /// `by UNKNOWN port 65535`) and suppressed when stderr also carries a + /// diagnostic marker for a non-recoverable proxy failure. + static func indicatesProxyCommandTransportClosed(_ stderr: String) -> Bool { + let lowered = stderr.lowercased() + let hasProxyPlaceholder = lowered.contains("to unknown port 65535") + || lowered.contains("by unknown port 65535") + guard hasProxyPlaceholder else { return false } + return !Self.nonRecoverableProxyMarkers.contains(where: { lowered.contains($0) }) + } + + /// Lowercase substrings that indicate a `ProxyCommand` / `ProxyJump` + /// closure was not silent, so an interactive ssh retry will not help. + private static let nonRecoverableProxyMarkers: [String] = [ + "connect failed:", // ssh -W target connection refused/timeout + ": open failed:", // channel N: open failed: ... + "stdio forwarding failed", // ProxyJump -W teardown + "port forwarding failed", + "connection refused", + "no route to host", + "network is unreachable", + "operation timed out", // BSD/macOS TCP connect timeout + "connection timed out", // Linux TCP connect timeout (nc / OpenSSH) + "could not resolve hostname", // OpenSSH DNS-resolution wrapper (all OSes) + "name or service not known", // Linux getaddrinfo NXDOMAIN + "nodename nor servname provided", // BSD/macOS getaddrinfo NXDOMAIN (e.g. ProxyCommand `nc`) + "temporary failure in name resolution", + "kex_exchange_identification:", // target spoke no SSH / closed during key exchange + "ssh_exchange_identification:", // target closed during banner exchange + "command not found", // bash/zsh: ProxyCommand binary missing + ": not found", // dash/busybox sh: ProxyCommand binary missing + "no such file or directory", // shell: ProxyCommand path does not exist + "exec format error", // shell: ProxyCommand binary for wrong architecture + ] + + /// Convenience predicate composing the recovery rule the controller's + /// BatchMode-discovery catch sites share: a failure where re-running ssh + /// interactively will open the shared master and let the next batch probe + /// succeed. + /// + /// All routing sites in ``RemoteTmuxController`` go through one name so a + /// future recovery signal does not silently regress any catch site that + /// spelled out only one constituent predicate. + static func indicatesInteractiveRetryWillHelp(_ stderr: String) -> Bool { + indicatesAuthRequired(stderr) + || indicatesProxyCommandTransportClosed(stderr) + } +} diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 8efd7d01663d..dd9f6338c557 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -809,6 +809,7 @@ 740FC5FDE4E13EAA440872C3 /* RemoteTmuxPaneHeader.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6F225C1630E640224D11E09 /* RemoteTmuxPaneHeader.swift */; }; 761639EDDD6C40883902D781 /* RemoteTmuxPostAttachAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE24906539C19AB10E4C1C71 /* RemoteTmuxPostAttachAction.swift */; }; A4C6F928D7E14B2AA924B47C /* RemoteTmuxProcessCancellation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 59B80C0A6FC64A59BD274E1E /* RemoteTmuxProcessCancellation.swift */; }; + 7020C0DE7020C0DE7020A002 /* RemoteTmuxProxyTransportRetryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7020C0DE7020C0DE7020A001 /* RemoteTmuxProxyTransportRetryTests.swift */; }; C9BA962E6CB31C9FD62505BF /* RemoteTmuxRawLayoutParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5294B6A6144B22BD3FEB8160 /* RemoteTmuxRawLayoutParser.swift */; }; A42DAD21BD502098E446999C /* RemoteTmuxSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = 541716FF0D90CB510EE3004A /* RemoteTmuxSession.swift */; }; 0A2A2FA17CD71DA5B4495DEE /* RemoteTmuxSessionEndAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9E3E94F6022485BB12789444 /* RemoteTmuxSessionEndAction.swift */; }; @@ -818,6 +819,7 @@ 1255599FA91128E5925D3983 /* RemoteTmuxSessionMirror.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7D5CC45BBE157F3EF880936 /* RemoteTmuxSessionMirror.swift */; }; 3AC9AB9046E742B93726A501 /* RemoteTmuxSessionRenameTitleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3F704ED4F177122D7DCD0B01 /* RemoteTmuxSessionRenameTitleTests.swift */; }; 9A5CF3AA2E6462E77144F9E6 /* RemoteTmuxSessionSnapshotTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 31FFFE3EFCDCCD6BDF405370 /* RemoteTmuxSessionSnapshotTests.swift */; }; + 7020C0DE7020C0DE7020B002 /* RemoteTmuxSSHTransport+InteractiveRetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7020C0DE7020C0DE7020B001 /* RemoteTmuxSSHTransport+InteractiveRetry.swift */; }; 6D3C19C2014FF9C754358EFF /* RemoteTmuxSSHTransport.swift in Sources */ = {isa = PBXBuildFile; fileRef = E8F5DB43CBC1B7DF31B7A0C9 /* RemoteTmuxSSHTransport.swift */; }; 4E9111628FAF490FBC51D350 /* RemoteTmuxStdoutPipeReader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CA9E692220B4E0C91499A05 /* RemoteTmuxStdoutPipeReader.swift */; }; 0E17C0DE0E17C0DE0E17C002 /* RemoteTmuxTransportRegistry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0E17C0DE0E17C0DE0E17C001 /* RemoteTmuxTransportRegistry.swift */; }; @@ -2045,6 +2047,7 @@ F6F225C1630E640224D11E09 /* RemoteTmuxPaneHeader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxPaneHeader.swift; sourceTree = ""; }; CE24906539C19AB10E4C1C71 /* RemoteTmuxPostAttachAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxPostAttachAction.swift; sourceTree = ""; }; 59B80C0A6FC64A59BD274E1E /* RemoteTmuxProcessCancellation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxProcessCancellation.swift; sourceTree = ""; }; + 7020C0DE7020C0DE7020A001 /* RemoteTmuxProxyTransportRetryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxProxyTransportRetryTests.swift; sourceTree = ""; }; 5294B6A6144B22BD3FEB8160 /* RemoteTmuxRawLayoutParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxRawLayoutParser.swift; sourceTree = ""; }; 541716FF0D90CB510EE3004A /* RemoteTmuxSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxSession.swift; sourceTree = ""; }; 9E3E94F6022485BB12789444 /* RemoteTmuxSessionEndAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxSessionEndAction.swift; sourceTree = ""; }; @@ -2054,6 +2057,7 @@ D7D5CC45BBE157F3EF880936 /* RemoteTmuxSessionMirror.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxSessionMirror.swift; sourceTree = ""; }; 3F704ED4F177122D7DCD0B01 /* RemoteTmuxSessionRenameTitleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxSessionRenameTitleTests.swift; sourceTree = ""; }; 31FFFE3EFCDCCD6BDF405370 /* RemoteTmuxSessionSnapshotTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxSessionSnapshotTests.swift; sourceTree = ""; }; + 7020C0DE7020C0DE7020B001 /* RemoteTmuxSSHTransport+InteractiveRetry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "RemoteTmuxSSHTransport+InteractiveRetry.swift"; sourceTree = ""; }; E8F5DB43CBC1B7DF31B7A0C9 /* RemoteTmuxSSHTransport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxSSHTransport.swift; sourceTree = ""; }; 5CA9E692220B4E0C91499A05 /* RemoteTmuxStdoutPipeReader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxStdoutPipeReader.swift; sourceTree = ""; }; 0E17C0DE0E17C0DE0E17C001 /* RemoteTmuxTransportRegistry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteTmuxTransportRegistry.swift; sourceTree = ""; }; @@ -3324,6 +3328,7 @@ 0E17C0DE0E17C0DE0E17C001 /* RemoteTmuxTransportRegistry.swift */, 0F17C0DE0F17C0DE0F17C001 /* RemoteTmuxWindowRegistry.swift */, E8F5DB43CBC1B7DF31B7A0C9 /* RemoteTmuxSSHTransport.swift */, + 7020C0DE7020C0DE7020B001 /* RemoteTmuxSSHTransport+InteractiveRetry.swift */, AF44822DC080B2110CFECF88 /* RemoteTmuxError.swift */, E833A06BDA073CA66A46D7FA /* RemoteTmuxSessionListParser.swift */, 541716FF0D90CB510EE3004A /* RemoteTmuxSession.swift */, @@ -3719,6 +3724,7 @@ F11ED7AB0002000200020002 /* RemoteTmuxMirrorNewTabPlacementTests.swift */, FA00C0DE0002BEEF0002CAFE /* RemoteTmuxWindowRegistryTests.swift */, 0A17C0DE0A17C0DE0A17C003 /* RemoteTmuxAuthTests.swift */, + 7020C0DE7020C0DE7020A001 /* RemoteTmuxProxyTransportRetryTests.swift */, 6732BEEF6732BEEF6732B001 /* RemoteTmuxMasterReadinessTests.swift */, 0C7D0CDD0C7D0CDD0C7D0001 /* RemoteTmuxNewWindowCwdTests.swift */, 5F5553CA5553CA5553CA0002 /* RemoteTmuxCapabilitiesTests.swift */, @@ -4703,11 +4709,12 @@ A4C6F928D7E14B2AA924B47C /* RemoteTmuxProcessCancellation.swift in Sources */, C9BA962E6CB31C9FD62505BF /* RemoteTmuxRawLayoutParser.swift in Sources */, A42DAD21BD502098E446999C /* RemoteTmuxSession.swift in Sources */, - 0A2A2FA17CD71DA5B4495DEE /* RemoteTmuxSessionEndAction.swift in Sources */, - 008540053079E1E9B08DF59C /* RemoteTmuxSessionListParser.swift in Sources */, - 7F023A000000000000007024 /* RemoteTmuxSessionMirror+Helpers.swift in Sources */, - 1255599FA91128E5925D3983 /* RemoteTmuxSessionMirror.swift in Sources */, - 6D3C19C2014FF9C754358EFF /* RemoteTmuxSSHTransport.swift in Sources */, + 0A2A2FA17CD71DA5B4495DEE /* RemoteTmuxSessionEndAction.swift in Sources */, + 008540053079E1E9B08DF59C /* RemoteTmuxSessionListParser.swift in Sources */, + 7F023A000000000000007024 /* RemoteTmuxSessionMirror+Helpers.swift in Sources */, + 1255599FA91128E5925D3983 /* RemoteTmuxSessionMirror.swift in Sources */, + 7020C0DE7020C0DE7020B002 /* RemoteTmuxSSHTransport+InteractiveRetry.swift in Sources */, + 6D3C19C2014FF9C754358EFF /* RemoteTmuxSSHTransport.swift in Sources */, 4E9111628FAF490FBC51D350 /* RemoteTmuxStdoutPipeReader.swift in Sources */, 0E17C0DE0E17C0DE0E17C002 /* RemoteTmuxTransportRegistry.swift in Sources */, 0A17C0DE0A17C0DE0A17C902 /* RemoteTmuxVersion.swift in Sources */, @@ -5288,6 +5295,7 @@ F11ED7AB0001000100010001 /* RemoteTmuxMirrorNewTabPlacementTests.swift in Sources */, D4F8A2E61C5B39707A8E6F12 /* RemoteTmuxMirrorSplitRoutingTests.swift in Sources */, 0C7D0CDD0C7D0CDD0C7D0002 /* RemoteTmuxNewWindowCwdTests.swift in Sources */, + 7020C0DE7020C0DE7020A002 /* RemoteTmuxProxyTransportRetryTests.swift in Sources */, 3AC9AB9046E742B93726A405 /* RemoteTmuxSessionListParserTests.swift in Sources */, 3AC9AB9046E742B93726A501 /* RemoteTmuxSessionRenameTitleTests.swift in Sources */, 9A5CF3AA2E6462E77144F9E6 /* RemoteTmuxSessionSnapshotTests.swift in Sources */, diff --git a/cmuxTests/RemoteTmuxProxyTransportRetryTests.swift b/cmuxTests/RemoteTmuxProxyTransportRetryTests.swift new file mode 100644 index 000000000000..4dbc14264d48 --- /dev/null +++ b/cmuxTests/RemoteTmuxProxyTransportRetryTests.swift @@ -0,0 +1,93 @@ +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +/// Tests for the proxy-transport stderr classifier used by remote-tmux +/// interactive retry routing. +@Suite struct RemoteTmuxProxyTransportRetryTests { + @Test(arguments: [ + "Connection closed by UNKNOWN port 65535", + "ssh_dispatch_run_fatal: Connection to UNKNOWN port 65535: Broken pipe", + "Connection closed by UnKnOwN port 65535", + ]) + func classifiesSilentProxyCommandClosures(_ stderr: String) { + #expect(RemoteTmuxSSHTransport.indicatesProxyCommandTransportClosed(stderr)) + } + + @Test(arguments: [ + "channel 0: open failed: connect failed: Connection refused\nstdio forwarding failed\nConnection closed by UNKNOWN port 65535", + "connect failed: Connection refused\nConnection closed by UNKNOWN port 65535", + "stdio forwarding failed\nssh_exchange_identification: Connection closed by remote host\nConnection closed by UNKNOWN port 65535", + "kex_exchange_identification: Connection closed by remote host\nConnection closed by UNKNOWN port 65535", + "ssh: Could not resolve hostname inner.invalid: nodename nor servname provided\nConnection closed by UNKNOWN port 65535", + "nc: getaddrinfo: name or service not known\nConnection closed by UNKNOWN port 65535", + "nc: getaddrinfo: nodename nor servname provided, or not known\nConnection closed by UNKNOWN port 65535", + "channel 1: open failed: administratively prohibited: open failed\nConnection closed by UNKNOWN port 65535", + "nc: connect to inner.invalid port 22 (tcp) failed: Connection timed out\nConnection closed by UNKNOWN port 65535", + "ssh_exchange_identification: Connection closed by remote host\nConnection closed by UNKNOWN port 65535", + "zsh:1: command not found: corp-proxy\nConnection closed by UNKNOWN port 65535", + "bash: line 1: corp-proxy: command not found\nConnection closed by UNKNOWN port 65535", + "sh: 1: corp-proxy: not found\nConnection closed by UNKNOWN port 65535", + "zsh:1: no such file or directory: /opt/corp/proxy\nConnection closed by UNKNOWN port 65535", + "bash: line 1: /opt/corp/proxy: No such file or directory\nConnection closed by UNKNOWN port 65535", + "bash: line 1: /opt/corp/proxy: cannot execute binary file: Exec format error\nConnection closed by UNKNOWN port 65535", + ]) + func doesNotClassifyExplainedProxyClosures(_ stderr: String) { + #expect(!RemoteTmuxSSHTransport.indicatesProxyCommandTransportClosed(stderr)) + } + + @Test(arguments: [ + "MOTD: lab name is UNKNOWN port 65535 status board", + "remote warning: process listening on port 65535 with unknown owner", + "user note: 'unknown port 65535' is reserved", + ]) + func anchorsProxyClosedMatchToOpenSSHPhrasing(_ stderr: String) { + #expect(!RemoteTmuxSSHTransport.indicatesProxyCommandTransportClosed(stderr)) + } + + @Test(arguments: [ + "ssh: connect to host bad.example.com port 22: Connection refused", + "ssh: connect to host bad.example.com port 2222: Operation timed out", + "Connection closed by 10.0.0.5 port 22", + ]) + func doesNotClassifyRealPortClosuresAsProxyTransport(_ stderr: String) { + #expect(!RemoteTmuxSSHTransport.indicatesProxyCommandTransportClosed(stderr)) + } + + @Test func proxyClosedAndAuthRequiredAreDisjoint() { + let proxyOnly = "ssh_dispatch_run_fatal: Connection to UNKNOWN port 65535: Broken pipe" + #expect(RemoteTmuxSSHTransport.indicatesProxyCommandTransportClosed(proxyOnly)) + #expect(!RemoteTmuxSSHTransport.indicatesAuthRequired(proxyOnly)) + + let authOnly = "user@host: Permission denied (publickey,password)." + #expect(RemoteTmuxSSHTransport.indicatesAuthRequired(authOnly)) + #expect(!RemoteTmuxSSHTransport.indicatesProxyCommandTransportClosed(authOnly)) + } + + @Test(arguments: [ + "user@host: Permission denied (publickey,password).", + "Host key verification failed.", + "Too many authentication failures", + "Connection closed by UNKNOWN port 65535", + "ssh_dispatch_run_fatal: Connection to UNKNOWN port 65535: Broken pipe", + ]) + func composedPredicateFiresForEitherRecoverableSignal(_ stderr: String) { + #expect(RemoteTmuxSSHTransport.indicatesInteractiveRetryWillHelp(stderr)) + } + + @Test(arguments: [ + "no server running on /tmp/tmux-501/default", + "no matching host key type found. their offer: ssh-rsa", + "ssh: connect to host bad.example.com port 22: Connection refused", + "", + "channel 0: open failed: connect failed: Connection refused\nstdio forwarding failed\nConnection closed by UNKNOWN port 65535", + "zsh:1: command not found: corp-proxy\nConnection closed by UNKNOWN port 65535", + ]) + func composedPredicateRejectsNonRecoverableFailures(_ stderr: String) { + #expect(!RemoteTmuxSSHTransport.indicatesInteractiveRetryWillHelp(stderr)) + } +}