From 4c5bf40299cf8455a4db48a5df540ffe2c80a6ba Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 17:24:40 -0700 Subject: [PATCH 01/39] test: keep SSH relay off shared ControlMaster (#8894) --- .../WorkspaceRemoteConnectionTests.swift | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 373b0ffed7ab..6e2b935206f3 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -2222,8 +2222,12 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { } @MainActor - func testPersistentReverseRelayCancelsStaleControlMasterForwardBeforeReusingRelayPort() throws { - let forwardInvoked = DispatchSemaphore(value: 0) + func testPersistentReverseRelayDoesNotAttachToSharedControlMaster() { + let daemonTransportStarted = DispatchSemaphore(value: 0) + let controlMasterTouched = expectation( + description: "workspace-scoped reverse relay must not mutate the host-shared ControlMaster" + ) + controlMasterTouched.isInverted = true let lock = NSLock() var controlOperations: [(command: String, spec: String)] = [] @@ -2239,9 +2243,7 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { lock.lock() controlOperations.append((command: operation, spec: spec)) lock.unlock() - if operation == "forward" { - forwardInvoked.signal() - } + controlMasterTouched.fulfill() return (status: 0, stdout: "", stderr: "") } @@ -2259,6 +2261,7 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { ) } if remoteDaemonServeCommand(command) { + daemonTransportStarted.signal() return ( status: 0, stdout: #"{"id":1,"ok":true,"result":{"name":"cmuxd-remote","version":"dev","capabilities":["proxy.stream.push","pty.session","pty.session.token","pty.write.notification","pty.resize.notification","pty.session.persistent_daemon"]}}"# + "\n", @@ -2272,13 +2275,13 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { workspace.remoteSessionProcessRunnerOverrideForTesting = ScriptedRemoteProcessRunner(script: remoteProcessScript) let config = WorkspaceRemoteConfiguration( - destination: "test@hpc.example", - port: 2222, + destination: "127.0.0.1", + port: 1, identityFile: nil, sshOptions: [ "ControlMaster=auto", "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-\(getuid())-64044-%C", + "ControlPath=/tmp/cmux-ssh-\(getuid())-issue-8894-%C", "StrictHostKeyChecking=accept-new", ], localProxyPort: nil, @@ -2294,18 +2297,15 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { workspace.configureRemoteConnection(config, autoConnect: true) - XCTAssertEqual(forwardInvoked.wait(timeout: .now() + 2), .success) + XCTAssertEqual(daemonTransportStarted.wait(timeout: .now() + 2), .success) + wait(for: [controlMasterTouched], timeout: 1) lock.lock() let operations = controlOperations lock.unlock() - XCTAssertGreaterThanOrEqual(operations.count, 2) - XCTAssertEqual(operations[0].command, "cancel") - XCTAssertEqual(operations[0].spec, "127.0.0.1:64044") - XCTAssertEqual(operations[1].command, "forward") XCTAssertTrue( - operations[1].spec.hasPrefix("127.0.0.1:64044:127.0.0.1:"), - "expected forward to reuse relay port after stale cancel, got \(operations[1].spec)" + operations.isEmpty, + "the relay must use its own app-owned ssh process, got ControlMaster operations: \(operations)" ) } From 54adb51804700dcb8f73a7d933b716f46ae6216b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 17:36:46 -0700 Subject: [PATCH 02/39] fix: give SSH reverse relay an app-owned transport (#8894) --- ...RemoteConfiguration+SSHBatchCommands.swift | 56 -- ...teConfigurationSSHBatchCommandsTests.swift | 43 -- ...SessionCoordinator+RelayProvisioning.swift | 125 ---- ...emoteSessionCoordinator+ReverseRelay.swift | 150 +---- .../Session/RemoteSessionCoordinator.swift | 1 - ...SessionSSHRemoteCommandOverrideTests.swift | 23 + .../WorkspaceRemoteConnectionTests.swift | 550 ------------------ 7 files changed, 31 insertions(+), 917 deletions(-) diff --git a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift index a6eafd2d11a3..f8d16023ef33 100644 --- a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift +++ b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift @@ -54,38 +54,6 @@ extension WorkspaceRemoteConfiguration { ] } - /// `ssh -O ` argv that drives a reverse forward on the - /// configured ControlMaster socket, or `nil` when no usable `ControlPath` - /// option is configured. Argument text is wire/process behavior; do not - /// alter. - public func reverseRelayControlMasterArguments( - controlCommand: String, - forwardSpec: String - ) -> [String]? { - guard let controlPath = firstSSHOptionValue(named: "ControlPath")? - .trimmingCharacters(in: .whitespacesAndNewlines), - !controlPath.isEmpty, - controlPath.lowercased() != "none" else { - return nil - } - - var args = batchSSHArguments() - args += ["-O", controlCommand, "-R", forwardSpec, destination] - return args - } - - /// ``reverseRelayControlMasterArguments(controlCommand:forwardSpec:)`` - /// specialized to `-O cancel` for the relay's remote listen port, or - /// `nil` for a non-positive port. Argument text is wire/process behavior; - /// do not alter. - public func reverseRelayControlMasterCancelArguments(relayPort: Int) -> [String]? { - guard relayPort > 0 else { return nil } - return reverseRelayControlMasterArguments( - controlCommand: "cancel", - forwardSpec: "127.0.0.1:\(relayPort)" - ) - } - // Shared batch-mode `ssh` options: keepalives, BatchMode, no new // ControlMaster (existing ControlPath sockets may be reused), port, // identity, then the configuration's options minus @@ -125,30 +93,6 @@ extension WorkspaceRemoteConfiguration { return !Self.batchSSHControlOptionKeys.contains(key) } } - - // First non-empty value for an option key, scanning forward. This - // deliberately differs from SSHAgentSocketResolver.optionValue(named:in:) - // (which scans in reverse for OpenSSH last-wins semantics): the legacy - // batch builder used first-match and the reverse-relay behavior is pinned - // to it. - private func firstSSHOptionValue(named key: String) -> String? { - let loweredKey = key.lowercased() - for option in Self.trimmedSSHOptions(sshOptions) { - let parts = option.split( - maxSplits: 1, - omittingEmptySubsequences: true, - whereSeparator: { $0 == "=" || $0.isWhitespace } - ) - guard parts.count == 2, parts[0].lowercased() == loweredKey else { - continue - } - let value = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) - if !value.isEmpty { - return value - } - } - return nil - } } extension String { diff --git a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift index 013230108f9c..eed03fa8f2a2 100644 --- a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift +++ b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift @@ -147,47 +147,4 @@ struct WorkspaceRemoteConfigurationSSHBatchCommandsTests { ) } - @Test("reverseRelayControlMasterArguments full argv with a configured ControlPath") - func reverseRelayControlMasterArguments() throws { - let arguments = try #require( - configuration().reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: "127.0.0.1:64007:127.0.0.1:54321" - ) - ) - #expect( - arguments == expectedBatchArguments - + ["-O", "forward", "-R", "127.0.0.1:64007:127.0.0.1:54321", "cmux-macmini"] - ) - } - - @Test("reverseRelayControlMasterCancelArguments full argv uses the remote listen port only") - func reverseRelayControlMasterCancelArguments() throws { - let arguments = try #require( - configuration().reverseRelayControlMasterCancelArguments(relayPort: 64007) - ) - #expect( - arguments == expectedBatchArguments - + ["-O", "cancel", "-R", "127.0.0.1:64007", "cmux-macmini"] - ) - } - - @Test("reverse relay requires a usable ControlPath") - func reverseRelayRequiresControlPath() { - #expect( - configuration(sshOptions: ["StrictHostKeyChecking=accept-new"]) - .reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: "127.0.0.1:64007:127.0.0.1:54321" - ) == nil - ) - #expect( - configuration(sshOptions: ["ControlPath=None"]) - .reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: "127.0.0.1:64007:127.0.0.1:54321" - ) == nil - ) - #expect(configuration().reverseRelayControlMasterCancelArguments(relayPort: 0) == nil) - } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift index 26eac087d9a4..192f5dbfb34e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift @@ -107,131 +107,6 @@ extension RemoteSessionCoordinator { return "[ ! -e \"$slot_file\" ] || exit 64" } - /// Script that kills a stale sshd listener (and its persistent - /// cmuxd-remote children for `persistentDaemonSlot`) still bound to - /// `relayPort`, or `nil` when the inputs cannot be matched safely. - public static func remoteStaleRelayListenerCleanupScript( - relayPort: Int, - persistentDaemonSlot: String? - ) -> String? { - guard relayPort > 0, relayPort <= 65535 else { return nil } - guard let persistentDaemonSlot = normalizedPersistentDaemonSlotForRemoteCleanup(persistentDaemonSlot) else { - return nil - } - - return """ - cmux_stale_relay_listener_cleanup=1 - cmux_relay_port='\(relayPort)' - cmux_persistent_slot=\(persistentDaemonSlot.shellSingleQuoted) - cmux_listener_pids='' - if command -v lsof >/dev/null 2>&1; then - cmux_listener_pids="$(lsof -nP -iTCP:"$cmux_relay_port" -sTCP:LISTEN -Fpn 2>/dev/null | awk -v port="$cmux_relay_port" ' - /^p/ { pid = substr($0, 2); next } - /^n/ { - name = substr($0, 2) - if (pid ~ /^[0-9]+$/ && name ~ ("(^|[^0-9])127[.]0[.]0[.]1:" port "$")) { - seen[pid] = 1 - } - } - END { - for (pid in seen) print pid - } - ')" - fi - [ -n "$cmux_listener_pids" ] || exit 0 - cmux_ps_output="$(ps -axo pid=,ppid=,command= 2>/dev/null || true)" - for cmux_listener_pid in $cmux_listener_pids; do - case "$cmux_listener_pid" in - ''|*[!0-9]*) continue ;; - esac - cmux_listener_command="$(printf '%s\\n' "$cmux_ps_output" | awk -v target="$cmux_listener_pid" '$1 == target { $1 = ""; $2 = ""; sub(/^[[:space:]]+/, ""); print; exit }')" - case "$cmux_listener_command" in - *sshd*|*ssh*) ;; - *) continue ;; - esac - cmux_child_pids="$(printf '%s\\n' "$cmux_ps_output" | awk -v parent="$cmux_listener_pid" -v slot="$cmux_persistent_slot" ' - function clean_token(value) { - gsub(/\\047/, "", value) - gsub(/"/, "", value) - gsub(/\\\\/, "", value) - return value - } - function has_token(target, i) { - for (i = 3; i <= NF; i++) { - if (clean_token($i) == target) return 1 - } - return 0 - } - function next_value(after, i, value) { - for (i = after + 1; i <= NF; i++) { - value = clean_token($i) - if (value != "") return value - } - return "" - } - function has_exact_slot(i, token, value) { - for (i = 3; i <= NF; i++) { - token = clean_token($i) - if (token == "--slot") { - return next_value(i) == slot - } - if (token ~ /^--slot=/) { - value = substr(token, 8) - if (value != "") return value == slot - return next_value(i) == slot - } - } - return 0 - } - $2 == parent && - index($0, "cmuxd-remote") && - has_token("serve") && - has_token("--stdio") && - has_token("--persistent") && - has_exact_slot() && - $1 ~ /^[0-9]+$/ { - print $1 - } - ')" - cmux_cleanup_reason=child - if [ -z "$cmux_child_pids" ]; then - cmux_cleanup_reason=metadata - cmux_metadata_ok=0 - cmux_slot_file="$HOME/.cmux/relay/${cmux_relay_port}.slot" - cmux_metadata_slot_ok=0 - if [ -r "$cmux_slot_file" ]; then - cmux_stored_slot="$(tr -d '\\r\\n' < "$cmux_slot_file")" - [ "$cmux_stored_slot" = "$cmux_persistent_slot" ] && cmux_metadata_slot_ok=1 - fi - if [ "$cmux_metadata_slot_ok" -eq 1 ]; then - cmux_daemon_map="$HOME/.cmux/relay/${cmux_relay_port}.daemon_path" - cmux_auth_file="$HOME/.cmux/relay/${cmux_relay_port}.auth" - if [ -r "$cmux_daemon_map" ]; then - cmux_daemon_path="$(tr -d '\\r\\n' < "$cmux_daemon_map")" - case "$cmux_daemon_path" in - *cmuxd-remote*) cmux_metadata_ok=1 ;; - esac - fi - if [ "$cmux_metadata_ok" -ne 1 ] && [ -r "$cmux_auth_file" ]; then - cmux_auth_payload="$(tr -d '\\r\\n' < "$cmux_auth_file")" - case "$cmux_auth_payload" in - *relay_id*relay_token*) cmux_metadata_ok=1 ;; - esac - fi - fi - [ "$cmux_metadata_ok" -eq 1 ] || continue - fi - kill -TERM "$cmux_listener_pid" $cmux_child_pids 2>/dev/null || true - for cmux_child_pid in $cmux_child_pids; do - kill -0 "$cmux_child_pid" 2>/dev/null && kill -KILL "$cmux_child_pid" 2>/dev/null || true - done - kill -0 "$cmux_listener_pid" 2>/dev/null && kill -KILL "$cmux_listener_pid" 2>/dev/null || true - cmux_child_list="$(printf '%s\\n' "$cmux_child_pids" | tr '\\n' ' ' | sed 's/[[:space:]]*$//')" - printf 'cmux_stale_relay_killed pid=%s children=%s port=%s reason=%s\\n' "$cmux_listener_pid" "$cmux_child_list" "$cmux_relay_port" "$cmux_cleanup_reason" - done - """ - } - static func normalizedPersistentDaemonSlotForRemoteCleanup(_ value: String?) -> String? { guard let value else { return nil } let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index dbac9d7046b2..e4bb1a7dae03 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -6,12 +6,10 @@ public import Foundation nonisolated private let remoteRelayLogger = Logger(subsystem: "com.cmuxterm.app", category: "RemoteRelay") // The reverse CLI relay: a remote `127.0.0.1:` listener forwarded -// back to the local CLI relay server, preferring an `ssh -O forward` on the -// user's existing ControlMaster and falling back to a standalone `ssh -N -R` -// transport. Faithful lift: argv composition, metadata install scripts, -// stderr capture caps, restart cadence (2s), and every debug-log line are -// pinned legacy behavior. The legacy restart `asyncAfter` work item became -// an injected-clock task with a token guard (strictly tighter cancel). +// back to the local CLI relay server by a dedicated `ssh -N -R` process. The +// relay targets an in-process server, so its SSH transport must share the app +// process's lifetime rather than a host-scoped ControlMaster's lifetime. +// Stderr capture caps and restart cadence (2s) are pinned legacy behavior. extension RemoteSessionCoordinator { func startReverseRelayLocked(remotePath: String) { guard !isStopping else { return } @@ -27,7 +25,6 @@ extension RemoteSessionCoordinator { return } guard reverseRelayProcess == nil else { return } - guard reverseRelayControlMasterForwardSpec == nil else { return } cancelReverseRelayRestartLocked() var relayServer: RemoteCLIRelayServer? @@ -44,31 +41,6 @@ extension RemoteSessionCoordinator { relayPort: relayPort, persistentDaemonSlot: configuration.persistentDaemonSlot ) - let forwardSpec = "127.0.0.1:\(relayPort):127.0.0.1:\(localRelayPort)" - - if startReverseRelayViaControlMasterLocked(forwardSpec: forwardSpec, relayPort: relayPort) { - cliRelayServer = relayServer - reverseRelayStderrBuffer = "" - do { - try installRemoteRelayMetadataLocked( - remotePath: remotePath, - relayPort: relayPort, - relayID: relayID, - relayToken: relayToken - ) - } catch { - debugLog("remote.relay.metadata.error \(error.localizedDescription)") - stopReverseRelayLocked() - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) - return - } - recordHeartbeatActivityLocked() - debugLog( - "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + - "target=\(configuration.displayTarget) controlMaster=1" - ) - return - } let process = Process() let stderrPipe = Pipe() @@ -130,7 +102,7 @@ extension RemoteSessionCoordinator { recordHeartbeatActivityLocked() debugLog( "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + - "target=\(configuration.displayTarget) controlMaster=0" + "target=\(configuration.displayTarget) transport=dedicated" ) } catch { debugLog( @@ -229,7 +201,6 @@ extension RemoteSessionCoordinator { reverseRelayProcess.terminate() } reverseRelayProcess = nil - stopReverseRelayViaControlMasterLocked() reverseRelayStderrPipe = nil reverseRelayStderrBuffer = "" cliRelayServer?.stop() @@ -238,10 +209,10 @@ extension RemoteSessionCoordinator { } func reverseRelayArguments(relayPort: Int, localRelayPort: Int) -> [String] { - // Fallback standalone transport when dynamic forwarding through an existing - // control master is unavailable. + // The relay's SSH process is deliberately app-owned. `-S none` also + // protects against a ControlPath inherited from the host's ssh_config. var args: [String] = ["-N", "-T", "-S", "none"] - args += sshCommonArguments(batchMode: true) + args += sshCommonArguments(batchMode: true, dropControlPath: true) args += [ "-o", "ExitOnForwardFailure=yes", "-o", "RequestTTY=no", @@ -251,111 +222,6 @@ extension RemoteSessionCoordinator { return args } - private func startReverseRelayViaControlMasterLocked(forwardSpec: String, relayPort: Int) -> Bool { - guard let arguments = configuration.reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: forwardSpec - ) else { - return false - } - - cancelStaleReverseRelayViaControlMasterLocked(relayPort: relayPort) - do { - var result = try sshExec(arguments: arguments, timeout: 6) - guard result.status == 0 else { - let detail = Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) - ?? "ssh exited \(result.status)" - debugLog("remote.relay.controlmaster.forwardFailed \(detail) \(debugConfigSummary())") - guard cleanupStaleRemoteRelayListenerLocked(relayPort: relayPort) else { - return false - } - - result = try sshExec(arguments: arguments, timeout: 6) - guard result.status == 0 else { - let retryDetail = Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) - ?? "ssh exited \(result.status)" - debugLog("remote.relay.controlmaster.forwardRetryFailed \(retryDetail) \(debugConfigSummary())") - return false - } - reverseRelayControlMasterForwardSpec = forwardSpec - return true - } - reverseRelayControlMasterForwardSpec = forwardSpec - return true - } catch { - debugLog("remote.relay.controlmaster.forwardFailed \(error.localizedDescription) \(debugConfigSummary())") - return false - } - } - - private func cancelStaleReverseRelayViaControlMasterLocked(relayPort: Int) { - guard let arguments = configuration.reverseRelayControlMasterCancelArguments(relayPort: relayPort) else { - return - } - do { - let result = try sshExec(arguments: arguments, timeout: 4) - guard result.status == 0 else { - let detail = Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) - ?? "ssh exited \(result.status)" - debugLog("remote.relay.controlmaster.cancelStaleIgnored \(detail) \(debugConfigSummary())") - return - } - debugLog("remote.relay.controlmaster.cancelStale relayPort=\(relayPort) \(debugConfigSummary())") - } catch { - debugLog("remote.relay.controlmaster.cancelStaleIgnored \(error.localizedDescription) \(debugConfigSummary())") - } - } - - private func cleanupStaleRemoteRelayListenerLocked(relayPort: Int) -> Bool { - guard let script = Self.remoteStaleRelayListenerCleanupScript( - relayPort: relayPort, - persistentDaemonSlot: configuration.persistentDaemonSlot - ) else { - debugLog("remote.relay.remoteListener.cleanupSkipped reason=no-persistent-slot relayPort=\(relayPort)") - return false - } - - let command = "sh -c \(script.shellSingleQuoted)" - do { - let result = try sshExec( - arguments: ["-S", "none"] + sshCommonArguments(batchMode: true, dropControlPath: true) + [ - configuration.destination, - command, - ], - timeout: 8 - ) - guard result.status == 0 else { - let detail = Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) - ?? "ssh exited \(result.status)" - debugLog("remote.relay.remoteListener.cleanupFailed relayPort=\(relayPort) \(detail) \(debugConfigSummary())") - return false - } - - let output = result.stdout.trimmingCharacters(in: .whitespacesAndNewlines) - if output.isEmpty { - debugLog("remote.relay.remoteListener.cleanupNoop relayPort=\(relayPort) \(debugConfigSummary())") - } else { - debugLog("remote.relay.remoteListener.cleanup relayPort=\(relayPort) \(output.debugLogSnippet()) \(debugConfigSummary())") - } - return true - } catch { - debugLog("remote.relay.remoteListener.cleanupFailed relayPort=\(relayPort) \(error.localizedDescription) \(debugConfigSummary())") - return false - } - } - - private func stopReverseRelayViaControlMasterLocked() { - guard let forwardSpec = reverseRelayControlMasterForwardSpec else { return } - reverseRelayControlMasterForwardSpec = nil - guard let arguments = configuration.reverseRelayControlMasterArguments( - controlCommand: "cancel", - forwardSpec: forwardSpec - ) else { - return - } - _ = try? sshExec(arguments: arguments, timeout: 4) - } - private func ensureCLIRelayServerLocked(localSocketPath: String, relayID: String, relayToken: String) throws -> RemoteCLIRelayServer { if let cliRelayServer { return cliRelayServer diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index a778dff079d1..a910565029ad 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -80,7 +80,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonBootstrapVersion: String? var daemonRemotePath: String? var reverseRelayProcess: Process? - var reverseRelayControlMasterForwardSpec: String? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] /// Stable publication state for best-effort remote TTY attribution scans. diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift index fbba3b0e8646..32432a7260c6 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift @@ -74,6 +74,29 @@ struct RemoteSessionSSHRemoteCommandOverrideTests { } } + @Test("Reverse relay disables shared ControlMaster transport") + func reverseRelayDisablesSharedControlMasterTransport() { + let coordinator = Self.makeCoordinator( + runner: RecordingProcessRunner(), + sshOptions: [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=/tmp/cmux-ssh-%C", + ] + ) + defer { coordinator.stop() } + + let arguments = coordinator.reverseRelayArguments( + relayPort: 64_007, + localRelayPort: 54_321 + ) + + #expect(arguments.starts(with: ["-N", "-T", "-S", "none"])) + #expect(!arguments.contains("-O")) + #expect(!arguments.contains(where: { $0.localizedCaseInsensitiveContains("ControlPath") })) + #expect(arguments.contains("127.0.0.1:64007:127.0.0.1:54321")) + } + @Test("File-backed SSH exec overrides a configured StdinNull") func fileBackedSSHExecOverridesConfiguredStdinNull() throws { let runner = RecordingProcessRunner() diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 6e2b935206f3..de136b02ae44 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -407,331 +407,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { XCTAssertFalse(fileManager.fileExists(atPath: ttyURL.path)) } - func testRemoteStaleRelayListenerCleanupScriptKillsMatchingPersistentRelayListener() throws { - let fileManager = FileManager.default - let root = fileManager.temporaryDirectory.appendingPathComponent("cmux-stale-relay-cleanup-\(UUID().uuidString)") - let bin = root.appendingPathComponent("bin") - let killLog = root.appendingPathComponent("kill.log") - try fileManager.createDirectory(at: bin, withIntermediateDirectories: true) - try "".write(to: killLog, atomically: true, encoding: .utf8) - defer { try? fileManager.removeItem(at: root) } - - try writeExecutableShellFile( - at: bin.appendingPathComponent("lsof"), - body: """ - #!/bin/sh - cat <<'EOF' - p33681 - f12 - n127.0.0.1:50446 - EOF - """ - ) - try writeExecutableShellFile( - at: bin.appendingPathComponent("ps"), - body: """ - #!/bin/sh - cat <<'EOF' - 33681 1 /usr/sbin/sshd-session - 34057 33681 /Users/cmux/.cmux/bin/cmuxd-remote/current/darwin-arm64/cmuxd-remote serve --stdio --persistent --slot ssh-c4ba8ab1 - 34058 33681 /bin/zsh - EOF - """ - ) - - let script = try XCTUnwrap( - RemoteSessionCoordinator.remoteStaleRelayListenerCleanupScript( - relayPort: 50446, - persistentDaemonSlot: "ssh-c4ba8ab1" - ) - ) - let result = runProcess( - executablePath: "/usr/bin/env", - arguments: [ - "PATH=\(bin.path):/usr/bin:/bin", - "CMUX_KILL_LOG=\(killLog.path)", - "/bin/sh", - "-c", - """ - kill() { printf '%s\\n' "$*" >> "$CMUX_KILL_LOG"; return 0; } - \(script) - """, - ], - timeout: 5 - ) - - XCTAssertFalse(result.timedOut, result.stderr) - XCTAssertEqual(result.status, 0, result.stderr) - XCTAssertTrue(result.stdout.contains("cmux_stale_relay_killed pid=33681 children=34057 port=50446"), result.stdout) - - let killOutput = try String(contentsOf: killLog, encoding: .utf8) - XCTAssertTrue(killOutput.contains("-TERM 33681 34057"), killOutput) - XCTAssertTrue(killOutput.contains("-KILL 33681"), killOutput) - XCTAssertTrue(killOutput.contains("-KILL 34057"), killOutput) - } - - func testRemoteStaleRelayListenerCleanupScriptPreservesDifferentPersistentSlot() throws { - let fileManager = FileManager.default - let root = fileManager.temporaryDirectory.appendingPathComponent("cmux-stale-relay-preserve-\(UUID().uuidString)") - let bin = root.appendingPathComponent("bin") - let killLog = root.appendingPathComponent("kill.log") - try fileManager.createDirectory(at: bin, withIntermediateDirectories: true) - try "".write(to: killLog, atomically: true, encoding: .utf8) - defer { try? fileManager.removeItem(at: root) } - - try writeExecutableShellFile( - at: bin.appendingPathComponent("lsof"), - body: """ - #!/bin/sh - cat <<'EOF' - p33681 - f12 - n127.0.0.1:50446 - EOF - """ - ) - try writeExecutableShellFile( - at: bin.appendingPathComponent("ps"), - body: """ - #!/bin/sh - cat <<'EOF' - 33681 1 /usr/sbin/sshd-session - 34057 33681 /Users/cmux/.cmux/bin/cmuxd-remote/current/darwin-arm64/cmuxd-remote serve --stdio --persistent --slot ssh-other - EOF - """ - ) - - let script = try XCTUnwrap( - RemoteSessionCoordinator.remoteStaleRelayListenerCleanupScript( - relayPort: 50446, - persistentDaemonSlot: "ssh-c4ba8ab1" - ) - ) - let result = runProcess( - executablePath: "/usr/bin/env", - arguments: [ - "PATH=\(bin.path):/usr/bin:/bin", - "CMUX_KILL_LOG=\(killLog.path)", - "/bin/sh", - "-c", - """ - kill() { printf '%s\\n' "$*" >> "$CMUX_KILL_LOG"; return 0; } - \(script) - """, - ], - timeout: 5 - ) - - XCTAssertFalse(result.timedOut, result.stderr) - XCTAssertEqual(result.status, 0, result.stderr) - XCTAssertEqual(result.stdout, "") - XCTAssertEqual(try String(contentsOf: killLog, encoding: .utf8), "") - } - - func testRemoteStaleRelayListenerCleanupScriptMatchesPersistentSlotExactly() throws { - let fileManager = FileManager.default - let root = fileManager.temporaryDirectory.appendingPathComponent("cmux-stale-relay-slot-prefix-\(UUID().uuidString)") - let bin = root.appendingPathComponent("bin") - let killLog = root.appendingPathComponent("kill.log") - try fileManager.createDirectory(at: bin, withIntermediateDirectories: true) - try "".write(to: killLog, atomically: true, encoding: .utf8) - defer { try? fileManager.removeItem(at: root) } - - try writeExecutableShellFile( - at: bin.appendingPathComponent("lsof"), - body: """ - #!/bin/sh - cat <<'EOF' - p33681 - f12 - n127.0.0.1:50446 - EOF - """ - ) - try writeExecutableShellFile( - at: bin.appendingPathComponent("ps"), - body: """ - #!/bin/sh - cat <<'EOF' - 33681 1 /usr/sbin/sshd-session - 34057 33681 /Users/cmux/.cmux/bin/cmuxd-remote/current/darwin-arm64/cmuxd-remote serve --stdio --persistent --slot ssh-ab - EOF - """ - ) - - let script = try XCTUnwrap( - RemoteSessionCoordinator.remoteStaleRelayListenerCleanupScript( - relayPort: 50446, - persistentDaemonSlot: "ssh-a" - ) - ) - let result = runProcess( - executablePath: "/usr/bin/env", - arguments: [ - "PATH=\(bin.path):/usr/bin:/bin", - "CMUX_KILL_LOG=\(killLog.path)", - "/bin/sh", - "-c", - """ - kill() { printf '%s\\n' "$*" >> "$CMUX_KILL_LOG"; return 0; } - \(script) - """, - ], - timeout: 5 - ) - - XCTAssertFalse(result.timedOut, result.stderr) - XCTAssertEqual(result.status, 0, result.stderr) - XCTAssertEqual(result.stdout, "") - XCTAssertEqual(try String(contentsOf: killLog, encoding: .utf8), "") - } - - func testRemoteStaleRelayListenerCleanupScriptKillsMetadataMatchedListenerWithoutChild() throws { - let fileManager = FileManager.default - let root = fileManager.temporaryDirectory.appendingPathComponent("cmux-stale-relay-metadata-\(UUID().uuidString)") - let bin = root.appendingPathComponent("bin") - let relayDir = root.appendingPathComponent(".cmux/relay") - let killLog = root.appendingPathComponent("kill.log") - try fileManager.createDirectory(at: bin, withIntermediateDirectories: true) - try fileManager.createDirectory(at: relayDir, withIntermediateDirectories: true) - try "/Users/cmux/.cmux/bin/cmuxd-remote/current/darwin-arm64/cmuxd-remote".write( - to: relayDir.appendingPathComponent("50446.daemon_path"), - atomically: true, - encoding: .utf8 - ) - try "ssh-c4ba8ab1".write( - to: relayDir.appendingPathComponent("50446.slot"), - atomically: true, - encoding: .utf8 - ) - try "".write(to: killLog, atomically: true, encoding: .utf8) - defer { try? fileManager.removeItem(at: root) } - - try writeExecutableShellFile( - at: bin.appendingPathComponent("lsof"), - body: """ - #!/bin/sh - cat <<'EOF' - p33681 - f12 - n127.0.0.1:50446 - EOF - """ - ) - try writeExecutableShellFile( - at: bin.appendingPathComponent("ps"), - body: """ - #!/bin/sh - cat <<'EOF' - 33681 1 /usr/sbin/sshd-session - EOF - """ - ) - - let script = try XCTUnwrap( - RemoteSessionCoordinator.remoteStaleRelayListenerCleanupScript( - relayPort: 50446, - persistentDaemonSlot: "ssh-c4ba8ab1" - ) - ) - let result = runProcess( - executablePath: "/usr/bin/env", - arguments: [ - "HOME=\(root.path)", - "PATH=\(bin.path):/usr/bin:/bin", - "CMUX_KILL_LOG=\(killLog.path)", - "/bin/sh", - "-c", - """ - kill() { printf '%s\\n' "$*" >> "$CMUX_KILL_LOG"; return 0; } - \(script) - """, - ], - timeout: 5 - ) - - XCTAssertFalse(result.timedOut, result.stderr) - XCTAssertEqual(result.status, 0, result.stderr) - XCTAssertTrue( - result.stdout.contains("cmux_stale_relay_killed pid=33681 children= port=50446 reason=metadata"), - result.stdout - ) - - let killOutput = try String(contentsOf: killLog, encoding: .utf8) - XCTAssertTrue(killOutput.contains("-TERM 33681"), killOutput) - XCTAssertTrue(killOutput.contains("-KILL 33681"), killOutput) - } - - func testRemoteStaleRelayListenerCleanupScriptPreservesMetadataMatchedDifferentPersistentSlot() throws { - let fileManager = FileManager.default - let root = fileManager.temporaryDirectory.appendingPathComponent("cmux-stale-relay-metadata-preserve-\(UUID().uuidString)") - let bin = root.appendingPathComponent("bin") - let relayDir = root.appendingPathComponent(".cmux/relay") - let killLog = root.appendingPathComponent("kill.log") - try fileManager.createDirectory(at: bin, withIntermediateDirectories: true) - try fileManager.createDirectory(at: relayDir, withIntermediateDirectories: true) - try "/Users/cmux/.cmux/bin/cmuxd-remote/current/darwin-arm64/cmuxd-remote".write( - to: relayDir.appendingPathComponent("50446.daemon_path"), - atomically: true, - encoding: .utf8 - ) - try "ssh-other-slot".write( - to: relayDir.appendingPathComponent("50446.slot"), - atomically: true, - encoding: .utf8 - ) - try "".write(to: killLog, atomically: true, encoding: .utf8) - defer { try? fileManager.removeItem(at: root) } - - try writeExecutableShellFile( - at: bin.appendingPathComponent("lsof"), - body: """ - #!/bin/sh - cat <<'EOF' - p33681 - f12 - n127.0.0.1:50446 - EOF - """ - ) - try writeExecutableShellFile( - at: bin.appendingPathComponent("ps"), - body: """ - #!/bin/sh - cat <<'EOF' - 33681 1 /usr/sbin/sshd-session - EOF - """ - ) - - let script = try XCTUnwrap( - RemoteSessionCoordinator.remoteStaleRelayListenerCleanupScript( - relayPort: 50446, - persistentDaemonSlot: "ssh-c4ba8ab1" - ) - ) - let result = runProcess( - executablePath: "/usr/bin/env", - arguments: [ - "HOME=\(root.path)", - "PATH=\(bin.path):/usr/bin:/bin", - "CMUX_KILL_LOG=\(killLog.path)", - "/bin/sh", - "-c", - """ - kill() { printf '%s\\n' "$*" >> "$CMUX_KILL_LOG"; return 0; } - \(script) - """, - ], - timeout: 5 - ) - - XCTAssertFalse(result.timedOut, result.stderr) - XCTAssertEqual(result.status, 0, result.stderr) - XCTAssertEqual(result.stdout, "") - XCTAssertEqual(try String(contentsOf: killLog, encoding: .utf8), "") - } - func testRelayZshBootstrapUsesRealHomeHistoryByDefault() throws { let histfile = try runRelayZshHistfile { home in try ":\n".write(to: home.appendingPathComponent(".zshenv"), atomically: true, encoding: .utf8) @@ -2309,126 +1984,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { ) } - @MainActor - func testPersistentReverseRelayCleansStaleRemoteListenerAndRetriesControlMasterForward() throws { - let retryForwardInvoked = DispatchSemaphore(value: 0) - let lock = NSLock() - var controlOperations: [(command: String, spec: String)] = [] - var forwardAttempts = 0 - var cleanupInvoked = false - var cleanupArguments: [String] = [] - - let remoteProcessScript: RemoteProcessScript = { executable, arguments, _, _ in - guard executable == "/usr/bin/ssh" else { - XCTFail("unexpected executable \(executable)") - return (status: 1, stdout: "", stderr: "unexpected executable") - } - - if let controlOperation = remoteReverseRelayControlOperation(from: arguments) { - let operation = controlOperation.command - let spec = controlOperation.spec - lock.lock() - controlOperations.append((command: operation, spec: spec)) - if operation == "forward" { - forwardAttempts += 1 - let attempt = forwardAttempts - lock.unlock() - if attempt == 1 { - return ( - status: 255, - stdout: "", - stderr: "remote port forwarding failed for listen port 64045" - ) - } - retryForwardInvoked.signal() - return (status: 0, stdout: "", stderr: "") - } - lock.unlock() - return (status: 0, stdout: "", stderr: "") - } - - let command = arguments.last ?? "" - if command.contains("cmux_stale_relay_listener_cleanup=1") { - lock.lock() - cleanupInvoked = true - cleanupArguments = arguments - lock.unlock() - return ( - status: 0, - stdout: "cmux_stale_relay_killed pid=33681 children=34057 port=64045\n", - stderr: "" - ) - } - if command.contains("uname -s") { - return ( - status: 0, - stdout: """ - __CMUX_REMOTE_HOME__=/home/test - __CMUX_REMOTE_OS__=Linux - __CMUX_REMOTE_ARCH__=x86_64 - __CMUX_REMOTE_EXISTS__=yes - """, - stderr: "" - ) - } - if remoteDaemonServeCommand(command) { - return ( - status: 0, - stdout: #"{"id":1,"ok":true,"result":{"name":"cmuxd-remote","version":"dev","capabilities":["proxy.stream.push","pty.session","pty.session.token","pty.write.notification","pty.resize.notification","pty.session.persistent_daemon"]}}"# + "\n", - stderr: "" - ) - } - return (status: 0, stdout: "", stderr: "") - } - - let workspace = Workspace() - workspace.remoteSessionProcessRunnerOverrideForTesting = - ScriptedRemoteProcessRunner(script: remoteProcessScript) - let config = WorkspaceRemoteConfiguration( - destination: "test@hpc.example", - port: 2222, - identityFile: nil, - sshOptions: [ - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-\(getuid())-64045-%C", - "StrictHostKeyChecking=accept-new", - ], - localProxyPort: nil, - relayPort: 64045, - relayID: "relay-stale-forward-retry", - relayToken: String(repeating: "d", count: 64), - localSocketPath: "/tmp/cmux-stale-forward-retry.sock", - terminalStartupCommand: "ssh-pty-attach", - preserveAfterTerminalExit: true, - persistentDaemonSlot: "ssh-stale-forward-retry" - ) - defer { workspace.disconnectRemoteConnection(clearConfiguration: true) } - - workspace.configureRemoteConnection(config, autoConnect: true) - - XCTAssertEqual(retryForwardInvoked.wait(timeout: .now() + 2), .success) - lock.lock() - let operations = controlOperations - let cleanupWasInvoked = cleanupInvoked - let capturedCleanupArguments = cleanupArguments - let attempts = forwardAttempts - lock.unlock() - - XCTAssertEqual(attempts, 2) - XCTAssertTrue(cleanupWasInvoked) - XCTAssertTrue(capturedCleanupArguments.contains("-S")) - XCTAssertTrue(capturedCleanupArguments.contains("none")) - XCTAssertFalse(capturedCleanupArguments.contains(where: { $0.hasPrefix("ControlPath=") })) - XCTAssertGreaterThanOrEqual(operations.count, 3) - XCTAssertEqual(operations[0].command, "cancel") - XCTAssertEqual(operations[0].spec, "127.0.0.1:64045") - XCTAssertEqual(operations[1].command, "forward") - XCTAssertEqual(operations[2].command, "forward") - XCTAssertEqual(operations[1].spec, operations[2].spec) - XCTAssertTrue(operations[2].spec.hasPrefix("127.0.0.1:64045:127.0.0.1:")) - } - @MainActor func testDetachAttachPreservesRemoteTerminalSurfaceTracking() throws { let workspace = Workspace() @@ -3250,111 +2805,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { XCTAssertTrue(arguments.contains(where: { $0 == "ControlPath /tmp/cmux-ssh-%C" || $0 == "ControlPath=/tmp/cmux-ssh-%C" })) } - func testReverseRelayControlMasterArgumentsReuseConfiguredControlSocket() throws { - let configuration = WorkspaceRemoteConfiguration( - destination: "cmux-macmini", - port: 2222, - identityFile: "/Users/test/.ssh/id_ed25519", - sshOptions: [ - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-%C", - "StrictHostKeyChecking=accept-new", - ], - localProxyPort: nil, - relayPort: 64007, - relayID: nil, - relayToken: nil, - localSocketPath: nil, - terminalStartupCommand: "ssh cmux-macmini" - ) - - let arguments = try XCTUnwrap( - configuration.reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: "127.0.0.1:64007:127.0.0.1:54321" - ) - ) - - XCTAssertFalse(arguments.contains("-S")) - XCTAssertTrue(arguments.contains("ControlMaster=no")) - XCTAssertTrue(arguments.contains("ControlPath=/tmp/cmux-ssh-%C")) - XCTAssertTrue(arguments.contains("-O")) - XCTAssertTrue(arguments.contains("forward")) - XCTAssertTrue(arguments.contains("-R")) - XCTAssertTrue(arguments.contains("127.0.0.1:64007:127.0.0.1:54321")) - XCTAssertTrue(arguments.contains("cmux-macmini")) - } - - func testReverseRelayControlMasterCancelArgumentsUseRemoteListenPortOnly() throws { - let configuration = WorkspaceRemoteConfiguration( - destination: "cmux-macmini", - port: 2222, - identityFile: "/Users/test/.ssh/id_ed25519", - sshOptions: [ - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-%C", - "StrictHostKeyChecking=accept-new", - ], - localProxyPort: nil, - relayPort: 64007, - relayID: nil, - relayToken: nil, - localSocketPath: nil, - terminalStartupCommand: "ssh cmux-macmini" - ) - - let arguments = try XCTUnwrap( - configuration.reverseRelayControlMasterCancelArguments( - relayPort: 64007 - ) - ) - - XCTAssertFalse(arguments.contains("-S")) - XCTAssertTrue(arguments.contains("ControlMaster=no")) - XCTAssertTrue(arguments.contains("ControlPath=/tmp/cmux-ssh-%C")) - XCTAssertTrue(arguments.contains("-O")) - XCTAssertTrue(arguments.contains("cancel")) - XCTAssertTrue(arguments.contains("-R")) - XCTAssertTrue(arguments.contains("127.0.0.1:64007")) - XCTAssertFalse(arguments.contains(where: { $0.hasPrefix("127.0.0.1:64007:127.0.0.1:") })) - XCTAssertTrue(arguments.contains("cmux-macmini")) - } - - func testReverseRelayControlMasterArgumentsReuseWhitespaceConfiguredControlSocket() throws { - let configuration = WorkspaceRemoteConfiguration( - destination: "cmux-macmini", - port: 2222, - identityFile: "/Users/test/.ssh/id_ed25519", - sshOptions: [ - "ControlMaster auto", - "ControlPersist 600", - "ControlPath /tmp/cmux-ssh-%C", - "StrictHostKeyChecking accept-new", - ], - localProxyPort: nil, - relayPort: 64033, - relayID: nil, - relayToken: nil, - localSocketPath: nil, - terminalStartupCommand: "ssh cmux-macmini" - ) - - let arguments = try XCTUnwrap( - configuration.reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: "127.0.0.1:64033:127.0.0.1:54321" - ) - ) - - XCTAssertFalse(arguments.contains("-S")) - XCTAssertTrue(arguments.contains("ControlMaster=no")) - XCTAssertTrue(arguments.contains(where: { $0 == "ControlPath /tmp/cmux-ssh-%C" || $0 == "ControlPath=/tmp/cmux-ssh-%C" })) - XCTAssertTrue(arguments.contains("-O")) - XCTAssertTrue(arguments.contains("forward")) - } - func testDetectedSSHSessionBracketsIPv6LiteralSCPDestination() { let session = DetectedSSHSession( destination: "lawrence@2001:db8::1", From 13a3d1d094eb0fd5cac2c768f81b66405fb7d4fd Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 17:48:16 -0700 Subject: [PATCH 03/39] test: prove dedicated relay startup is reached (#8894) --- cmuxTests/WorkspaceRemoteConnectionTests.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index de136b02ae44..ca1e847473ff 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -1760,6 +1760,14 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { let workspace = Workspace() workspace.remoteSessionProcessRunnerOverrideForTesting = ScriptedRemoteProcessRunner(script: remoteProcessScript) + let relayStartupReached = expectation( + description: "dedicated reverse-relay transport reached startup" + ) + let relayStatusObservation = workspace.$remoteDaemonStatus.sink { status in + if status.detail?.contains("Remote SSH relay unavailable") == true { + relayStartupReached.fulfill() + } + } let config = WorkspaceRemoteConfiguration( destination: "test@hpc.example", port: 2222, @@ -1973,7 +1981,9 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { workspace.configureRemoteConnection(config, autoConnect: true) XCTAssertEqual(daemonTransportStarted.wait(timeout: .now() + 2), .success) + wait(for: [relayStartupReached], timeout: 2) wait(for: [controlMasterTouched], timeout: 1) + relayStatusObservation.cancel() lock.lock() let operations = controlOperations lock.unlock() From 6acd19671e9398d984f48cf74406f44a9392931b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 18:09:53 -0700 Subject: [PATCH 04/39] test: observe relay startup in regression scope (#8894) --- cmuxTests/WorkspaceRemoteConnectionTests.swift | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index ca1e847473ff..4f814adbd9ec 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -1760,14 +1760,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { let workspace = Workspace() workspace.remoteSessionProcessRunnerOverrideForTesting = ScriptedRemoteProcessRunner(script: remoteProcessScript) - let relayStartupReached = expectation( - description: "dedicated reverse-relay transport reached startup" - ) - let relayStatusObservation = workspace.$remoteDaemonStatus.sink { status in - if status.detail?.contains("Remote SSH relay unavailable") == true { - relayStartupReached.fulfill() - } - } let config = WorkspaceRemoteConfiguration( destination: "test@hpc.example", port: 2222, @@ -1957,6 +1949,14 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { let workspace = Workspace() workspace.remoteSessionProcessRunnerOverrideForTesting = ScriptedRemoteProcessRunner(script: remoteProcessScript) + let relayStartupReached = expectation( + description: "dedicated reverse-relay transport reached startup" + ) + let relayStatusObservation = workspace.$remoteDaemonStatus.sink { status in + if status.detail?.contains("Remote SSH relay unavailable") == true { + relayStartupReached.fulfill() + } + } let config = WorkspaceRemoteConfiguration( destination: "127.0.0.1", port: 1, From f2acd126e991371971c98cc169e9fb84ffa31c58 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 20:26:05 -0700 Subject: [PATCH 05/39] test: capture dedicated reverse relay launch argv (#8894) --- ...emoteSessionCoordinator+ReverseRelay.swift | 7 ++- .../Session/RemoteSessionCoordinator.swift | 5 ++ .../Workspace+RemoteSessionLifecycle.swift | 6 +- Sources/Workspace.swift | 3 + .../WorkspaceRemoteConnectionTests.swift | 55 +++++++++++++++---- 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index e4bb1a7dae03..625a353c89e1 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -45,7 +45,12 @@ extension RemoteSessionCoordinator { let process = Process() let stderrPipe = Pipe() process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") - process.arguments = reverseRelayArguments(relayPort: relayPort, localRelayPort: localRelayPort) + let relayArguments = reverseRelayArguments( + relayPort: relayPort, + localRelayPort: localRelayPort + ) + process.arguments = relayArguments + reverseRelayLaunchObserver?(relayArguments) process.environment = configuration.sshProcessEnvironment process.standardInput = FileHandle.nullDevice process.standardOutput = FileHandle.nullDevice diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index a910565029ad..c746c256ef34 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -63,6 +63,9 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { let buildInfo: any RemoteSessionBuildInfoProviding let daemonStrings: RemoteDaemonStrings let strings: RemoteSessionStrings + /// Optional launch observer used by integration tests to verify the exact + /// argv of the long-lived reverse-relay process. + let reverseRelayLaunchObserver: (@Sendable ([String]) -> Void)? /// Sleep seam for every legacy `asyncAfter` delay (reconnect backoff, /// relay restart, bootstrap-TTY retry, port-scan coalesce and burst). let clock: any RemoteProxyRetryClock @@ -166,6 +169,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { buildInfo: any RemoteSessionBuildInfoProviding, daemonStrings: RemoteDaemonStrings, strings: RemoteSessionStrings, + reverseRelayLaunchObserver: (@Sendable ([String]) -> Void)? = nil, clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock() ) { self.host = host @@ -179,6 +183,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { self.buildInfo = buildInfo self.daemonStrings = daemonStrings self.strings = strings + self.reverseRelayLaunchObserver = reverseRelayLaunchObserver self.clock = clock queue.setSpecific(key: queueKey, value: ()) } diff --git a/Sources/Workspace+RemoteSessionLifecycle.swift b/Sources/Workspace+RemoteSessionLifecycle.swift index f93d8b687110..b37240c707b4 100644 --- a/Sources/Workspace+RemoteSessionLifecycle.swift +++ b/Sources/Workspace+RemoteSessionLifecycle.swift @@ -104,6 +104,9 @@ extension Workspace { var processRunner: any RemoteSessionProcessRunning = RemoteSessionProcessRunner() #if DEBUG if let override = remoteSessionProcessRunnerOverrideForTesting { processRunner = override } + let reverseRelayLaunchObserver = remoteSessionReverseRelayLaunchObserverForTesting +#else + let reverseRelayLaunchObserver: (@Sendable ([String]) -> Void)? = nil #endif let controller = RemoteSessionCoordinator( host: WorkspaceRemoteSessionHostAdapter(workspace: self, controllerID: controllerID), @@ -121,7 +124,8 @@ extension Workspace { ), buildInfo: WorkspaceRemoteSessionBuildInfo(), daemonStrings: RemoteDaemonStrings.appLocalized, - strings: RemoteSessionStrings.appLocalized + strings: RemoteSessionStrings.appLocalized, + reverseRelayLaunchObserver: reverseRelayLaunchObserver ) activeRemoteSessionControllerID = controllerID remoteSessionController = controller diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index f679f58ffd9b..e4d926b0fc10 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -2433,6 +2433,9 @@ final class Workspace: Identifiable, ObservableObject { /// the package process-runner seam (replaces the legacy process-wide /// `WorkspaceRemoteSessionController.runProcessOverrideForTesting` static). var remoteSessionProcessRunnerOverrideForTesting: (any RemoteSessionProcessRunning)? + /// XCTest seam for observing the exact argv passed to the long-lived + /// reverse-relay `Process`. + var remoteSessionReverseRelayLaunchObserverForTesting: (@Sendable ([String]) -> Void)? #endif /// The shell-activity classification per panel id; stored in the /// surface-registry sub-model. diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 4f814adbd9ec..63419e169b5b 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -31,6 +31,34 @@ private struct ScriptedRemoteProcessRunner: RemoteSessionProcessRunning, @unchec } } +private final class ReverseRelayLaunchRecorder: @unchecked Sendable { + private let lock = NSLock() + private let observed = DispatchSemaphore(value: 0) + private var arguments: [String]? + + func record(_ arguments: [String]) { + lock.lock() + let isFirstLaunch = self.arguments == nil + if isFirstLaunch { + self.arguments = arguments + } + lock.unlock() + if isFirstLaunch { + observed.signal() + } + } + + func wait(timeout: DispatchTime) -> DispatchTimeoutResult { + observed.wait(timeout: timeout) + } + + func snapshot() -> [String]? { + lock.lock() + defer { lock.unlock() } + return arguments + } +} + private func remoteDaemonServeCommand(_ command: String) -> Bool { command.contains("serve") && command.contains("--stdio") } @@ -1897,8 +1925,9 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { } @MainActor - func testPersistentReverseRelayDoesNotAttachToSharedControlMaster() { + func testPersistentReverseRelayDoesNotAttachToSharedControlMaster() throws { let daemonTransportStarted = DispatchSemaphore(value: 0) + let relayLaunch = ReverseRelayLaunchRecorder() let controlMasterTouched = expectation( description: "workspace-scoped reverse relay must not mutate the host-shared ControlMaster" ) @@ -1949,13 +1978,8 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { let workspace = Workspace() workspace.remoteSessionProcessRunnerOverrideForTesting = ScriptedRemoteProcessRunner(script: remoteProcessScript) - let relayStartupReached = expectation( - description: "dedicated reverse-relay transport reached startup" - ) - let relayStatusObservation = workspace.$remoteDaemonStatus.sink { status in - if status.detail?.contains("Remote SSH relay unavailable") == true { - relayStartupReached.fulfill() - } + workspace.remoteSessionReverseRelayLaunchObserverForTesting = { arguments in + relayLaunch.record(arguments) } let config = WorkspaceRemoteConfiguration( destination: "127.0.0.1", @@ -1981,13 +2005,24 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { workspace.configureRemoteConnection(config, autoConnect: true) XCTAssertEqual(daemonTransportStarted.wait(timeout: .now() + 2), .success) - wait(for: [relayStartupReached], timeout: 2) + XCTAssertEqual(relayLaunch.wait(timeout: .now() + 2), .success) wait(for: [controlMasterTouched], timeout: 1) - relayStatusObservation.cancel() lock.lock() let operations = controlOperations lock.unlock() + let arguments = try XCTUnwrap(relayLaunch.snapshot()) + XCTAssertEqual(Array(arguments.prefix(4)), ["-N", "-T", "-S", "none"]) + XCTAssertFalse(arguments.contains("-O")) + XCTAssertFalse( + arguments.contains(where: { $0.localizedCaseInsensitiveContains("ControlPath") }) + ) + let reverseForwardIndex = try XCTUnwrap(arguments.firstIndex(of: "-R")) + let reverseForwardSpec = try XCTUnwrap(arguments.dropFirst(reverseForwardIndex + 1).first) + XCTAssertTrue( + reverseForwardSpec.hasPrefix("127.0.0.1:64044:127.0.0.1:"), + "expected dedicated relay forwarding argv, got \(arguments)" + ) XCTAssertTrue( operations.isEmpty, "the relay must use its own app-owned ssh process, got ControlMaster operations: \(operations)" From 4a66fd03190bbace27773930aaf4a0196be009e8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 20:45:47 -0700 Subject: [PATCH 06/39] fix: cancel inherited SSH relay forwards (#8894) --- ...emoteSessionCoordinator+ReverseRelay.swift | 35 +++++++++++++++++++ .../WorkspaceRemoteConnectionTests.swift | 25 +++++++------ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 625a353c89e1..5d706760f4cc 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -41,6 +41,7 @@ extension RemoteSessionCoordinator { relayPort: relayPort, persistentDaemonSlot: configuration.persistentDaemonSlot ) + cancelInheritedReverseRelayForwardLocked(relayPort: relayPort) let process = Process() let stderrPipe = Pipe() @@ -227,6 +228,40 @@ extension RemoteSessionCoordinator { return args } + /// Removes a reverse forward left on a ControlPersist master by a + /// pre-dedicated-relay app instance. Do not require an explicit + /// `ControlPath`: `ssh -O` can resolve one from the user's host config, + /// and exits without creating a connection when no master is available. + private func cancelInheritedReverseRelayForwardLocked(relayPort: Int) { + let forwardSpec = "127.0.0.1:\(relayPort)" + let arguments = sshCommonArguments(batchMode: true) + [ + "-O", "cancel", + "-R", forwardSpec, + configuration.destination, + ] + do { + let result = try sshExec(arguments: arguments, timeout: 4) + guard result.status == 0 else { + let detail = Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) + ?? "ssh exited \(result.status)" + debugLog( + "remote.relay.inheritedForward.cancelIgnored " + + "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" + ) + return + } + debugLog( + "remote.relay.inheritedForward.cancelled " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + } catch { + debugLog( + "remote.relay.inheritedForward.cancelIgnored " + + "relayPort=\(relayPort) \(error.localizedDescription) \(debugConfigSummary())" + ) + } + } + private func ensureCLIRelayServerLocked(localSocketPath: String, relayID: String, relayToken: String) throws -> RemoteCLIRelayServer { if let cliRelayServer { return cliRelayServer diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 63419e169b5b..c584e5cdecf4 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -1925,13 +1925,10 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { } @MainActor - func testPersistentReverseRelayDoesNotAttachToSharedControlMaster() throws { + func testPersistentReverseRelayCancelsInheritedForwardBeforeDedicatedLaunch() throws { let daemonTransportStarted = DispatchSemaphore(value: 0) let relayLaunch = ReverseRelayLaunchRecorder() - let controlMasterTouched = expectation( - description: "workspace-scoped reverse relay must not mutate the host-shared ControlMaster" - ) - controlMasterTouched.isInverted = true + let inheritedForwardCancelled = DispatchSemaphore(value: 0) let lock = NSLock() var controlOperations: [(command: String, spec: String)] = [] @@ -1947,7 +1944,9 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { lock.lock() controlOperations.append((command: operation, spec: spec)) lock.unlock() - controlMasterTouched.fulfill() + if operation == "cancel" { + inheritedForwardCancelled.signal() + } return (status: 0, stdout: "", stderr: "") } @@ -1986,9 +1985,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { port: 1, identityFile: nil, sshOptions: [ - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-\(getuid())-issue-8894-%C", "StrictHostKeyChecking=accept-new", ], localProxyPort: nil, @@ -2005,8 +2001,8 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { workspace.configureRemoteConnection(config, autoConnect: true) XCTAssertEqual(daemonTransportStarted.wait(timeout: .now() + 2), .success) + XCTAssertEqual(inheritedForwardCancelled.wait(timeout: .now() + 2), .success) XCTAssertEqual(relayLaunch.wait(timeout: .now() + 2), .success) - wait(for: [controlMasterTouched], timeout: 1) lock.lock() let operations = controlOperations lock.unlock() @@ -2023,9 +2019,12 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { reverseForwardSpec.hasPrefix("127.0.0.1:64044:127.0.0.1:"), "expected dedicated relay forwarding argv, got \(arguments)" ) - XCTAssertTrue( - operations.isEmpty, - "the relay must use its own app-owned ssh process, got ControlMaster operations: \(operations)" + XCTAssertEqual(operations.count, 1) + XCTAssertEqual(operations.first?.command, "cancel") + XCTAssertEqual(operations.first?.spec, "127.0.0.1:64044") + XCTAssertFalse( + operations.contains(where: { $0.command == "forward" }), + "the relay must never install its forward on the shared ControlMaster: \(operations)" ) } From dd17886b340d37a14b9cf5ae807600997e3e6a90 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 21:18:15 -0700 Subject: [PATCH 07/39] test: remove reverse relay launch seam (#8894) --- ...emoteSessionCoordinator+ReverseRelay.swift | 1 - .../Session/RemoteSessionCoordinator.swift | 5 -- .../Workspace+RemoteSessionLifecycle.swift | 6 +-- Sources/Workspace.swift | 3 -- .../WorkspaceRemoteConnectionTests.swift | 47 +------------------ 5 files changed, 2 insertions(+), 60 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 5d706760f4cc..147a291a6695 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -51,7 +51,6 @@ extension RemoteSessionCoordinator { localRelayPort: localRelayPort ) process.arguments = relayArguments - reverseRelayLaunchObserver?(relayArguments) process.environment = configuration.sshProcessEnvironment process.standardInput = FileHandle.nullDevice process.standardOutput = FileHandle.nullDevice diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index c746c256ef34..a910565029ad 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -63,9 +63,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { let buildInfo: any RemoteSessionBuildInfoProviding let daemonStrings: RemoteDaemonStrings let strings: RemoteSessionStrings - /// Optional launch observer used by integration tests to verify the exact - /// argv of the long-lived reverse-relay process. - let reverseRelayLaunchObserver: (@Sendable ([String]) -> Void)? /// Sleep seam for every legacy `asyncAfter` delay (reconnect backoff, /// relay restart, bootstrap-TTY retry, port-scan coalesce and burst). let clock: any RemoteProxyRetryClock @@ -169,7 +166,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { buildInfo: any RemoteSessionBuildInfoProviding, daemonStrings: RemoteDaemonStrings, strings: RemoteSessionStrings, - reverseRelayLaunchObserver: (@Sendable ([String]) -> Void)? = nil, clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock() ) { self.host = host @@ -183,7 +179,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { self.buildInfo = buildInfo self.daemonStrings = daemonStrings self.strings = strings - self.reverseRelayLaunchObserver = reverseRelayLaunchObserver self.clock = clock queue.setSpecific(key: queueKey, value: ()) } diff --git a/Sources/Workspace+RemoteSessionLifecycle.swift b/Sources/Workspace+RemoteSessionLifecycle.swift index b37240c707b4..f93d8b687110 100644 --- a/Sources/Workspace+RemoteSessionLifecycle.swift +++ b/Sources/Workspace+RemoteSessionLifecycle.swift @@ -104,9 +104,6 @@ extension Workspace { var processRunner: any RemoteSessionProcessRunning = RemoteSessionProcessRunner() #if DEBUG if let override = remoteSessionProcessRunnerOverrideForTesting { processRunner = override } - let reverseRelayLaunchObserver = remoteSessionReverseRelayLaunchObserverForTesting -#else - let reverseRelayLaunchObserver: (@Sendable ([String]) -> Void)? = nil #endif let controller = RemoteSessionCoordinator( host: WorkspaceRemoteSessionHostAdapter(workspace: self, controllerID: controllerID), @@ -124,8 +121,7 @@ extension Workspace { ), buildInfo: WorkspaceRemoteSessionBuildInfo(), daemonStrings: RemoteDaemonStrings.appLocalized, - strings: RemoteSessionStrings.appLocalized, - reverseRelayLaunchObserver: reverseRelayLaunchObserver + strings: RemoteSessionStrings.appLocalized ) activeRemoteSessionControllerID = controllerID remoteSessionController = controller diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index e4d926b0fc10..f679f58ffd9b 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -2433,9 +2433,6 @@ final class Workspace: Identifiable, ObservableObject { /// the package process-runner seam (replaces the legacy process-wide /// `WorkspaceRemoteSessionController.runProcessOverrideForTesting` static). var remoteSessionProcessRunnerOverrideForTesting: (any RemoteSessionProcessRunning)? - /// XCTest seam for observing the exact argv passed to the long-lived - /// reverse-relay `Process`. - var remoteSessionReverseRelayLaunchObserverForTesting: (@Sendable ([String]) -> Void)? #endif /// The shell-activity classification per panel id; stored in the /// surface-registry sub-model. diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index c584e5cdecf4..6e100f089a88 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -31,34 +31,6 @@ private struct ScriptedRemoteProcessRunner: RemoteSessionProcessRunning, @unchec } } -private final class ReverseRelayLaunchRecorder: @unchecked Sendable { - private let lock = NSLock() - private let observed = DispatchSemaphore(value: 0) - private var arguments: [String]? - - func record(_ arguments: [String]) { - lock.lock() - let isFirstLaunch = self.arguments == nil - if isFirstLaunch { - self.arguments = arguments - } - lock.unlock() - if isFirstLaunch { - observed.signal() - } - } - - func wait(timeout: DispatchTime) -> DispatchTimeoutResult { - observed.wait(timeout: timeout) - } - - func snapshot() -> [String]? { - lock.lock() - defer { lock.unlock() } - return arguments - } -} - private func remoteDaemonServeCommand(_ command: String) -> Bool { command.contains("serve") && command.contains("--stdio") } @@ -1925,9 +1897,8 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { } @MainActor - func testPersistentReverseRelayCancelsInheritedForwardBeforeDedicatedLaunch() throws { + func testPersistentReverseRelayCancelsInheritedForwardWithoutExplicitControlPath() { let daemonTransportStarted = DispatchSemaphore(value: 0) - let relayLaunch = ReverseRelayLaunchRecorder() let inheritedForwardCancelled = DispatchSemaphore(value: 0) let lock = NSLock() var controlOperations: [(command: String, spec: String)] = [] @@ -1977,9 +1948,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { let workspace = Workspace() workspace.remoteSessionProcessRunnerOverrideForTesting = ScriptedRemoteProcessRunner(script: remoteProcessScript) - workspace.remoteSessionReverseRelayLaunchObserverForTesting = { arguments in - relayLaunch.record(arguments) - } let config = WorkspaceRemoteConfiguration( destination: "127.0.0.1", port: 1, @@ -2002,23 +1970,10 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { XCTAssertEqual(daemonTransportStarted.wait(timeout: .now() + 2), .success) XCTAssertEqual(inheritedForwardCancelled.wait(timeout: .now() + 2), .success) - XCTAssertEqual(relayLaunch.wait(timeout: .now() + 2), .success) lock.lock() let operations = controlOperations lock.unlock() - let arguments = try XCTUnwrap(relayLaunch.snapshot()) - XCTAssertEqual(Array(arguments.prefix(4)), ["-N", "-T", "-S", "none"]) - XCTAssertFalse(arguments.contains("-O")) - XCTAssertFalse( - arguments.contains(where: { $0.localizedCaseInsensitiveContains("ControlPath") }) - ) - let reverseForwardIndex = try XCTUnwrap(arguments.firstIndex(of: "-R")) - let reverseForwardSpec = try XCTUnwrap(arguments.dropFirst(reverseForwardIndex + 1).first) - XCTAssertTrue( - reverseForwardSpec.hasPrefix("127.0.0.1:64044:127.0.0.1:"), - "expected dedicated relay forwarding argv, got \(arguments)" - ) XCTAssertEqual(operations.count, 1) XCTAssertEqual(operations.first?.command, "cancel") XCTAssertEqual(operations.first?.spec, "127.0.0.1:64044") From 3917d9bf0e425fe1ba30767f34ff7e5248c8e3dc Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 21:42:20 -0700 Subject: [PATCH 08/39] fix: keep relay cleanup off coordinator queue (#8894) --- .../RemoteProcessCancellationOperation.swift | 62 ++++++++ ...emoteSessionCoordinator+ReverseRelay.swift | 55 +++---- ...ssionCoordinator+ReverseRelayStartup.swift | 142 ++++++++++++++++++ .../Session/RemoteSessionCoordinator.swift | 1 + .../Values/ReverseRelayStartupPhase.swift | 25 +++ ...emoteSessionReverseRelayStartupTests.swift | 120 +++++++++++++++ 6 files changed, 370 insertions(+), 35 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift new file mode 100644 index 000000000000..43c42ea0fc19 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift @@ -0,0 +1,62 @@ +internal import Foundation + +/// Cancellation token used to terminate a blocking process runner from a +/// coordinator-owned task. +/// +/// `@unchecked Sendable` is safe because the lock protects the complete +/// mutable state, and handlers are always invoked after releasing the lock. +final class RemoteProcessCancellationOperation: RemoteTransferCancelling, @unchecked Sendable { + private let lock = NSLock() + private var cancelled = false + private var cancellationHandler: (() -> Void)? + + var isCancelled: Bool { + lock.lock() + defer { lock.unlock() } + return cancelled + } + + var cancellationError: any Error { + CancellationError() + } + + func throwIfCancelled() throws { + if isCancelled { + throw CancellationError() + } + } + + func installCancellationHandler(_ handler: @escaping () -> Void) { + lock.lock() + let invokeImmediately = cancelled + if !cancelled { + cancellationHandler = handler + } + lock.unlock() + + if invokeImmediately { + handler() + } + } + + func clearCancellationHandler() { + lock.lock() + cancellationHandler = nil + lock.unlock() + } + + func cancel() { + let handler: (() -> Void)? + lock.lock() + guard !cancelled else { + lock.unlock() + return + } + cancelled = true + handler = cancellationHandler + cancellationHandler = nil + lock.unlock() + + handler?() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 147a291a6695..0749d7bdc702 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -25,8 +25,27 @@ extension RemoteSessionCoordinator { return } guard reverseRelayProcess == nil else { return } + guard reverseRelayStartupPhase.isIdle else { return } cancelReverseRelayRestartLocked() + beginInheritedReverseRelayCancellationLocked( + remotePath: remotePath, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + localSocketPath: localSocketPath + ) + } + + func launchReverseRelayLocked( + remotePath: String, + relayPort: Int, + relayID: String, + relayToken: String, + localSocketPath: String + ) { + guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } + var relayServer: RemoteCLIRelayServer? do { let server = try ensureCLIRelayServerLocked( @@ -41,7 +60,6 @@ extension RemoteSessionCoordinator { relayPort: relayPort, persistentDaemonSlot: configuration.persistentDaemonSlot ) - cancelInheritedReverseRelayForwardLocked(relayPort: relayPort) let process = Process() let stderrPipe = Pipe() @@ -201,6 +219,7 @@ extension RemoteSessionCoordinator { @discardableResult func stopReverseRelayLocked(cleanupScope: RemoteRelayCleanupScope = .transport) -> Bool { + cancelReverseRelayStartupLocked() reverseRelayStderrPipe?.fileHandleForReading.readabilityHandler = nil if let reverseRelayProcess, reverseRelayProcess.isRunning { reverseRelayProcess.terminate() @@ -227,40 +246,6 @@ extension RemoteSessionCoordinator { return args } - /// Removes a reverse forward left on a ControlPersist master by a - /// pre-dedicated-relay app instance. Do not require an explicit - /// `ControlPath`: `ssh -O` can resolve one from the user's host config, - /// and exits without creating a connection when no master is available. - private func cancelInheritedReverseRelayForwardLocked(relayPort: Int) { - let forwardSpec = "127.0.0.1:\(relayPort)" - let arguments = sshCommonArguments(batchMode: true) + [ - "-O", "cancel", - "-R", forwardSpec, - configuration.destination, - ] - do { - let result = try sshExec(arguments: arguments, timeout: 4) - guard result.status == 0 else { - let detail = Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) - ?? "ssh exited \(result.status)" - debugLog( - "remote.relay.inheritedForward.cancelIgnored " + - "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" - ) - return - } - debugLog( - "remote.relay.inheritedForward.cancelled " + - "relayPort=\(relayPort) \(debugConfigSummary())" - ) - } catch { - debugLog( - "remote.relay.inheritedForward.cancelIgnored " + - "relayPort=\(relayPort) \(error.localizedDescription) \(debugConfigSummary())" - ) - } - } - private func ensureCLIRelayServerLocked(localSocketPath: String, relayID: String, relayToken: String) throws -> RemoteCLIRelayServer { if let cliRelayServer { return cliRelayServer diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift new file mode 100644 index 000000000000..eacb59d05eb4 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -0,0 +1,142 @@ +internal import Foundation + +private enum InheritedReverseRelayCancellationOutcome: Sendable { + case cancelled + case ignored(String) +} + +extension RemoteSessionCoordinator { + /// Removes a reverse forward left on a ControlPersist master by a + /// pre-dedicated-relay app instance, then continues startup on `queue`. + /// + /// Do not require an explicit `ControlPath`: `ssh -O` can resolve one from + /// the user's host config, and exits without creating a connection when no + /// master is available. + func beginInheritedReverseRelayCancellationLocked( + remotePath: String, + relayPort: Int, + relayID: String, + relayToken: String, + localSocketPath: String + ) { + let forwardSpec = "127.0.0.1:\(relayPort)" + let arguments = sshCommonArguments(batchMode: true) + [ + "-O", "cancel", + "-R", forwardSpec, + configuration.destination, + ] + let request = RemoteProcessRequest( + executable: "/usr/bin/ssh", + arguments: arguments, + environment: configuration.sshProcessEnvironment, + timeout: 4 + ) + let token = UUID() + let cancellation = RemoteProcessCancellationOperation() + let processRunner = self.processRunner + + let task = Task { [weak self] in + let outcome = await withTaskCancellationHandler { + await Self.runInheritedReverseRelayCancellation( + request: request, + processRunner: processRunner, + cancellation: cancellation + ) + } onCancel: { + cancellation.cancel() + } + guard !Task.isCancelled else { return } + self?.queue.async { [weak self] in + self?.finishInheritedReverseRelayCancellationLocked( + token: token, + outcome: outcome, + remotePath: remotePath, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + localSocketPath: localSocketPath + ) + } + } + reverseRelayStartupPhase = .cancellingInheritedForward( + token: token, + task: task, + cancellation: cancellation + ) + } + + private static func runInheritedReverseRelayCancellation( + request: RemoteProcessRequest, + processRunner: any RemoteSessionProcessRunning, + cancellation: RemoteProcessCancellationOperation + ) async -> InheritedReverseRelayCancellationOutcome { + await withCheckedContinuation { continuation in + // The process runner is intentionally blocking. Keep that legacy + // boundary on a utility thread while the owning Task suspends. + DispatchQueue.global(qos: .utility).async { + let outcome: InheritedReverseRelayCancellationOutcome + do { + let result = try processRunner.run(request, operation: cancellation) + if result.status == 0 { + outcome = .cancelled + } else { + let detail = bestErrorLine(stderr: result.stderr, stdout: result.stdout) + ?? "ssh exited \(result.status)" + outcome = .ignored(detail) + } + } catch { + outcome = .ignored(error.localizedDescription) + } + continuation.resume(returning: outcome) + } + } + } + + private func finishInheritedReverseRelayCancellationLocked( + token: UUID, + outcome: InheritedReverseRelayCancellationOutcome, + remotePath: String, + relayPort: Int, + relayID: String, + relayToken: String, + localSocketPath: String + ) { + guard reverseRelayStartupPhase.token == token else { return } + reverseRelayStartupPhase = .idle + + switch outcome { + case .cancelled: + debugLog( + "remote.relay.inheritedForward.cancelled " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + case .ignored(let detail): + debugLog( + "remote.relay.inheritedForward.cancelIgnored " + + "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" + ) + } + + guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } + launchReverseRelayLocked( + remotePath: remotePath, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + localSocketPath: localSocketPath + ) + } + + func cancelReverseRelayStartupLocked() { + guard case .cancellingInheritedForward( + _, + let task, + let cancellation + ) = reverseRelayStartupPhase else { + return + } + reverseRelayStartupPhase = .idle + task.cancel() + cancellation.cancel() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index a910565029ad..614b2d51e775 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -79,6 +79,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonReady = false var daemonBootstrapVersion: String? var daemonRemotePath: String? + var reverseRelayStartupPhase = ReverseRelayStartupPhase.idle var reverseRelayProcess: Process? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift new file mode 100644 index 000000000000..4da9364ce6d1 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift @@ -0,0 +1,25 @@ +internal import Foundation + +/// Queue-confined phase for the asynchronous pre-launch relay cleanup. +enum ReverseRelayStartupPhase: Sendable { + case idle + case cancellingInheritedForward( + token: UUID, + task: Task, + cancellation: RemoteProcessCancellationOperation + ) + + var isIdle: Bool { + if case .idle = self { + return true + } + return false + } + + var token: UUID? { + guard case .cancellingInheritedForward(let token, _, _) = self else { + return nil + } + return token + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift new file mode 100644 index 000000000000..101cae04f3c9 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -0,0 +1,120 @@ +import Foundation +import Testing +import CmuxCore +import CmuxRemoteDaemon +@testable import CmuxRemoteSession +@testable import CmuxRemoteWorkspace + +@Suite("Reverse relay startup lifecycle") +struct RemoteSessionReverseRelayStartupTests { + @Test( + "Stop cancels inherited-forward cleanup without waiting on its timeout", + .timeLimit(.minutes(1)) + ) + func stopCancelsInheritedForwardCleanup() async { + let runner = BlockingInheritedForwardCancellationRunner() + let coordinator = Self.makeCoordinator(runner: runner) + + coordinator.queue.async { + coordinator.daemonReady = true + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + + var started = runner.started.makeAsyncIterator() + #expect(await started.next() != nil) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + + var cancelled = runner.cancelled.makeAsyncIterator() + #expect(await cancelled.next() != nil) + let startupCleared = coordinator.queue.sync { + coordinator.reverseRelayStartupPhase.isIdle && + coordinator.reverseRelayProcess == nil + } + #expect(startupCleared) + } + + private static func makeCoordinator( + runner: BlockingInheritedForwardCancellationRunner + ) -> RemoteSessionCoordinator { + let configuration = WorkspaceRemoteConfiguration( + destination: "user@example.test", + port: nil, + identityFile: nil, + sshOptions: ["StrictHostKeyChecking=accept-new"], + localProxyPort: nil, + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock", + terminalStartupCommand: nil, + preserveAfterTerminalExit: false, + persistentDaemonSlot: nil + ) + return RemoteSessionCoordinator( + host: NoopRemoteSessionHost(), + configuration: configuration, + proxyBroker: SSHOverrideUnusedRemoteProxyBroker(), + connectionBroker: NativeSSHConnectionBroker(), + manifestRepository: RemoteDaemonManifestRepository( + homeDirectory: FileManager.default.temporaryDirectory + ), + processRunner: runner, + reachabilityProbe: SSHOverrideNoopReachabilityProbe(), + relayCommandRewriter: SSHOverridePassthroughRelayCommandRewriter(), + buildInfo: SSHOverrideStubBuildInfo(), + daemonStrings: RemoteDaemonStrings( + missingPersistentPTYCapability: "", + missingRequiredFunctionality: "" + ), + strings: RemoteSessionStrings( + connectedVMNoProxyFormat: "%@", + suspendedDetailFormat: "%@" + ) + ) + } +} + +private final class BlockingInheritedForwardCancellationRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let started: AsyncStream + let cancelled: AsyncStream + + private let startedContinuation: AsyncStream.Continuation + private let cancelledContinuation: AsyncStream.Continuation + private let release = DispatchSemaphore(value: 0) + + init() { + (started, startedContinuation) = AsyncStream.makeStream() + (cancelled, cancelledContinuation) = AsyncStream.makeStream() + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + guard request.arguments.contains("-O"), + request.arguments.contains("cancel") else { + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + guard let operation else { + throw BlockingInheritedForwardCancellationError.missingCancellationOperation + } + + try operation.throwIfCancelled() + operation.installCancellationHandler { [cancelledContinuation, release] in + cancelledContinuation.yield() + release.signal() + } + defer { operation.clearCancellationHandler() } + startedContinuation.yield() + release.wait() + throw operation.cancellationError + } +} + +private enum BlockingInheritedForwardCancellationError: Error { + case missingCancellationOperation +} From 5c99979a8d726bd3c535fad7934cac61f78fb986 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 21:51:49 -0700 Subject: [PATCH 09/39] fix: recover relay from inherited ControlMaster lease (#8894) --- .../RemoteProcessCancellationOperation.swift | 2 + ...emoteSessionCoordinator+ReverseRelay.swift | 23 ++- ...ssionCoordinator+ReverseRelayStartup.swift | 85 +++++++---- .../Session/RemoteSessionCoordinator.swift | 2 +- .../Values/ReverseRelayStartupPhase.swift | 27 +++- ...emoteSessionReverseRelayStartupTests.swift | 138 ++++++++++++++++-- .../WorkspaceRemoteConnectionTests.swift | 97 ------------ 7 files changed, 226 insertions(+), 148 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift index 43c42ea0fc19..427789cef476 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift @@ -6,6 +6,8 @@ internal import Foundation /// `@unchecked Sendable` is safe because the lock protects the complete /// mutable state, and handlers are always invoked after releasing the lock. final class RemoteProcessCancellationOperation: RemoteTransferCancelling, @unchecked Sendable { + // lint:allow lock - Process termination handlers are synchronous and the + // critical region only exchanges a cancellation bit and one callback. private let lock = NSLock() private var cancelled = false private var cancellationHandler: (() -> Void)? diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 0749d7bdc702..9142786a9589 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -25,10 +25,10 @@ extension RemoteSessionCoordinator { return } guard reverseRelayProcess == nil else { return } - guard reverseRelayStartupPhase.isIdle else { return } + guard reverseRelayStartupPhase.allowsRelayLaunch else { return } cancelReverseRelayRestartLocked() - beginInheritedReverseRelayCancellationLocked( + launchReverseRelayLocked( remotePath: remotePath, relayPort: relayPort, relayID: relayID, @@ -37,6 +37,7 @@ extension RemoteSessionCoordinator { ) } + /// Launches the app-owned relay without adopting a shared ControlMaster. func launchReverseRelayLocked( remotePath: String, relayPort: Int, @@ -97,6 +98,16 @@ extension RemoteSessionCoordinator { cliRelayServer = nil } } + if beginConflictedControlMasterExitIfNeededLocked( + startupFailure: startupFailure, + remotePath: remotePath, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + localSocketPath: localSocketPath + ) { + return + } publishDaemonStatus( .error, detail: "Remote SSH relay unavailable: \(startupFailure) (retry in \(retrySeconds)s)" @@ -406,4 +417,12 @@ extension RemoteSessionCoordinator { let stderr = String(data: stderrData, encoding: .utf8) ?? "" return bestErrorLine(stderr: stderr) ?? "status=\(process.terminationStatus)" } + + /// Returns whether OpenSSH reported that this relay's remote listener is + /// already bound. The optional `Error:` prefix varies across OpenSSH builds. + static func isReverseRelayPortBindingFailure(_ detail: String, relayPort: Int) -> Bool { + let expected = "remote port forwarding failed for listen port \(relayPort)" + let normalized = detail.trimmingCharacters(in: .whitespacesAndNewlines) + return normalized == expected || normalized == "Error: \(expected)" + } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index eacb59d05eb4..db90398f816f 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -1,30 +1,37 @@ internal import Foundation -private enum InheritedReverseRelayCancellationOutcome: Sendable { - case cancelled +private enum ConflictedControlMasterExitOutcome: Sendable { + case exited case ignored(String) } extension RemoteSessionCoordinator { - /// Removes a reverse forward left on a ControlPersist master by a - /// pre-dedicated-relay app instance, then continues startup on `queue`. + /// Exits a legacy ControlPersist master only after the dedicated relay + /// proves that its configured remote port is already bound. /// - /// Do not require an explicit `ControlPath`: `ssh -O` can resolve one from - /// the user's host config, and exits without creating a connection when no - /// master is available. - func beginInheritedReverseRelayCancellationLocked( + /// OpenSSH cannot cancel an inherited reverse forward without its original + /// full target specification. Exiting the owning master is the only + /// deterministic migration path; persistent remote PTYs survive and active + /// transports reconnect. + @discardableResult + func beginConflictedControlMasterExitIfNeededLocked( + startupFailure: String, remotePath: String, relayPort: Int, relayID: String, relayToken: String, localSocketPath: String - ) { - let forwardSpec = "127.0.0.1:\(relayPort)" - let arguments = sshCommonArguments(batchMode: true) + [ - "-O", "cancel", - "-R", forwardSpec, - configuration.destination, - ] + ) -> Bool { + guard reverseRelayStartupPhase.canAttemptRecovery, + Self.isReverseRelayPortBindingFailure( + startupFailure, + relayPort: relayPort + ) else { + return false + } + + let arguments = RemoteControlMasterCleanup() + .cleanupArguments(configuration: configuration) let request = RemoteProcessRequest( executable: "/usr/bin/ssh", arguments: arguments, @@ -37,7 +44,7 @@ extension RemoteSessionCoordinator { let task = Task { [weak self] in let outcome = await withTaskCancellationHandler { - await Self.runInheritedReverseRelayCancellation( + await Self.runConflictedControlMasterExit( request: request, processRunner: processRunner, cancellation: cancellation @@ -47,9 +54,10 @@ extension RemoteSessionCoordinator { } guard !Task.isCancelled else { return } self?.queue.async { [weak self] in - self?.finishInheritedReverseRelayCancellationLocked( + self?.finishConflictedControlMasterExitLocked( token: token, outcome: outcome, + startupFailure: startupFailure, remotePath: remotePath, relayPort: relayPort, relayID: relayID, @@ -58,27 +66,34 @@ extension RemoteSessionCoordinator { ) } } - reverseRelayStartupPhase = .cancellingInheritedForward( + reverseRelayStartupPhase = .exitingConflictedControlMaster( token: token, task: task, cancellation: cancellation ) + debugLog( + "remote.relay.conflictedMaster.exitBegin " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + return true } - private static func runInheritedReverseRelayCancellation( + /// Runs blocking `ssh -O exit` without occupying the coordinator queue or + /// Swift's cooperative executor. + private static func runConflictedControlMasterExit( request: RemoteProcessRequest, processRunner: any RemoteSessionProcessRunning, cancellation: RemoteProcessCancellationOperation - ) async -> InheritedReverseRelayCancellationOutcome { + ) async -> ConflictedControlMasterExitOutcome { await withCheckedContinuation { continuation in // The process runner is intentionally blocking. Keep that legacy // boundary on a utility thread while the owning Task suspends. DispatchQueue.global(qos: .utility).async { - let outcome: InheritedReverseRelayCancellationOutcome + let outcome: ConflictedControlMasterExitOutcome do { let result = try processRunner.run(request, operation: cancellation) if result.status == 0 { - outcome = .cancelled + outcome = .exited } else { let detail = bestErrorLine(stderr: result.stderr, stdout: result.stdout) ?? "ssh exited \(result.status)" @@ -92,9 +107,12 @@ extension RemoteSessionCoordinator { } } - private func finishInheritedReverseRelayCancellationLocked( + /// Re-enters queue confinement and retries only after `ssh -O exit` + /// completes and this recovery phase still owns the token. + private func finishConflictedControlMasterExitLocked( token: UUID, - outcome: InheritedReverseRelayCancellationOutcome, + outcome: ConflictedControlMasterExitOutcome, + startupFailure: String, remotePath: String, relayPort: Int, relayID: String, @@ -102,19 +120,25 @@ extension RemoteSessionCoordinator { localSocketPath: String ) { guard reverseRelayStartupPhase.token == token else { return } - reverseRelayStartupPhase = .idle + reverseRelayStartupPhase = .recoveryAttempted switch outcome { - case .cancelled: + case .exited: debugLog( - "remote.relay.inheritedForward.cancelled " + + "remote.relay.conflictedMaster.exited " + "relayPort=\(relayPort) \(debugConfigSummary())" ) case .ignored(let detail): debugLog( - "remote.relay.inheritedForward.cancelIgnored " + + "remote.relay.conflictedMaster.exitIgnored " + "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" ) + publishDaemonStatus( + .error, + detail: "Remote SSH relay unavailable: \(startupFailure) (retry in 2s)" + ) + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) + return } guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } @@ -127,15 +151,16 @@ extension RemoteSessionCoordinator { ) } + /// Cancels the in-flight OpenSSH recovery and invalidates its continuation. func cancelReverseRelayStartupLocked() { - guard case .cancellingInheritedForward( + guard case .exitingConflictedControlMaster( _, let task, let cancellation ) = reverseRelayStartupPhase else { return } - reverseRelayStartupPhase = .idle + reverseRelayStartupPhase = .recoveryAttempted task.cancel() cancellation.cancel() } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index 614b2d51e775..d3c029e2fa19 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -79,7 +79,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonReady = false var daemonBootstrapVersion: String? var daemonRemotePath: String? - var reverseRelayStartupPhase = ReverseRelayStartupPhase.idle + var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable var reverseRelayProcess: Process? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift index 4da9364ce6d1..044a36b7346b 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift @@ -1,23 +1,38 @@ internal import Foundation -/// Queue-confined phase for the asynchronous pre-launch relay cleanup. +/// Queue-confined phase for one conflict-triggered legacy-master recovery. enum ReverseRelayStartupPhase: Sendable { - case idle - case cancellingInheritedForward( + case recoveryAvailable + case exitingConflictedControlMaster( token: UUID, task: Task, cancellation: RemoteProcessCancellationOperation ) + case recoveryAttempted - var isIdle: Bool { - if case .idle = self { + var allowsRelayLaunch: Bool { + if case .exitingConflictedControlMaster = self { + return false + } + return true + } + + var canAttemptRecovery: Bool { + if case .recoveryAvailable = self { + return true + } + return false + } + + var isRecovering: Bool { + if case .exitingConflictedControlMaster = self { return true } return false } var token: UUID? { - guard case .cancellingInheritedForward(let token, _, _) = self else { + guard case .exitingConflictedControlMaster(let token, _, _) = self else { return nil } return token diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 101cae04f3c9..4b41c624e921 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -7,17 +7,112 @@ import CmuxRemoteDaemon @Suite("Reverse relay startup lifecycle") struct RemoteSessionReverseRelayStartupTests { + @Test("Only the configured OpenSSH remote-bind error triggers migration recovery") + func identifiesConfiguredPortBindingFailure() { + #expect(RemoteSessionCoordinator.isReverseRelayPortBindingFailure( + "remote port forwarding failed for listen port 64044", + relayPort: 64_044 + )) + #expect(RemoteSessionCoordinator.isReverseRelayPortBindingFailure( + "Error: remote port forwarding failed for listen port 64044", + relayPort: 64_044 + )) + #expect(!RemoteSessionCoordinator.isReverseRelayPortBindingFailure( + "remote port forwarding failed for listen port 64045", + relayPort: 64_044 + )) + #expect(!RemoteSessionCoordinator.isReverseRelayPortBindingFailure( + "Connection refused", + relayPort: 64_044 + )) + } + + @Test("Confirmed bind conflict exits the configured master once") + func confirmedConflictExitsConfiguredMaster() async { + let host = ReverseRelayRecoveryHost() + let runner = RecordingProcessRunner { _ in + RemoteCommandResult( + status: 255, + stdout: "", + stderr: "Control socket connect: No such file or directory" + ) + } + let coordinator = Self.makeCoordinator(host: host, runner: runner) + + let ignoredUnrelatedFailure = coordinator.queue.sync { + coordinator.beginConflictedControlMasterExitIfNeededLocked( + startupFailure: "Connection refused", + remotePath: "/tmp/cmuxd-remote", + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + ) + } + #expect(!ignoredUnrelatedFailure) + #expect(runner.requests.isEmpty) + + let beganRecovery = coordinator.queue.sync { + coordinator.beginConflictedControlMasterExitIfNeededLocked( + startupFailure: "Error: remote port forwarding failed for listen port 64044", + remotePath: "/tmp/cmuxd-remote", + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + ) + } + #expect(beganRecovery) + + var statuses = host.daemonStatuses.makeAsyncIterator() + let status = await statuses.next() + #expect(status?.detail?.contains("retry in 2s") == true) + + let request = runner.requests.first + #expect(request?.executable == "/usr/bin/ssh") + #expect(request?.arguments.contains("-O") == true) + #expect(request?.arguments.contains("exit") == true) + #expect(request?.arguments.contains("-R") == false) + #expect(request?.arguments.contains("StrictHostKeyChecking=accept-new") == true) + #expect(request?.arguments.last == "user@example.test") + + let recoveryAttempted = coordinator.queue.sync { + !coordinator.reverseRelayStartupPhase.isRecovering && + coordinator.reverseRelayRestartTask != nil + } + #expect(recoveryAttempted) + let beganSecondRecovery = coordinator.queue.sync { + coordinator.beginConflictedControlMasterExitIfNeededLocked( + startupFailure: "remote port forwarding failed for listen port 64044", + remotePath: "/tmp/cmuxd-remote", + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + ) + } + #expect(!beganSecondRecovery) + #expect(runner.requests.count == 1) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test( - "Stop cancels inherited-forward cleanup without waiting on its timeout", + "Stop cancels conflicted-master exit without waiting on its timeout", .timeLimit(.minutes(1)) ) - func stopCancelsInheritedForwardCleanup() async { - let runner = BlockingInheritedForwardCancellationRunner() + func stopCancelsConflictedMasterExit() async { + let runner = BlockingConflictedMasterExitRunner() let coordinator = Self.makeCoordinator(runner: runner) coordinator.queue.async { - coordinator.daemonReady = true - coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( + startupFailure: "remote port forwarding failed for listen port 64044", + remotePath: "/tmp/cmuxd-remote", + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + ) } var started = runner.started.makeAsyncIterator() @@ -28,14 +123,15 @@ struct RemoteSessionReverseRelayStartupTests { var cancelled = runner.cancelled.makeAsyncIterator() #expect(await cancelled.next() != nil) let startupCleared = coordinator.queue.sync { - coordinator.reverseRelayStartupPhase.isIdle && + !coordinator.reverseRelayStartupPhase.isRecovering && coordinator.reverseRelayProcess == nil } #expect(startupCleared) } private static func makeCoordinator( - runner: BlockingInheritedForwardCancellationRunner + host: any RemoteSessionHosting = NoopRemoteSessionHost(), + runner: any RemoteSessionProcessRunning ) -> RemoteSessionCoordinator { let configuration = WorkspaceRemoteConfiguration( destination: "user@example.test", @@ -52,7 +148,7 @@ struct RemoteSessionReverseRelayStartupTests { persistentDaemonSlot: nil ) return RemoteSessionCoordinator( - host: NoopRemoteSessionHost(), + host: host, configuration: configuration, proxyBroker: SSHOverrideUnusedRemoteProxyBroker(), connectionBroker: NativeSSHConnectionBroker(), @@ -75,7 +171,7 @@ struct RemoteSessionReverseRelayStartupTests { } } -private final class BlockingInheritedForwardCancellationRunner: +private final class BlockingConflictedMasterExitRunner: RemoteSessionProcessRunning, @unchecked Sendable { @@ -96,11 +192,11 @@ private final class BlockingInheritedForwardCancellationRunner: operation: (any RemoteTransferCancelling)? ) throws -> RemoteCommandResult { guard request.arguments.contains("-O"), - request.arguments.contains("cancel") else { + request.arguments.contains("exit") else { return RemoteCommandResult(status: 0, stdout: "", stderr: "") } guard let operation else { - throw BlockingInheritedForwardCancellationError.missingCancellationOperation + throw BlockingConflictedMasterExitError.missingCancellationOperation } try operation.throwIfCancelled() @@ -115,6 +211,24 @@ private final class BlockingInheritedForwardCancellationRunner: } } -private enum BlockingInheritedForwardCancellationError: Error { +private enum BlockingConflictedMasterExitError: Error { case missingCancellationOperation } + +private final class ReverseRelayRecoveryHost: RemoteSessionHosting, @unchecked Sendable { + let daemonStatuses: AsyncStream + private let daemonStatusContinuation: AsyncStream.Continuation + + init() { + (daemonStatuses, daemonStatusContinuation) = AsyncStream.makeStream() + } + + func publishConnectionState(_ state: WorkspaceRemoteConnectionState, detail: String?) {} + func publishDaemonStatus(_ status: WorkspaceRemoteDaemonStatus) { + daemonStatusContinuation.yield(status) + } + func publishProxyEndpoint(_ endpoint: BrowserProxyEndpoint?) {} + func publishPortsSnapshot(detectedByPanel: [UUID: [Int]], detected: [Int]) {} + func publishHeartbeat(count: Int, lastSeenAt: Date?) {} + func publishBootstrapRemoteTTY(_ ttyName: String) {} +} diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 6e100f089a88..293b8396a48b 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -35,16 +35,6 @@ private func remoteDaemonServeCommand(_ command: String) -> Bool { command.contains("serve") && command.contains("--stdio") } -private func remoteReverseRelayControlOperation(from arguments: [String]) -> (command: String, spec: String)? { - guard let operationIndex = arguments.firstIndex(of: "-O"), - operationIndex + 1 < arguments.count, - let reverseIndex = arguments.firstIndex(of: "-R"), - reverseIndex + 1 < arguments.count else { - return nil - } - return (arguments[operationIndex + 1], arguments[reverseIndex + 1]) -} - @MainActor private final class NativeSSHCleanupRecorder { var arguments: [[String]] = [] @@ -1896,93 +1886,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { ) } - @MainActor - func testPersistentReverseRelayCancelsInheritedForwardWithoutExplicitControlPath() { - let daemonTransportStarted = DispatchSemaphore(value: 0) - let inheritedForwardCancelled = DispatchSemaphore(value: 0) - let lock = NSLock() - var controlOperations: [(command: String, spec: String)] = [] - - let remoteProcessScript: RemoteProcessScript = { executable, arguments, _, _ in - guard executable == "/usr/bin/ssh" else { - XCTFail("unexpected executable \(executable)") - return (status: 1, stdout: "", stderr: "unexpected executable") - } - - if let controlOperation = remoteReverseRelayControlOperation(from: arguments) { - let operation = controlOperation.command - let spec = controlOperation.spec - lock.lock() - controlOperations.append((command: operation, spec: spec)) - lock.unlock() - if operation == "cancel" { - inheritedForwardCancelled.signal() - } - return (status: 0, stdout: "", stderr: "") - } - - let command = arguments.last ?? "" - if command.contains("uname -s") { - return ( - status: 0, - stdout: """ - __CMUX_REMOTE_HOME__=/home/test - __CMUX_REMOTE_OS__=Linux - __CMUX_REMOTE_ARCH__=x86_64 - __CMUX_REMOTE_EXISTS__=yes - """, - stderr: "" - ) - } - if remoteDaemonServeCommand(command) { - daemonTransportStarted.signal() - return ( - status: 0, - stdout: #"{"id":1,"ok":true,"result":{"name":"cmuxd-remote","version":"dev","capabilities":["proxy.stream.push","pty.session","pty.session.token","pty.write.notification","pty.resize.notification","pty.session.persistent_daemon"]}}"# + "\n", - stderr: "" - ) - } - return (status: 0, stdout: "", stderr: "") - } - - let workspace = Workspace() - workspace.remoteSessionProcessRunnerOverrideForTesting = - ScriptedRemoteProcessRunner(script: remoteProcessScript) - let config = WorkspaceRemoteConfiguration( - destination: "127.0.0.1", - port: 1, - identityFile: nil, - sshOptions: [ - "StrictHostKeyChecking=accept-new", - ], - localProxyPort: nil, - relayPort: 64044, - relayID: "relay-stale-forward", - relayToken: String(repeating: "c", count: 64), - localSocketPath: "/tmp/cmux-stale-forward-test.sock", - terminalStartupCommand: "ssh-pty-attach", - preserveAfterTerminalExit: true, - persistentDaemonSlot: "ssh-stale-forward-test" - ) - defer { workspace.disconnectRemoteConnection(clearConfiguration: true) } - - workspace.configureRemoteConnection(config, autoConnect: true) - - XCTAssertEqual(daemonTransportStarted.wait(timeout: .now() + 2), .success) - XCTAssertEqual(inheritedForwardCancelled.wait(timeout: .now() + 2), .success) - lock.lock() - let operations = controlOperations - lock.unlock() - - XCTAssertEqual(operations.count, 1) - XCTAssertEqual(operations.first?.command, "cancel") - XCTAssertEqual(operations.first?.spec, "127.0.0.1:64044") - XCTAssertFalse( - operations.contains(where: { $0.command == "forward" }), - "the relay must never install its forward on the shared ControlMaster: \(operations)" - ) - } - @MainActor func testDetachAttachPreservesRemoteTerminalSurfaceTracking() throws { let workspace = Workspace() From 9ddb4a9d5495a0be92811b910538b4276b8ee9a0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 22:34:10 -0700 Subject: [PATCH 10/39] docs: justify synchronous relay cancellation bridge (#8894) --- .../Process/RemoteProcessCancellationOperation.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift index 427789cef476..16521cf51c68 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteProcessCancellationOperation.swift @@ -3,6 +3,10 @@ internal import Foundation /// Cancellation token used to terminate a blocking process runner from a /// coordinator-owned task. /// +/// `installCancellationHandler` and `cancel` run synchronously from +/// non-isolated contexts that cannot await, so actor-backed state cannot serve +/// this bridge. +/// /// `@unchecked Sendable` is safe because the lock protects the complete /// mutable state, and handlers are always invoked after releasing the lock. final class RemoteProcessCancellationOperation: RemoteTransferCancelling, @unchecked Sendable { @@ -12,6 +16,8 @@ final class RemoteProcessCancellationOperation: RemoteTransferCancelling, @unche private var cancelled = false private var cancellationHandler: (() -> Void)? + deinit {} + var isCancelled: Bool { lock.lock() defer { lock.unlock() } From 0f4abbad184b1136a7fe04602b72500ef00c7a7f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 22:43:35 -0700 Subject: [PATCH 11/39] test: isolate relay recovery and sanitize status (#8894) --- ...ssionCoordinator+ReverseRelayStartup.swift | 7 +-- ...emoteSessionReverseRelayStartupTests.swift | 44 +++++++++++++------ Resources/Localizable.xcstrings | 17 +++++++ 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index db90398f816f..d235153ca5b6 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -57,7 +57,6 @@ extension RemoteSessionCoordinator { self?.finishConflictedControlMasterExitLocked( token: token, outcome: outcome, - startupFailure: startupFailure, remotePath: remotePath, relayPort: relayPort, relayID: relayID, @@ -112,7 +111,6 @@ extension RemoteSessionCoordinator { private func finishConflictedControlMasterExitLocked( token: UUID, outcome: ConflictedControlMasterExitOutcome, - startupFailure: String, remotePath: String, relayPort: Int, relayID: String, @@ -135,7 +133,10 @@ extension RemoteSessionCoordinator { ) publishDaemonStatus( .error, - detail: "Remote SSH relay unavailable: \(startupFailure) (retry in 2s)" + detail: String( + localized: "remoteSession.reverseRelay.portUnavailableRetrying", + defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" + ) ) scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) return diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 4b41c624e921..c9344eab62e1 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -28,7 +28,7 @@ struct RemoteSessionReverseRelayStartupTests { } @Test("Confirmed bind conflict exits the configured master once") - func confirmedConflictExitsConfiguredMaster() async { + func confirmedConflictExitsConfiguredMaster() async throws { let host = ReverseRelayRecoveryHost() let runner = RecordingProcessRunner { _ in RemoteCommandResult( @@ -37,7 +37,9 @@ struct RemoteSessionReverseRelayStartupTests { stderr: "Control socket connect: No such file or directory" ) } - let coordinator = Self.makeCoordinator(host: host, runner: runner) + let fixture = try Self.makeCoordinator(host: host, runner: runner) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } let ignoredUnrelatedFailure = coordinator.queue.sync { coordinator.beginConflictedControlMasterExitIfNeededLocked( @@ -46,7 +48,7 @@ struct RemoteSessionReverseRelayStartupTests { relayPort: 64_044, relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), - localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + localSocketPath: coordinator.configuration.localSocketPath ?? "" ) } #expect(!ignoredUnrelatedFailure) @@ -59,14 +61,17 @@ struct RemoteSessionReverseRelayStartupTests { relayPort: 64_044, relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), - localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + localSocketPath: coordinator.configuration.localSocketPath ?? "" ) } #expect(beganRecovery) var statuses = host.daemonStatuses.makeAsyncIterator() let status = await statuses.next() - #expect(status?.detail?.contains("retry in 2s") == true) + #expect(status?.detail == String( + localized: "remoteSession.reverseRelay.portUnavailableRetrying", + defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" + )) let request = runner.requests.first #expect(request?.executable == "/usr/bin/ssh") @@ -88,7 +93,7 @@ struct RemoteSessionReverseRelayStartupTests { relayPort: 64_044, relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), - localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + localSocketPath: coordinator.configuration.localSocketPath ?? "" ) } #expect(!beganSecondRecovery) @@ -100,9 +105,11 @@ struct RemoteSessionReverseRelayStartupTests { "Stop cancels conflicted-master exit without waiting on its timeout", .timeLimit(.minutes(1)) ) - func stopCancelsConflictedMasterExit() async { + func stopCancelsConflictedMasterExit() async throws { let runner = BlockingConflictedMasterExitRunner() - let coordinator = Self.makeCoordinator(runner: runner) + let fixture = try Self.makeCoordinator(runner: runner) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } coordinator.queue.async { _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( @@ -111,7 +118,7 @@ struct RemoteSessionReverseRelayStartupTests { relayPort: 64_044, relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), - localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock" + localSocketPath: coordinator.configuration.localSocketPath ?? "" ) } @@ -124,6 +131,7 @@ struct RemoteSessionReverseRelayStartupTests { #expect(await cancelled.next() != nil) let startupCleared = coordinator.queue.sync { !coordinator.reverseRelayStartupPhase.isRecovering && + coordinator.reverseRelayStartupPhase.allowsRelayLaunch && coordinator.reverseRelayProcess == nil } #expect(startupCleared) @@ -132,7 +140,16 @@ struct RemoteSessionReverseRelayStartupTests { private static func makeCoordinator( host: any RemoteSessionHosting = NoopRemoteSessionHost(), runner: any RemoteSessionProcessRunning - ) -> RemoteSessionCoordinator { + ) throws -> (coordinator: RemoteSessionCoordinator, scratchDirectory: URL) { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-reverse-relay-startup-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory( + at: scratchDirectory, + withIntermediateDirectories: true + ) let configuration = WorkspaceRemoteConfiguration( destination: "user@example.test", port: nil, @@ -142,18 +159,18 @@ struct RemoteSessionReverseRelayStartupTests { relayPort: 64_044, relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), - localSocketPath: "/tmp/cmux-relay-startup-cancellation.sock", + localSocketPath: scratchDirectory.appendingPathComponent("relay.sock").path, terminalStartupCommand: nil, preserveAfterTerminalExit: false, persistentDaemonSlot: nil ) - return RemoteSessionCoordinator( + let coordinator = RemoteSessionCoordinator( host: host, configuration: configuration, proxyBroker: SSHOverrideUnusedRemoteProxyBroker(), connectionBroker: NativeSSHConnectionBroker(), manifestRepository: RemoteDaemonManifestRepository( - homeDirectory: FileManager.default.temporaryDirectory + homeDirectory: scratchDirectory ), processRunner: runner, reachabilityProbe: SSHOverrideNoopReachabilityProbe(), @@ -168,6 +185,7 @@ struct RemoteSessionReverseRelayStartupTests { suspendedDetailFormat: "%@" ) ) + return (coordinator, scratchDirectory) } } diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index 40590e122216..952b3418bc71 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -139907,6 +139907,23 @@ } } }, + "remoteSession.reverseRelay.portUnavailableRetrying": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Remote SSH relay port unavailable; retrying in 2 seconds" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "リモート SSH リレーポートを利用できません。2秒後に再試行します" + } + } + } + }, "remoteTmux.error.commandFailed": { "extractionState": "manual", "localizations": { From 04cb2c5fbe92d21caf616a3b65fcedd25c442841 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 22:49:23 -0700 Subject: [PATCH 12/39] test: cover successful relay conflict recovery (#8894) --- ...emoteSessionReverseRelayStartupTests.swift | 56 +++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index c9344eab62e1..ea74ea1de292 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -101,6 +101,51 @@ struct RemoteSessionReverseRelayStartupTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } + @Test( + "Successful master exit retries the dedicated relay", + .timeLimit(.minutes(1)) + ) + func successfulMasterExitRetriesDedicatedRelay() async throws { + let host = ReverseRelayRecoveryHost() + let runner = RecordingProcessRunner() + let fixture = try Self.makeCoordinator( + host: host, + runner: runner, + destination: "127.0.0.1", + port: 1, + relayPort: 64_046 + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + coordinator.queue.async { + coordinator.daemonReady = true + _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( + startupFailure: "Error: remote port forwarding failed for listen port 64046", + remotePath: "/tmp/cmuxd-remote", + relayPort: 64_046, + relayID: "relay-successful-recovery", + relayToken: String(repeating: "b", count: 64), + localSocketPath: coordinator.configuration.localSocketPath ?? "" + ) + } + + var statuses = host.daemonStatuses.makeAsyncIterator() + let retryStatus = await statuses.next() + #expect(retryStatus?.detail?.contains("Remote SSH relay unavailable") == true) + + let recoveryRequest = runner.requests.first + #expect(recoveryRequest?.arguments.contains("-O") == true) + #expect(recoveryRequest?.arguments.contains("exit") == true) + #expect(runner.requests.count == 1) + let retriedAfterRecovery = coordinator.queue.sync { + coordinator.reverseRelayStartupPhase.allowsRelayLaunch && + coordinator.reverseRelayRestartTask != nil + } + #expect(retriedAfterRecovery) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test( "Stop cancels conflicted-master exit without waiting on its timeout", .timeLimit(.minutes(1)) @@ -139,7 +184,10 @@ struct RemoteSessionReverseRelayStartupTests { private static func makeCoordinator( host: any RemoteSessionHosting = NoopRemoteSessionHost(), - runner: any RemoteSessionProcessRunning + runner: any RemoteSessionProcessRunning, + destination: String = "user@example.test", + port: Int? = nil, + relayPort: Int = 64_044 ) throws -> (coordinator: RemoteSessionCoordinator, scratchDirectory: URL) { let scratchDirectory = FileManager.default.temporaryDirectory .appendingPathComponent( @@ -151,12 +199,12 @@ struct RemoteSessionReverseRelayStartupTests { withIntermediateDirectories: true ) let configuration = WorkspaceRemoteConfiguration( - destination: "user@example.test", - port: nil, + destination: destination, + port: port, identityFile: nil, sshOptions: ["StrictHostKeyChecking=accept-new"], localProxyPort: nil, - relayPort: 64_044, + relayPort: relayPort, relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), localSocketPath: scratchDirectory.appendingPathComponent("relay.sock").path, From 3272ae619c865fa60645673c426398a864a46aa7 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 23:35:37 -0700 Subject: [PATCH 13/39] test: isolate reverse relay recovery launch (#8894) --- .../FoundationRemoteReverseRelayProcess.swift | 53 +++++++++ .../Process/RemoteReverseRelayLauncher.swift | 34 ++++++ .../Process/RemoteReverseRelayLaunching.swift | 16 +++ .../Process/RemoteReverseRelayProcess.swift | 26 +++++ ...emoteSessionCoordinator+ReverseRelay.swift | 61 ++++------ .../Session/RemoteSessionCoordinator.swift | 6 +- ...emoteSessionReverseRelayStartupTests.swift | 108 ++++++++++++++---- 7 files changed, 240 insertions(+), 64 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift new file mode 100644 index 000000000000..1962df89c677 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -0,0 +1,53 @@ +internal import CmuxFoundation +internal import Foundation + +/// Foundation-backed handle for one dedicated SSH reverse-relay process. +/// +/// `Process` and `Pipe` callbacks cross executor boundaries; all mutation is +/// owned by Foundation while the coordinator serializes its handle access. +final class FoundationRemoteReverseRelayProcess: + RemoteReverseRelayProcess, + @unchecked Sendable +{ + let stderrPipe: Pipe + + private let process: Process + + init(process: Process, stderrPipe: Pipe) { + self.process = process + self.stderrPipe = stderrPipe + } + + var isRunning: Bool { + process.isRunning + } + + var terminationStatus: Int32 { + process.terminationStatus + } + + func startupFailureDetail(gracePeriod: TimeInterval) -> String? { + if process.isRunning { + let originalTerminationHandler = process.terminationHandler + let exitSemaphore = DispatchSemaphore(value: 0) + process.terminationHandler = { terminated in + originalTerminationHandler?(terminated) + exitSemaphore.signal() + } + if !process.isRunning { + exitSemaphore.signal() + } + guard exitSemaphore.wait(timeout: .now() + max(0, gracePeriod)) == .success else { + return nil + } + } + let stderrData = stderrPipe.fileHandleForReading.readDataToEndOfFileOrEmpty() + let stderr = String(data: stderrData, encoding: .utf8) ?? "" + return RemoteSessionCoordinator.bestErrorLine(stderr: stderr) + ?? "status=\(process.terminationStatus)" + } + + func terminate() { + process.terminate() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift new file mode 100644 index 000000000000..b7b0b4ffdd73 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift @@ -0,0 +1,34 @@ +internal import Foundation + +/// Production launcher for a standalone SSH reverse-relay transport. +public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { + /// Creates a production reverse-relay launcher. + public init() {} + + /// Launches `/usr/bin/ssh` with null stdin/stdout and captured stderr. + public func launch( + arguments: [String], + environment: [String: String]?, + terminationHandler: @escaping @Sendable (any RemoteReverseRelayProcess) -> Void + ) throws -> any RemoteReverseRelayProcess { + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") + process.arguments = arguments + process.environment = environment + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe + ) + process.terminationHandler = { [weak relayProcess] _ in + guard let relayProcess else { return } + terminationHandler(relayProcess) + } + try process.run() + return relayProcess + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift new file mode 100644 index 000000000000..a43e32f4d8fc --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift @@ -0,0 +1,16 @@ +/// Launches the standalone SSH process that owns one reverse relay. +public protocol RemoteReverseRelayLaunching: Sendable { + /// Starts `/usr/bin/ssh` with the supplied dedicated-relay configuration. + /// + /// - Parameters: + /// - arguments: SSH arguments for the reverse-relay transport. + /// - environment: Process environment, or `nil` to inherit. + /// - terminationHandler: Called when the launched transport exits. + /// - Returns: A coordinator-owned handle for the running transport. + /// - Throws: A Foundation process-launch error. + func launch( + arguments: [String], + environment: [String: String]?, + terminationHandler: @escaping @Sendable (any RemoteReverseRelayProcess) -> Void + ) throws -> any RemoteReverseRelayProcess +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift new file mode 100644 index 000000000000..7f5949b7ddaf --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift @@ -0,0 +1,26 @@ +public import Foundation + +/// A running dedicated SSH reverse-relay transport. +/// +/// The coordinator owns this handle, terminates it during normal teardown, and +/// uses its stderr stream to diagnose startup and later transport failures. +public protocol RemoteReverseRelayProcess: AnyObject, Sendable { + /// The process's standard-error pipe. + var stderrPipe: Pipe { get } + + /// Whether the transport process is still running. + var isRunning: Bool { get } + + /// The process's exit status after termination. + var terminationStatus: Int32 { get } + + /// Waits for an immediate startup failure and returns its best diagnostic. + /// + /// - Parameter gracePeriod: Maximum time to wait for an early process exit. + /// - Returns: The best failure line when the process exits, or `nil` when it + /// remains running through the grace period. + func startupFailureDetail(gracePeriod: TimeInterval) -> String? + + /// Requests termination of the transport process. + func terminate() +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 9142786a9589..78f4c2c41b61 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -7,8 +7,10 @@ nonisolated private let remoteRelayLogger = Logger(subsystem: "com.cmuxterm.app" // The reverse CLI relay: a remote `127.0.0.1:` listener forwarded // back to the local CLI relay server by a dedicated `ssh -N -R` process. The -// relay targets an in-process server, so its SSH transport must share the app -// process's lifetime rather than a host-scoped ControlMaster's lifetime. +// relay targets an in-process server, so its SSH transport is coordinator-owned +// and standalone rather than attached to a host-scoped ControlMaster. Normal +// stop terminates it; a later connection attempt reaps a PPID-1 orphan left by +// a crash using its destination and pinned relay-port argv. // Stderr capture caps and restart cadence (2s) are pinned legacy behavior. extension RemoteSessionCoordinator { func startReverseRelayLocked(remotePath: String) { @@ -62,29 +64,22 @@ extension RemoteSessionCoordinator { persistentDaemonSlot: configuration.persistentDaemonSlot ) - let process = Process() - let stderrPipe = Pipe() - process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") let relayArguments = reverseRelayArguments( relayPort: relayPort, localRelayPort: localRelayPort ) - process.arguments = relayArguments - process.environment = configuration.sshProcessEnvironment - process.standardInput = FileHandle.nullDevice - process.standardOutput = FileHandle.nullDevice - process.standardError = stderrPipe - - process.terminationHandler = { [weak self] terminated in - self?.queue.async { - self?.handleReverseRelayTerminationLocked(process: terminated) + let process = try reverseRelayLauncher.launch( + arguments: relayArguments, + environment: configuration.sshProcessEnvironment + ) { [weak self] terminated in + guard let coordinator = self else { return } + coordinator.queue.async { + coordinator.handleReverseRelayTerminationLocked(process: terminated) } } - - try process.run() - if let startupFailure = Self.reverseRelayStartupFailureDetail( - process: process, - stderrPipe: stderrPipe + let stderrPipe = process.stderrPipe + if let startupFailure = process.startupFailureDetail( + gracePeriod: Self.reverseRelayStartupGracePeriod ) { let retryDelay = 2.0 let retrySeconds = max(1, Int(retryDelay.rounded())) @@ -174,7 +169,7 @@ extension RemoteSessionCoordinator { } } - private func handleReverseRelayTerminationLocked(process: Process) { + private func handleReverseRelayTerminationLocked(process: any RemoteReverseRelayProcess) { guard reverseRelayProcess === process else { return } let stderrDetail = Self.bestErrorLine(stderr: reverseRelayStderrBuffer) reverseRelayStderrPipe?.fileHandleForReading.readabilityHandler = nil @@ -244,8 +239,9 @@ extension RemoteSessionCoordinator { } func reverseRelayArguments(relayPort: Int, localRelayPort: Int) -> [String] { - // The relay's SSH process is deliberately app-owned. `-S none` also - // protects against a ControlPath inherited from the host's ssh_config. + // The relay's SSH process is deliberately standalone and coordinator + // owned. `-S none` also protects against a ControlPath inherited from + // the host's ssh_config and leaves argv that crash recovery can match. var args: [String] = ["-N", "-T", "-S", "none"] args += sshCommonArguments(batchMode: true, dropControlPath: true) args += [ @@ -399,23 +395,10 @@ extension RemoteSessionCoordinator { stderrPipe: Pipe, gracePeriod: TimeInterval = reverseRelayStartupGracePeriod ) -> String? { - if process.isRunning { - let originalTerminationHandler = process.terminationHandler - let exitSemaphore = DispatchSemaphore(value: 0) - process.terminationHandler = { terminated in - originalTerminationHandler?(terminated) - exitSemaphore.signal() - } - if !process.isRunning { - exitSemaphore.signal() - } - guard exitSemaphore.wait(timeout: .now() + max(0, gracePeriod)) == .success else { - return nil - } - } - let stderrData = stderrPipe.fileHandleForReading.readDataToEndOfFileOrEmpty() - let stderr = String(data: stderrData, encoding: .utf8) ?? "" - return bestErrorLine(stderr: stderr) ?? "status=\(process.terminationStatus)" + FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe + ).startupFailureDetail(gracePeriod: gracePeriod) } /// Returns whether OpenSSH reported that this relay's remote listener is diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index d3c029e2fa19..9fbd49ec8e7f 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -58,6 +58,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { let connectionBroker: NativeSSHConnectionBroker let manifestRepository: RemoteDaemonManifestRepository let processRunner: any RemoteSessionProcessRunning + let reverseRelayLauncher: any RemoteReverseRelayLaunching let reachabilityProbe: any RemoteHostReachabilityProbing let relayCommandRewriter: any RemoteRelayCommandRewriting let buildInfo: any RemoteSessionBuildInfoProviding @@ -80,7 +81,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonBootstrapVersion: String? var daemonRemotePath: String? var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable - var reverseRelayProcess: Process? + var reverseRelayProcess: (any RemoteReverseRelayProcess)? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] /// Stable publication state for best-effort remote TTY attribution scans. @@ -147,6 +148,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { /// connection-attempt broker. /// - manifestRepository: cmuxd-remote manifest/binary-cache repository. /// - processRunner: Blocking subprocess seam (ssh/scp/dev go build). + /// - reverseRelayLauncher: Standalone SSH reverse-relay launch seam. /// - reachabilityProbe: SSH endpoint reachability seam for the /// reconnect-suspend policy. /// - relayCommandRewriter: Alias-aware CLI relay command rewriter. @@ -162,6 +164,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { connectionBroker: NativeSSHConnectionBroker, manifestRepository: RemoteDaemonManifestRepository, processRunner: any RemoteSessionProcessRunning, + reverseRelayLauncher: any RemoteReverseRelayLaunching = RemoteReverseRelayLauncher(), reachabilityProbe: any RemoteHostReachabilityProbing, relayCommandRewriter: any RemoteRelayCommandRewriting, buildInfo: any RemoteSessionBuildInfoProviding, @@ -175,6 +178,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { self.connectionBroker = connectionBroker self.manifestRepository = manifestRepository self.processRunner = processRunner + self.reverseRelayLauncher = reverseRelayLauncher self.reachabilityProbe = reachabilityProbe self.relayCommandRewriter = relayCommandRewriter self.buildInfo = buildInfo diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index ea74ea1de292..978ad64edec7 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -101,55 +101,58 @@ struct RemoteSessionReverseRelayStartupTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test( - "Successful master exit retries the dedicated relay", - .timeLimit(.minutes(1)) - ) + @Test("Successful master exit retries the dedicated relay") func successfulMasterExitRetriesDedicatedRelay() async throws { - let host = ReverseRelayRecoveryHost() let runner = RecordingProcessRunner() + let launcher = RecordingReverseRelayLauncher() + let relayPort = 64_046 let fixture = try Self.makeCoordinator( - host: host, runner: runner, - destination: "127.0.0.1", - port: 1, - relayPort: 64_046 + reverseRelayLauncher: launcher, + relayPort: relayPort ) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + var launches = launcher.launches.makeAsyncIterator() coordinator.queue.async { coordinator.daemonReady = true _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( - startupFailure: "Error: remote port forwarding failed for listen port 64046", + startupFailure: "Error: remote port forwarding failed for listen port \(relayPort)", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_046, + relayPort: relayPort, relayID: "relay-successful-recovery", relayToken: String(repeating: "b", count: 64), localSocketPath: coordinator.configuration.localSocketPath ?? "" ) } - var statuses = host.daemonStatuses.makeAsyncIterator() - let retryStatus = await statuses.next() - #expect(retryStatus?.detail?.contains("Remote SSH relay unavailable") == true) + let launch = try #require(await launches.next()) let recoveryRequest = runner.requests.first #expect(recoveryRequest?.arguments.contains("-O") == true) #expect(recoveryRequest?.arguments.contains("exit") == true) - #expect(runner.requests.count == 1) + #expect(launch.arguments.starts(with: ["-N", "-T", "-S", "none"])) + #expect(launch.arguments.contains("-R")) + #expect(launch.arguments.contains( + "127.0.0.1:\(relayPort):127.0.0.1:\(launch.localRelayPort)" + )) + #expect(!launch.arguments.contains(where: { $0.hasPrefix("ControlPath=") })) + let retriedAfterRecovery = coordinator.queue.sync { coordinator.reverseRelayStartupPhase.allowsRelayLaunch && - coordinator.reverseRelayRestartTask != nil + coordinator.reverseRelayProcess === launcher.process } #expect(retriedAfterRecovery) + #expect(RemoteSessionCoordinator.orphanedCMUXRemoteSSHPIDs( + psOutput: "909 1 /usr/bin/ssh \(launch.arguments.joined(separator: " "))", + destination: "user@example.test", + relayPort: relayPort + ) == [909]) _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test( - "Stop cancels conflicted-master exit without waiting on its timeout", - .timeLimit(.minutes(1)) - ) + @Test("Stop cancels conflicted-master exit without waiting on its timeout") func stopCancelsConflictedMasterExit() async throws { let runner = BlockingConflictedMasterExitRunner() let fixture = try Self.makeCoordinator(runner: runner) @@ -185,8 +188,7 @@ struct RemoteSessionReverseRelayStartupTests { private static func makeCoordinator( host: any RemoteSessionHosting = NoopRemoteSessionHost(), runner: any RemoteSessionProcessRunning, - destination: String = "user@example.test", - port: Int? = nil, + reverseRelayLauncher: any RemoteReverseRelayLaunching = RemoteReverseRelayLauncher(), relayPort: Int = 64_044 ) throws -> (coordinator: RemoteSessionCoordinator, scratchDirectory: URL) { let scratchDirectory = FileManager.default.temporaryDirectory @@ -199,8 +201,8 @@ struct RemoteSessionReverseRelayStartupTests { withIntermediateDirectories: true ) let configuration = WorkspaceRemoteConfiguration( - destination: destination, - port: port, + destination: "user@example.test", + port: nil, identityFile: nil, sshOptions: ["StrictHostKeyChecking=accept-new"], localProxyPort: nil, @@ -221,6 +223,7 @@ struct RemoteSessionReverseRelayStartupTests { homeDirectory: scratchDirectory ), processRunner: runner, + reverseRelayLauncher: reverseRelayLauncher, reachabilityProbe: SSHOverrideNoopReachabilityProbe(), relayCommandRewriter: SSHOverridePassthroughRelayCommandRewriter(), buildInfo: SSHOverrideStubBuildInfo(), @@ -237,6 +240,63 @@ struct RemoteSessionReverseRelayStartupTests { } } +private struct RecordedReverseRelayLaunch: Sendable { + let arguments: [String] + let localRelayPort: Int +} + +/// Immutable recorder; `AsyncStream.Continuation` owns synchronized delivery. +private final class RecordingReverseRelayLauncher: + RemoteReverseRelayLaunching, + @unchecked Sendable +{ + let launches: AsyncStream + let process = StubReverseRelayProcess() + + private let launchContinuation: AsyncStream.Continuation + + init() { + (launches, launchContinuation) = AsyncStream.makeStream() + } + + func launch( + arguments: [String], + environment: [String: String]?, + terminationHandler: @escaping @Sendable (any RemoteReverseRelayProcess) -> Void + ) throws -> any RemoteReverseRelayProcess { + let reverseArgumentIndex = try #require(arguments.firstIndex(of: "-R")) + let reverseArgument = try #require( + arguments.indices.contains(arguments.index(after: reverseArgumentIndex)) + ? arguments[arguments.index(after: reverseArgumentIndex)] + : nil + ) + let localRelayPort = try #require( + Int(reverseArgument.split(separator: ":").last ?? "") + ) + launchContinuation.yield(RecordedReverseRelayLaunch( + arguments: arguments, + localRelayPort: localRelayPort + )) + return process + } +} + +/// Immutable fake process; the pipe is never concurrently read or written. +private final class StubReverseRelayProcess: + RemoteReverseRelayProcess, + @unchecked Sendable +{ + let stderrPipe = Pipe() + let isRunning = true + let terminationStatus: Int32 = 0 + + func startupFailureDetail(gracePeriod: TimeInterval) -> String? { + nil + } + + func terminate() {} +} + private final class BlockingConflictedMasterExitRunner: RemoteSessionProcessRunning, @unchecked Sendable From 946cea6b78f6ca029dde1bb4041c499577651052 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 00:25:06 -0700 Subject: [PATCH 14/39] fix: recover relay through the shared SSH master (#8894) --- ...RemoteConfiguration+SSHBatchCommands.swift | 62 +++- ...teConfigurationSSHBatchCommandsTests.swift | 39 +++ .../FoundationRemoteReverseRelayProcess.swift | 66 ++-- .../Process/RemoteReverseRelayLauncher.swift | 11 +- .../Process/RemoteReverseRelayLaunching.swift | 5 +- .../Process/RemoteReverseRelayProcess.swift | 14 +- .../Session/RemoteControlMasterCleanup.swift | 14 +- ...SessionCoordinator+RelayProvisioning.swift | 2 +- ...emoteSessionCoordinator+ReverseRelay.swift | 185 ++++++----- ...oordinator+ReverseRelayControlMaster.swift | 78 +++++ ...ssionCoordinator+ReverseRelayStartup.swift | 72 +++-- .../Session/RemoteSessionCoordinator.swift | 8 +- ...emoteSessionReverseRelayStartupTests.swift | 104 +++---- ...oteSessionReverseRelayTransportTests.swift | 289 ++++++++++++++++++ ...SessionSSHRemoteCommandOverrideTests.swift | 4 +- .../WorkspaceRemoteConnectionTests.swift | 20 -- 16 files changed, 715 insertions(+), 258 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift diff --git a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift index f8d16023ef33..aee555d68d7d 100644 --- a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift +++ b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift @@ -54,12 +54,45 @@ extension WorkspaceRemoteConfiguration { ] } + /// `ssh -O ` argv that drives a reverse forward on the + /// configured ControlMaster socket, or `nil` when no usable `ControlPath` + /// option is present in the effective SSH options. + /// + /// - Parameters: + /// - controlCommand: OpenSSH multiplexing command, such as `forward` or `cancel`. + /// - forwardSpec: Exact reverse-forward specification. + /// - effectiveSSHOptions: Options used by the foreground SSH connection. + /// - Returns: Arguments for `/usr/bin/ssh`, or `nil` without a usable control socket. + public func reverseRelayControlMasterArguments( + controlCommand: String, + forwardSpec: String, + effectiveSSHOptions: [String] + ) -> [String]? { + guard let controlPath = Self.firstSSHOptionValue( + named: "ControlPath", + in: effectiveSSHOptions + )? + .trimmingCharacters(in: .whitespacesAndNewlines), + !controlPath.isEmpty, + controlPath.lowercased() != "none" else { + return nil + } + + var arguments = batchSSHArguments(sshOptions: effectiveSSHOptions) + arguments += ["-O", controlCommand, "-R", forwardSpec, destination] + return arguments + } + // Shared batch-mode `ssh` options: keepalives, BatchMode, no new // ControlMaster (existing ControlPath sockets may be reused), port, // identity, then the configuration's options minus // ControlMaster/ControlPersist. private func batchSSHArguments() -> [String] { - let effectiveSSHOptions = backgroundSSHOptions() + batchSSHArguments(sshOptions: sshOptions) + } + + private func batchSSHArguments(sshOptions: [String]) -> [String] { + let effectiveSSHOptions = backgroundSSHOptions(sshOptions) var args: [String] = [ "-o", "ConnectTimeout=6", "-o", "ServerAliveInterval=20", @@ -86,13 +119,36 @@ extension WorkspaceRemoteConfiguration { // Trimmed options minus ControlMaster/ControlPersist (ControlPath is // kept so batch helpers can reuse an existing master's socket). - private func backgroundSSHOptions() -> [String] { + private func backgroundSSHOptions(_ options: [String]) -> [String] { let resolver = SSHAgentSocketResolver() - return Self.trimmedSSHOptions(sshOptions).filter { option in + return Self.trimmedSSHOptions(options).filter { option in guard let key = resolver.optionKey(option) else { return false } return !Self.batchSSHControlOptionKeys.contains(key) } } + + // OpenSSH uses the first obtained value for these command-line options. + private static func firstSSHOptionValue( + named key: String, + in options: [String] + ) -> String? { + let loweredKey = key.lowercased() + for option in trimmedSSHOptions(options) { + let parts = option.split( + maxSplits: 1, + omittingEmptySubsequences: true, + whereSeparator: { $0 == "=" || $0.isWhitespace } + ) + guard parts.count == 2, parts[0].lowercased() == loweredKey else { + continue + } + let value = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) + if !value.isEmpty { + return value + } + } + return nil + } } extension String { diff --git a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift index eed03fa8f2a2..dd20c8283421 100644 --- a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift +++ b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift @@ -147,4 +147,43 @@ struct WorkspaceRemoteConfigurationSSHBatchCommandsTests { ) } + @Test("reverseRelayControlMasterArguments uses the configured ControlPath") + func reverseRelayControlMasterArguments() throws { + let configuration = configuration() + let arguments = try #require( + configuration.reverseRelayControlMasterArguments( + controlCommand: "forward", + forwardSpec: "127.0.0.1:64007:127.0.0.1:54321", + effectiveSSHOptions: configuration.sshOptions + ) + ) + #expect( + arguments == expectedBatchArguments + + [ + "-O", "forward", + "-R", "127.0.0.1:64007:127.0.0.1:54321", + "cmux-macmini", + ] + ) + } + + @Test("reverse relay ControlMaster commands require a usable ControlPath") + func reverseRelayRequiresControlPath() { + #expect( + configuration(sshOptions: ["StrictHostKeyChecking=accept-new"]) + .reverseRelayControlMasterArguments( + controlCommand: "forward", + forwardSpec: "127.0.0.1:64007:127.0.0.1:54321", + effectiveSSHOptions: ["StrictHostKeyChecking=accept-new"] + ) == nil + ) + #expect( + configuration(sshOptions: ["ControlPath=None"]) + .reverseRelayControlMasterArguments( + controlCommand: "forward", + forwardSpec: "127.0.0.1:64007:127.0.0.1:54321", + effectiveSSHOptions: ["ControlPath=None"] + ) == nil + ) + } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift index 1962df89c677..3b7a84702da4 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -1,4 +1,4 @@ -internal import CmuxFoundation +internal import Darwin internal import Foundation /// Foundation-backed handle for one dedicated SSH reverse-relay process. @@ -9,9 +9,8 @@ final class FoundationRemoteReverseRelayProcess: RemoteReverseRelayProcess, @unchecked Sendable { - let stderrPipe: Pipe - private let process: Process + private let stderrPipe: Pipe init(process: Process, stderrPipe: Pipe) { self.process = process @@ -26,28 +25,53 @@ final class FoundationRemoteReverseRelayProcess: process.terminationStatus } - func startupFailureDetail(gracePeriod: TimeInterval) -> String? { - if process.isRunning { - let originalTerminationHandler = process.terminationHandler - let exitSemaphore = DispatchSemaphore(value: 0) - process.terminationHandler = { terminated in - originalTerminationHandler?(terminated) - exitSemaphore.signal() - } - if !process.isRunning { - exitSemaphore.signal() - } - guard exitSemaphore.wait(timeout: .now() + max(0, gracePeriod)) == .success else { - return nil - } + /// Drains stderr from launch through EOF before reporting termination. + func captureTermination( + _ handler: @escaping @Sendable (String?) -> Void + ) { + let stderrDescriptor = stderrPipe.fileHandleForReading.fileDescriptor + // Blocking pipe drainage is a Foundation/Process bridge. It owns the + // raw descriptor on a utility thread so SSH can never fill the pipe, + // and the callback cannot outrun the final stderr bytes. + DispatchQueue.global(qos: .utility).async { [self] in + let stderr = Self.readStderrTail(fileDescriptor: stderrDescriptor) + process.waitUntilExit() + handler( + RemoteSessionCoordinator.bestErrorLine(stderr: stderr) + ?? "status=\(process.terminationStatus)" + ) } - let stderrData = stderrPipe.fileHandleForReading.readDataToEndOfFileOrEmpty() - let stderr = String(data: stderrData, encoding: .utf8) ?? "" - return RemoteSessionCoordinator.bestErrorLine(stderr: stderr) - ?? "status=\(process.terminationStatus)" } func terminate() { process.terminate() } + + private static func readStderrTail( + fileDescriptor: Int32, + byteLimit: Int = 8192 + ) -> String { + let chunkSize = 4096 + var bytes = [UInt8](repeating: 0, count: chunkSize) + var tail = Data() + + while true { + let count = bytes.withUnsafeMutableBytes { buffer -> Int in + guard let baseAddress = buffer.baseAddress else { return 0 } + return Darwin.read(fileDescriptor, baseAddress, chunkSize) + } + if count > 0 { + tail.append(contentsOf: bytes.prefix(count)) + if tail.count > byteLimit { + tail.removeFirst(tail.count - byteLimit) + } + continue + } + if count < 0, errno == EINTR { + continue + } + break + } + return String(data: tail, encoding: .utf8) ?? "" + } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift index b7b0b4ffdd73..ac166d5e9e69 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift @@ -9,7 +9,10 @@ public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { public func launch( arguments: [String], environment: [String: String]?, - terminationHandler: @escaping @Sendable (any RemoteReverseRelayProcess) -> Void + terminationHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess, + String? + ) -> Void ) throws -> any RemoteReverseRelayProcess { let process = Process() let stderrPipe = Pipe() @@ -24,11 +27,11 @@ public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { process: process, stderrPipe: stderrPipe ) - process.terminationHandler = { [weak relayProcess] _ in + try process.run() + relayProcess.captureTermination { [weak relayProcess] detail in guard let relayProcess else { return } - terminationHandler(relayProcess) + terminationHandler(relayProcess, detail) } - try process.run() return relayProcess } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift index a43e32f4d8fc..a186c0053670 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift @@ -11,6 +11,9 @@ public protocol RemoteReverseRelayLaunching: Sendable { func launch( arguments: [String], environment: [String: String]?, - terminationHandler: @escaping @Sendable (any RemoteReverseRelayProcess) -> Void + terminationHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess, + String? + ) -> Void ) throws -> any RemoteReverseRelayProcess } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift index 7f5949b7ddaf..ebff3552d6c7 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayProcess.swift @@ -1,26 +1,14 @@ -public import Foundation - /// A running dedicated SSH reverse-relay transport. /// /// The coordinator owns this handle, terminates it during normal teardown, and -/// uses its stderr stream to diagnose startup and later transport failures. +/// receives its fully drained diagnostic from the launcher at termination. public protocol RemoteReverseRelayProcess: AnyObject, Sendable { - /// The process's standard-error pipe. - var stderrPipe: Pipe { get } - /// Whether the transport process is still running. var isRunning: Bool { get } /// The process's exit status after termination. var terminationStatus: Int32 { get } - /// Waits for an immediate startup failure and returns its best diagnostic. - /// - /// - Parameter gracePeriod: Maximum time to wait for an early process exit. - /// - Returns: The best failure line when the process exits, or `nil` when it - /// remains running through the grace period. - func startupFailureDetail(gracePeriod: TimeInterval) -> String? - /// Requests termination of the transport process. func terminate() } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteControlMasterCleanup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteControlMasterCleanup.swift index a5f787157116..b491c93af5be 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteControlMasterCleanup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteControlMasterCleanup.swift @@ -12,9 +12,15 @@ public struct RemoteControlMasterCleanup: Sendable { /// find the existing socket. `ControlMaster` and `ControlPersist` are /// replaced by the leading reuse-only settings. /// - /// - Parameter configuration: Native SSH workspace configuration. + /// - Parameters: + /// - configuration: Native SSH workspace configuration. + /// - sshOptionsOverride: Effective SSH options that identify the master, + /// or `nil` to use the configuration's stored options. /// - Returns: Arguments for `/usr/bin/ssh`. - public func cleanupArguments(configuration: WorkspaceRemoteConfiguration) -> [String] { + public func cleanupArguments( + configuration: WorkspaceRemoteConfiguration, + sshOptionsOverride: [String]? = nil + ) -> [String] { var arguments = [ "-o", "BatchMode=yes", "-o", "ControlMaster=no", @@ -26,7 +32,9 @@ public struct RemoteControlMasterCleanup: Sendable { !identityFile.isEmpty { arguments += ["-i", identityFile] } - for option in normalizedCleanupOptions(configuration.sshOptions) { + for option in normalizedCleanupOptions( + sshOptionsOverride ?? configuration.sshOptions + ) { arguments += ["-o", option] } arguments += ["-O", "exit", configuration.destination] diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift index 192f5dbfb34e..e45e50190a5e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift @@ -1,4 +1,4 @@ -public import Foundation +internal import Foundation // Remote-side relay provisioning script builders. Static because they // compose pure script text from raw inputs independent of a session instance diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 78f4c2c41b61..736481d3d507 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -1,17 +1,15 @@ internal import CmuxFoundation +internal import CmuxRemoteWorkspace internal import OSLog -public import CmuxRemoteWorkspace -public import Foundation +internal import Foundation nonisolated private let remoteRelayLogger = Logger(subsystem: "com.cmuxterm.app", category: "RemoteRelay") // The reverse CLI relay: a remote `127.0.0.1:` listener forwarded -// back to the local CLI relay server by a dedicated `ssh -N -R` process. The -// relay targets an in-process server, so its SSH transport is coordinator-owned -// and standalone rather than attached to a host-scoped ControlMaster. Normal -// stop terminates it; a later connection attempt reaps a PPID-1 orphan left by -// a crash using its destination and pinned relay-port argv. -// Stderr capture caps and restart cadence (2s) are pinned legacy behavior. +// back to the local CLI relay server. It prefers `ssh -O forward` on the +// already-authenticated shared ControlMaster, preserving password/MFA hosts, +// and falls back to a coordinator-owned standalone `ssh -N -R` transport. +// Standalone stderr capture caps and restart cadence (2s) are pinned behavior. extension RemoteSessionCoordinator { func startReverseRelayLocked(remotePath: String) { guard !isStopping else { return } @@ -27,6 +25,7 @@ extension RemoteSessionCoordinator { return } guard reverseRelayProcess == nil else { return } + guard reverseRelayControlMasterForwardSpec == nil else { return } guard reverseRelayStartupPhase.allowsRelayLaunch else { return } cancelReverseRelayRestartLocked() @@ -39,7 +38,7 @@ extension RemoteSessionCoordinator { ) } - /// Launches the app-owned relay without adopting a shared ControlMaster. + /// Starts the relay on the authenticated shared master or its standalone fallback. func launchReverseRelayLocked( remotePath: String, relayPort: Int, @@ -48,6 +47,8 @@ extension RemoteSessionCoordinator { localSocketPath: String ) { guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } + guard reverseRelayControlMasterForwardSpec == nil else { return } + guard reverseRelayStartupPhase.allowsRelayLaunch else { return } var relayServer: RemoteCLIRelayServer? do { @@ -64,57 +65,70 @@ extension RemoteSessionCoordinator { persistentDaemonSlot: configuration.persistentDaemonSlot ) - let relayArguments = reverseRelayArguments( - relayPort: relayPort, - localRelayPort: localRelayPort - ) - let process = try reverseRelayLauncher.launch( - arguments: relayArguments, - environment: configuration.sshProcessEnvironment - ) { [weak self] terminated in - guard let coordinator = self else { return } - coordinator.queue.async { - coordinator.handleReverseRelayTerminationLocked(process: terminated) - } - } - let stderrPipe = process.stderrPipe - if let startupFailure = process.startupFailureDetail( - gracePeriod: Self.reverseRelayStartupGracePeriod + let forwardSpec = "127.0.0.1:\(relayPort):127.0.0.1:\(localRelayPort)" + switch startReverseRelayViaControlMasterLocked( + forwardSpec: forwardSpec, + relayPort: relayPort ) { - let retryDelay = 2.0 - let retrySeconds = max(1, Int(retryDelay.rounded())) + case .started: + cliRelayServer = relayServer + do { + try installRemoteRelayMetadataLocked( + remotePath: remotePath, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken + ) + } catch { + debugLog("remote.relay.metadata.error \(error.localizedDescription)") + stopReverseRelayLocked() + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) + return + } + recordHeartbeatActivityLocked() debugLog( - "remote.relay.startFailed relayPort=\(relayPort) " + - "error=\(startupFailure)" + "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + + "target=\(configuration.displayTarget) controlMaster=1" + ) + return + case .bindingConflict(let detail): + debugLog( + "remote.relay.startFailed relayPort=\(relayPort) error=\(detail)" ) - if let relayServer { - relayServer.stop() - if cliRelayServer === relayServer { - cliRelayServer = nil - } - } if beginConflictedControlMasterExitIfNeededLocked( - startupFailure: startupFailure, + startupFailure: detail, remotePath: remotePath, - relayPort: relayPort, - relayID: relayID, - relayToken: relayToken, - localSocketPath: localSocketPath + relayPort: relayPort ) { return } - publishDaemonStatus( - .error, - detail: "Remote SSH relay unavailable: \(startupFailure) (retry in \(retrySeconds)s)" + publishReverseRelayFailureLocked( + detail: detail, + remotePath: remotePath ) - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: retryDelay) return + case .unavailable: + break + } + + let relayArguments = reverseRelayArguments( + relayPort: relayPort, + localRelayPort: localRelayPort + ) + let process = try reverseRelayLauncher.launch( + arguments: relayArguments, + environment: configuration.sshProcessEnvironment + ) { [weak self] terminated, stderrDetail in + guard let coordinator = self else { return } + coordinator.queue.async { + coordinator.handleReverseRelayTerminationLocked( + process: terminated, + stderrDetail: stderrDetail + ) + } } - installReverseRelayStderrHandlerLocked(stderrPipe) reverseRelayProcess = process cliRelayServer = relayServer - reverseRelayStderrPipe = stderrPipe - reverseRelayStderrBuffer = "" do { try installRemoteRelayMetadataLocked( remotePath: remotePath, @@ -131,7 +145,7 @@ extension RemoteSessionCoordinator { recordHeartbeatActivityLocked() debugLog( "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + - "target=\(configuration.displayTarget) transport=dedicated" + "target=\(configuration.displayTarget) controlMaster=0" ) } catch { debugLog( @@ -148,33 +162,12 @@ extension RemoteSessionCoordinator { } } - private func installReverseRelayStderrHandlerLocked(_ stderrPipe: Pipe) { - stderrPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in - switch handle.readAvailableDataOrEndOfFile() { - case .data(let data): - self?.queue.async { - guard let self else { return } - if let chunk = String(data: data, encoding: .utf8), !chunk.isEmpty { - self.reverseRelayStderrBuffer.append(chunk) - if self.reverseRelayStderrBuffer.count > 8192 { - self.reverseRelayStderrBuffer.removeFirst(self.reverseRelayStderrBuffer.count - 8192) - } - } - } - case .wouldBlock: - return - case .endOfFile: - handle.readabilityHandler = nil - } - } - } - - private func handleReverseRelayTerminationLocked(process: any RemoteReverseRelayProcess) { + func handleReverseRelayTerminationLocked( + process: any RemoteReverseRelayProcess, + stderrDetail: String? + ) { guard reverseRelayProcess === process else { return } - let stderrDetail = Self.bestErrorLine(stderr: reverseRelayStderrBuffer) - reverseRelayStderrPipe?.fileHandleForReading.readabilityHandler = nil reverseRelayProcess = nil - reverseRelayStderrPipe = nil guard !isStopping else { return } guard let remotePath = daemonRemotePath, @@ -182,9 +175,30 @@ extension RemoteSessionCoordinator { let detail = stderrDetail ?? "status=\(process.terminationStatus)" debugLog("remote.relay.exit \(detail)") + if let relayPort = configuration.relayPort, + beginConflictedControlMasterExitIfNeededLocked( + startupFailure: detail, + remotePath: remotePath, + relayPort: relayPort + ) { + return + } scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) } + private func publishReverseRelayFailureLocked( + detail: String, + remotePath: String + ) { + let retryDelay = 2.0 + let retrySeconds = max(1, Int(retryDelay.rounded())) + publishDaemonStatus( + .error, + detail: "Remote SSH relay unavailable: \(detail) (retry in \(retrySeconds)s)" + ) + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: retryDelay) + } + func scheduleReverseRelayRestartLocked(remotePath: String, delay: TimeInterval) { guard !isStopping else { return } reverseRelayRestartTask?.cancel() @@ -226,22 +240,19 @@ extension RemoteSessionCoordinator { @discardableResult func stopReverseRelayLocked(cleanupScope: RemoteRelayCleanupScope = .transport) -> Bool { cancelReverseRelayStartupLocked() - reverseRelayStderrPipe?.fileHandleForReading.readabilityHandler = nil if let reverseRelayProcess, reverseRelayProcess.isRunning { reverseRelayProcess.terminate() } reverseRelayProcess = nil - reverseRelayStderrPipe = nil - reverseRelayStderrBuffer = "" + stopReverseRelayViaControlMasterLocked() cliRelayServer?.stop() cliRelayServer = nil return removeRemoteRelayMetadataLocked(cleanupScope: cleanupScope) } func reverseRelayArguments(relayPort: Int, localRelayPort: Int) -> [String] { - // The relay's SSH process is deliberately standalone and coordinator - // owned. `-S none` also protects against a ControlPath inherited from - // the host's ssh_config and leaves argv that crash recovery can match. + // Fallback only: `-S none` prevents accidental adoption of a shared + // transport after `-O forward` proved unavailable. var args: [String] = ["-N", "-T", "-S", "none"] args += sshCommonArguments(batchMode: true, dropControlPath: true) args += [ @@ -385,22 +396,6 @@ extension RemoteSessionCoordinator { } } - /// Waits a short grace period for an `ssh -N -R` relay transport that may - /// fail immediately (port already bound, auth failure); returns the best - /// stderr line when it exited within the grace period, or `nil` while it - /// keeps running. Static and pinned by tests; the bounded semaphore wait - /// rides the real termination signal. - public static func reverseRelayStartupFailureDetail( - process: Process, - stderrPipe: Pipe, - gracePeriod: TimeInterval = reverseRelayStartupGracePeriod - ) -> String? { - FoundationRemoteReverseRelayProcess( - process: process, - stderrPipe: stderrPipe - ).startupFailureDetail(gracePeriod: gracePeriod) - } - /// Returns whether OpenSSH reported that this relay's remote listener is /// already bound. The optional `Error:` prefix varies across OpenSSH builds. static func isReverseRelayPortBindingFailure(_ detail: String, relayPort: Int) -> Bool { diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift new file mode 100644 index 000000000000..b153cd136a31 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -0,0 +1,78 @@ +internal import CmuxCore +internal import CmuxFoundation +internal import Foundation + +/// Result of trying to install a relay channel on an existing SSH master. +enum ReverseRelayControlMasterStartOutcome: Sendable { + case started + case unavailable + case bindingConflict(String) +} + +extension RemoteSessionCoordinator { + /// Matches the connection-sharing defaults used by foreground authentication. + var reverseRelayControlMasterSSHOptions: [String] { + SSHConnectionSharingOptions().mergingDefaults( + into: configuration.sshOptions + ) + } + + /// Prefers the already-authenticated shared transport without creating one. + func startReverseRelayViaControlMasterLocked( + forwardSpec: String, + relayPort: Int + ) -> ReverseRelayControlMasterStartOutcome { + let effectiveSSHOptions = reverseRelayControlMasterSSHOptions + guard let arguments = configuration.reverseRelayControlMasterArguments( + controlCommand: "forward", + forwardSpec: forwardSpec, + effectiveSSHOptions: effectiveSSHOptions + ) else { + return .unavailable + } + + do { + let result = try sshExec(arguments: arguments, timeout: 6) + guard result.status == 0 else { + let detail = Self.bestErrorLine( + stderr: result.stderr, + stdout: result.stdout + ) ?? "ssh exited \(result.status)" + debugLog( + "remote.relay.controlmaster.forwardFailed \(detail) " + + debugConfigSummary() + ) + if Self.isReverseRelayPortBindingFailure( + detail, + relayPort: relayPort + ) { + return .bindingConflict(detail) + } + return .unavailable + } + reverseRelayControlMasterForwardSpec = forwardSpec + return .started + } catch { + debugLog( + "remote.relay.controlmaster.forwardFailed " + + "\(error.localizedDescription) \(debugConfigSummary())" + ) + return .unavailable + } + } + + /// Cancels only the exact forward this coordinator successfully installed. + func stopReverseRelayViaControlMasterLocked() { + guard let forwardSpec = reverseRelayControlMasterForwardSpec else { return } + reverseRelayControlMasterForwardSpec = nil + let effectiveSSHOptions = reverseRelayControlMasterSSHOptions + guard let arguments = configuration.reverseRelayControlMasterArguments( + controlCommand: "cancel", + forwardSpec: forwardSpec, + effectiveSSHOptions: effectiveSSHOptions + ) else { + return + } + _ = try? sshExec(arguments: arguments, timeout: 4) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index d235153ca5b6..96a2009944a2 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -1,3 +1,4 @@ +internal import CmuxFoundation internal import Foundation private enum ConflictedControlMasterExitOutcome: Sendable { @@ -6,32 +7,53 @@ private enum ConflictedControlMasterExitOutcome: Sendable { } extension RemoteSessionCoordinator { - /// Exits a legacy ControlPersist master only after the dedicated relay - /// proves that its configured remote port is already bound. + /// Exits a cmux-owned ControlPersist master only after OpenSSH proves that + /// the configured relay port is already bound. /// /// OpenSSH cannot cancel an inherited reverse forward without its original - /// full target specification. Exiting the owning master is the only - /// deterministic migration path; persistent remote PTYs survive and active - /// transports reconnect. + /// full target specification. Custom ControlPaths fail closed here: cmux + /// never terminates a master it did not create. @discardableResult func beginConflictedControlMasterExitIfNeededLocked( startupFailure: String, remotePath: String, - relayPort: Int, - relayID: String, - relayToken: String, - localSocketPath: String + relayPort: Int ) -> Bool { - guard reverseRelayStartupPhase.canAttemptRecovery, - Self.isReverseRelayPortBindingFailure( - startupFailure, - relayPort: relayPort - ) else { + guard Self.isReverseRelayPortBindingFailure( + startupFailure, + relayPort: relayPort + ) else { + return false + } + guard reverseRelayStartupPhase.canAttemptRecovery else { + return false + } + let effectiveSSHOptions = reverseRelayControlMasterSSHOptions + guard SSHConnectionSharingOptions().cmuxOwnedControlPath( + in: effectiveSSHOptions + ) != nil else { + debugLog( + "remote.relay.conflictedMaster.exitSkipped " + + "reason=control-path-not-owned relayPort=\(relayPort) " + + debugConfigSummary() + ) + return false + } + + guard reverseRelayControlMasterForwardSpec == nil else { + debugLog( + "remote.relay.conflictedMaster.exitSkipped " + + "reason=current-forward-owned relayPort=\(relayPort) " + + debugConfigSummary() + ) return false } let arguments = RemoteControlMasterCleanup() - .cleanupArguments(configuration: configuration) + .cleanupArguments( + configuration: configuration, + sshOptionsOverride: effectiveSSHOptions + ) let request = RemoteProcessRequest( executable: "/usr/bin/ssh", arguments: arguments, @@ -58,10 +80,7 @@ extension RemoteSessionCoordinator { token: token, outcome: outcome, remotePath: remotePath, - relayPort: relayPort, - relayID: relayID, - relayToken: relayToken, - localSocketPath: localSocketPath + relayPort: relayPort ) } } @@ -112,10 +131,7 @@ extension RemoteSessionCoordinator { token: UUID, outcome: ConflictedControlMasterExitOutcome, remotePath: String, - relayPort: Int, - relayID: String, - relayToken: String, - localSocketPath: String + relayPort: Int ) { guard reverseRelayStartupPhase.token == token else { return } reverseRelayStartupPhase = .recoveryAttempted @@ -142,14 +158,8 @@ extension RemoteSessionCoordinator { return } - guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } - launchReverseRelayLocked( - remotePath: remotePath, - relayPort: relayPort, - relayID: relayID, - relayToken: relayToken, - localSocketPath: localSocketPath - ) + guard !isStopping else { return } + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) } /// Cancels the in-flight OpenSSH recovery and invalidates its continuation. diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index 9fbd49ec8e7f..583912477945 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -82,6 +82,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonRemotePath: String? var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable var reverseRelayProcess: (any RemoteReverseRelayProcess)? + var reverseRelayControlMasterForwardSpec: String? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] /// Stable publication state for best-effort remote TTY attribution scans. @@ -109,10 +110,8 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var bootstrapRemoteTTYRetryToken: UUID? var bootstrapRemoteTTYFetchInFlight = false var bootstrapRemoteTTYRetryCount = 0 - var reverseRelayStderrPipe: Pipe? var reverseRelayRestartTask: Task? var reverseRelayRestartToken: UUID? - var reverseRelayStderrBuffer = "" var reconnectRetryCount = 0 var reconnectTask: Task? var reconnectToken: UUID? @@ -131,11 +130,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { /// `.some(nil)` = computed and unavailable (legacy process-wide /// `static let` cache, made per-coordinator with the build-info seam). var remoteDaemonSourceFingerprintCache: String?? - /// Grace period the relay-startup failure probe waits for an `ssh -N -R` - /// transport that may exit immediately (public because it is the default - /// argument of the test-pinned ``reverseRelayStartupFailureDetail(process:stderrPipe:gracePeriod:)``). - public static let reverseRelayStartupGracePeriod: TimeInterval = 0.5 - /// Creates a coordinator for one remote-workspace connection attempt. /// /// - Parameters: diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 978ad64edec7..9eb5999f6f09 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -45,10 +45,7 @@ struct RemoteSessionReverseRelayStartupTests { coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "Connection refused", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - relayID: "relay-startup-cancellation", - relayToken: String(repeating: "a", count: 64), - localSocketPath: coordinator.configuration.localSocketPath ?? "" + relayPort: 64_044 ) } #expect(!ignoredUnrelatedFailure) @@ -58,10 +55,7 @@ struct RemoteSessionReverseRelayStartupTests { coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "Error: remote port forwarding failed for listen port 64044", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - relayID: "relay-startup-cancellation", - relayToken: String(repeating: "a", count: 64), - localSocketPath: coordinator.configuration.localSocketPath ?? "" + relayPort: 64_044 ) } #expect(beganRecovery) @@ -90,10 +84,7 @@ struct RemoteSessionReverseRelayStartupTests { coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "remote port forwarding failed for listen port 64044", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - relayID: "relay-startup-cancellation", - relayToken: String(repeating: "a", count: 64), - localSocketPath: coordinator.configuration.localSocketPath ?? "" + relayPort: 64_044 ) } #expect(!beganSecondRecovery) @@ -101,54 +92,39 @@ struct RemoteSessionReverseRelayStartupTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test("Successful master exit retries the dedicated relay") - func successfulMasterExitRetriesDedicatedRelay() async throws { + @Test("Successful master exit schedules relay retry after reconnect can recreate it") + func successfulMasterExitSchedulesRetry() async throws { let runner = RecordingProcessRunner() let launcher = RecordingReverseRelayLauncher() + let clock = ManualBrokerClock() let relayPort = 64_046 let fixture = try Self.makeCoordinator( runner: runner, reverseRelayLauncher: launcher, - relayPort: relayPort + relayPort: relayPort, + clock: clock ) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } - var launches = launcher.launches.makeAsyncIterator() coordinator.queue.async { coordinator.daemonReady = true _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "Error: remote port forwarding failed for listen port \(relayPort)", remotePath: "/tmp/cmuxd-remote", - relayPort: relayPort, - relayID: "relay-successful-recovery", - relayToken: String(repeating: "b", count: 64), - localSocketPath: coordinator.configuration.localSocketPath ?? "" + relayPort: relayPort ) } - let launch = try #require(await launches.next()) - + #expect(await clock.nextRequestedDelay() == 2_000) let recoveryRequest = runner.requests.first #expect(recoveryRequest?.arguments.contains("-O") == true) #expect(recoveryRequest?.arguments.contains("exit") == true) - #expect(launch.arguments.starts(with: ["-N", "-T", "-S", "none"])) - #expect(launch.arguments.contains("-R")) - #expect(launch.arguments.contains( - "127.0.0.1:\(relayPort):127.0.0.1:\(launch.localRelayPort)" - )) - #expect(!launch.arguments.contains(where: { $0.hasPrefix("ControlPath=") })) - - let retriedAfterRecovery = coordinator.queue.sync { + #expect(launcher.launchCount == 0) + #expect(coordinator.queue.sync { coordinator.reverseRelayStartupPhase.allowsRelayLaunch && - coordinator.reverseRelayProcess === launcher.process - } - #expect(retriedAfterRecovery) - #expect(RemoteSessionCoordinator.orphanedCMUXRemoteSSHPIDs( - psOutput: "909 1 /usr/bin/ssh \(launch.arguments.joined(separator: " "))", - destination: "user@example.test", - relayPort: relayPort - ) == [909]) + coordinator.reverseRelayProcess == nil + }) _ = await coordinator.stopAndWait(cleanupScope: .transport) } @@ -163,10 +139,7 @@ struct RemoteSessionReverseRelayStartupTests { _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "remote port forwarding failed for listen port 64044", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - relayID: "relay-startup-cancellation", - relayToken: String(repeating: "a", count: 64), - localSocketPath: coordinator.configuration.localSocketPath ?? "" + relayPort: 64_044 ) } @@ -185,11 +158,13 @@ struct RemoteSessionReverseRelayStartupTests { #expect(startupCleared) } - private static func makeCoordinator( + static func makeCoordinator( host: any RemoteSessionHosting = NoopRemoteSessionHost(), runner: any RemoteSessionProcessRunning, reverseRelayLauncher: any RemoteReverseRelayLaunching = RemoteReverseRelayLauncher(), - relayPort: Int = 64_044 + relayPort: Int = 64_044, + sshOptions: [String]? = nil, + clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock() ) throws -> (coordinator: RemoteSessionCoordinator, scratchDirectory: URL) { let scratchDirectory = FileManager.default.temporaryDirectory .appendingPathComponent( @@ -204,7 +179,7 @@ struct RemoteSessionReverseRelayStartupTests { destination: "user@example.test", port: nil, identityFile: nil, - sshOptions: ["StrictHostKeyChecking=accept-new"], + sshOptions: sshOptions ?? ["StrictHostKeyChecking=accept-new"], localProxyPort: nil, relayPort: relayPort, relayID: "relay-startup-cancellation", @@ -234,25 +209,29 @@ struct RemoteSessionReverseRelayStartupTests { strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@" - ) + ), + clock: clock ) return (coordinator, scratchDirectory) } } -private struct RecordedReverseRelayLaunch: Sendable { +struct RecordedReverseRelayLaunch: Sendable { let arguments: [String] let localRelayPort: Int } -/// Immutable recorder; `AsyncStream.Continuation` owns synchronized delivery. -private final class RecordingReverseRelayLauncher: +/// Synchronous launcher callbacks cannot await; the lock protects only callback snapshots and a counter. +final class RecordingReverseRelayLauncher: RemoteReverseRelayLaunching, @unchecked Sendable { let launches: AsyncStream let process = StubReverseRelayProcess() + private let lock = NSLock() + private var _launchCount = 0 + private var terminationHandler: (@Sendable (any RemoteReverseRelayProcess, String?) -> Void)? private let launchContinuation: AsyncStream.Continuation init() { @@ -262,7 +241,10 @@ private final class RecordingReverseRelayLauncher: func launch( arguments: [String], environment: [String: String]?, - terminationHandler: @escaping @Sendable (any RemoteReverseRelayProcess) -> Void + terminationHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess, + String? + ) -> Void ) throws -> any RemoteReverseRelayProcess { let reverseArgumentIndex = try #require(arguments.firstIndex(of: "-R")) let reverseArgument = try #require( @@ -277,23 +259,31 @@ private final class RecordingReverseRelayLauncher: arguments: arguments, localRelayPort: localRelayPort )) + lock.withLock { + _launchCount += 1 + self.terminationHandler = terminationHandler + } return process } + + var launchCount: Int { + lock.withLock { _launchCount } + } + + func emitTermination(detail: String?) { + let handler = lock.withLock { terminationHandler } + handler?(process, detail) + } } -/// Immutable fake process; the pipe is never concurrently read or written. -private final class StubReverseRelayProcess: +/// Immutable fake process used by the injected launcher. +final class StubReverseRelayProcess: RemoteReverseRelayProcess, @unchecked Sendable { - let stderrPipe = Pipe() let isRunning = true let terminationStatus: Int32 = 0 - func startupFailureDetail(gracePeriod: TimeInterval) -> String? { - nil - } - func terminate() {} } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift new file mode 100644 index 000000000000..20b1baef70d1 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -0,0 +1,289 @@ +import Foundation +import Testing +import CmuxCore +import CmuxFoundation +@testable import CmuxRemoteSession + +@Suite("Reverse relay SSH transport selection") +struct RemoteSessionReverseRelayTransportTests { + @Test("An authenticated shared ControlMaster carries the relay") + func sharedControlMasterIsPreferred() async throws { + let runner = RecordingProcessRunner() + let launcher = RecordingReverseRelayLauncher() + let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + + let forwardRequest = try #require(runner.requests.first(where: { + Self.isControlCommand("forward", in: $0.arguments) + })) + #expect(forwardRequest.arguments.contains("-R")) + #expect(forwardRequest.arguments.contains("BatchMode=yes")) + #expect( + forwardRequest.arguments.contains( + "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)" + ) + ) + #expect(launcher.launchCount == 0) + #expect(coordinator.queue.sync { + coordinator.reverseRelayControlMasterForwardSpec != nil && + coordinator.reverseRelayProcess == nil + }) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + let cancelRequest = try #require(runner.requests.first(where: { + Self.isControlCommand("cancel", in: $0.arguments) + })) + #expect( + cancelRequest.arguments.contains( + "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)" + ) + ) + #expect( + Self.reverseForward(in: cancelRequest.arguments) + == Self.reverseForward(in: forwardRequest.arguments) + ) + } + + @Test("An explicitly disabled ControlMaster uses the standalone fallback") + func disabledControlMasterUsesStandaloneFallback() async throws { + let runner = RecordingProcessRunner() + let launcher = RecordingReverseRelayLauncher() + let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher, + sshOptions: [ + "StrictHostKeyChecking=accept-new", + "ControlMaster=no", + ] + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + var launches = launcher.launches.makeAsyncIterator() + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + + let launch = try #require(await launches.next()) + #expect(launch.arguments.starts(with: ["-N", "-T", "-S", "none"])) + #expect(!runner.requests.contains(where: { + Self.isControlCommand("forward", in: $0.arguments) + })) + #expect(coordinator.queue.sync { + coordinator.reverseRelayControlMasterForwardSpec == nil && + coordinator.reverseRelayProcess === launcher.process + }) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("A bind conflict exits a cmux-owned master") + func ownedConflictExitsMaster() async throws { + let clock = ManualBrokerClock() + let runner = RecordingProcessRunner { request in + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "Error: remote port forwarding failed for listen port 64044" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let launcher = RecordingReverseRelayLauncher() + let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher, + clock: clock + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + + #expect(await clock.nextRequestedDelay() == 2_000) + #expect(runner.requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + let exitRequest = try #require(runner.requests.first(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + #expect( + exitRequest.arguments.contains( + "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)" + ) + ) + #expect(launcher.launchCount == 0) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("A custom ControlPath is never exited") + func customControlPathFailsClosed() async throws { + let clock = ManualBrokerClock() + let runner = RecordingProcessRunner { request in + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "remote port forwarding failed for listen port 64044" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let launcher = RecordingReverseRelayLauncher() + let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher, + sshOptions: [ + "StrictHostKeyChecking=accept-new", + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=~/.ssh/custom-%C", + ], + clock: clock + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + + #expect(await clock.nextRequestedDelay() == 2_000) + #expect(!runner.requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + #expect(coordinator.queue.sync { + coordinator.reverseRelayStartupPhase.canAttemptRecovery + }) + #expect(launcher.launchCount == 0) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("A late standalone bind failure triggers owned-master recovery") + func lateStandaloneConflictTriggersRecovery() async throws { + let relayPort = 64_047 + let clock = ManualBrokerClock() + let runner = RecordingProcessRunner { request in + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "Control socket connect: No such file or directory" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let launcher = RecordingReverseRelayLauncher() + let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher, + relayPort: relayPort, + clock: clock + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + var launches = launcher.launches.makeAsyncIterator() + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + let launch = try #require(await launches.next()) + #expect(launch.arguments.starts(with: ["-N", "-T", "-S", "none"])) + #expect(!launch.arguments.contains(where: { + $0.localizedCaseInsensitiveContains("ControlPath") + })) + #expect(coordinator.queue.sync { + coordinator.reverseRelayProcess === launcher.process + }) + + launcher.emitTermination( + detail: "Error: remote port forwarding failed for listen port \(relayPort)" + ) + + #expect(await clock.nextRequestedDelay() == 2_000) + #expect(runner.requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + #expect(coordinator.queue.sync { + coordinator.reverseRelayProcess == nil + }) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("Standalone termination waits for the complete stderr tail") + func standaloneTerminationDrainsStderr() async throws { + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + """ + i=0 + while [ "$i" -lt 1000 ]; do + printf 'diagnostic-noise-%s\n' "$i" >&2 + i=$((i + 1)) + done + printf 'Error: remote port forwarding failed for listen port 64044\n' >&2 + exit 255 + """, + ] + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe + ) + let (details, continuation) = AsyncStream.makeStream() + + try process.run() + relayProcess.captureTermination { detail in + continuation.yield(detail) + continuation.finish() + } + + var iterator = details.makeAsyncIterator() + #expect( + await iterator.next() + == "Error: remote port forwarding failed for listen port 64044" + ) + #expect(process.terminationStatus == 255) + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } + + private static func reverseForward(in arguments: [String]) -> String? { + guard let reverseIndex = arguments.firstIndex(of: "-R") else { + return nil + } + let valueIndex = arguments.index(after: reverseIndex) + return arguments.indices.contains(valueIndex) + ? arguments[valueIndex] + : nil + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift index 32432a7260c6..b0c053acf183 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift @@ -74,8 +74,8 @@ struct RemoteSessionSSHRemoteCommandOverrideTests { } } - @Test("Reverse relay disables shared ControlMaster transport") - func reverseRelayDisablesSharedControlMasterTransport() { + @Test("Reverse relay standalone fallback disables ControlMaster transport") + func reverseRelayStandaloneFallbackDisablesControlMasterTransport() { let coordinator = Self.makeCoordinator( runner: RecordingProcessRunner(), sshOptions: [ diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 293b8396a48b..5fe6c18d3f70 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -707,26 +707,6 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { workspace.disconnectRemoteConnection(clearConfiguration: true) } - func testReverseRelayStartupFailureDetailCapturesImmediateForwardingFailure() throws { - let process = Process() - let stderrPipe = Pipe() - process.executableURL = URL(fileURLWithPath: "/bin/sh") - process.arguments = ["-c", "echo 'remote port forwarding failed for listen port 64009' >&2; exit 1"] - process.standardInput = FileHandle.nullDevice - process.standardOutput = FileHandle.nullDevice - process.standardError = stderrPipe - - try process.run() - - let detail = RemoteSessionCoordinator.reverseRelayStartupFailureDetail( - process: process, - stderrPipe: stderrPipe, - gracePeriod: 1.0 - ) - - XCTAssertEqual(detail, "remote port forwarding failed for listen port 64009") - } - func testExecutableSearchPathsIncludesHomebrewAndHomeFallbacks() { let paths = RemoteSessionCoordinator.executableSearchPaths( environment: [ From dbfb2da278a869039db1eb4b2ea27faad5845912 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 00:45:30 -0700 Subject: [PATCH 15/39] fix: coordinate conflicted SSH master recovery --- ...RemoteConfiguration+SSHBatchCommands.swift | 7 + ...teConfigurationSSHBatchCommandsTests.swift | 14 ++ .../SSHConnectionSharingOptions.swift | 35 +++- .../SSHConnectionSharingOptionsTests.swift | 20 +++ .../NativeSSHConnectionBroker.swift | 61 ++++++- ...tiveSSHControlMasterResetCoordinator.swift | 159 ++++++++++++++++++ .../NativeSSHControlMasterResetKey.swift | 41 +++++ .../FoundationRemoteReverseRelayProcess.swift | 115 +++++++++---- ...emoteSessionCoordinator+ReverseRelay.swift | 10 +- ...ssionCoordinator+ReverseRelayStartup.swift | 84 +-------- .../Values/ReverseRelayStartupPhase.swift | 5 +- .../BlockingControlMasterResetRunner.swift | 35 ++++ .../NativeSSHConnectionBrokerTests.swift | 52 +++++- ...emoteSessionReverseRelayStartupTests.swift | 53 +++--- ...oteSessionReverseRelayTransportTests.swift | 64 ++++++- Resources/Localizable.xcstrings | 17 ++ 16 files changed, 603 insertions(+), 169 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift diff --git a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift index aee555d68d7d..9ed953d3d926 100644 --- a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift +++ b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift @@ -68,6 +68,13 @@ extension WorkspaceRemoteConfiguration { forwardSpec: String, effectiveSSHOptions: [String] ) -> [String]? { + if let controlMaster = Self.firstSSHOptionValue( + named: "ControlMaster", + in: effectiveSSHOptions + )?.lowercased(), + ["no", "false", "off", "0"].contains(controlMaster) { + return nil + } guard let controlPath = Self.firstSSHOptionValue( named: "ControlPath", in: effectiveSSHOptions diff --git a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift index dd20c8283421..ae9848b40edc 100644 --- a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift +++ b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift @@ -185,5 +185,19 @@ struct WorkspaceRemoteConfigurationSSHBatchCommandsTests { effectiveSSHOptions: ["ControlPath=None"] ) == nil ) + #expect( + configuration(sshOptions: [ + "ControlMaster=no", + "ControlPath=~/.ssh/custom-%C", + ]) + .reverseRelayControlMasterArguments( + controlCommand: "forward", + forwardSpec: "127.0.0.1:64007:127.0.0.1:54321", + effectiveSSHOptions: [ + "ControlMaster=no", + "ControlPath=~/.ssh/custom-%C", + ] + ) == nil + ) } } diff --git a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift index 75119e632ca8..c5753ff3306f 100644 --- a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift +++ b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift @@ -89,10 +89,10 @@ public struct SSHConnectionSharingOptions: Sendable { } } } - let controlMaster = resolver.optionValue(named: "ControlMaster", in: merged) + let controlMaster = firstOptionValue(named: "ControlMaster", in: merged) let controlMasterDisabled = isDisabled(controlMaster) if !controlMasterDisabled, - let controlPath = resolver.optionValue(named: "ControlPath", in: merged), + let controlPath = firstOptionValue(named: "ControlPath", in: merged), isLegacyRelayScopedControlPath(controlPath) { merged = merged.map { option in guard resolver.optionKey(option) == "controlpath" else { return option } @@ -158,11 +158,10 @@ public struct SSHConnectionSharingOptions: Sendable { /// - Parameter options: OpenSSH `-o` values to inspect. /// - Returns: The cmux-owned path, or `nil` for user-managed paths. public func cmuxOwnedControlPath(in options: [String]) -> String? { - let resolver = SSHAgentSocketResolver() - guard !isDisabled(resolver.optionValue(named: "ControlMaster", in: options)) else { + guard !isDisabled(firstOptionValue(named: "ControlMaster", in: options)) else { return nil } - guard let rawPath = resolver.optionValue(named: "ControlPath", in: options) else { + guard let rawPath = firstOptionValue(named: "ControlPath", in: options) else { return nil } let path = rawPath.trimmingCharacters(in: .whitespacesAndNewlines) @@ -293,7 +292,31 @@ public struct SSHConnectionSharingOptions: Sendable { guard let value = rawValue?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() else { return false } - return ["no", "false", "off"].contains(value) + return ["no", "false", "off", "0"].contains(value) + } + + /// OpenSSH keeps the first value obtained for command-line configuration + /// options. Ownership decisions must inspect the same value that `ssh` + /// will use, especially when callers supplied a duplicate option. + private func firstOptionValue(named key: String, in options: [String]) -> String? { + let loweredKey = key.lowercased() + for option in options { + let trimmed = option.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { continue } + let parts = trimmed.split( + maxSplits: 1, + omittingEmptySubsequences: true, + whereSeparator: { $0 == "=" || $0.isWhitespace } + ) + guard parts.count == 2, parts[0].lowercased() == loweredKey else { + continue + } + let value = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) + if !value.isEmpty { + return value + } + } + return nil } private func shellQuote(_ value: String) -> String { diff --git a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift index 47adcd6830d9..c8f824617cf0 100644 --- a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift +++ b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift @@ -65,6 +65,26 @@ struct SSHConnectionSharingOptionsTests { #expect(options.cmuxOwnedControlPath(in: supplied) == nil) } + @Test("Ownership follows OpenSSH's first repeated ControlPath") + func ownershipUsesFirstControlPath() { + let customFirst = [ + "ControlMaster=auto", + "ControlPath=~/.ssh/custom-%C", + "ControlPath=/tmp/cmux-ssh-501-%C", + ] + let ownedFirst = [ + "ControlMaster=auto", + "ControlPath=/tmp/cmux-ssh-501-%C", + "ControlPath=~/.ssh/custom-%C", + ] + + #expect(options.cmuxOwnedControlPath(in: customFirst) == nil) + #expect( + options.cmuxOwnedControlPath(in: ownedFirst) + == "/tmp/cmux-ssh-501-%C" + ) + } + @Test("Effective custom ssh_config control settings replace cmux defaults") func preservesResolvedSSHConfigSettings() { let output = """ diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 31138db149a0..672b9051bf6b 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -23,6 +23,7 @@ public final class NativeSSHConnectionBroker { private let clock: any RemoteProxyRetryClock private let jitterMilliseconds: @MainActor @Sendable () -> Int private let cleanupLauncherOverride: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)? + private let conflictedMasterResetCoordinator: NativeSSHControlMasterResetCoordinator private var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] private var ownersByControlMaster: [NativeSSHControlMasterKey: Set] = [:] @@ -45,6 +46,10 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = nil + self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( + sharingOptions: sharingOptions, + processRunner: RemoteSessionProcessRunner() + ) } /// Creates a broker with an injected cleanup launcher. @@ -63,18 +68,27 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = cleanupLauncher + self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( + sharingOptions: sharingOptions, + processRunner: RemoteSessionProcessRunner() + ) } nonisolated init( sharingOptions: SSHConnectionSharingOptions, clock: any RemoteProxyRetryClock, jitterMilliseconds: @escaping @MainActor @Sendable () -> Int, - cleanupLauncher: @escaping @MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void + cleanupLauncher: @escaping @MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void, + conflictedMasterResetRunner: any RemoteSessionProcessRunning = RemoteSessionProcessRunner() ) { self.sharingOptions = sharingOptions self.clock = clock self.jitterMilliseconds = jitterMilliseconds self.cleanupLauncherOverride = cleanupLauncher + self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( + sharingOptions: sharingOptions, + processRunner: conflictedMasterResetRunner + ) } /// Retains the cmux-owned master used by a configured workspace. @@ -91,9 +105,21 @@ public final class NativeSSHConnectionBroker { configuration: configuration, sharingOptions: sharingOptions ) - guard let nextKey else { return configuration } - cancelCleanup(for: nextKey) + let resetKey = NativeSSHControlMasterResetKey( + configuration: configuration, + sharingOptions: sharingOptions + ) + guard nextKey != nil || resetKey != nil else { return configuration } let leasedConfiguration = configuration.withSSHControlMasterLeaseGeneration(UUID()) + if let resetKey { + conflictedMasterResetCoordinator.retainWorkspace( + leasedConfiguration, + ownerWorkspaceID: ownerWorkspaceID, + key: resetKey + ) + } + guard let nextKey else { return leasedConfiguration } + cancelCleanup(for: nextKey) var leases = ownerLeases[ownerWorkspaceID] ?? [:] let isNewMaster = leases[nextKey] == nil leases[nextKey] = leasedConfiguration @@ -112,17 +138,36 @@ public final class NativeSSHConnectionBroker { /// - Parameter configuration: Exact owner-scoped configuration being released. public func releaseWorkspace(_ configuration: WorkspaceRemoteConfiguration) { guard let ownerWorkspaceID = configuration.ownerWorkspaceID, - let generation = configuration.sshControlMasterLeaseGeneration, - let key = NativeSSHControlMasterKey( - configuration: configuration, - sharingOptions: sharingOptions - ), + let generation = configuration.sshControlMasterLeaseGeneration else { + return + } + if let resetKey = NativeSSHControlMasterResetKey( + configuration: configuration, + sharingOptions: sharingOptions + ) { + conflictedMasterResetCoordinator.releaseWorkspace( + ownerWorkspaceID: ownerWorkspaceID, + generation: generation, + key: resetKey + ) + } + guard let key = NativeSSHControlMasterKey( + configuration: configuration, + sharingOptions: sharingOptions + ), ownerLeases[ownerWorkspaceID]?[key]?.sshControlMasterLeaseGeneration == generation else { return } removeLease(ownerWorkspaceID: ownerWorkspaceID, key: key) } + /// Coalesces an inherited-master reset after OpenSSH confirms a relay bind conflict. + func resetConflictedControlMaster( + for configuration: WorkspaceRemoteConfiguration + ) async -> NativeSSHControlMasterResetOutcome { + await conflictedMasterResetCoordinator.reset(for: configuration) + } + /// Runs one connection attempt after acquiring the endpoint's FIFO permit. /// /// Same-endpoint attempts are separated by 100–350 ms of injected-clock diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift new file mode 100644 index 000000000000..7398dc832763 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -0,0 +1,159 @@ +internal import CmuxCore +internal import CmuxFoundation +internal import Foundation + +/// Outcome of one broker-authorized conflicted-master migration. +enum NativeSSHControlMasterResetOutcome: Sendable, Equatable { + case reset + case ignored(String) +} + +/// Broker-owned state for disruptive ControlMaster resets. +/// +/// The parent ``NativeSSHConnectionBroker`` calls every mutating method on the +/// main actor. Keeping this state in one collaborator makes reset authorization +/// and coalescing independent from the normal last-owner cleanup lifecycle. +@MainActor +final class NativeSSHControlMasterResetCoordinator { + private struct InFlightReset { + let id: UUID + let task: Task + } + + private let sharingOptions: SSHConnectionSharingOptions + private let processRunner: any RemoteSessionProcessRunning + private var leases: [ + UUID: [NativeSSHControlMasterResetKey: WorkspaceRemoteConfiguration] + ] = [:] + private var inFlightResets: [ + NativeSSHControlMasterResetKey: InFlightReset + ] = [:] + + nonisolated init( + sharingOptions: SSHConnectionSharingOptions, + processRunner: any RemoteSessionProcessRunning + ) { + self.sharingOptions = sharingOptions + self.processRunner = processRunner + } + + func retainWorkspace( + _ configuration: WorkspaceRemoteConfiguration, + ownerWorkspaceID: UUID, + key: NativeSSHControlMasterResetKey + ) { + var ownerLeases = leases[ownerWorkspaceID] ?? [:] + ownerLeases[key] = configuration + leases[ownerWorkspaceID] = ownerLeases + } + + func releaseWorkspace( + ownerWorkspaceID: UUID, + generation: UUID, + key: NativeSSHControlMasterResetKey + ) { + guard var ownerLeases = leases[ownerWorkspaceID], + ownerLeases[key]?.sshControlMasterLeaseGeneration == generation else { + return + } + ownerLeases.removeValue(forKey: key) + if ownerLeases.isEmpty { + leases.removeValue(forKey: ownerWorkspaceID) + } else { + leases[ownerWorkspaceID] = ownerLeases + } + } + + func reset( + for configuration: WorkspaceRemoteConfiguration + ) async -> NativeSSHControlMasterResetOutcome { + guard let ownerWorkspaceID = configuration.ownerWorkspaceID, + let generation = configuration.sshControlMasterLeaseGeneration, + let key = NativeSSHControlMasterResetKey( + configuration: configuration, + sharingOptions: sharingOptions + ), + leases[ownerWorkspaceID]?[key]?.sshControlMasterLeaseGeneration == generation else { + return .ignored("workspace no longer owns this cmux SSH master") + } + if let inFlight = inFlightResets[key] { + return await inFlight.task.value + } + + let effectiveOptions = sharingOptions.mergingDefaults( + into: configuration.sshOptions + ) + let arguments = RemoteControlMasterCleanup().cleanupArguments( + configuration: configuration, + sshOptionsOverride: effectiveOptions + ) + let authenticationLockPath = sharingOptions.foregroundAuthenticationLockPath( + destination: configuration.destination, + port: configuration.port, + options: effectiveOptions + ) + let request = NativeSSHControlMasterCleanupRequest( + arguments: arguments, + environment: configuration.sshProcessEnvironment, + authenticationLockPath: authenticationLockPath + ) + let resetID = UUID() + let processRunner = self.processRunner + let task = Task { + await Self.runReset(request: request, processRunner: processRunner) + } + inFlightResets[key] = InFlightReset(id: resetID, task: task) + let outcome = await task.value + if inFlightResets[key]?.id == resetID { + inFlightResets.removeValue(forKey: key) + } + return outcome + } + + private nonisolated static func runReset( + request: NativeSSHControlMasterCleanupRequest, + processRunner: any RemoteSessionProcessRunning + ) async -> NativeSSHControlMasterResetOutcome { + await withCheckedContinuation { continuation in + DispatchQueue.global(qos: .utility).async { + let invocation = request.processInvocation + let processRequest = RemoteProcessRequest( + executable: invocation.executableURL.path, + arguments: invocation.arguments, + environment: request.environment, + timeout: 5 + ) + do { + let result = try processRunner.run(processRequest, operation: nil) + if result.status == 0 { + continuation.resume(returning: .reset) + } else { + continuation.resume(returning: .ignored( + bestErrorLine(stderr: result.stderr, stdout: result.stdout) + ?? "ssh exited \(result.status)" + )) + } + } catch { + continuation.resume(returning: .ignored(error.localizedDescription)) + } + } + } + } + + private nonisolated static func bestErrorLine( + stderr: String, + stdout: String + ) -> String? { + for text in [stderr, stdout] { + if let line = text + .split(whereSeparator: \.isNewline) + .map(String.init) + .last(where: { + !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + }) { + return line.trimmingCharacters(in: .whitespacesAndNewlines) + } + } + return nil + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift new file mode 100644 index 000000000000..e9137752f80c --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift @@ -0,0 +1,41 @@ +internal import CmuxCore +internal import CmuxFoundation +internal import Foundation + +/// Identifies the effective cmux-owned master that may be reset after an +/// OpenSSH-confirmed reverse-forward bind conflict. +/// +/// Resolved socket paths are globally stable. An unresolved `%` template also +/// includes the configured endpoint so independent hosts do not share reset +/// state before OpenSSH expands the template. +struct NativeSSHControlMasterResetKey: Hashable, Sendable { + let controlPath: String + let destination: String? + let port: Int? + + init?( + configuration: WorkspaceRemoteConfiguration, + sharingOptions: SSHConnectionSharingOptions + ) { + guard configuration.transport == .ssh else { return nil } + let effectiveOptions = sharingOptions.mergingDefaults( + into: configuration.sshOptions + ) + guard let controlPath = sharingOptions.cmuxOwnedControlPath( + in: effectiveOptions + ) else { + return nil + } + self.controlPath = controlPath + if controlPath.contains("%") { + let destination = configuration.destination + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !destination.isEmpty else { return nil } + self.destination = destination + self.port = configuration.port + } else { + self.destination = nil + self.port = nil + } + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift index 3b7a84702da4..ad05c97bcaf4 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -1,10 +1,10 @@ -internal import Darwin internal import Foundation /// Foundation-backed handle for one dedicated SSH reverse-relay process. /// /// `Process` and `Pipe` callbacks cross executor boundaries; all mutation is -/// owned by Foundation while the coordinator serializes its handle access. +/// protected by the capture state's lock while the coordinator serializes its +/// handle access. final class FoundationRemoteReverseRelayProcess: RemoteReverseRelayProcess, @unchecked Sendable @@ -25,53 +25,100 @@ final class FoundationRemoteReverseRelayProcess: process.terminationStatus } - /// Drains stderr from launch through EOF before reporting termination. + /// Drains stderr through EOF without parking one utility worker for the + /// lifetime of the relay. Completion waits for both EOF and termination, + /// so the final diagnostic bytes always precede the callback. func captureTermination( _ handler: @escaping @Sendable (String?) -> Void ) { - let stderrDescriptor = stderrPipe.fileHandleForReading.fileDescriptor - // Blocking pipe drainage is a Foundation/Process bridge. It owns the - // raw descriptor on a utility thread so SSH can never fill the pipe, - // and the callback cannot outrun the final stderr bytes. - DispatchQueue.global(qos: .utility).async { [self] in - let stderr = Self.readStderrTail(fileDescriptor: stderrDescriptor) - process.waitUntilExit() - handler( - RemoteSessionCoordinator.bestErrorLine(stderr: stderr) - ?? "status=\(process.terminationStatus)" - ) + let readHandle = stderrPipe.fileHandleForReading + let capture = ReverseRelayStderrCapture( + readHandle: readHandle, + handler: handler + ) + readHandle.readabilityHandler = { handle in + capture.receive(handle.availableData) + } + process.terminationHandler = { terminatedProcess in + capture.processDidTerminate(status: terminatedProcess.terminationStatus) + } + if !process.isRunning { + capture.processDidTerminate(status: process.terminationStatus) } } func terminate() { process.terminate() } +} - private static func readStderrTail( - fileDescriptor: Int32, - byteLimit: Int = 8192 - ) -> String { - let chunkSize = 4096 - var bytes = [UInt8](repeating: 0, count: chunkSize) - var tail = Data() +/// Event-driven stderr tail and process-lifecycle rendezvous. +private final class ReverseRelayStderrCapture: @unchecked Sendable { + private struct Completion { + let stderr: String + let status: Int32 + } - while true { - let count = bytes.withUnsafeMutableBytes { buffer -> Int in - guard let baseAddress = buffer.baseAddress else { return 0 } - return Darwin.read(fileDescriptor, baseAddress, chunkSize) - } - if count > 0 { - tail.append(contentsOf: bytes.prefix(count)) + // lint:allow lock - FileHandle and Process callbacks are synchronous; the + // critical sections only append bounded data and update lifecycle bits. + private let lock = NSLock() + private let readHandle: FileHandle + private let handler: @Sendable (String?) -> Void + private let byteLimit: Int + private var tail = Data() + private var sawEOF = false + private var terminationStatus: Int32? + private var completed = false + + init( + readHandle: FileHandle, + byteLimit: Int = 8192, + handler: @escaping @Sendable (String?) -> Void + ) { + self.readHandle = readHandle + self.byteLimit = byteLimit + self.handler = handler + } + + func receive(_ data: Data) { + let completion = lock.withLock { () -> Completion? in + if data.isEmpty { + sawEOF = true + } else { + tail.append(data) if tail.count > byteLimit { tail.removeFirst(tail.count - byteLimit) } - continue } - if count < 0, errno == EINTR { - continue - } - break + return takeCompletionIfReady() + } + finish(completion) + } + + func processDidTerminate(status: Int32) { + let completion = lock.withLock { () -> Completion? in + terminationStatus = status + return takeCompletionIfReady() } - return String(data: tail, encoding: .utf8) ?? "" + finish(completion) + } + + private func takeCompletionIfReady() -> Completion? { + guard !completed, sawEOF, let terminationStatus else { return nil } + completed = true + return Completion( + stderr: String(data: tail, encoding: .utf8) ?? "", + status: terminationStatus + ) + } + + private func finish(_ completion: Completion?) { + guard let completion else { return } + readHandle.readabilityHandler = nil + try? readHandle.close() + handler( + RemoteSessionCoordinator.bestErrorLine(stderr: completion.stderr) + ?? "status=\(completion.status)" + ) } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 736481d3d507..e6d41b9e8955 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -103,7 +103,6 @@ extension RemoteSessionCoordinator { return } publishReverseRelayFailureLocked( - detail: detail, remotePath: remotePath ) return @@ -183,18 +182,19 @@ extension RemoteSessionCoordinator { ) { return } - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) + publishReverseRelayFailureLocked(remotePath: remotePath) } private func publishReverseRelayFailureLocked( - detail: String, remotePath: String ) { let retryDelay = 2.0 - let retrySeconds = max(1, Int(retryDelay.rounded())) publishDaemonStatus( .error, - detail: "Remote SSH relay unavailable: \(detail) (retry in \(retrySeconds)s)" + detail: String( + localized: "remoteSession.reverseRelay.unavailableRetrying", + defaultValue: "Remote SSH relay unavailable; retrying in 2 seconds" + ) ) scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: retryDelay) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index 96a2009944a2..032cae606972 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -1,11 +1,6 @@ internal import CmuxFoundation internal import Foundation -private enum ConflictedControlMasterExitOutcome: Sendable { - case exited - case ignored(String) -} - extension RemoteSessionCoordinator { /// Exits a cmux-owned ControlPersist master only after OpenSSH proves that /// the configured relay port is already bound. @@ -28,18 +23,6 @@ extension RemoteSessionCoordinator { guard reverseRelayStartupPhase.canAttemptRecovery else { return false } - let effectiveSSHOptions = reverseRelayControlMasterSSHOptions - guard SSHConnectionSharingOptions().cmuxOwnedControlPath( - in: effectiveSSHOptions - ) != nil else { - debugLog( - "remote.relay.conflictedMaster.exitSkipped " + - "reason=control-path-not-owned relayPort=\(relayPort) " + - debugConfigSummary() - ) - return false - } - guard reverseRelayControlMasterForwardSpec == nil else { debugLog( "remote.relay.conflictedMaster.exitSkipped " + @@ -49,31 +32,14 @@ extension RemoteSessionCoordinator { return false } - let arguments = RemoteControlMasterCleanup() - .cleanupArguments( - configuration: configuration, - sshOptionsOverride: effectiveSSHOptions - ) - let request = RemoteProcessRequest( - executable: "/usr/bin/ssh", - arguments: arguments, - environment: configuration.sshProcessEnvironment, - timeout: 4 - ) let token = UUID() - let cancellation = RemoteProcessCancellationOperation() - let processRunner = self.processRunner + let configuration = self.configuration + let connectionBroker = self.connectionBroker let task = Task { [weak self] in - let outcome = await withTaskCancellationHandler { - await Self.runConflictedControlMasterExit( - request: request, - processRunner: processRunner, - cancellation: cancellation - ) - } onCancel: { - cancellation.cancel() - } + let outcome = await connectionBroker.resetConflictedControlMaster( + for: configuration + ) guard !Task.isCancelled else { return } self?.queue.async { [weak self] in self?.finishConflictedControlMasterExitLocked( @@ -86,8 +52,7 @@ extension RemoteSessionCoordinator { } reverseRelayStartupPhase = .exitingConflictedControlMaster( token: token, - task: task, - cancellation: cancellation + task: task ) debugLog( "remote.relay.conflictedMaster.exitBegin " + @@ -96,40 +61,11 @@ extension RemoteSessionCoordinator { return true } - /// Runs blocking `ssh -O exit` without occupying the coordinator queue or - /// Swift's cooperative executor. - private static func runConflictedControlMasterExit( - request: RemoteProcessRequest, - processRunner: any RemoteSessionProcessRunning, - cancellation: RemoteProcessCancellationOperation - ) async -> ConflictedControlMasterExitOutcome { - await withCheckedContinuation { continuation in - // The process runner is intentionally blocking. Keep that legacy - // boundary on a utility thread while the owning Task suspends. - DispatchQueue.global(qos: .utility).async { - let outcome: ConflictedControlMasterExitOutcome - do { - let result = try processRunner.run(request, operation: cancellation) - if result.status == 0 { - outcome = .exited - } else { - let detail = bestErrorLine(stderr: result.stderr, stdout: result.stdout) - ?? "ssh exited \(result.status)" - outcome = .ignored(detail) - } - } catch { - outcome = .ignored(error.localizedDescription) - } - continuation.resume(returning: outcome) - } - } - } - /// Re-enters queue confinement and retries only after `ssh -O exit` /// completes and this recovery phase still owns the token. private func finishConflictedControlMasterExitLocked( token: UUID, - outcome: ConflictedControlMasterExitOutcome, + outcome: NativeSSHControlMasterResetOutcome, remotePath: String, relayPort: Int ) { @@ -137,7 +73,7 @@ extension RemoteSessionCoordinator { reverseRelayStartupPhase = .recoveryAttempted switch outcome { - case .exited: + case .reset: debugLog( "remote.relay.conflictedMaster.exited " + "relayPort=\(relayPort) \(debugConfigSummary())" @@ -166,13 +102,11 @@ extension RemoteSessionCoordinator { func cancelReverseRelayStartupLocked() { guard case .exitingConflictedControlMaster( _, - let task, - let cancellation + let task ) = reverseRelayStartupPhase else { return } reverseRelayStartupPhase = .recoveryAttempted task.cancel() - cancellation.cancel() } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift index 044a36b7346b..d1f9aa3a0adf 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift @@ -5,8 +5,7 @@ enum ReverseRelayStartupPhase: Sendable { case recoveryAvailable case exitingConflictedControlMaster( token: UUID, - task: Task, - cancellation: RemoteProcessCancellationOperation + task: Task ) case recoveryAttempted @@ -32,7 +31,7 @@ enum ReverseRelayStartupPhase: Sendable { } var token: UUID? { - guard case .exitingConflictedControlMaster(let token, _, _) = self else { + guard case .exitingConflictedControlMaster(let token, _) = self else { return nil } return token diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift new file mode 100644 index 000000000000..7a1afebf3620 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift @@ -0,0 +1,35 @@ +import Foundation +@testable import CmuxRemoteSession + +final class BlockingControlMasterResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let starts: AsyncStream + private let startsContinuation: AsyncStream.Continuation + private let lock = NSLock() + private var _requests: [RemoteProcessRequest] = [] + private let release = DispatchSemaphore(value: 0) + + init() { + (starts, startsContinuation) = AsyncStream.makeStream() + } + + var requests: [RemoteProcessRequest] { + lock.withLock { _requests } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { _requests.append(request) } + startsContinuation.yield() + release.wait() + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + + func finish() { + release.signal() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index 378f3ce0b031..c216fe2d6721 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -68,8 +68,8 @@ struct NativeSSHConnectionBrokerTests { #expect(recorder.requests.isEmpty) } - @Test("Unresolved cmux templates remain unowned until ssh -G resolves them") - func unresolvedTemplatesAreNotOwned() { + @Test("Unresolved templates authorize reset but not last-owner cleanup") + func unresolvedTemplatesOnlyAuthorizeReset() { let recorder = CleanupRequestRecorder() let broker = makeBroker(cleanupRecorder: recorder) let templateOptions = sharingOptions.mergingDefaults(into: []) @@ -86,8 +86,8 @@ struct NativeSSHConnectionBrokerTests { let firstLease = broker.retainWorkspace(first) let secondLease = broker.retainWorkspace(second) - #expect(firstLease.sshControlMasterLeaseGeneration == nil) - #expect(secondLease.sshControlMasterLeaseGeneration == nil) + #expect(firstLease.sshControlMasterLeaseGeneration != nil) + #expect(secondLease.sshControlMasterLeaseGeneration != nil) broker.releaseWorkspace(firstLease) broker.releaseWorkspace(secondLease) @@ -166,6 +166,50 @@ struct NativeSSHConnectionBrokerTests { #expect(recorder.requests[1].arguments.contains(replacement.sshOptions[2])) } + @Test("Concurrent owners coalesce one conflicted-master reset") + func concurrentOwnersCoalesceConflictReset() async throws { + let runner = BlockingControlMasterResetRunner() + let broker = NativeSSHConnectionBroker( + sharingOptions: sharingOptions, + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + conflictedMasterResetRunner: runner + ) + let firstLease = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "first-alias", + sshOptions: resolvedOwnedSSHOptions + )) + let secondLease = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "second-alias", + sshOptions: resolvedOwnedSSHOptions + )) + + let firstReset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: firstLease) + } + var starts = runner.starts.makeAsyncIterator() + _ = try #require(await starts.next()) + + let secondReset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: secondLease) + } + await Task.yield() + #expect(runner.requests.count == 1) + + runner.finish() + #expect(await firstReset.value == .reset) + #expect(await secondReset.value == .reset) + #expect(runner.requests.count == 1) + let request = try #require(runner.requests.first) + #expect(request.executable == "/bin/zsh") + #expect(request.arguments.contains("-O")) + #expect(request.arguments.contains("exit")) + #expect(request.arguments.contains(resolvedOwnedSSHOptions[2])) + } + @Test("Cleanup reuses the shared path without negotiating a replacement master") func cleanupArgumentsAreReuseOnly() { let configuration = configuration( diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 9eb5999f6f09..4dcb4b85c5ed 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -2,6 +2,7 @@ import Foundation import Testing import CmuxCore import CmuxRemoteDaemon +import CmuxFoundation @testable import CmuxRemoteSession @testable import CmuxRemoteWorkspace @@ -37,7 +38,7 @@ struct RemoteSessionReverseRelayStartupTests { stderr: "Control socket connect: No such file or directory" ) } - let fixture = try Self.makeCoordinator(host: host, runner: runner) + let fixture = try await Self.makeCoordinator(host: host, runner: runner) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } @@ -68,7 +69,7 @@ struct RemoteSessionReverseRelayStartupTests { )) let request = runner.requests.first - #expect(request?.executable == "/usr/bin/ssh") + #expect(request?.executable == "/bin/zsh") #expect(request?.arguments.contains("-O") == true) #expect(request?.arguments.contains("exit") == true) #expect(request?.arguments.contains("-R") == false) @@ -98,7 +99,7 @@ struct RemoteSessionReverseRelayStartupTests { let launcher = RecordingReverseRelayLauncher() let clock = ManualBrokerClock() let relayPort = 64_046 - let fixture = try Self.makeCoordinator( + let fixture = try await Self.makeCoordinator( runner: runner, reverseRelayLauncher: launcher, relayPort: relayPort, @@ -128,10 +129,10 @@ struct RemoteSessionReverseRelayStartupTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test("Stop cancels conflicted-master exit without waiting on its timeout") - func stopCancelsConflictedMasterExit() async throws { + @Test("Stop detaches from a broker-owned reset without waiting") + func stopDetachesFromConflictedMasterReset() async throws { let runner = BlockingConflictedMasterExitRunner() - let fixture = try Self.makeCoordinator(runner: runner) + let fixture = try await Self.makeCoordinator(runner: runner) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } @@ -148,16 +149,16 @@ struct RemoteSessionReverseRelayStartupTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) - var cancelled = runner.cancelled.makeAsyncIterator() - #expect(await cancelled.next() != nil) let startupCleared = coordinator.queue.sync { !coordinator.reverseRelayStartupPhase.isRecovering && coordinator.reverseRelayStartupPhase.allowsRelayLaunch && coordinator.reverseRelayProcess == nil } #expect(startupCleared) + runner.finish() } + @MainActor static func makeCoordinator( host: any RemoteSessionHosting = NoopRemoteSessionHost(), runner: any RemoteSessionProcessRunning, @@ -175,7 +176,7 @@ struct RemoteSessionReverseRelayStartupTests { at: scratchDirectory, withIntermediateDirectories: true ) - let configuration = WorkspaceRemoteConfiguration( + let rawConfiguration = WorkspaceRemoteConfiguration( destination: "user@example.test", port: nil, identityFile: nil, @@ -185,15 +186,24 @@ struct RemoteSessionReverseRelayStartupTests { relayID: "relay-startup-cancellation", relayToken: String(repeating: "a", count: 64), localSocketPath: scratchDirectory.appendingPathComponent("relay.sock").path, + ownerWorkspaceID: UUID(), terminalStartupCommand: nil, preserveAfterTerminalExit: false, persistentDaemonSlot: nil ) + let connectionBroker = NativeSSHConnectionBroker( + sharingOptions: SSHConnectionSharingOptions(), + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + conflictedMasterResetRunner: runner + ) + let configuration = connectionBroker.retainWorkspace(rawConfiguration) let coordinator = RemoteSessionCoordinator( host: host, configuration: configuration, proxyBroker: SSHOverrideUnusedRemoteProxyBroker(), - connectionBroker: NativeSSHConnectionBroker(), + connectionBroker: connectionBroker, manifestRepository: RemoteDaemonManifestRepository( homeDirectory: scratchDirectory ), @@ -292,15 +302,12 @@ private final class BlockingConflictedMasterExitRunner: @unchecked Sendable { let started: AsyncStream - let cancelled: AsyncStream private let startedContinuation: AsyncStream.Continuation - private let cancelledContinuation: AsyncStream.Continuation private let release = DispatchSemaphore(value: 0) init() { (started, startedContinuation) = AsyncStream.makeStream() - (cancelled, cancelledContinuation) = AsyncStream.makeStream() } func run( @@ -311,27 +318,17 @@ private final class BlockingConflictedMasterExitRunner: request.arguments.contains("exit") else { return RemoteCommandResult(status: 0, stdout: "", stderr: "") } - guard let operation else { - throw BlockingConflictedMasterExitError.missingCancellationOperation - } - - try operation.throwIfCancelled() - operation.installCancellationHandler { [cancelledContinuation, release] in - cancelledContinuation.yield() - release.signal() - } - defer { operation.clearCancellationHandler() } startedContinuation.yield() release.wait() - throw operation.cancellationError + return RemoteCommandResult(status: 0, stdout: "", stderr: "") } -} -private enum BlockingConflictedMasterExitError: Error { - case missingCancellationOperation + func finish() { + release.signal() + } } -private final class ReverseRelayRecoveryHost: RemoteSessionHosting, @unchecked Sendable { +final class ReverseRelayRecoveryHost: RemoteSessionHosting, @unchecked Sendable { let daemonStatuses: AsyncStream private let daemonStatusContinuation: AsyncStream.Continuation diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index 20b1baef70d1..0282a15e5fca 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -10,7 +10,7 @@ struct RemoteSessionReverseRelayTransportTests { func sharedControlMasterIsPreferred() async throws { let runner = RecordingProcessRunner() let launcher = RecordingReverseRelayLauncher() - let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( runner: runner, reverseRelayLauncher: launcher ) @@ -57,12 +57,13 @@ struct RemoteSessionReverseRelayTransportTests { func disabledControlMasterUsesStandaloneFallback() async throws { let runner = RecordingProcessRunner() let launcher = RecordingReverseRelayLauncher() - let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( runner: runner, reverseRelayLauncher: launcher, sshOptions: [ "StrictHostKeyChecking=accept-new", "ControlMaster=no", + "ControlPath=~/.ssh/custom-%C", ] ) let coordinator = fixture.coordinator @@ -101,7 +102,7 @@ struct RemoteSessionReverseRelayTransportTests { return RemoteCommandResult(status: 0, stdout: "", stderr: "") } let launcher = RecordingReverseRelayLauncher() - let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( runner: runner, reverseRelayLauncher: launcher, clock: clock @@ -145,7 +146,7 @@ struct RemoteSessionReverseRelayTransportTests { return RemoteCommandResult(status: 0, stdout: "", stderr: "") } let launcher = RecordingReverseRelayLauncher() - let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( runner: runner, reverseRelayLauncher: launcher, sshOptions: [ @@ -153,6 +154,7 @@ struct RemoteSessionReverseRelayTransportTests { "ControlMaster=auto", "ControlPersist=600", "ControlPath=~/.ssh/custom-%C", + "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)", ], clock: clock ) @@ -169,13 +171,63 @@ struct RemoteSessionReverseRelayTransportTests { #expect(!runner.requests.contains(where: { Self.isControlCommand("exit", in: $0.arguments) })) + let forwardRequest = try #require(runner.requests.first(where: { + Self.isControlCommand("forward", in: $0.arguments) + })) + #expect(forwardRequest.arguments.contains("ControlPath=~/.ssh/custom-%C")) #expect(coordinator.queue.sync { - coordinator.reverseRelayStartupPhase.canAttemptRecovery + !coordinator.reverseRelayStartupPhase.isRecovering && + coordinator.reverseRelayStartupPhase.allowsRelayLaunch }) #expect(launcher.launchCount == 0) _ = await coordinator.stopAndWait(cleanupScope: .transport) } + @Test("A standalone non-bind failure publishes only a generic retry status") + func standaloneFailurePublishesSanitizedStatus() async throws { + let rawFailure = "Permission denied: secret diagnostic" + let host = ReverseRelayRecoveryHost() + let clock = ManualBrokerClock() + let runner = RecordingProcessRunner { request in + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "Control socket connect: No such file or directory" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let launcher = RecordingReverseRelayLauncher() + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( + host: host, + runner: runner, + reverseRelayLauncher: launcher, + clock: clock + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + var launches = launcher.launches.makeAsyncIterator() + var statuses = host.daemonStatuses.makeAsyncIterator() + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + _ = try #require(await launches.next()) + launcher.emitTermination(detail: rawFailure) + + let status = try #require(await statuses.next()) + #expect(status.detail == String( + localized: "remoteSession.reverseRelay.unavailableRetrying", + defaultValue: "Remote SSH relay unavailable; retrying in 2 seconds" + )) + #expect(status.detail?.contains(rawFailure) == false) + #expect(await clock.nextRequestedDelay() == 2_000) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test("A late standalone bind failure triggers owned-master recovery") func lateStandaloneConflictTriggersRecovery() async throws { let relayPort = 64_047 @@ -191,7 +243,7 @@ struct RemoteSessionReverseRelayTransportTests { return RemoteCommandResult(status: 0, stdout: "", stderr: "") } let launcher = RecordingReverseRelayLauncher() - let fixture = try RemoteSessionReverseRelayStartupTests.makeCoordinator( + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( runner: runner, reverseRelayLauncher: launcher, relayPort: relayPort, diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index 952b3418bc71..3896bd75e0f6 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -139924,6 +139924,23 @@ } } }, + "remoteSession.reverseRelay.unavailableRetrying": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Remote SSH relay unavailable; retrying in 2 seconds" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "リモート SSH リレーを利用できません。2秒後に再試行します" + } + } + } + }, "remoteTmux.error.commandFailed": { "extractionState": "manual", "localizations": { From 93abf0b0d698d003b49d7e960d9024a7dc1c8889 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 01:02:53 -0700 Subject: [PATCH 16/39] fix: make shared master reset recoverable --- .../NativeSSHConnectionBroker.swift | 38 +++- ...tiveSSHControlMasterResetCoordinator.swift | 121 +++++++++-- .../NativeSSHControlMasterResetEventHub.swift | 63 ++++++ .../NativeSSHControlMasterResetKey.swift | 39 +++- ...emoteSessionCoordinator+ReverseRelay.swift | 22 +- ...oordinator+ReverseRelayControlMaster.swift | 33 ++- ...ssionCoordinator+ReverseRelayStartup.swift | 31 ++- .../Session/RemoteSessionCoordinator.swift | 9 + .../NativeSSHControlMasterResetTests.swift | 192 ++++++++++++++++++ ...emoteSessionReverseRelayStartupTests.swift | 7 + ...oteSessionReverseRelayTransportTests.swift | 5 +- 11 files changed, 512 insertions(+), 48 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 672b9051bf6b..177f4a67fa60 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -19,10 +19,11 @@ private enum NativeSSHCleanupPolicy { /// remain independent. @MainActor public final class NativeSSHConnectionBroker { - private let sharingOptions: SSHConnectionSharingOptions + private nonisolated let sharingOptions: SSHConnectionSharingOptions private let clock: any RemoteProxyRetryClock private let jitterMilliseconds: @MainActor @Sendable () -> Int private let cleanupLauncherOverride: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)? + private nonisolated let conflictedMasterResetEventHub: NativeSSHControlMasterResetEventHub private let conflictedMasterResetCoordinator: NativeSSHControlMasterResetCoordinator private var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] @@ -46,9 +47,13 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = nil + let eventHub = NativeSSHControlMasterResetEventHub() + self.conflictedMasterResetEventHub = eventHub self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( sharingOptions: sharingOptions, - processRunner: RemoteSessionProcessRunner() + processRunner: RemoteSessionProcessRunner(), + clock: clock, + eventHub: eventHub ) } @@ -68,9 +73,13 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = cleanupLauncher + let eventHub = NativeSSHControlMasterResetEventHub() + self.conflictedMasterResetEventHub = eventHub self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( sharingOptions: sharingOptions, - processRunner: RemoteSessionProcessRunner() + processRunner: RemoteSessionProcessRunner(), + clock: clock, + eventHub: eventHub ) } @@ -85,9 +94,13 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = jitterMilliseconds self.cleanupLauncherOverride = cleanupLauncher + let eventHub = NativeSSHControlMasterResetEventHub() + self.conflictedMasterResetEventHub = eventHub self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( sharingOptions: sharingOptions, - processRunner: conflictedMasterResetRunner + processRunner: conflictedMasterResetRunner, + clock: clock, + eventHub: eventHub ) } @@ -168,6 +181,23 @@ public final class NativeSSHConnectionBroker { await conflictedMasterResetCoordinator.reset(for: configuration) } + /// Observes resets that can invalidate this configuration's relay forward. + nonisolated func observeControlMasterResets( + for configuration: WorkspaceRemoteConfiguration, + handler: @escaping @Sendable () -> Void + ) -> NativeSSHControlMasterResetObservation? { + guard let scope = NativeSSHControlMasterResetKey( + configuration: configuration, + sharingOptions: sharingOptions + )?.impactScope else { + return nil + } + return conflictedMasterResetEventHub.observe( + scope: scope, + handler: handler + ) + } + /// Runs one connection attempt after acquiring the endpoint's FIFO permit. /// /// Same-endpoint attempts are separated by 100–350 ms of injected-clock diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index 7398dc832763..1f1cc8f430cb 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -1,10 +1,12 @@ internal import CmuxCore internal import CmuxFoundation +internal import CmuxRemoteWorkspace internal import Foundation /// Outcome of one broker-authorized conflicted-master migration. enum NativeSSHControlMasterResetOutcome: Sendable, Equatable { case reset + case deferred(String) case ignored(String) } @@ -22,6 +24,8 @@ final class NativeSSHControlMasterResetCoordinator { private let sharingOptions: SSHConnectionSharingOptions private let processRunner: any RemoteSessionProcessRunning + private let clock: any RemoteProxyRetryClock + private let eventHub: NativeSSHControlMasterResetEventHub private var leases: [ UUID: [NativeSSHControlMasterResetKey: WorkspaceRemoteConfiguration] ] = [:] @@ -31,10 +35,14 @@ final class NativeSSHControlMasterResetCoordinator { nonisolated init( sharingOptions: SSHConnectionSharingOptions, - processRunner: any RemoteSessionProcessRunning + processRunner: any RemoteSessionProcessRunning, + clock: any RemoteProxyRetryClock, + eventHub: NativeSSHControlMasterResetEventHub ) { self.sharingOptions = sharingOptions self.processRunner = processRunner + self.clock = clock + self.eventHub = eventHub } func retainWorkspace( @@ -62,6 +70,10 @@ final class NativeSSHControlMasterResetCoordinator { } else { leases[ownerWorkspaceID] = ownerLeases } + let remainsOwned = leases.values.contains { $0[key] != nil } + if !remainsOwned { + inFlightResets[key]?.task.cancel() + } } func reset( @@ -99,8 +111,19 @@ final class NativeSSHControlMasterResetCoordinator { ) let resetID = UUID() let processRunner = self.processRunner + let clock = self.clock + let eventHub = self.eventHub + let impactScope = key.impactScope let task = Task { - await Self.runReset(request: request, processRunner: processRunner) + let outcome = await Self.runReset( + request: request, + processRunner: processRunner, + clock: clock + ) + if case .reset = outcome { + eventHub.emit(scope: impactScope) + } + return outcome } inFlightResets[key] = InFlightReset(id: resetID, task: task) let outcome = await task.value @@ -112,31 +135,79 @@ final class NativeSSHControlMasterResetCoordinator { private nonisolated static func runReset( request: NativeSSHControlMasterCleanupRequest, - processRunner: any RemoteSessionProcessRunning + processRunner: any RemoteSessionProcessRunning, + clock: any RemoteProxyRetryClock ) async -> NativeSSHControlMasterResetOutcome { - await withCheckedContinuation { continuation in - DispatchQueue.global(qos: .utility).async { - let invocation = request.processInvocation - let processRequest = RemoteProcessRequest( - executable: invocation.executableURL.path, - arguments: invocation.arguments, - environment: request.environment, - timeout: 5 - ) + let maximumAttempts = 3 + for attemptIndex in 0.. ResetAttemptOutcome { + let cancellation = RemoteProcessCancellationOperation() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + DispatchQueue.global(qos: .utility).async { + let invocation = request.processInvocation + let processRequest = RemoteProcessRequest( + executable: invocation.executableURL.path, + arguments: invocation.arguments, + environment: request.environment, + timeout: 5 + ) + do { + let result = try processRunner.run( + processRequest, + operation: cancellation + ) + let detail = bestErrorLine( + stderr: result.stderr, + stdout: result.stdout + ) ?? "ssh exited \(result.status)" + if result.status == 0 { + continuation.resume(returning: .reset) + } else if result.status == + NativeSSHControlMasterCleanupRequest.retryExitStatus { + continuation.resume(returning: .retry(detail)) + } else { + continuation.resume(returning: .ignored(detail)) + } + } catch { + continuation.resume(returning: .ignored(error.localizedDescription)) + } } } + } onCancel: { + cancellation.cancel() } } @@ -157,3 +228,9 @@ final class NativeSSHControlMasterResetCoordinator { return nil } } + +private enum ResetAttemptOutcome: Sendable { + case reset + case retry(String) + case ignored(String) +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift new file mode 100644 index 000000000000..438501472426 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift @@ -0,0 +1,63 @@ +internal import Foundation + +/// Process-local fanout for disruptive shared-ControlMaster resets. +final class NativeSSHControlMasterResetEventHub: @unchecked Sendable { + // lint:allow lock - subscription and synchronous event snapshots are tiny. + private let lock = NSLock() + private var observers: [ + UUID: ( + scope: NativeSSHControlMasterResetImpactScope, + handler: @Sendable () -> Void + ) + ] = [:] + + func observe( + scope: NativeSSHControlMasterResetImpactScope, + handler: @escaping @Sendable () -> Void + ) -> NativeSSHControlMasterResetObservation { + let id = UUID() + lock.withLock { + observers[id] = (scope: scope, handler: handler) + } + return NativeSSHControlMasterResetObservation { [weak self] in + self?.removeObserver(id) + } + } + + func emit(scope: NativeSSHControlMasterResetImpactScope) { + let handlers = lock.withLock { + observers.values.compactMap { observer in + observer.scope == scope ? observer.handler : nil + } + } + for handler in handlers { + handler() + } + } + + private func removeObserver(_ id: UUID) { + lock.withLock { + _ = observers.removeValue(forKey: id) + } + } +} + +/// Lifetime token for one ControlMaster-reset observer. +final class NativeSSHControlMasterResetObservation: @unchecked Sendable { + // lint:allow lock - cancellation exchanges one closure exactly once. + private let lock = NSLock() + private var cancellation: (@Sendable () -> Void)? + + init(cancellation: @escaping @Sendable () -> Void) { + self.cancellation = cancellation + } + + deinit { + let cancellation = lock.withLock { + let value = self.cancellation + self.cancellation = nil + return value + } + cancellation?() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift index e9137752f80c..4a47b12cc68e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift @@ -5,13 +5,23 @@ internal import Foundation /// Identifies the effective cmux-owned master that may be reset after an /// OpenSSH-confirmed reverse-forward bind conflict. /// -/// Resolved socket paths are globally stable. An unresolved `%` template also -/// includes the configured endpoint so independent hosts do not share reset -/// state before OpenSSH expands the template. +/// Resolved socket paths are globally stable. An unresolved `%` template is +/// intentionally scoped to one owner and its full explicit SSH identity: +/// without `ssh -G` expansion, coalescing separate owners could falsely claim +/// that an exit for one effective socket reset another. struct NativeSSHControlMasterResetKey: Hashable, Sendable { let controlPath: String let destination: String? let port: Int? + let identityFile: String? + let effectiveOptions: [String] + let ownerWorkspaceID: UUID? + + var impactScope: NativeSSHControlMasterResetImpactScope { + controlPath.contains("%") + ? .unresolvedTemplate(controlPath) + : .resolvedPath(controlPath) + } init?( configuration: WorkspaceRemoteConfiguration, @@ -30,12 +40,33 @@ struct NativeSSHControlMasterResetKey: Hashable, Sendable { if controlPath.contains("%") { let destination = configuration.destination .trimmingCharacters(in: .whitespacesAndNewlines) - guard !destination.isEmpty else { return nil } + guard !destination.isEmpty, + let ownerWorkspaceID = configuration.ownerWorkspaceID else { + return nil + } self.destination = destination self.port = configuration.port + self.identityFile = configuration.identityFile? + .trimmingCharacters(in: .whitespacesAndNewlines) + self.effectiveOptions = effectiveOptions + self.ownerWorkspaceID = ownerWorkspaceID } else { self.destination = nil self.port = nil + self.identityFile = nil + self.effectiveOptions = [] + self.ownerWorkspaceID = nil } } } + +/// Scope potentially disrupted by a successful `ssh -O exit`. +/// +/// A resolved path is exact. With an unresolved template, aliases can expand +/// to the same socket, so the conservative event scope covers every live +/// coordinator using that cmux-owned template. Extra restart notifications +/// are safe; missing one would leave a sibling relay marked alive. +enum NativeSSHControlMasterResetImpactScope: Hashable, Sendable { + case resolvedPath(String) + case unresolvedTemplate(String) +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index e6d41b9e8955..bc30fbfcc76d 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -397,10 +397,26 @@ extension RemoteSessionCoordinator { } /// Returns whether OpenSSH reported that this relay's remote listener is - /// already bound. The optional `Error:` prefix varies across OpenSSH builds. + /// already bound. static func isReverseRelayPortBindingFailure(_ detail: String, relayPort: Int) -> Bool { + reverseRelayPortBindingFailureLine(in: detail, relayPort: relayPort) != nil + } + + /// Extracts the exact bind diagnostic from standalone or multiplexed + /// OpenSSH stderr. Multiplexing adds a prefix and may append a later + /// summary line, so classification must inspect every line. + static func reverseRelayPortBindingFailureLine( + in detail: String, + relayPort: Int + ) -> String? { let expected = "remote port forwarding failed for listen port \(relayPort)" - let normalized = detail.trimmingCharacters(in: .whitespacesAndNewlines) - return normalized == expected || normalized == "Error: \(expected)" + return detail + .split(whereSeparator: \.isNewline) + .map { + $0.trimmingCharacters(in: .whitespacesAndNewlines) + } + .first(where: { + $0 == expected || $0.hasSuffix(": \(expected)") + }) } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index b153cd136a31..e4cb464a90b8 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -34,6 +34,15 @@ extension RemoteSessionCoordinator { do { let result = try sshExec(arguments: arguments, timeout: 6) guard result.status == 0 else { + let bindingConflict = [ + result.stderr, + result.stdout, + ].compactMap { + Self.reverseRelayPortBindingFailureLine( + in: $0, + relayPort: relayPort + ) + }.first let detail = Self.bestErrorLine( stderr: result.stderr, stdout: result.stdout @@ -42,11 +51,8 @@ extension RemoteSessionCoordinator { "remote.relay.controlmaster.forwardFailed \(detail) " + debugConfigSummary() ) - if Self.isReverseRelayPortBindingFailure( - detail, - relayPort: relayPort - ) { - return .bindingConflict(detail) + if let bindingConflict { + return .bindingConflict(bindingConflict) } return .unavailable } @@ -75,4 +81,21 @@ extension RemoteSessionCoordinator { } _ = try? sshExec(arguments: arguments, timeout: 4) } + + /// Invalidates a relay installed on a shared master that another owner + /// exited during conflict recovery. + func sharedControlMasterDidResetLocked() { + guard reverseRelayControlMasterForwardSpec != nil else { return } + reverseRelayControlMasterForwardSpec = nil + debugLog( + "remote.relay.controlmaster.resetObserved \(debugConfigSummary())" + ) + guard !isStopping, + daemonReady, + let remotePath = daemonRemotePath, + !remotePath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + return + } + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) + } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index 032cae606972..c92d159fec02 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -70,26 +70,29 @@ extension RemoteSessionCoordinator { relayPort: Int ) { guard reverseRelayStartupPhase.token == token else { return } - reverseRelayStartupPhase = .recoveryAttempted - switch outcome { case .reset: + reverseRelayStartupPhase = .recoveryAttempted debugLog( "remote.relay.conflictedMaster.exited " + "relayPort=\(relayPort) \(debugConfigSummary())" ) + case .deferred(let detail): + reverseRelayStartupPhase = .recoveryAvailable + debugLog( + "remote.relay.conflictedMaster.exitDeferred " + + "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" + ) + publishReverseRelayPortUnavailableLocked() + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) + return case .ignored(let detail): + reverseRelayStartupPhase = .recoveryAttempted debugLog( "remote.relay.conflictedMaster.exitIgnored " + "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" ) - publishDaemonStatus( - .error, - detail: String( - localized: "remoteSession.reverseRelay.portUnavailableRetrying", - defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" - ) - ) + publishReverseRelayPortUnavailableLocked() scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) return } @@ -98,6 +101,16 @@ extension RemoteSessionCoordinator { scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) } + private func publishReverseRelayPortUnavailableLocked() { + publishDaemonStatus( + .error, + detail: String( + localized: "remoteSession.reverseRelay.portUnavailableRetrying", + defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" + ) + ) + } + /// Cancels the in-flight OpenSSH recovery and invalidates its continuation. func cancelReverseRelayStartupLocked() { guard case .exitingConflictedControlMaster( diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index 583912477945..f23efc5e239d 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -83,6 +83,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? + var conflictedControlMasterResetObservation: NativeSSHControlMasterResetObservation? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] /// Stable publication state for best-effort remote TTY attribution scans. @@ -180,6 +181,14 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { self.strings = strings self.clock = clock queue.setSpecific(key: queueKey, value: ()) + conflictedControlMasterResetObservation = + connectionBroker.observeControlMasterResets( + for: configuration + ) { [weak self] in + self?.queue.async { [weak self] in + self?.sharedControlMasterDidResetLocked() + } + } } /// The capabilities advertised by the cmuxd-remote baked into the Freestyle snapshot diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift new file mode 100644 index 000000000000..599f4e0fe6c7 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -0,0 +1,192 @@ +import CmuxCore +import CmuxFoundation +import CmuxRemoteWorkspace +import Foundation +import Testing +@testable import CmuxRemoteSession + +@MainActor +@Suite("Native SSH conflicted-master reset") +struct NativeSSHControlMasterResetTests { + private let sharingOptions = SSHConnectionSharingOptions(userID: 501) + private let resolvedOptions = [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567", + ] + + @Test("A successful global exit notifies every shared-master owner") + func successfulExitNotifiesSiblingOwners() async throws { + let recorder = ResetEventRecorder() + let broker = makeBroker(processRunner: RecordingProcessRunner()) + let first = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "first-alias", + options: resolvedOptions + )) + let second = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "second-alias", + options: resolvedOptions + )) + let firstObservation = try #require( + broker.observeControlMasterResets(for: first) { + recorder.record() + } + ) + let secondObservation = try #require( + broker.observeControlMasterResets(for: second) { + recorder.record() + } + ) + + #expect(await broker.resetConflictedControlMaster(for: first) == .reset) + #expect(recorder.count == 2) + _ = firstObservation + _ = secondObservation + } + + @Test("Authentication-lock deferrals retry before resetting") + func authenticationDeferralRetries() async { + let clock = ManualBrokerClock() + let runner = RetryThenSuccessResetRunner(retryCount: 2) + let broker = makeBroker(clock: clock, processRunner: runner) + let lease = broker.retainWorkspace(configuration( + owner: UUID(), + options: resolvedOptions + )) + + let reset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: lease) + } + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + + #expect( + await reset.value == NativeSSHControlMasterResetOutcome.reset + ) + #expect(runner.requestCount == 3) + } + + @Test("Unresolved templates never coalesce distinct effective identities") + func unresolvedTemplatesUseConservativeKeys() throws { + let first = configuration( + owner: UUID(), + destination: "shared-alias", + options: [ + "ControlMaster=auto", + "ControlPath=/tmp/cmux-ssh-501-%C", + "User=alice", + ] + ) + let second = configuration( + owner: UUID(), + destination: "shared-alias", + options: [ + "ControlMaster=auto", + "ControlPath=/tmp/cmux-ssh-501-%C", + "User=bob", + ] + ) + let firstKey = try #require(NativeSSHControlMasterResetKey( + configuration: first, + sharingOptions: sharingOptions + )) + let secondKey = try #require(NativeSSHControlMasterResetKey( + configuration: second, + sharingOptions: sharingOptions + )) + + #expect(firstKey != secondKey) + #expect(firstKey.impactScope == secondKey.impactScope) + } + + private func makeBroker( + clock: any RemoteProxyRetryClock = RecordingImmediateClock(), + processRunner: any RemoteSessionProcessRunning + ) -> NativeSSHConnectionBroker { + NativeSSHConnectionBroker( + sharingOptions: sharingOptions, + clock: clock, + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + conflictedMasterResetRunner: processRunner + ) + } + + private func configuration( + owner: UUID, + destination: String = "alice@example.test", + options: [String] + ) -> WorkspaceRemoteConfiguration { + WorkspaceRemoteConfiguration( + destination: destination, + port: nil, + identityFile: nil, + sshOptions: options, + localProxyPort: nil, + relayPort: 64_001, + relayID: "relay-id", + relayToken: "token", + localSocketPath: "/tmp/cmux-test.sock", + ownerWorkspaceID: owner, + terminalStartupCommand: nil, + preserveAfterTerminalExit: true, + persistentDaemonSlot: "ssh-test" + ) + } +} + +private final class ResetEventRecorder: @unchecked Sendable { + // lint:allow lock - event callbacks increment one test counter. + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func record() { + lock.withLock { + value += 1 + } + } +} + +private final class RetryThenSuccessResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - process calls consume one scripted test counter. + private let lock = NSLock() + private let retryCount: Int + private var count = 0 + + init(retryCount: Int) { + self.retryCount = retryCount + } + + var requestCount: Int { + lock.withLock { count } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + let attempt = lock.withLock { + count += 1 + return count + } + if attempt <= retryCount { + return RemoteCommandResult( + status: NativeSSHControlMasterCleanupRequest.retryExitStatus, + stdout: "", + stderr: "foreground authentication still active" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 4dcb4b85c5ed..d8f7addda79c 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -18,6 +18,13 @@ struct RemoteSessionReverseRelayStartupTests { "Error: remote port forwarding failed for listen port 64044", relayPort: 64_044 )) + #expect(RemoteSessionCoordinator.isReverseRelayPortBindingFailure( + """ + mux_client_forward: forwarding request failed: remote port forwarding failed for listen port 64044 + muxclient: master forward request failed + """, + relayPort: 64_044 + )) #expect(!RemoteSessionCoordinator.isReverseRelayPortBindingFailure( "remote port forwarding failed for listen port 64045", relayPort: 64_044 diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index 0282a15e5fca..d6c6a17424d3 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -96,7 +96,10 @@ struct RemoteSessionReverseRelayTransportTests { return RemoteCommandResult( status: 255, stdout: "", - stderr: "Error: remote port forwarding failed for listen port 64044" + stderr: """ + mux_client_forward: forwarding request failed: remote port forwarding failed for listen port 64044 + muxclient: master forward request failed + """ ) } return RemoteCommandResult(status: 0, stdout: "", stderr: "") From dc3de787440f83fb679bc4a36794ab6303badab6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 01:10:13 -0700 Subject: [PATCH 17/39] fix: scope unresolved master reset events --- .../NativeSSHControlMasterResetKey.swift | 29 +++++++--- .../NativeSSHControlMasterResetTests.swift | 54 ++++++++++++++++++- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift index 4a47b12cc68e..c075523d037b 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift @@ -18,9 +18,16 @@ struct NativeSSHControlMasterResetKey: Hashable, Sendable { let ownerWorkspaceID: UUID? var impactScope: NativeSSHControlMasterResetImpactScope { - controlPath.contains("%") - ? .unresolvedTemplate(controlPath) - : .resolvedPath(controlPath) + if controlPath.contains("%") { + return .unresolvedTemplate( + controlPath: controlPath, + destination: destination ?? "", + port: port, + identityFile: identityFile, + effectiveOptions: effectiveOptions + ) + } + return .resolvedPath(controlPath) } init?( @@ -62,11 +69,17 @@ struct NativeSSHControlMasterResetKey: Hashable, Sendable { /// Scope potentially disrupted by a successful `ssh -O exit`. /// -/// A resolved path is exact. With an unresolved template, aliases can expand -/// to the same socket, so the conservative event scope covers every live -/// coordinator using that cmux-owned template. Extra restart notifications -/// are safe; missing one would leave a sibling relay marked alive. +/// A resolved path is exact. An unresolved template uses the complete explicit +/// connection identity while excluding workspace ownership, so identical +/// sibling configurations receive the event without cascading resets to +/// unrelated hosts that share cmux's default `%C` template. enum NativeSSHControlMasterResetImpactScope: Hashable, Sendable { case resolvedPath(String) - case unresolvedTemplate(String) + case unresolvedTemplate( + controlPath: String, + destination: String, + port: Int?, + identityFile: String?, + effectiveOptions: [String] + ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift index 599f4e0fe6c7..f23251c0b180 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -70,6 +70,43 @@ struct NativeSSHControlMasterResetTests { #expect(runner.requestCount == 3) } + @Test("An unresolved reset does not invalidate a different host") + func unresolvedResetDoesNotNotifyDifferentHost() async throws { + let firstRecorder = ResetEventRecorder() + let secondRecorder = ResetEventRecorder() + let broker = makeBroker(processRunner: RecordingProcessRunner()) + let unresolvedOptions = [ + "ControlMaster=auto", + "ControlPath=/tmp/cmux-ssh-501-%C", + ] + let first = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "first.example.test", + options: unresolvedOptions + )) + let second = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "second.example.test", + options: unresolvedOptions + )) + let firstObservation = try #require( + broker.observeControlMasterResets(for: first) { + firstRecorder.record() + } + ) + let secondObservation = try #require( + broker.observeControlMasterResets(for: second) { + secondRecorder.record() + } + ) + + #expect(await broker.resetConflictedControlMaster(for: first) == .reset) + #expect(firstRecorder.count == 1) + #expect(secondRecorder.count == 0) + _ = firstObservation + _ = secondObservation + } + @Test("Unresolved templates never coalesce distinct effective identities") func unresolvedTemplatesUseConservativeKeys() throws { let first = configuration( @@ -90,6 +127,15 @@ struct NativeSSHControlMasterResetTests { "User=bob", ] ) + let matchingSibling = configuration( + owner: UUID(), + destination: "shared-alias", + options: [ + "ControlMaster=auto", + "ControlPath=/tmp/cmux-ssh-501-%C", + "User=alice", + ] + ) let firstKey = try #require(NativeSSHControlMasterResetKey( configuration: first, sharingOptions: sharingOptions @@ -98,9 +144,15 @@ struct NativeSSHControlMasterResetTests { configuration: second, sharingOptions: sharingOptions )) + let matchingSiblingKey = try #require(NativeSSHControlMasterResetKey( + configuration: matchingSibling, + sharingOptions: sharingOptions + )) #expect(firstKey != secondKey) - #expect(firstKey.impactScope == secondKey.impactScope) + #expect(firstKey.impactScope != secondKey.impactScope) + #expect(firstKey != matchingSiblingKey) + #expect(firstKey.impactScope == matchingSiblingKey.impactScope) } private func makeBroker( From 3792bebb7d19fec28d59b1ae591867a298ea77e9 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 01:29:59 -0700 Subject: [PATCH 18/39] fix: resolve SSH master paths before reset --- .../NativeSSHConnectionBroker.swift | 15 +- ...NativeSSHControlMasterCleanupRequest.swift | 11 +- ...tiveSSHControlMasterResetCoordinator.swift | 132 ++++++++- .../NativeSSHControlMasterResetEventHub.swift | 10 +- .../NativeSSHControlMasterResetKey.swift | 30 -- .../NativeSSHControlPathResolver.swift | 66 +++++ ...oordinator+ReverseRelayControlMaster.swift | 88 +++++- .../Session/RemoteSessionCoordinator.swift | 10 +- ...tiveSSHControlMasterResetTestSupport.swift | 181 ++++++++++++ .../NativeSSHControlMasterResetTests.swift | 273 +++++++++++++----- ...emoteSessionReverseRelayStartupTests.swift | 13 +- ...oteSessionReverseRelayTransportTests.swift | 45 ++- .../ResolvedControlPathProcessRunner.swift | 36 +++ 13 files changed, 765 insertions(+), 145 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlPathResolver.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 177f4a67fa60..ee74bfe135dd 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -181,19 +181,20 @@ public final class NativeSSHConnectionBroker { await conflictedMasterResetCoordinator.reset(for: configuration) } - /// Observes resets that can invalidate this configuration's relay forward. + /// Observes resets that invalidate an exact cmux-owned control socket. nonisolated func observeControlMasterResets( - for configuration: WorkspaceRemoteConfiguration, + controlPath: String, handler: @escaping @Sendable () -> Void ) -> NativeSSHControlMasterResetObservation? { - guard let scope = NativeSSHControlMasterResetKey( - configuration: configuration, - sharingOptions: sharingOptions - )?.impactScope else { + guard !controlPath.contains("%"), + sharingOptions.cmuxOwnedControlPath(in: [ + "ControlMaster=auto", + "ControlPath=\(controlPath)", + ]) == controlPath else { return nil } return conflictedMasterResetEventHub.observe( - scope: scope, + controlPath: controlPath, handler: handler ) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift index 757a91cc6a27..17bc0b21d932 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift @@ -30,16 +30,23 @@ public struct NativeSSHControlMasterCleanupRequest: Sendable { extension NativeSSHControlMasterCleanupRequest { static let retryExitStatus: Int32 = 75 + static let resetSkippedExitStatus: Int32 = 76 var processInvocation: (executableURL: URL, arguments: [String]) { + processInvocation(noOpExitStatus: 0) + } + + func processInvocation( + noOpExitStatus: Int32 + ) -> (executableURL: URL, arguments: [String]) { guard let authenticationLockPath else { return (URL(fileURLWithPath: "/usr/bin/ssh"), arguments) } let inFlightPath = authenticationLockPath + ".inflight" let script = """ umask 077 - : >> "$1" || exit 0 - zmodload zsh/system || exit 0 + : >> "$1" || exit \(noOpExitStatus) + zmodload zsh/system || exit \(noOpExitStatus) zsystem flock -t 4 -e -f cmux_ssh_auth_lock_fd "$1" || exit \(Self.retryExitStatus) cmux_auth_pid="$(/bin/cat -- "$2" 2>/dev/null || true)" case "$cmux_auth_pid" in diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index 1f1cc8f430cb..b926b98bf88a 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -19,6 +19,7 @@ enum NativeSSHControlMasterResetOutcome: Sendable, Equatable { final class NativeSSHControlMasterResetCoordinator { private struct InFlightReset { let id: UUID + let authorizationKey: NativeSSHControlMasterResetKey let task: Task } @@ -30,7 +31,7 @@ final class NativeSSHControlMasterResetCoordinator { UUID: [NativeSSHControlMasterResetKey: WorkspaceRemoteConfiguration] ] = [:] private var inFlightResets: [ - NativeSSHControlMasterResetKey: InFlightReset + String: InFlightReset ] = [:] nonisolated init( @@ -72,7 +73,10 @@ final class NativeSSHControlMasterResetCoordinator { } let remainsOwned = leases.values.contains { $0[key] != nil } if !remainsOwned { - inFlightResets[key]?.task.cancel() + for reset in inFlightResets.values + where reset.authorizationKey == key { + reset.task.cancel() + } } } @@ -85,24 +89,62 @@ final class NativeSSHControlMasterResetCoordinator { configuration: configuration, sharingOptions: sharingOptions ), - leases[ownerWorkspaceID]?[key]?.sshControlMasterLeaseGeneration == generation else { + ownsLease( + ownerWorkspaceID: ownerWorkspaceID, + generation: generation, + key: key + ) else { return .ignored("workspace no longer owns this cmux SSH master") } - if let inFlight = inFlightResets[key] { - return await inFlight.task.value - } let effectiveOptions = sharingOptions.mergingDefaults( into: configuration.sshOptions ) + let pathResolver = NativeSSHControlPathResolver( + sharingOptions: sharingOptions + ) + let resolvedControlPath: String? + if let exactPath = pathResolver.resolvedControlPath( + effectiveOptions: effectiveOptions + ) { + resolvedControlPath = exactPath + } else { + resolvedControlPath = await Self.resolveControlPath( + configuration: configuration, + effectiveOptions: effectiveOptions, + resolver: pathResolver, + processRunner: processRunner + ) + } + guard !Task.isCancelled else { + return .deferred("control-master reset cancelled") + } + guard let resolvedControlPath else { + return .ignored("could not resolve the cmux SSH master socket") + } + guard ownsLease( + ownerWorkspaceID: ownerWorkspaceID, + generation: generation, + key: key + ) else { + return .ignored("workspace no longer owns this cmux SSH master") + } + if let inFlight = inFlightResets[resolvedControlPath] { + return await inFlight.task.value + } + + let resolvedOptions = pathResolver.replacingControlPath( + in: effectiveOptions, + with: resolvedControlPath + ) let arguments = RemoteControlMasterCleanup().cleanupArguments( configuration: configuration, - sshOptionsOverride: effectiveOptions + sshOptionsOverride: resolvedOptions ) let authenticationLockPath = sharingOptions.foregroundAuthenticationLockPath( destination: configuration.destination, port: configuration.port, - options: effectiveOptions + options: resolvedOptions ) let request = NativeSSHControlMasterCleanupRequest( arguments: arguments, @@ -113,7 +155,6 @@ final class NativeSSHControlMasterResetCoordinator { let processRunner = self.processRunner let clock = self.clock let eventHub = self.eventHub - let impactScope = key.impactScope let task = Task { let outcome = await Self.runReset( request: request, @@ -121,18 +162,72 @@ final class NativeSSHControlMasterResetCoordinator { clock: clock ) if case .reset = outcome { - eventHub.emit(scope: impactScope) + eventHub.emit(controlPath: resolvedControlPath) } return outcome } - inFlightResets[key] = InFlightReset(id: resetID, task: task) + inFlightResets[resolvedControlPath] = InFlightReset( + id: resetID, + authorizationKey: key, + task: task + ) let outcome = await task.value - if inFlightResets[key]?.id == resetID { - inFlightResets.removeValue(forKey: key) + if inFlightResets[resolvedControlPath]?.id == resetID { + inFlightResets.removeValue(forKey: resolvedControlPath) } return outcome } + private func ownsLease( + ownerWorkspaceID: UUID, + generation: UUID, + key: NativeSSHControlMasterResetKey + ) -> Bool { + leases[ownerWorkspaceID]?[key]?.sshControlMasterLeaseGeneration == generation + } + + private nonisolated static func resolveControlPath( + configuration: WorkspaceRemoteConfiguration, + effectiveOptions: [String], + resolver: NativeSSHControlPathResolver, + processRunner: any RemoteSessionProcessRunning + ) async -> String? { + let cancellation = RemoteProcessCancellationOperation() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + DispatchQueue.global(qos: .utility).async { + let request = RemoteProcessRequest( + executable: "/usr/bin/ssh", + arguments: resolver.resolutionArguments( + configuration: configuration, + effectiveOptions: effectiveOptions + ), + environment: configuration.sshProcessEnvironment, + timeout: 5 + ) + do { + let result = try processRunner.run( + request, + operation: cancellation + ) + guard result.status == 0 else { + continuation.resume(returning: nil) + return + } + continuation.resume(returning: resolver.resolvedControlPath( + effectiveOptions: effectiveOptions, + sshConfigOutput: result.stdout + )) + } catch { + continuation.resume(returning: nil) + } + } + } + } onCancel: { + cancellation.cancel() + } + } + private nonisolated static func runReset( request: NativeSSHControlMasterCleanupRequest, processRunner: any RemoteSessionProcessRunning, @@ -177,7 +272,10 @@ final class NativeSSHControlMasterResetCoordinator { return await withTaskCancellationHandler { await withCheckedContinuation { continuation in DispatchQueue.global(qos: .utility).async { - let invocation = request.processInvocation + let invocation = request.processInvocation( + noOpExitStatus: + NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus + ) let processRequest = RemoteProcessRequest( executable: invocation.executableURL.path, arguments: invocation.arguments, @@ -195,8 +293,10 @@ final class NativeSSHControlMasterResetCoordinator { ) ?? "ssh exited \(result.status)" if result.status == 0 { continuation.resume(returning: .reset) - } else if result.status == - NativeSSHControlMasterCleanupRequest.retryExitStatus { + } else if [ + NativeSSHControlMasterCleanupRequest.retryExitStatus, + NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus, + ].contains(result.status) { continuation.resume(returning: .retry(detail)) } else { continuation.resume(returning: .ignored(detail)) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift index 438501472426..bdff7806e6db 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift @@ -6,28 +6,28 @@ final class NativeSSHControlMasterResetEventHub: @unchecked Sendable { private let lock = NSLock() private var observers: [ UUID: ( - scope: NativeSSHControlMasterResetImpactScope, + controlPath: String, handler: @Sendable () -> Void ) ] = [:] func observe( - scope: NativeSSHControlMasterResetImpactScope, + controlPath: String, handler: @escaping @Sendable () -> Void ) -> NativeSSHControlMasterResetObservation { let id = UUID() lock.withLock { - observers[id] = (scope: scope, handler: handler) + observers[id] = (controlPath: controlPath, handler: handler) } return NativeSSHControlMasterResetObservation { [weak self] in self?.removeObserver(id) } } - func emit(scope: NativeSSHControlMasterResetImpactScope) { + func emit(controlPath: String) { let handlers = lock.withLock { observers.values.compactMap { observer in - observer.scope == scope ? observer.handler : nil + observer.controlPath == controlPath ? observer.handler : nil } } for handler in handlers { diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift index c075523d037b..7a9938bf22d3 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift @@ -17,19 +17,6 @@ struct NativeSSHControlMasterResetKey: Hashable, Sendable { let effectiveOptions: [String] let ownerWorkspaceID: UUID? - var impactScope: NativeSSHControlMasterResetImpactScope { - if controlPath.contains("%") { - return .unresolvedTemplate( - controlPath: controlPath, - destination: destination ?? "", - port: port, - identityFile: identityFile, - effectiveOptions: effectiveOptions - ) - } - return .resolvedPath(controlPath) - } - init?( configuration: WorkspaceRemoteConfiguration, sharingOptions: SSHConnectionSharingOptions @@ -66,20 +53,3 @@ struct NativeSSHControlMasterResetKey: Hashable, Sendable { } } } - -/// Scope potentially disrupted by a successful `ssh -O exit`. -/// -/// A resolved path is exact. An unresolved template uses the complete explicit -/// connection identity while excluding workspace ownership, so identical -/// sibling configurations receive the event without cascading resets to -/// unrelated hosts that share cmux's default `%C` template. -enum NativeSSHControlMasterResetImpactScope: Hashable, Sendable { - case resolvedPath(String) - case unresolvedTemplate( - controlPath: String, - destination: String, - port: Int?, - identityFile: String?, - effectiveOptions: [String] - ) -} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlPathResolver.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlPathResolver.swift new file mode 100644 index 000000000000..bde541d9f610 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlPathResolver.swift @@ -0,0 +1,66 @@ +internal import CmuxCore +internal import CmuxFoundation +internal import Foundation + +/// Resolves a cmux-owned OpenSSH ControlPath to the exact local socket path. +struct NativeSSHControlPathResolver: Sendable { + let sharingOptions: SSHConnectionSharingOptions + + func resolutionArguments( + configuration: WorkspaceRemoteConfiguration, + effectiveOptions: [String] + ) -> [String] { + var arguments = ["-G"] + if let port = configuration.port { + arguments += ["-p", String(port)] + } + if let identityFile = configuration.identityFile? + .trimmingCharacters(in: .whitespacesAndNewlines), + !identityFile.isEmpty { + arguments += ["-i", identityFile] + } + for option in effectiveOptions { + arguments += ["-o", option] + } + arguments.append(configuration.destination) + return arguments + } + + func resolvedControlPath( + effectiveOptions: [String], + sshConfigOutput: String? = nil + ) -> String? { + guard let ownedPath = sharingOptions.cmuxOwnedControlPath( + in: effectiveOptions + ) else { + return nil + } + guard ownedPath.contains("%") else { return ownedPath } + guard let sshConfigOutput else { return nil } + for line in sshConfigOutput.split(whereSeparator: \.isNewline) { + let parts = line.split(maxSplits: 1, whereSeparator: \.isWhitespace) + guard parts.count == 2, + parts[0].lowercased() == "controlpath" else { + continue + } + let path = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) + guard !path.contains("%") else { return nil } + let resolvedOptions = replacingControlPath( + in: effectiveOptions, + with: path + ) + return sharingOptions.cmuxOwnedControlPath(in: resolvedOptions) + } + return nil + } + + func replacingControlPath( + in effectiveOptions: [String], + with resolvedControlPath: String + ) -> [String] { + let optionResolver = SSHAgentSocketResolver() + return ["ControlPath=\(resolvedControlPath)"] + effectiveOptions.filter { + optionResolver.optionKey($0) != "controlpath" + } + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index e4cb464a90b8..ece7e0e6c214 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -22,7 +22,10 @@ extension RemoteSessionCoordinator { forwardSpec: String, relayPort: Int ) -> ReverseRelayControlMasterStartOutcome { - let effectiveSSHOptions = reverseRelayControlMasterSSHOptions + guard let effectiveSSHOptions = + resolvedReverseRelayControlMasterSSHOptionsLocked() else { + return .unavailable + } guard let arguments = configuration.reverseRelayControlMasterArguments( controlCommand: "forward", forwardSpec: forwardSpec, @@ -71,7 +74,10 @@ extension RemoteSessionCoordinator { func stopReverseRelayViaControlMasterLocked() { guard let forwardSpec = reverseRelayControlMasterForwardSpec else { return } reverseRelayControlMasterForwardSpec = nil - let effectiveSSHOptions = reverseRelayControlMasterSSHOptions + guard let effectiveSSHOptions = + reverseRelayResolvedControlMasterSSHOptions else { + return + } guard let arguments = configuration.reverseRelayControlMasterArguments( controlCommand: "cancel", forwardSpec: forwardSpec, @@ -82,6 +88,84 @@ extension RemoteSessionCoordinator { _ = try? sshExec(arguments: arguments, timeout: 4) } + /// Resolves cmux's `%C` template before adopting a shared master. + /// + /// The exact socket path is both the command target and the reset-event + /// identity. If OpenSSH cannot produce that identity, relay startup falls + /// back to its standalone transport without touching the unresolved + /// master. + private func resolvedReverseRelayControlMasterSSHOptionsLocked() -> [String]? { + if reverseRelayControlMasterResolutionAttempted { + return reverseRelayResolvedControlMasterSSHOptions + } + reverseRelayControlMasterResolutionAttempted = true + + let effectiveOptions = reverseRelayControlMasterSSHOptions + let sharingOptions = SSHConnectionSharingOptions() + let resolver = NativeSSHControlPathResolver( + sharingOptions: sharingOptions + ) + guard let ownedPath = sharingOptions.cmuxOwnedControlPath( + in: effectiveOptions + ) else { + reverseRelayResolvedControlMasterSSHOptions = effectiveOptions + return effectiveOptions + } + + let resolvedPath: String? + if ownedPath.contains("%") { + do { + let result = try sshExec( + arguments: resolver.resolutionArguments( + configuration: configuration, + effectiveOptions: effectiveOptions + ), + timeout: 5 + ) + guard result.status == 0 else { + return nil + } + resolvedPath = resolver.resolvedControlPath( + effectiveOptions: effectiveOptions, + sshConfigOutput: result.stdout + ) + } catch { + debugLog( + "remote.relay.controlmaster.resolveFailed " + + "\(error.localizedDescription) \(debugConfigSummary())" + ) + return nil + } + } else { + resolvedPath = ownedPath + } + guard let resolvedPath else { + debugLog( + "remote.relay.controlmaster.resolveFailed " + + "missing-owned-path \(debugConfigSummary())" + ) + return nil + } + + let resolvedOptions = resolver.replacingControlPath( + in: effectiveOptions, + with: resolvedPath + ) + guard let observation = connectionBroker.observeControlMasterResets( + controlPath: resolvedPath, + handler: { [weak self] in + self?.queue.async { [weak self] in + self?.sharedControlMasterDidResetLocked() + } + } + ) else { + return nil + } + conflictedControlMasterResetObservation = observation + reverseRelayResolvedControlMasterSSHOptions = resolvedOptions + return resolvedOptions + } + /// Invalidates a relay installed on a shared master that another owner /// exited during conflict recovery. func sharedControlMasterDidResetLocked() { diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index f23efc5e239d..134e856577ac 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -83,6 +83,8 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? + var reverseRelayResolvedControlMasterSSHOptions: [String]? + var reverseRelayControlMasterResolutionAttempted = false var conflictedControlMasterResetObservation: NativeSSHControlMasterResetObservation? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] @@ -181,14 +183,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { self.strings = strings self.clock = clock queue.setSpecific(key: queueKey, value: ()) - conflictedControlMasterResetObservation = - connectionBroker.observeControlMasterResets( - for: configuration - ) { [weak self] in - self?.queue.async { [weak self] in - self?.sharedControlMasterDidResetLocked() - } - } } /// The capabilities advertised by the cmuxd-remote baked into the Freestyle snapshot diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift new file mode 100644 index 000000000000..d4a9b04d263e --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift @@ -0,0 +1,181 @@ +import CmuxFoundation +import Foundation +@testable import CmuxRemoteSession + +final class ResetEventRecorder: @unchecked Sendable { + // lint:allow lock - event callbacks increment one test counter. + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func record() { + lock.withLock { + value += 1 + } + } +} + +final class RetryThenSuccessResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - process calls consume one scripted test counter. + private let lock = NSLock() + private let retryCount: Int + private var count = 0 + + init(retryCount: Int) { + self.retryCount = retryCount + } + + var requestCount: Int { + lock.withLock { count } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + let attempt = lock.withLock { + count += 1 + return count + } + if attempt <= retryCount { + return RemoteCommandResult( + status: NativeSSHControlMasterCleanupRequest.retryExitStatus, + stdout: "", + stderr: "foreground authentication still active" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } +} + +final class FixedStatusResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - process calls increment one test counter. + private let lock = NSLock() + private let status: Int32 + private var count = 0 + + init(status: Int32) { + self.status = status + } + + var requestCount: Int { + lock.withLock { count } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { + count += 1 + } + return RemoteCommandResult( + status: status, + stdout: "", + stderr: "cleanup wrapper skipped ssh" + ) + } +} + +final class ResolvingResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - process calls append test request snapshots. + private let lock = NSLock() + private let pathsByDestination: [String: String] + private var requests: [RemoteProcessRequest] = [] + + init(pathsByDestination: [String: String]) { + self.pathsByDestination = pathsByDestination + } + + var exitRequests: [RemoteProcessRequest] { + lock.withLock { + requests.filter { + $0.arguments.contains("-O") && $0.arguments.contains("exit") + } + } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { + requests.append(request) + } + if request.arguments.contains("-G"), + let destination = request.arguments.last, + let path = pathsByDestination[destination] { + return RemoteCommandResult( + status: 0, + stdout: "controlpath \(path)\n", + stderr: "" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } +} + +final class BlockingResolvingResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let resolutions: AsyncStream + let exits: AsyncStream + + // lint:allow lock - process calls increment one test counter. + private let lock = NSLock() + private let resolvedPath: String + private let resolutionContinuation: AsyncStream.Continuation + private let exitContinuation: AsyncStream.Continuation + private let exitRelease = DispatchSemaphore(value: 0) + private var _exitCount = 0 + + init(resolvedPath: String) { + self.resolvedPath = resolvedPath + (resolutions, resolutionContinuation) = AsyncStream.makeStream() + (exits, exitContinuation) = AsyncStream.makeStream() + } + + var exitCount: Int { + lock.withLock { _exitCount } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + if request.arguments.contains("-G") { + resolutionContinuation.yield() + return RemoteCommandResult( + status: 0, + stdout: "controlpath \(resolvedPath)\n", + stderr: "" + ) + } + if request.arguments.contains("-O"), + request.arguments.contains("exit") { + lock.withLock { + _exitCount += 1 + } + exitContinuation.yield() + exitRelease.wait() + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + + func finishExit() { + exitRelease.signal() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift index f23251c0b180..29332ec45661 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -9,11 +9,20 @@ import Testing @Suite("Native SSH conflicted-master reset") struct NativeSSHControlMasterResetTests { private let sharingOptions = SSHConnectionSharingOptions(userID: 501) + private let firstResolvedPath = + "/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567" + private let secondResolvedPath = + "/tmp/cmux-ssh-501-89abcdef0123456789abcdef0123456789abcdef" private let resolvedOptions = [ "ControlMaster=auto", "ControlPersist=600", "ControlPath=/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567", ] + private let unresolvedOptions = [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=/tmp/cmux-ssh-501-%C", + ] @Test("A successful global exit notifies every shared-master owner") func successfulExitNotifiesSiblingOwners() async throws { @@ -24,18 +33,18 @@ struct NativeSSHControlMasterResetTests { destination: "first-alias", options: resolvedOptions )) - let second = broker.retainWorkspace(configuration( + _ = broker.retainWorkspace(configuration( owner: UUID(), destination: "second-alias", options: resolvedOptions )) let firstObservation = try #require( - broker.observeControlMasterResets(for: first) { + broker.observeControlMasterResets(controlPath: firstResolvedPath) { recorder.record() } ) let secondObservation = try #require( - broker.observeControlMasterResets(for: second) { + broker.observeControlMasterResets(controlPath: firstResolvedPath) { recorder.record() } ) @@ -70,32 +79,32 @@ struct NativeSSHControlMasterResetTests { #expect(runner.requestCount == 3) } - @Test("An unresolved reset does not invalidate a different host") - func unresolvedResetDoesNotNotifyDifferentHost() async throws { + @Test("Expanded paths keep unrelated hosts isolated") + func expandedPathsDoNotNotifyDifferentHost() async throws { let firstRecorder = ResetEventRecorder() let secondRecorder = ResetEventRecorder() - let broker = makeBroker(processRunner: RecordingProcessRunner()) - let unresolvedOptions = [ - "ControlMaster=auto", - "ControlPath=/tmp/cmux-ssh-501-%C", - ] + let runner = ResolvingResetRunner(pathsByDestination: [ + "first.example.test": firstResolvedPath, + "second.example.test": secondResolvedPath, + ]) + let broker = makeBroker(processRunner: runner) let first = broker.retainWorkspace(configuration( owner: UUID(), destination: "first.example.test", options: unresolvedOptions )) - let second = broker.retainWorkspace(configuration( + _ = broker.retainWorkspace(configuration( owner: UUID(), destination: "second.example.test", options: unresolvedOptions )) let firstObservation = try #require( - broker.observeControlMasterResets(for: first) { + broker.observeControlMasterResets(controlPath: firstResolvedPath) { firstRecorder.record() } ) let secondObservation = try #require( - broker.observeControlMasterResets(for: second) { + broker.observeControlMasterResets(controlPath: secondResolvedPath) { secondRecorder.record() } ) @@ -103,12 +112,192 @@ struct NativeSSHControlMasterResetTests { #expect(await broker.resetConflictedControlMaster(for: first) == .reset) #expect(firstRecorder.count == 1) #expect(secondRecorder.count == 0) + #expect(runner.exitRequests.count == 1) + #expect(runner.exitRequests[0].arguments.contains( + "ControlPath=\(firstResolvedPath)" + )) + #expect(!runner.exitRequests[0].arguments.contains( + "ControlPath=/tmp/cmux-ssh-501-%C" + )) + _ = firstObservation + _ = secondObservation + } + + @Test("Different aliases resolving to one socket share reset fanout") + func aliasesResolvingToSamePathShareFanout() async throws { + let recorder = ResetEventRecorder() + let runner = ResolvingResetRunner(pathsByDestination: [ + "first-alias": firstResolvedPath, + "second-alias": firstResolvedPath, + ]) + let broker = makeBroker(processRunner: runner) + let first = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "first-alias", + options: unresolvedOptions + )) + _ = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "second-alias", + options: unresolvedOptions + )) + let firstObservation = try #require( + broker.observeControlMasterResets(controlPath: firstResolvedPath) { + recorder.record() + } + ) + let secondObservation = try #require( + broker.observeControlMasterResets(controlPath: firstResolvedPath) { + recorder.record() + } + ) + + #expect(await broker.resetConflictedControlMaster(for: first) == .reset) + #expect(recorder.count == 2) + #expect(runner.exitRequests.count == 1) _ = firstObservation _ = secondObservation } - @Test("Unresolved templates never coalesce distinct effective identities") - func unresolvedTemplatesUseConservativeKeys() throws { + @Test("Concurrent aliases coalesce only after exact path resolution") + func concurrentAliasesCoalesceByResolvedPath() async { + let runner = BlockingResolvingResetRunner( + resolvedPath: firstResolvedPath + ) + let broker = makeBroker(processRunner: runner) + let first = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "first-alias", + options: unresolvedOptions + )) + let second = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "second-alias", + options: unresolvedOptions + )) + + let firstReset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: first) + } + let secondReset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: second) + } + var resolutions = runner.resolutions.makeAsyncIterator() + #expect(await resolutions.next() != nil) + #expect(await resolutions.next() != nil) + var exits = runner.exits.makeAsyncIterator() + #expect(await exits.next() != nil) + runner.finishExit() + + #expect(await firstReset.value == .reset) + #expect(await secondReset.value == .reset) + #expect(runner.exitCount == 1) + } + + @Test("ControlPath resolution failure never exits a master") + func resolutionFailureFailsClosed() async { + let runner = RecordingProcessRunner { request in + if request.arguments.contains("-G") { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "configuration resolution failed" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let broker = makeBroker(processRunner: runner) + let lease = broker.retainWorkspace(configuration( + owner: UUID(), + options: unresolvedOptions + )) + + let outcome = await broker.resetConflictedControlMaster(for: lease) + + guard case .ignored = outcome else { + Issue.record("Expected resolution failure to be ignored") + return + } + #expect(!runner.requests.contains(where: { + $0.arguments.contains("-O") && $0.arguments.contains("exit") + })) + } + + @Test("A reset-wrapper no-op remains deferred and emits no reset") + func resetWrapperNoOpDoesNotCountAsReset() async throws { + let clock = ManualBrokerClock() + let runner = FixedStatusResetRunner( + status: NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus + ) + let recorder = ResetEventRecorder() + let broker = makeBroker(clock: clock, processRunner: runner) + let lease = broker.retainWorkspace(configuration( + owner: UUID(), + options: resolvedOptions + )) + let observation = try #require( + broker.observeControlMasterResets(controlPath: firstResolvedPath) { + recorder.record() + } + ) + + let reset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: lease) + } + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + + guard case .deferred = await reset.value else { + Issue.record("Expected skipped reset attempts to remain deferred") + return + } + #expect(runner.requestCount == 3) + #expect(recorder.count == 0) + _ = observation + } + + @Test("Cleanup wrapper distinguishes ordinary and reset-only no-ops") + func cleanupWrapperUsesResetOnlySkippedStatus() throws { + let request = NativeSSHControlMasterCleanupRequest( + arguments: ["-V"], + environment: nil, + authenticationLockPath: "/dev/null/cmux-test.lock" + ) + let normalInvocation = request.processInvocation + let resetInvocation = request.processInvocation( + noOpExitStatus: + NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus + ) + let runner = RemoteSessionProcessRunner() + + let normal = try runner.run( + RemoteProcessRequest( + executable: normalInvocation.executableURL.path, + arguments: normalInvocation.arguments, + timeout: 2 + ), + operation: nil + ) + let reset = try runner.run( + RemoteProcessRequest( + executable: resetInvocation.executableURL.path, + arguments: resetInvocation.arguments, + timeout: 2 + ), + operation: nil + ) + + #expect(normal.status == 0) + #expect( + reset.status == + NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus + ) + } + + @Test("Unresolved authorization keys remain owner-scoped") + func unresolvedTemplatesUseConservativeAuthorizationKeys() throws { let first = configuration( owner: UUID(), destination: "shared-alias", @@ -150,9 +339,7 @@ struct NativeSSHControlMasterResetTests { )) #expect(firstKey != secondKey) - #expect(firstKey.impactScope != secondKey.impactScope) #expect(firstKey != matchingSiblingKey) - #expect(firstKey.impactScope == matchingSiblingKey.impactScope) } private func makeBroker( @@ -190,55 +377,3 @@ struct NativeSSHControlMasterResetTests { ) } } - -private final class ResetEventRecorder: @unchecked Sendable { - // lint:allow lock - event callbacks increment one test counter. - private let lock = NSLock() - private var value = 0 - - var count: Int { - lock.withLock { value } - } - - func record() { - lock.withLock { - value += 1 - } - } -} - -private final class RetryThenSuccessResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - // lint:allow lock - process calls consume one scripted test counter. - private let lock = NSLock() - private let retryCount: Int - private var count = 0 - - init(retryCount: Int) { - self.retryCount = retryCount - } - - var requestCount: Int { - lock.withLock { count } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - let attempt = lock.withLock { - count += 1 - return count - } - if attempt <= retryCount { - return RemoteCommandResult( - status: NativeSSHControlMasterCleanupRequest.retryExitStatus, - stdout: "", - stderr: "foreground authentication still active" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index d8f7addda79c..f490e9bf0d3d 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -172,7 +172,8 @@ struct RemoteSessionReverseRelayStartupTests { reverseRelayLauncher: any RemoteReverseRelayLaunching = RemoteReverseRelayLauncher(), relayPort: Int = 64_044, sshOptions: [String]? = nil, - clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock() + clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock(), + providesResolvedControlPath: Bool = true ) throws -> (coordinator: RemoteSessionCoordinator, scratchDirectory: URL) { let scratchDirectory = FileManager.default.temporaryDirectory .appendingPathComponent( @@ -198,12 +199,18 @@ struct RemoteSessionReverseRelayStartupTests { preserveAfterTerminalExit: false, persistentDaemonSlot: nil ) + let effectiveRunner: any RemoteSessionProcessRunning + if providesResolvedControlPath { + effectiveRunner = ResolvedControlPathProcessRunner(base: runner) + } else { + effectiveRunner = runner + } let connectionBroker = NativeSSHConnectionBroker( sharingOptions: SSHConnectionSharingOptions(), clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, - conflictedMasterResetRunner: runner + conflictedMasterResetRunner: effectiveRunner ) let configuration = connectionBroker.retainWorkspace(rawConfiguration) let coordinator = RemoteSessionCoordinator( @@ -214,7 +221,7 @@ struct RemoteSessionReverseRelayStartupTests { manifestRepository: RemoteDaemonManifestRepository( homeDirectory: scratchDirectory ), - processRunner: runner, + processRunner: effectiveRunner, reverseRelayLauncher: reverseRelayLauncher, reachabilityProbe: SSHOverrideNoopReachabilityProbe(), relayCommandRewriter: SSHOverridePassthroughRelayCommandRewriter(), diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index d6c6a17424d3..a435b575c437 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -30,7 +30,7 @@ struct RemoteSessionReverseRelayTransportTests { #expect(forwardRequest.arguments.contains("BatchMode=yes")) #expect( forwardRequest.arguments.contains( - "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)" + "ControlPath=\(ResolvedControlPathFixture.path)" ) ) #expect(launcher.launchCount == 0) @@ -44,7 +44,7 @@ struct RemoteSessionReverseRelayTransportTests { })) #expect( cancelRequest.arguments.contains( - "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)" + "ControlPath=\(ResolvedControlPathFixture.path)" ) ) #expect( @@ -88,6 +88,45 @@ struct RemoteSessionReverseRelayTransportTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } + @Test("An unresolved cmux ControlPath fails closed to standalone") + func unresolvedOwnedControlPathUsesStandaloneFallback() async throws { + let runner = RecordingProcessRunner { request in + if request.arguments.first == "-G" { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "could not resolve configuration" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let launcher = RecordingReverseRelayLauncher() + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher, + providesResolvedControlPath: false + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + + var launches = launcher.launches.makeAsyncIterator() + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") + } + + let launch = try #require(await launches.next()) + #expect(launch.arguments.starts(with: ["-N", "-T", "-S", "none"])) + #expect(runner.requests.contains(where: { + $0.arguments.first == "-G" + })) + #expect(!runner.requests.contains(where: { + Self.isControlCommand("forward", in: $0.arguments) + })) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test("A bind conflict exits a cmux-owned master") func ownedConflictExitsMaster() async throws { let clock = ManualBrokerClock() @@ -128,7 +167,7 @@ struct RemoteSessionReverseRelayTransportTests { })) #expect( exitRequest.arguments.contains( - "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)" + "ControlPath=\(ResolvedControlPathFixture.path)" ) ) #expect(launcher.launchCount == 0) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift new file mode 100644 index 000000000000..e930d54fb98c --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift @@ -0,0 +1,36 @@ +import CmuxFoundation +@testable import CmuxRemoteSession + +enum ResolvedControlPathFixture { + static let path = + "/tmp/cmux-ssh-\(SSHConnectionSharingOptions().userID)-" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} + +/// Gives relay tests deterministic `ssh -G` expansion while preserving their +/// existing process-runner scripts for forward, cancel, and reset commands. +final class ResolvedControlPathProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + private let base: any RemoteSessionProcessRunning + + init(base: any RemoteSessionProcessRunning) { + self.base = base + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + if request.executable == "/usr/bin/ssh", + request.arguments.first == "-G" { + return RemoteCommandResult( + status: 0, + stdout: "controlpath \(ResolvedControlPathFixture.path)\n", + stderr: "" + ) + } + return try base.run(request, operation: operation) + } +} From a656a952117c6f9df73b41641e5b01f3e75f0d61 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 01:40:51 -0700 Subject: [PATCH 19/39] fix: preserve retryable SSH reset state --- ...tiveSSHControlMasterResetCoordinator.swift | 36 +++++++++++++---- ...tiveSSHControlMasterResetTestSupport.swift | 35 +++++++++++++++++ .../NativeSSHControlMasterResetTests.swift | 39 +++++++++++++++++++ 3 files changed, 102 insertions(+), 8 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index b926b98bf88a..4691a6cf6f28 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -109,12 +109,20 @@ final class NativeSSHControlMasterResetCoordinator { ) { resolvedControlPath = exactPath } else { - resolvedControlPath = await Self.resolveControlPath( + let resolution = await Self.resolveControlPath( configuration: configuration, effectiveOptions: effectiveOptions, resolver: pathResolver, processRunner: processRunner ) + switch resolution { + case .resolved(let path): + resolvedControlPath = path + case .unavailable: + return .ignored("could not resolve the cmux SSH master socket") + case .retry(let detail): + return .deferred(detail) + } } guard !Task.isCancelled else { return .deferred("control-master reset cancelled") @@ -144,7 +152,7 @@ final class NativeSSHControlMasterResetCoordinator { let authenticationLockPath = sharingOptions.foregroundAuthenticationLockPath( destination: configuration.destination, port: configuration.port, - options: resolvedOptions + options: effectiveOptions ) let request = NativeSSHControlMasterCleanupRequest( arguments: arguments, @@ -191,7 +199,7 @@ final class NativeSSHControlMasterResetCoordinator { effectiveOptions: [String], resolver: NativeSSHControlPathResolver, processRunner: any RemoteSessionProcessRunning - ) async -> String? { + ) async -> ControlPathResolutionOutcome { let cancellation = RemoteProcessCancellationOperation() return await withTaskCancellationHandler { await withCheckedContinuation { continuation in @@ -211,15 +219,21 @@ final class NativeSSHControlMasterResetCoordinator { operation: cancellation ) guard result.status == 0 else { - continuation.resume(returning: nil) + continuation.resume(returning: .unavailable) return } - continuation.resume(returning: resolver.resolvedControlPath( + if let path = resolver.resolvedControlPath( effectiveOptions: effectiveOptions, sshConfigOutput: result.stdout - )) + ) { + continuation.resume(returning: .resolved(path)) + } else { + continuation.resume(returning: .unavailable) + } } catch { - continuation.resume(returning: nil) + continuation.resume(returning: .retry( + error.localizedDescription + )) } } } @@ -302,7 +316,7 @@ final class NativeSSHControlMasterResetCoordinator { continuation.resume(returning: .ignored(detail)) } } catch { - continuation.resume(returning: .ignored(error.localizedDescription)) + continuation.resume(returning: .retry(error.localizedDescription)) } } } @@ -334,3 +348,9 @@ private enum ResetAttemptOutcome: Sendable { case retry(String) case ignored(String) } + +private enum ControlPathResolutionOutcome: Sendable { + case resolved(String) + case unavailable + case retry(String) +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift index d4a9b04d263e..66669bac02f3 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift @@ -54,6 +54,41 @@ final class RetryThenSuccessResetRunner: } } +final class ThrowThenSuccessResetRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - process calls consume one scripted test counter. + private let lock = NSLock() + private let throwCount: Int + private var count = 0 + + init(throwCount: Int) { + self.throwCount = throwCount + } + + var requestCount: Int { + lock.withLock { count } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + let attempt = lock.withLock { + count += 1 + return count + } + if attempt <= throwCount { + throw NSError( + domain: "NativeSSHControlMasterResetTests", + code: attempt + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } +} + final class FixedStatusResetRunner: RemoteSessionProcessRunning, @unchecked Sendable diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift index 29332ec45661..6ff7a118aca6 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -79,6 +79,28 @@ struct NativeSSHControlMasterResetTests { #expect(runner.requestCount == 3) } + @Test("Transient reset runner errors retry before resetting") + func transientRunnerErrorsRetry() async { + let clock = ManualBrokerClock() + let runner = ThrowThenSuccessResetRunner(throwCount: 2) + let broker = makeBroker(clock: clock, processRunner: runner) + let lease = broker.retainWorkspace(configuration( + owner: UUID(), + options: resolvedOptions + )) + + let reset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: lease) + } + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + + #expect(await reset.value == .reset) + #expect(runner.requestCount == 3) + } + @Test("Expanded paths keep unrelated hosts isolated") func expandedPathsDoNotNotifyDifferentHost() async throws { let firstRecorder = ResetEventRecorder() @@ -119,6 +141,23 @@ struct NativeSSHControlMasterResetTests { #expect(!runner.exitRequests[0].arguments.contains( "ControlPath=/tmp/cmux-ssh-501-%C" )) + let unresolvedLock = try #require( + sharingOptions.foregroundAuthenticationLockPath( + destination: "first.example.test", + port: nil, + options: unresolvedOptions + ) + ) + let resolvedLock = try #require( + sharingOptions.foregroundAuthenticationLockPath( + destination: "first.example.test", + port: nil, + options: resolvedOptions + ) + ) + #expect(unresolvedLock != resolvedLock) + #expect(runner.exitRequests[0].arguments.contains(unresolvedLock)) + #expect(!runner.exitRequests[0].arguments.contains(resolvedLock)) _ = firstObservation _ = secondObservation } From 6450e5f59bf7a806735b55ee6960d46125bfe2dc Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 01:58:43 -0700 Subject: [PATCH 20/39] fix: bound reverse relay startup lifecycle --- .../FoundationRemoteReverseRelayProcess.swift | 58 +++++++++-- .../Process/RemoteReverseRelayLauncher.swift | 11 +++ .../Process/RemoteReverseRelayLaunching.swift | 5 + ...emoteSessionCoordinator+ReverseRelay.swift | 84 ++++++++++------ ...oordinator+ReverseRelayControlMaster.swift | 3 +- .../Session/RemoteSessionCoordinator.swift | 1 - ...emoteSessionReverseRelayStartupTests.swift | 10 ++ ...oteSessionReverseRelayTransportTests.swift | 95 +++++++++++++++++++ .../ResolvedControlPathProcessRunner.swift | 50 ++++++++++ 9 files changed, 278 insertions(+), 39 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift index ad05c97bcaf4..282e3d3164a1 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -11,10 +11,16 @@ final class FoundationRemoteReverseRelayProcess: { private let process: Process private let stderrPipe: Pipe + private let stderrDrainGracePeriod: TimeInterval - init(process: Process, stderrPipe: Pipe) { + init( + process: Process, + stderrPipe: Pipe, + stderrDrainGracePeriod: TimeInterval = 0.5 + ) { self.process = process self.stderrPipe = stderrPipe + self.stderrDrainGracePeriod = stderrDrainGracePeriod } var isRunning: Bool { @@ -25,15 +31,16 @@ final class FoundationRemoteReverseRelayProcess: process.terminationStatus } - /// Drains stderr through EOF without parking one utility worker for the - /// lifetime of the relay. Completion waits for both EOF and termination, - /// so the final diagnostic bytes always precede the callback. + /// Drains stderr without parking one utility worker for the relay's + /// lifetime. EOF completes immediately after termination; an inherited + /// writer that outlives ssh is cut off after a bounded final grace period. func captureTermination( _ handler: @escaping @Sendable (String?) -> Void ) { let readHandle = stderrPipe.fileHandleForReading let capture = ReverseRelayStderrCapture( readHandle: readHandle, + drainGracePeriod: stderrDrainGracePeriod, handler: handler ) readHandle.readabilityHandler = { handle in @@ -52,7 +59,7 @@ final class FoundationRemoteReverseRelayProcess: } } -/// Event-driven stderr tail and process-lifecycle rendezvous. +/// Event-driven stderr tail with a bounded post-termination drain. private final class ReverseRelayStderrCapture: @unchecked Sendable { private struct Completion { let stderr: String @@ -65,18 +72,22 @@ private final class ReverseRelayStderrCapture: @unchecked Sendable { private let readHandle: FileHandle private let handler: @Sendable (String?) -> Void private let byteLimit: Int + private let drainGracePeriod: TimeInterval private var tail = Data() private var sawEOF = false private var terminationStatus: Int32? + private var drainDeadlineScheduled = false private var completed = false init( readHandle: FileHandle, byteLimit: Int = 8192, + drainGracePeriod: TimeInterval, handler: @escaping @Sendable (String?) -> Void ) { self.readHandle = readHandle self.byteLimit = byteLimit + self.drainGracePeriod = drainGracePeriod self.handler = handler } @@ -96,15 +107,44 @@ private final class ReverseRelayStderrCapture: @unchecked Sendable { } func processDidTerminate(status: Int32) { - let completion = lock.withLock { () -> Completion? in + let result = lock.withLock { () -> ( + completion: Completion?, + scheduleDeadline: Bool + ) in terminationStatus = status - return takeCompletionIfReady() + let completion = takeCompletionIfReady() + if let completion { + return (completion, false) + } + guard !drainDeadlineScheduled else { + return (nil, false) + } + drainDeadlineScheduled = true + return (nil, true) + } + finish(result.completion) + if result.scheduleDeadline { + DispatchQueue.global(qos: .utility).asyncAfter( + deadline: .now() + max(0, drainGracePeriod) + ) { [self] in + drainDeadlineElapsed() + } + } + } + + private func drainDeadlineElapsed() { + let completion = lock.withLock { + takeCompletionIfReady(force: true) } finish(completion) } - private func takeCompletionIfReady() -> Completion? { - guard !completed, sawEOF, let terminationStatus else { return nil } + private func takeCompletionIfReady(force: Bool = false) -> Completion? { + guard !completed, + (sawEOF || force), + let terminationStatus else { + return nil + } completed = true return Completion( stderr: String(data: tail, encoding: .utf8) ?? "", diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift index ac166d5e9e69..dd8e63e869cf 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift @@ -2,6 +2,8 @@ internal import Foundation /// Production launcher for a standalone SSH reverse-relay transport. public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { + private static let startupGracePeriod: TimeInterval = 0.5 + /// Creates a production reverse-relay launcher. public init() {} @@ -9,6 +11,9 @@ public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { public func launch( arguments: [String], environment: [String: String]?, + startupHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess + ) -> Void, terminationHandler: @escaping @Sendable ( any RemoteReverseRelayProcess, String? @@ -32,6 +37,12 @@ public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { guard let relayProcess else { return } terminationHandler(relayProcess, detail) } + DispatchQueue.global(qos: .utility).asyncAfter( + deadline: .now() + Self.startupGracePeriod + ) { [weak relayProcess] in + guard let relayProcess, relayProcess.isRunning else { return } + startupHandler(relayProcess) + } return relayProcess } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift index a186c0053670..4c188e842dcb 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift @@ -5,12 +5,17 @@ public protocol RemoteReverseRelayLaunching: Sendable { /// - Parameters: /// - arguments: SSH arguments for the reverse-relay transport. /// - environment: Process environment, or `nil` to inherit. + /// - startupHandler: Called after the transport survives its bounded + /// `ExitOnForwardFailure` startup window. /// - terminationHandler: Called when the launched transport exits. /// - Returns: A coordinator-owned handle for the running transport. /// - Throws: A Foundation process-launch error. func launch( arguments: [String], environment: [String: String]?, + startupHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess + ) -> Void, terminationHandler: @escaping @Sendable ( any RemoteReverseRelayProcess, String? diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index bc30fbfcc76d..fc14985469e5 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -116,36 +116,32 @@ extension RemoteSessionCoordinator { ) let process = try reverseRelayLauncher.launch( arguments: relayArguments, - environment: configuration.sshProcessEnvironment - ) { [weak self] terminated, stderrDetail in - guard let coordinator = self else { return } - coordinator.queue.async { - coordinator.handleReverseRelayTerminationLocked( - process: terminated, - stderrDetail: stderrDetail - ) + environment: configuration.sshProcessEnvironment, + startupHandler: { [weak self] readyProcess in + guard let coordinator = self else { return } + coordinator.queue.async { + coordinator.handleStandaloneReverseRelayReadyLocked( + process: readyProcess, + remotePath: remotePath, + relayPort: relayPort, + localRelayPort: localRelayPort, + relayID: relayID, + relayToken: relayToken + ) + } + }, + terminationHandler: { [weak self] terminated, stderrDetail in + guard let coordinator = self else { return } + coordinator.queue.async { + coordinator.handleReverseRelayTerminationLocked( + process: terminated, + stderrDetail: stderrDetail + ) + } } - } + ) reverseRelayProcess = process cliRelayServer = relayServer - do { - try installRemoteRelayMetadataLocked( - remotePath: remotePath, - relayPort: relayPort, - relayID: relayID, - relayToken: relayToken - ) - } catch { - debugLog("remote.relay.metadata.error \(error.localizedDescription)") - stopReverseRelayLocked() - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) - return - } - recordHeartbeatActivityLocked() - debugLog( - "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + - "target=\(configuration.displayTarget) controlMaster=0" - ) } catch { debugLog( "remote.relay.startFailed relayPort=\(relayPort) " + @@ -161,6 +157,40 @@ extension RemoteSessionCoordinator { } } + func handleStandaloneReverseRelayReadyLocked( + process: any RemoteReverseRelayProcess, + remotePath: String, + relayPort: Int, + localRelayPort: Int, + relayID: String, + relayToken: String + ) { + guard reverseRelayProcess === process, + process.isRunning, + !isStopping, + daemonReady else { + return + } + do { + try installRemoteRelayMetadataLocked( + remotePath: remotePath, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken + ) + } catch { + debugLog("remote.relay.metadata.error \(error.localizedDescription)") + stopReverseRelayLocked() + scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) + return + } + recordHeartbeatActivityLocked() + debugLog( + "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + + "target=\(configuration.displayTarget) controlMaster=0" + ) + } + func handleReverseRelayTerminationLocked( process: any RemoteReverseRelayProcess, stderrDetail: String? diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index ece7e0e6c214..f8d0d7c42aa4 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -95,10 +95,9 @@ extension RemoteSessionCoordinator { /// back to its standalone transport without touching the unresolved /// master. private func resolvedReverseRelayControlMasterSSHOptionsLocked() -> [String]? { - if reverseRelayControlMasterResolutionAttempted { + if let reverseRelayResolvedControlMasterSSHOptions { return reverseRelayResolvedControlMasterSSHOptions } - reverseRelayControlMasterResolutionAttempted = true let effectiveOptions = reverseRelayControlMasterSSHOptions let sharingOptions = SSHConnectionSharingOptions() diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index 134e856577ac..5b94196818a7 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -84,7 +84,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? var reverseRelayResolvedControlMasterSSHOptions: [String]? - var reverseRelayControlMasterResolutionAttempted = false var conflictedControlMasterResetObservation: NativeSSHControlMasterResetObservation? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index f490e9bf0d3d..d5d8c16b3e10 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -255,6 +255,7 @@ final class RecordingReverseRelayLauncher: private let lock = NSLock() private var _launchCount = 0 + private var startupHandler: (@Sendable (any RemoteReverseRelayProcess) -> Void)? private var terminationHandler: (@Sendable (any RemoteReverseRelayProcess, String?) -> Void)? private let launchContinuation: AsyncStream.Continuation @@ -265,6 +266,9 @@ final class RecordingReverseRelayLauncher: func launch( arguments: [String], environment: [String: String]?, + startupHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess + ) -> Void, terminationHandler: @escaping @Sendable ( any RemoteReverseRelayProcess, String? @@ -285,6 +289,7 @@ final class RecordingReverseRelayLauncher: )) lock.withLock { _launchCount += 1 + self.startupHandler = startupHandler self.terminationHandler = terminationHandler } return process @@ -294,6 +299,11 @@ final class RecordingReverseRelayLauncher: lock.withLock { _launchCount } } + func emitStartupReady() { + let handler = lock.withLock { startupHandler } + handler?(process) + } + func emitTermination(detail: String?) { let handler = lock.withLock { terminationHandler } handler?(process, detail) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index a435b575c437..e568b9877d15 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -124,6 +124,55 @@ struct RemoteSessionReverseRelayTransportTests { #expect(!runner.requests.contains(where: { Self.isControlCommand("forward", in: $0.arguments) })) + #expect(!runner.requests.contains(where: Self.isMetadataInstallRequest)) + + launcher.emitStartupReady() + coordinator.queue.sync {} + + #expect(runner.requests.contains(where: Self.isMetadataInstallRequest)) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("A transient ControlPath resolution failure is retried") + func transientControlPathResolutionFailureRetries() async throws { + let baseRunner = RecordingProcessRunner() + let runner = FlakyResolvedControlPathProcessRunner( + base: baseRunner, + failureCount: 1 + ) + let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( + runner: runner, + providesResolvedControlPath: false + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + let forwardSpec = "127.0.0.1:64044:127.0.0.1:55001" + + let first = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: forwardSpec, + relayPort: 64_044 + ) + } + guard case .unavailable = first else { + Issue.record("Expected the transient resolution failure to fall back") + return + } + let second = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: forwardSpec, + relayPort: 64_044 + ) + } + + guard case .started = second else { + Issue.record("Expected the next attempt to retry ControlPath resolution") + return + } + #expect(runner.resolutionAttempts == 2) + #expect(baseRunner.requests.contains(where: { + Self.isControlCommand("forward", in: $0.arguments) + })) _ = await coordinator.stopAndWait(cleanupScope: .transport) } @@ -308,12 +357,16 @@ struct RemoteSessionReverseRelayTransportTests { #expect(coordinator.queue.sync { coordinator.reverseRelayProcess === launcher.process }) + #expect(!runner.requests.contains(where: Self.isMetadataInstallRequest)) launcher.emitTermination( detail: "Error: remote port forwarding failed for listen port \(relayPort)" ) + launcher.emitStartupReady() #expect(await clock.nextRequestedDelay() == 2_000) + coordinator.queue.sync {} + #expect(!runner.requests.contains(where: Self.isMetadataInstallRequest)) #expect(runner.requests.contains(where: { Self.isControlCommand("exit", in: $0.arguments) })) @@ -362,6 +415,42 @@ struct RemoteSessionReverseRelayTransportTests { #expect(process.terminationStatus == 255) } + @Test("Standalone termination bounds draining inherited stderr writers") + func standaloneTerminationBoundsInheritedStderr() async throws { + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + """ + sleep 3 & + printf 'proxy diagnostic\n' >&2 + exit 23 + """, + ] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe, + stderrDrainGracePeriod: 0.05 + ) + let (details, continuation) = AsyncStream.makeStream() + let startedAt = Date() + + try process.run() + relayProcess.captureTermination { detail in + continuation.yield(detail) + continuation.finish() + } + + var iterator = details.makeAsyncIterator() + #expect(await iterator.next() == "proxy diagnostic") + #expect(Date().timeIntervalSince(startedAt) < 1) + #expect(process.terminationStatus == 23) + } + private static func isControlCommand( _ command: String, in arguments: [String] @@ -380,4 +469,10 @@ struct RemoteSessionReverseRelayTransportTests { ? arguments[valueIndex] : nil } + + private static func isMetadataInstallRequest( + _ request: RemoteProcessRequest + ) -> Bool { + request.arguments.last?.contains("CMUXRELAYAUTH") == true + } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift index e930d54fb98c..80ec537066d4 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift @@ -1,4 +1,5 @@ import CmuxFoundation +import Foundation @testable import CmuxRemoteSession enum ResolvedControlPathFixture { @@ -34,3 +35,52 @@ final class ResolvedControlPathProcessRunner: return try base.run(request, operation: operation) } } + +final class FlakyResolvedControlPathProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - resolution calls consume one scripted test counter. + private let lock = NSLock() + private let base: any RemoteSessionProcessRunning + private let failureCount: Int + private var attempts = 0 + + init( + base: any RemoteSessionProcessRunning, + failureCount: Int + ) { + self.base = base + self.failureCount = failureCount + } + + var resolutionAttempts: Int { + lock.withLock { attempts } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + if request.executable == "/usr/bin/ssh", + request.arguments.first == "-G" { + let attempt = lock.withLock { + attempts += 1 + return attempts + } + guard attempt > failureCount else { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "temporary configuration failure" + ) + } + return RemoteCommandResult( + status: 0, + stdout: "controlpath \(ResolvedControlPathFixture.path)\n", + stderr: "" + ) + } + return try base.run(request, operation: operation) + } +} From 0bdcb2d1b615bd127b6092100e1694fdc111443c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 02:13:35 -0700 Subject: [PATCH 21/39] fix: wait for SSH forward confirmation --- .../FoundationRemoteReverseRelayProcess.swift | 125 ++++++++++++- .../Process/RemoteReverseRelayLauncher.swift | 25 +-- .../Process/RemoteReverseRelayLaunching.swift | 6 +- ...emoteSessionCoordinator+ReverseRelay.swift | 14 +- ...dationRemoteReverseRelayProcessTests.swift | 171 ++++++++++++++++++ ...emoteSessionReverseRelayStartupTests.swift | 5 +- ...oteSessionReverseRelayTransportTests.swift | 83 +-------- 7 files changed, 330 insertions(+), 99 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift index 282e3d3164a1..dd5865b96009 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -36,12 +36,49 @@ final class FoundationRemoteReverseRelayProcess: /// writer that outlives ssh is cut off after a bounded final grace period. func captureTermination( _ handler: @escaping @Sendable (String?) -> Void + ) { + installLifecycleCapture( + startupMarker: nil, + startupHandler: nil, + terminationHandler: handler + ) + } + + /// Reports exact forward confirmation and eventual process termination + /// from the same event-driven stderr stream. + func captureLifecycle( + startupMarker: String, + startupTimeout: TimeInterval, + startupHandler: @escaping @Sendable () -> Void, + terminationHandler: @escaping @Sendable (String?) -> Void + ) { + installLifecycleCapture( + startupMarker: startupMarker, + startupTimeout: startupTimeout, + startupTimeoutHandler: { [weak self] in + self?.terminate() + }, + startupHandler: startupHandler, + terminationHandler: terminationHandler + ) + } + + private func installLifecycleCapture( + startupMarker: String?, + startupTimeout: TimeInterval? = nil, + startupTimeoutHandler: (@Sendable () -> Void)? = nil, + startupHandler: (@Sendable () -> Void)?, + terminationHandler: @escaping @Sendable (String?) -> Void ) { let readHandle = stderrPipe.fileHandleForReading let capture = ReverseRelayStderrCapture( readHandle: readHandle, drainGracePeriod: stderrDrainGracePeriod, - handler: handler + startupMarker: startupMarker, + startupTimeout: startupTimeout, + startupTimeoutHandler: startupTimeoutHandler, + startupHandler: startupHandler, + terminationHandler: terminationHandler ) readHandle.readabilityHandler = { handle in capture.receive(handle.availableData) @@ -52,6 +89,7 @@ final class FoundationRemoteReverseRelayProcess: if !process.isRunning { capture.processDidTerminate(status: process.terminationStatus) } + capture.startStartupDeadline() } func terminate() { @@ -70,10 +108,16 @@ private final class ReverseRelayStderrCapture: @unchecked Sendable { // critical sections only append bounded data and update lifecycle bits. private let lock = NSLock() private let readHandle: FileHandle - private let handler: @Sendable (String?) -> Void + private let startupMarker: Data? + private let startupTimeout: TimeInterval? + private let startupTimeoutHandler: (@Sendable () -> Void)? + private let startupHandler: (@Sendable () -> Void)? + private let terminationHandler: @Sendable (String?) -> Void private let byteLimit: Int private let drainGracePeriod: TimeInterval private var tail = Data() + private var startupReported = false + private var startupExpired = false private var sawEOF = false private var terminationStatus: Int32? private var drainDeadlineScheduled = false @@ -83,27 +127,59 @@ private final class ReverseRelayStderrCapture: @unchecked Sendable { readHandle: FileHandle, byteLimit: Int = 8192, drainGracePeriod: TimeInterval, - handler: @escaping @Sendable (String?) -> Void + startupMarker: String?, + startupTimeout: TimeInterval?, + startupTimeoutHandler: (@Sendable () -> Void)?, + startupHandler: (@Sendable () -> Void)?, + terminationHandler: @escaping @Sendable (String?) -> Void ) { self.readHandle = readHandle self.byteLimit = byteLimit self.drainGracePeriod = drainGracePeriod - self.handler = handler + self.startupMarker = startupMarker?.data(using: .utf8) + self.startupTimeout = startupTimeout + self.startupTimeoutHandler = startupTimeoutHandler + self.startupHandler = startupHandler + self.terminationHandler = terminationHandler + } + + func startStartupDeadline() { + guard let startupTimeout else { return } + DispatchQueue.global(qos: .utility).asyncAfter( + deadline: .now() + max(0, startupTimeout) + ) { [weak self] in + self?.startupDeadlineElapsed() + } } func receive(_ data: Data) { - let completion = lock.withLock { () -> Completion? in + let result = lock.withLock { () -> ( + completion: Completion?, + reportStartup: Bool + ) in if data.isEmpty { sawEOF = true } else { tail.append(data) + let reportStartup = + !completed && + !startupReported && + !startupExpired && + startupMarker.map { tail.range(of: $0) != nil } == true + if reportStartup { + startupReported = true + } if tail.count > byteLimit { tail.removeFirst(tail.count - byteLimit) } + return (takeCompletionIfReady(), reportStartup) } - return takeCompletionIfReady() + return (takeCompletionIfReady(), false) } - finish(completion) + if result.reportStartup { + startupHandler?() + } + finish(result.completion) } func processDidTerminate(status: Int32) { @@ -139,6 +215,22 @@ private final class ReverseRelayStderrCapture: @unchecked Sendable { finish(completion) } + private func startupDeadlineElapsed() { + let shouldTerminate = lock.withLock { + guard !startupReported, + !startupExpired, + !completed, + terminationStatus == nil else { + return false + } + startupExpired = true + return true + } + if shouldTerminate { + startupTimeoutHandler?() + } + } + private func takeCompletionIfReady(force: Bool = false) -> Completion? { guard !completed, (sawEOF || force), @@ -156,9 +248,24 @@ private final class ReverseRelayStderrCapture: @unchecked Sendable { guard let completion else { return } readHandle.readabilityHandler = nil try? readHandle.close() - handler( - RemoteSessionCoordinator.bestErrorLine(stderr: completion.stderr) + terminationHandler( + Self.preferredTerminationDetail(stderr: completion.stderr) ?? "status=\(completion.status)" ) } + + private static func preferredTerminationDetail(stderr: String) -> String? { + let lines = stderr + .split(separator: "\n") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + if let forwardFailure = lines.last(where: { + $0.localizedCaseInsensitiveContains( + "remote port forwarding failed for listen" + ) + }) { + return forwardFailure + } + return RemoteSessionCoordinator.bestErrorLine(stderr: stderr) + } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift index dd8e63e869cf..0f5c757ff682 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLauncher.swift @@ -2,7 +2,7 @@ internal import Foundation /// Production launcher for a standalone SSH reverse-relay transport. public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { - private static let startupGracePeriod: TimeInterval = 0.5 + private static let startupTimeout: TimeInterval = 10 /// Creates a production reverse-relay launcher. public init() {} @@ -11,6 +11,7 @@ public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { public func launch( arguments: [String], environment: [String: String]?, + startupMarker: String, startupHandler: @escaping @Sendable ( any RemoteReverseRelayProcess ) -> Void, @@ -33,16 +34,18 @@ public struct RemoteReverseRelayLauncher: RemoteReverseRelayLaunching { stderrPipe: stderrPipe ) try process.run() - relayProcess.captureTermination { [weak relayProcess] detail in - guard let relayProcess else { return } - terminationHandler(relayProcess, detail) - } - DispatchQueue.global(qos: .utility).asyncAfter( - deadline: .now() + Self.startupGracePeriod - ) { [weak relayProcess] in - guard let relayProcess, relayProcess.isRunning else { return } - startupHandler(relayProcess) - } + relayProcess.captureLifecycle( + startupMarker: startupMarker, + startupTimeout: Self.startupTimeout, + startupHandler: { [weak relayProcess] in + guard let relayProcess else { return } + startupHandler(relayProcess) + }, + terminationHandler: { [weak relayProcess] detail in + guard let relayProcess else { return } + terminationHandler(relayProcess, detail) + } + ) return relayProcess } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift index 4c188e842dcb..67216247528e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/RemoteReverseRelayLaunching.swift @@ -5,14 +5,16 @@ public protocol RemoteReverseRelayLaunching: Sendable { /// - Parameters: /// - arguments: SSH arguments for the reverse-relay transport. /// - environment: Process environment, or `nil` to inherit. - /// - startupHandler: Called after the transport survives its bounded - /// `ExitOnForwardFailure` startup window. + /// - startupMarker: Exact OpenSSH DEBUG1 diagnostic emitted after the + /// requested remote forward is confirmed. + /// - startupHandler: Called after stderr contains `startupMarker`. /// - terminationHandler: Called when the launched transport exits. /// - Returns: A coordinator-owned handle for the running transport. /// - Throws: A Foundation process-launch error. func launch( arguments: [String], environment: [String: String]?, + startupMarker: String, startupHandler: @escaping @Sendable ( any RemoteReverseRelayProcess ) -> Void, diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index fc14985469e5..4d383ba30a22 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -117,6 +117,10 @@ extension RemoteSessionCoordinator { let process = try reverseRelayLauncher.launch( arguments: relayArguments, environment: configuration.sshProcessEnvironment, + startupMarker: Self.reverseRelayForwardSuccessMarker( + relayPort: relayPort, + localRelayPort: localRelayPort + ), startupHandler: { [weak self] readyProcess in guard let coordinator = self else { return } coordinator.queue.async { @@ -283,7 +287,7 @@ extension RemoteSessionCoordinator { func reverseRelayArguments(relayPort: Int, localRelayPort: Int) -> [String] { // Fallback only: `-S none` prevents accidental adoption of a shared // transport after `-O forward` proved unavailable. - var args: [String] = ["-N", "-T", "-S", "none"] + var args: [String] = ["-N", "-T", "-S", "none", "-v"] args += sshCommonArguments(batchMode: true, dropControlPath: true) args += [ "-o", "ExitOnForwardFailure=yes", @@ -294,6 +298,14 @@ extension RemoteSessionCoordinator { return args } + static func reverseRelayForwardSuccessMarker( + relayPort: Int, + localRelayPort: Int + ) -> String { + "remote forward success for: listen 127.0.0.1:\(relayPort), " + + "connect 127.0.0.1:\(localRelayPort)" + } + private func ensureCLIRelayServerLocked(localSocketPath: String, relayID: String, relayToken: String) throws -> RemoteCLIRelayServer { if let cliRelayServer { return cliRelayServer diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift new file mode 100644 index 000000000000..94152a763131 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift @@ -0,0 +1,171 @@ +import CmuxFoundation +import Foundation +import Testing +@testable import CmuxRemoteSession + +@Suite("Foundation reverse relay process") +struct FoundationRemoteReverseRelayProcessTests { + @Test("Termination waits for the complete stderr tail") + func terminationDrainsStderr() async throws { + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + """ + i=0 + while [ "$i" -lt 1000 ]; do + printf 'diagnostic-noise-%s\n' "$i" >&2 + i=$((i + 1)) + done + printf 'Error: remote port forwarding failed for listen port 64044\n' >&2 + printf 'Connection to example.com closed.\n' >&2 + printf 'debug1: cleanup\n' >&2 + exit 255 + """, + ] + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe + ) + let (details, continuation) = AsyncStream.makeStream() + + try process.run() + relayProcess.captureTermination { detail in + continuation.yield(detail) + continuation.finish() + } + + var iterator = details.makeAsyncIterator() + #expect( + await iterator.next() + == "Error: remote port forwarding failed for listen port 64044" + ) + #expect(process.terminationStatus == 255) + } + + @Test("Termination bounds draining inherited stderr writers") + func terminationBoundsInheritedStderr() async throws { + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + """ + sleep 3 & + printf 'proxy diagnostic\n' >&2 + exit 23 + """, + ] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe, + stderrDrainGracePeriod: 0.05 + ) + let (details, continuation) = AsyncStream.makeStream() + let startedAt = Date() + + try process.run() + relayProcess.captureTermination { detail in + continuation.yield(detail) + continuation.finish() + } + + var iterator = details.makeAsyncIterator() + #expect(await iterator.next() == "proxy diagnostic") + #expect(Date().timeIntervalSince(startedAt) < 1) + #expect(process.terminationStatus == 23) + } + + @Test("Exact OpenSSH forward confirmation reports startup") + func forwardConfirmationReportsStartup() async throws { + let marker = RemoteSessionCoordinator.reverseRelayForwardSuccessMarker( + relayPort: 64_044, + localRelayPort: 55_001 + ) + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + """ + printf 'debug1: %s\n' \(marker.shellSingleQuoted) >&2 + sleep 0.1 + exit 0 + """, + ] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe + ) + let (startups, startupContinuation) = AsyncStream.makeStream() + let (terminations, terminationContinuation) = + AsyncStream.makeStream() + + try process.run() + relayProcess.captureLifecycle( + startupMarker: marker, + startupTimeout: 1, + startupHandler: { + startupContinuation.yield() + startupContinuation.finish() + }, + terminationHandler: { detail in + terminationContinuation.yield(detail) + terminationContinuation.finish() + } + ) + + var startupIterator = startups.makeAsyncIterator() + #expect(await startupIterator.next() != nil) + var terminationIterator = terminations.makeAsyncIterator() + #expect(await terminationIterator.next() != nil) + #expect(process.terminationStatus == 0) + } + + @Test("Missing forward confirmation hits a bounded startup deadline") + func missingForwardConfirmationTerminates() async throws { + let process = Process() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = ["-c", "sleep 3"] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe, + stderrDrainGracePeriod: 0.05 + ) + let startupRecorder = ResetEventRecorder() + let (terminations, continuation) = AsyncStream.makeStream() + let startedAt = Date() + + try process.run() + relayProcess.captureLifecycle( + startupMarker: "marker-that-never-arrives", + startupTimeout: 0.05, + startupHandler: { + startupRecorder.record() + }, + terminationHandler: { detail in + continuation.yield(detail) + continuation.finish() + } + ) + + var iterator = terminations.makeAsyncIterator() + #expect(await iterator.next() != nil) + #expect(startupRecorder.count == 0) + #expect(Date().timeIntervalSince(startedAt) < 1) + #expect(!process.isRunning) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index d5d8c16b3e10..545098d2bdad 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -243,6 +243,7 @@ struct RemoteSessionReverseRelayStartupTests { struct RecordedReverseRelayLaunch: Sendable { let arguments: [String] let localRelayPort: Int + let startupMarker: String } /// Synchronous launcher callbacks cannot await; the lock protects only callback snapshots and a counter. @@ -266,6 +267,7 @@ final class RecordingReverseRelayLauncher: func launch( arguments: [String], environment: [String: String]?, + startupMarker: String, startupHandler: @escaping @Sendable ( any RemoteReverseRelayProcess ) -> Void, @@ -285,7 +287,8 @@ final class RecordingReverseRelayLauncher: ) launchContinuation.yield(RecordedReverseRelayLaunch( arguments: arguments, - localRelayPort: localRelayPort + localRelayPort: localRelayPort, + startupMarker: startupMarker )) lock.withLock { _launchCount += 1 diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index e568b9877d15..dbf8e5b71333 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -118,6 +118,14 @@ struct RemoteSessionReverseRelayTransportTests { let launch = try #require(await launches.next()) #expect(launch.arguments.starts(with: ["-N", "-T", "-S", "none"])) + #expect(launch.arguments.contains("-v")) + #expect( + launch.startupMarker == + RemoteSessionCoordinator.reverseRelayForwardSuccessMarker( + relayPort: 64_044, + localRelayPort: launch.localRelayPort + ) + ) #expect(runner.requests.contains(where: { $0.arguments.first == "-G" })) @@ -376,81 +384,6 @@ struct RemoteSessionReverseRelayTransportTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test("Standalone termination waits for the complete stderr tail") - func standaloneTerminationDrainsStderr() async throws { - let process = Process() - let stderrPipe = Pipe() - process.executableURL = URL(fileURLWithPath: "/bin/sh") - process.arguments = [ - "-c", - """ - i=0 - while [ "$i" -lt 1000 ]; do - printf 'diagnostic-noise-%s\n' "$i" >&2 - i=$((i + 1)) - done - printf 'Error: remote port forwarding failed for listen port 64044\n' >&2 - exit 255 - """, - ] - process.standardOutput = FileHandle.nullDevice - process.standardError = stderrPipe - let relayProcess = FoundationRemoteReverseRelayProcess( - process: process, - stderrPipe: stderrPipe - ) - let (details, continuation) = AsyncStream.makeStream() - - try process.run() - relayProcess.captureTermination { detail in - continuation.yield(detail) - continuation.finish() - } - - var iterator = details.makeAsyncIterator() - #expect( - await iterator.next() - == "Error: remote port forwarding failed for listen port 64044" - ) - #expect(process.terminationStatus == 255) - } - - @Test("Standalone termination bounds draining inherited stderr writers") - func standaloneTerminationBoundsInheritedStderr() async throws { - let process = Process() - let stderrPipe = Pipe() - process.executableURL = URL(fileURLWithPath: "/bin/sh") - process.arguments = [ - "-c", - """ - sleep 3 & - printf 'proxy diagnostic\n' >&2 - exit 23 - """, - ] - process.standardInput = FileHandle.nullDevice - process.standardOutput = FileHandle.nullDevice - process.standardError = stderrPipe - let relayProcess = FoundationRemoteReverseRelayProcess( - process: process, - stderrPipe: stderrPipe, - stderrDrainGracePeriod: 0.05 - ) - let (details, continuation) = AsyncStream.makeStream() - let startedAt = Date() - - try process.run() - relayProcess.captureTermination { detail in - continuation.yield(detail) - continuation.finish() - } - - var iterator = details.makeAsyncIterator() - #expect(await iterator.next() == "proxy diagnostic") - #expect(Date().timeIntervalSince(startedAt) < 1) - #expect(process.terminationStatus == 23) - } - private static func isControlCommand( _ command: String, in arguments: [String] From 710a641865d01f344a726fc568e11f9683133cd8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 02:13:40 -0700 Subject: [PATCH 22/39] fix: retain shared SSH reset ownership --- ...tiveSSHControlMasterResetCoordinator.swift | 30 +++++--- .../NativeSSHControlMasterResetTests.swift | 72 +++++++++++++++++++ 2 files changed, 92 insertions(+), 10 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index 4691a6cf6f28..01d322afab29 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -19,7 +19,7 @@ enum NativeSSHControlMasterResetOutcome: Sendable, Equatable { final class NativeSSHControlMasterResetCoordinator { private struct InFlightReset { let id: UUID - let authorizationKey: NativeSSHControlMasterResetKey + var authorizationKeys: Set let task: Task } @@ -71,10 +71,12 @@ final class NativeSSHControlMasterResetCoordinator { } else { leases[ownerWorkspaceID] = ownerLeases } - let remainsOwned = leases.values.contains { $0[key] != nil } - if !remainsOwned { - for reset in inFlightResets.values - where reset.authorizationKey == key { + for reset in inFlightResets.values { + let remainsOwned = reset.authorizationKeys.contains { + authorizationKey in + leases.values.contains { $0[authorizationKey] != nil } + } + if !remainsOwned { reset.task.cancel() } } @@ -137,7 +139,9 @@ final class NativeSSHControlMasterResetCoordinator { ) else { return .ignored("workspace no longer owns this cmux SSH master") } - if let inFlight = inFlightResets[resolvedControlPath] { + if var inFlight = inFlightResets[resolvedControlPath] { + inFlight.authorizationKeys.insert(key) + inFlightResets[resolvedControlPath] = inFlight return await inFlight.task.value } @@ -176,7 +180,7 @@ final class NativeSSHControlMasterResetCoordinator { } inFlightResets[resolvedControlPath] = InFlightReset( id: resetID, - authorizationKey: key, + authorizationKeys: [key], task: task ) let outcome = await task.value @@ -256,15 +260,21 @@ final class NativeSSHControlMasterResetCoordinator { request: request, processRunner: processRunner ) - guard !Task.isCancelled else { - return .deferred("control-master reset cancelled") - } switch attempt { case .reset: + // The exit command already succeeded. Always publish + // invalidation even if the initiating lease disappeared while + // the process was completing. return .reset case .ignored(let detail): + guard !Task.isCancelled else { + return .deferred("control-master reset cancelled") + } return .ignored(detail) case .retry(let detail): + guard !Task.isCancelled else { + return .deferred("control-master reset cancelled") + } guard attemptIndex + 1 < maximumAttempts else { return .deferred(detail) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift index 6ff7a118aca6..bf91a5330a7d 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -233,6 +233,78 @@ struct NativeSSHControlMasterResetTests { #expect(runner.exitCount == 1) } + @Test("A coalesced alias keeps its resolved-socket reset alive") + func coalescedAliasKeepsResetAlive() async throws { + let runner = BlockingResolvingResetRunner( + resolvedPath: firstResolvedPath + ) + let recorder = ResetEventRecorder() + let broker = makeBroker(processRunner: runner) + let first = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "first-alias", + options: unresolvedOptions + )) + let second = broker.retainWorkspace(configuration( + owner: UUID(), + destination: "second-alias", + options: unresolvedOptions + )) + let observation = try #require( + broker.observeControlMasterResets(controlPath: firstResolvedPath) { + recorder.record() + } + ) + + let firstReset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: first) + } + var resolutions = runner.resolutions.makeAsyncIterator() + #expect(await resolutions.next() != nil) + var exits = runner.exits.makeAsyncIterator() + #expect(await exits.next() != nil) + let secondReset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: second) + } + #expect(await resolutions.next() != nil) + await Task.yield() + broker.releaseWorkspace(first) + runner.finishExit() + + #expect(await firstReset.value == .reset) + #expect(await secondReset.value == .reset) + #expect(recorder.count == 1) + _ = observation + } + + @Test("A successful exit still invalidates after its lease is released") + func successfulExitAfterReleaseStillInvalidates() async throws { + let runner = BlockingControlMasterResetRunner() + let recorder = ResetEventRecorder() + let broker = makeBroker(processRunner: runner) + let lease = broker.retainWorkspace(configuration( + owner: UUID(), + options: resolvedOptions + )) + let observation = try #require( + broker.observeControlMasterResets(controlPath: firstResolvedPath) { + recorder.record() + } + ) + + let reset = Task { @MainActor in + await broker.resetConflictedControlMaster(for: lease) + } + var starts = runner.starts.makeAsyncIterator() + #expect(await starts.next() != nil) + broker.releaseWorkspace(lease) + runner.finish() + + #expect(await reset.value == .reset) + #expect(recorder.count == 1) + _ = observation + } + @Test("ControlPath resolution failure never exits a master") func resolutionFailureFailsClosed() async { let runner = RecordingProcessRunner { request in From 46d16a7390243573918cb751a610fa822463c4c6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 03:11:35 -0700 Subject: [PATCH 23/39] fix: gate SSH master resets across processes --- .../ControlCommandCoordinator+Workspace.swift | 10 +- .../Workspace/ControlWorkspaceContext.swift | 4 +- .../ControlWorkspaceRemoteResolution.swift | 3 + .../ControlCommandContextTestStubs.swift | 3 +- ...trolCommandCoordinatorWorkspaceTests.swift | 32 +++ .../FakeWorkspaceControlCommandContext.swift | 20 ++ .../SSHConnectionSharingOptions.swift | 59 ++++ .../SSHConnectionSharingOptionsTests.swift | 22 +- .../NativeSSHConnectionBroker+Cleanup.swift | 205 ++++++++++++++ ...nectionBroker+ControlMasterOwnership.swift | 79 ++++++ .../NativeSSHConnectionBroker.swift | 249 ++++------------ ...ativeSSHControlMasterAdoptionHandoff.swift | 45 +++ .../NativeSSHControlMasterLeaseIdentity.swift | 23 ++ ...iveSSHControlMasterOwnershipRegistry.swift | 266 ++++++++++++++++++ ...iveSSHControlMasterOwnershipTracking.swift | 11 + ...veSSHControlMasterResetAuthorization.swift | 25 ++ ...tiveSSHControlMasterResetCoordinator.swift | 33 ++- .../Hosting/RemoteSessionStrings.swift | 23 +- ...emoteSessionCoordinator+ReverseRelay.swift | 5 +- ...oordinator+ReverseRelayControlMaster.swift | 20 ++ ...ssionCoordinator+ReverseRelayStartup.swift | 5 +- .../NativeSSHConnectionBrokerTests.swift | 22 +- ...HControlMasterOwnershipRecoveryTests.swift | 142 ++++++++++ ...HControlMasterOwnershipRegistryTests.swift | 172 +++++++++++ .../NativeSSHControlMasterResetTests.swift | 21 +- ...iveSSHControlMasterOwnershipRegistry.swift | 23 ++ .../RemoteDaemonUploadTests.swift | 4 +- .../RemotePTYIntentionalCleanupTests.swift | 4 +- .../RemotePortScanGatingTests.swift | 4 +- .../RemoteReconnectPolicyTests.swift | 4 +- .../RemoteRelaySlotTeardownTests.swift | 4 +- ...emoteSessionReverseRelayStartupTests.swift | 17 +- ...oteSessionReverseRelayTransportTests.swift | 5 +- ...SessionSSHRemoteCommandOverrideTests.swift | 4 +- Resources/Localizable.xcstrings | 17 ++ Sources/RemoteSessionStrings+App.swift | 10 + .../SSHPTYAttachStartupCommandBuilder.swift | 122 +++++++- ...alController+ControlWorkspaceContext.swift | 18 +- .../Workspace+RemoteSessionLifecycle.swift | 62 +++- Sources/Workspace.swift | 55 +++- ...oundAuthenticationMarkerCleanupTests.swift | 29 +- .../WorkspaceRemoteConnectionTests.swift | 13 +- 42 files changed, 1613 insertions(+), 281 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterLeaseIdentity.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift diff --git a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift index e5333be47bd2..4c0a150011a2 100644 --- a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift +++ b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift @@ -710,6 +710,11 @@ extension ControlCommandCoordinator { "workspace_id": .string(workspaceID.uuidString), "workspace_ref": ref(.workspace, workspaceID), ])) + case .unavailable(let workspaceID, let message): + return .err(code: "unavailable", message: message, data: .object([ + "workspace_id": .string(workspaceID.uuidString), + "workspace_ref": ref(.workspace, workspaceID), + ])) case .resolved(let windowID, let workspaceID, let remoteStatus): return .ok(.object([ "window_id": orNull(windowID?.uuidString), @@ -798,9 +803,12 @@ extension ControlCommandCoordinator { // empty-to-nil variant. let token = rawString(params, "foreground_auth_token")? .trimmingCharacters(in: .whitespacesAndNewlines) + let controlPath = rawString(params, "control_path")? + .trimmingCharacters(in: .whitespacesAndNewlines) return workspaceRemoteResult(context?.controlWorkspaceRemoteForegroundAuthReady( workspaceID: workspaceID, - foregroundAuthToken: token + foregroundAuthToken: token, + resolvedControlPath: controlPath )) } diff --git a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceContext.swift b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceContext.swift index cf79e017134d..783d78cba377 100644 --- a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceContext.swift +++ b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceContext.swift @@ -230,10 +230,12 @@ public protocol ControlWorkspaceContext: AnyObject { /// - Parameters: /// - workspaceID: The resolved workspace id. /// - foregroundAuthToken: The trimmed token, if any. + /// - resolvedControlPath: Exact cmux-owned socket authenticated by SSH. /// - Returns: The remote resolution. func controlWorkspaceRemoteForegroundAuthReady( workspaceID: UUID, - foregroundAuthToken: String? + foregroundAuthToken: String?, + resolvedControlPath: String? ) -> ControlWorkspaceRemoteResolution /// Reads remote status for `workspace.remote.status`. diff --git a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceRemoteResolution.swift b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceRemoteResolution.swift index 1974e88fb79d..db12793f68e5 100644 --- a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceRemoteResolution.swift +++ b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlWorkspaceRemoteResolution.swift @@ -22,6 +22,9 @@ public enum ControlWorkspaceRemoteResolution: Sendable, Equatable { /// "Remote workspace is not configured", `reconnect` only). Carries the /// resolved workspace id for that payload. case notConfigured(workspaceID: UUID) + /// The requested ownership handoff could not be acquired without + /// disrupting another live cmux process. + case unavailable(workspaceID: UUID, message: String) /// The mutation succeeded. Carries the owning window id (may be absent), the /// resolved workspace id, and the bridged `remoteStatusPayload()`. case resolved(windowID: UUID?, workspaceID: UUID, remoteStatus: JSONValue) diff --git a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandContextTestStubs.swift b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandContextTestStubs.swift index 14c5b101463d..09641d7a3642 100644 --- a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandContextTestStubs.swift +++ b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandContextTestStubs.swift @@ -346,7 +346,8 @@ extension ControlWorkspaceContext { func controlWorkspaceRemoteForegroundAuthReady( workspaceID: UUID, - foregroundAuthToken: String? + foregroundAuthToken: String?, + resolvedControlPath: String? ) -> ControlWorkspaceRemoteResolution { .notFound(workspaceID: workspaceID) } func controlWorkspaceRemoteStatus(workspaceID: UUID) -> ControlWorkspaceRemoteResolution { diff --git a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift index 95efd066a434..656e4e67ede3 100644 --- a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift +++ b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift @@ -234,4 +234,36 @@ struct ControlCommandCoordinatorWorkspaceTests { #expect(code == "invalid_params") #expect(context.terminalSessionEndCall == nil) } + + @Test func foregroundAuthenticationForwardsResolvedControlPath() { + let (coordinator, context) = coordinator() + let workspaceID = UUID() + context.foregroundAuthResolution = .unavailable( + workspaceID: workspaceID, + message: "localized ownership unavailable" + ) + + guard case .err(let code, let message, _) = coordinator.handle(request( + "workspace.remote.foreground_auth_ready", + [ + "workspace_id": .string(workspaceID.uuidString), + "foreground_auth_token": .string(" auth-token "), + "control_path": .string( + " /tmp/cmux-ssh-501-0123456789abcdef " + ), + ] + )) else { + Issue.record("unexpected foreground-auth result") + return + } + + #expect(code == "unavailable") + #expect(message == "localized ownership unavailable") + #expect(context.foregroundAuthCall?.workspaceID == workspaceID) + #expect(context.foregroundAuthCall?.token == "auth-token") + #expect( + context.foregroundAuthCall?.controlPath == + "/tmp/cmux-ssh-501-0123456789abcdef" + ) + } } diff --git a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/FakeWorkspaceControlCommandContext.swift b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/FakeWorkspaceControlCommandContext.swift index 2868cfc18bfa..0d77963cb22c 100644 --- a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/FakeWorkspaceControlCommandContext.swift +++ b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/FakeWorkspaceControlCommandContext.swift @@ -19,6 +19,13 @@ final class FakeWorkspaceControlCommandContext: ControlCommandContext { workspaceID: UUID, surfaceID: UUID, relayPort: Int?, sessionID: String?, lifecycleID: String?, lifecycleOnly: Bool )? + var foregroundAuthResolution: + ControlWorkspaceRemoteResolution = .missingWorkspaceID + var foregroundAuthCall: ( + workspaceID: UUID, + token: String?, + controlPath: String? + )? func controlWindowSummaries() -> [ControlWindowSummary] { [] } func controlResolveCurrentWindow(routing: ControlRoutingSelectors) -> ControlCurrentWindowResolution { @@ -86,4 +93,17 @@ final class FakeWorkspaceControlCommandContext: ControlCommandContext { terminalSessionEndCall = (workspaceID, surfaceID, relayPort, sessionID, lifecycleID, lifecycleOnly) return terminalSessionEndResolution } + + func controlWorkspaceRemoteForegroundAuthReady( + workspaceID: UUID, + foregroundAuthToken: String?, + resolvedControlPath: String? + ) -> ControlWorkspaceRemoteResolution { + foregroundAuthCall = ( + workspaceID, + foregroundAuthToken, + resolvedControlPath + ) + return foregroundAuthResolution + } } diff --git a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift index c5753ff3306f..a76d564b2e89 100644 --- a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift +++ b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift @@ -45,6 +45,11 @@ public struct SSHConnectionSharingOptions: Sendable { "/tmp/cmux-ssh-\(userID)-%C" } + /// User-private directory used for cross-process ControlMaster locks. + public var controlMasterLockDirectoryPath: String { + authenticationLockDirectory.path + } + /// Adds missing sharing defaults while preserving every supplied value. /// /// A caller that disables `ControlMaster` keeps a standalone connection; @@ -211,6 +216,40 @@ public struct SSHConnectionSharingOptions: Sendable { .path } + /// Returns the shared authentication lock for one exact cmux-owned socket. + /// + /// Unlike ``foregroundAuthenticationLockPath(destination:port:options:)``, + /// this identity is stable across different SSH aliases that OpenSSH + /// expands to the same `ControlPath`. + public func resolvedControlMasterAuthenticationLockPath( + controlPath: String + ) -> String? { + guard let basename = resolvedControlPathBasename(controlPath) else { + return nil + } + return authenticationLockDirectory + .appendingPathComponent( + "cmux-ssh-\(userID)-resolved-auth-\(basename).lock", + isDirectory: false + ) + .path + } + + /// Returns the process-ownership gate for one exact cmux-owned socket. + public func resolvedControlMasterOwnershipLockPath( + controlPath: String + ) -> String? { + guard let basename = resolvedControlPathBasename(controlPath) else { + return nil + } + return authenticationLockDirectory + .appendingPathComponent( + "cmux-ssh-\(userID)-owner-\(basename).lock", + isDirectory: false + ) + .path + } + /// Returns the shell commands that finish successful foreground authentication. /// /// The marker must be cleared before the advisory lock is released so a @@ -221,6 +260,7 @@ public struct SSHConnectionSharingOptions: Sendable { public func successfulForegroundAuthenticationCleanupShellLines() -> [String] { [ "cmux_ssh_clear_auth_inflight", + "if [ -n \"${cmux_ssh_resolved_auth_lock_fd:-}\" ]; then zsystem flock -u \"$cmux_ssh_resolved_auth_lock_fd\" || exit 255; fi", "zsystem flock -u \"$cmux_ssh_auth_lock_fd\" || exit 255", "trap - EXIT HUP INT TERM", ] @@ -288,6 +328,25 @@ public struct SSHConnectionSharingOptions: Sendable { return hash.count == 40 && hash.allSatisfy(\.isHexDigit) } + private func resolvedControlPathBasename(_ controlPath: String) -> String? { + let path = controlPath.trimmingCharacters(in: .whitespacesAndNewlines) + guard !path.contains("%"), + cmuxOwnedControlPath(in: [ + "ControlMaster=auto", + "ControlPath=\(path)", + ]) == path else { + return nil + } + let basename = URL(fileURLWithPath: path).lastPathComponent + guard !basename.isEmpty, + basename.allSatisfy({ + $0.isLetter || $0.isNumber || $0 == "." || $0 == "_" || $0 == "-" + }) else { + return nil + } + return basename + } + private func isDisabled(_ rawValue: String?) -> Bool { guard let value = rawValue?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() else { return false diff --git a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift index c8f824617cf0..945048ad432c 100644 --- a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift +++ b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift @@ -210,9 +210,27 @@ struct SSHConnectionSharingOptionsTests { #expect(resolvedLock.map { URL(fileURLWithPath: $0).deletingLastPathComponent() } == lockDirectory) #expect(resolvedLock.map { URL(fileURLWithPath: $0).lastPathComponent.hasPrefix("cmux-ssh-501-auth-") } == true) #expect(resolvedLock != first) - #expect(options.cmuxOwnedControlPath(in: resolvedOwned) == String( + let resolvedControlPath = String( resolvedOwned[1].dropFirst("ControlPath=".count) - )) + ) + #expect(options.cmuxOwnedControlPath(in: resolvedOwned) == resolvedControlPath) + let aliasIndependentLock = + options.resolvedControlMasterAuthenticationLockPath( + controlPath: resolvedControlPath + ) + #expect(aliasIndependentLock.map { + URL(fileURLWithPath: $0).deletingLastPathComponent() + } == lockDirectory) + #expect(aliasIndependentLock?.contains("resolved-auth") == true) + #expect(options.resolvedControlMasterOwnershipLockPath( + controlPath: resolvedControlPath + )?.contains("-owner-") == true) + #expect(options.resolvedControlMasterAuthenticationLockPath( + controlPath: options.defaultControlPath + ) == nil) + #expect(options.resolvedControlMasterOwnershipLockPath( + controlPath: "~/.ssh/custom-control" + ) == nil) #expect(options.foregroundAuthenticationLockPath( destination: "alice@example.test", port: 2222, diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift new file mode 100644 index 000000000000..00712848e669 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift @@ -0,0 +1,205 @@ +internal import CmuxCore +internal import Darwin +internal import Foundation + +@MainActor +extension NativeSSHConnectionBroker { + private static let cleanupProcessTimeoutMilliseconds = 5_000 + private static let cleanupForcedTerminationDelayMilliseconds = 1_000 + private static let cleanupRetryDelayMilliseconds = 31_000 + + func removeLease( + ownerWorkspaceID: UUID, + key: NativeSSHControlMasterKey + ) { + guard var leases = ownerLeases[ownerWorkspaceID], + let previousConfiguration = leases.removeValue(forKey: key) else { + return + } + if leases.isEmpty { + ownerLeases.removeValue(forKey: ownerWorkspaceID) + } else { + ownerLeases[ownerWorkspaceID] = leases + } + var owners = ownersByControlMaster[key] ?? [] + owners.remove(ownerWorkspaceID) + guard owners.isEmpty else { + ownersByControlMaster[key] = owners + return + } + ownersByControlMaster.removeValue(forKey: key) + let arguments = RemoteControlMasterCleanup().cleanupArguments( + configuration: previousConfiguration + ) + let authenticationLockPath = + sharingOptions.resolvedControlMasterAuthenticationLockPath( + controlPath: key.controlPath + ) + let request = NativeSSHControlMasterCleanupRequest( + arguments: arguments, + environment: previousConfiguration.sshProcessEnvironment, + authenticationLockPath: authenticationLockPath + ) + beginCleanup(request, for: key) + } + + func beginCleanup( + _ request: NativeSSHControlMasterCleanupRequest, + for key: NativeSSHControlMasterKey + ) { + if let cleanupLauncherOverride { + cleanupLauncherOverride(request) + cleanupRequestsByControlMaster.removeValue(forKey: key) + } else { + cleanupRequestsByControlMaster[key] = request + launchCleanup(request, for: key) + } + } + + func cancelCleanup(for key: NativeSSHControlMasterKey) { + cleanupRequestsByControlMaster.removeValue(forKey: key) + cleanupRetryTasks.removeValue(forKey: key)?.cancel() + guard let cleanupID = cleanupProcessIDByControlMaster[key], + let process = cleanupProcesses[cleanupID], + process.isRunning else { + return + } + cleanupTerminationRequested.insert(cleanupID) + process.terminate() + } + + private func launchCleanup( + _ request: NativeSSHControlMasterCleanupRequest, + for key: NativeSSHControlMasterKey + ) { + guard cleanupRequestsByControlMaster[key] != nil, + ownersByControlMaster[key]?.isEmpty != false, + cleanupProcessIDByControlMaster[key] == nil else { + return + } + guard let authorization = controlMasterOwnershipRegistry.beginReset( + controlPath: key.controlPath + ) else { + scheduleCleanupRetry(for: key) + return + } + let cleanupID = UUID() + let process = Process() + let invocation = NativeSSHControlMasterCleanupRequest( + arguments: request.arguments, + environment: request.environment, + authenticationLockPath: nil + ).processInvocation + process.executableURL = invocation.executableURL + process.arguments = invocation.arguments + process.environment = request.environment + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + process.terminationHandler = { [weak self] _ in + Task { @MainActor in + self?.cleanupProcessDidTerminate(cleanupID) + } + } + do { + try process.run() + } catch { + authorization.release() + scheduleCleanupRetry(for: key) + return + } + cleanupAuthorizations[cleanupID] = authorization + cleanupProcesses[cleanupID] = process + cleanupControlMasterKeysByProcessID[cleanupID] = key + cleanupProcessIDByControlMaster[key] = cleanupID + scheduleCleanupTimeout( + cleanupID, + afterMilliseconds: Self.cleanupProcessTimeoutMilliseconds + ) + } + + private func scheduleCleanupRetry(for key: NativeSSHControlMasterKey) { + guard cleanupRequestsByControlMaster[key] != nil, + ownersByControlMaster[key]?.isEmpty != false, + cleanupRetryTasks[key] == nil else { + return + } + let clock = self.clock + cleanupRetryTasks[key] = Task { @MainActor [weak self] in + guard (try? await clock.sleep( + forMilliseconds: Self.cleanupRetryDelayMilliseconds + )) != nil, + !Task.isCancelled else { + return + } + self?.retryCleanup(for: key) + } + } + + private func retryCleanup(for key: NativeSSHControlMasterKey) { + cleanupRetryTasks.removeValue(forKey: key) + guard let request = cleanupRequestsByControlMaster[key], + ownersByControlMaster[key]?.isEmpty != false else { + cleanupRequestsByControlMaster.removeValue(forKey: key) + return + } + launchCleanup(request, for: key) + } + + private func scheduleCleanupTimeout( + _ cleanupID: UUID, + afterMilliseconds delay: Int + ) { + let clock = self.clock + cleanupTimeoutTasks[cleanupID] = Task { @MainActor [weak self] in + guard (try? await clock.sleep(forMilliseconds: delay)) != nil else { + return + } + self?.cleanupProcessTimedOut(cleanupID) + } + } + + private func cleanupProcessTimedOut(_ cleanupID: UUID) { + guard let process = cleanupProcesses[cleanupID], process.isRunning else { + cleanupProcessDidTerminate(cleanupID) + return + } + if cleanupTerminationRequested.insert(cleanupID).inserted { + process.terminate() + scheduleCleanupTimeout( + cleanupID, + afterMilliseconds: Self.cleanupForcedTerminationDelayMilliseconds + ) + } else { + _ = Darwin.kill(process.processIdentifier, SIGKILL) + } + } + + private func cleanupProcessDidTerminate(_ cleanupID: UUID) { + cleanupTimeoutTasks.removeValue(forKey: cleanupID)?.cancel() + cleanupAuthorizations.removeValue(forKey: cleanupID)?.release() + let terminationWasRequested = + cleanupTerminationRequested.remove(cleanupID) != nil + let process = cleanupProcesses.removeValue(forKey: cleanupID) + guard let key = + cleanupControlMasterKeysByProcessID.removeValue( + forKey: cleanupID + ) else { + return + } + if cleanupProcessIDByControlMaster[key] == cleanupID { + cleanupProcessIDByControlMaster.removeValue(forKey: key) + } + guard cleanupRequestsByControlMaster[key] != nil, + ownersByControlMaster[key]?.isEmpty != false else { + return + } + if terminationWasRequested || + process?.terminationStatus == + NativeSSHControlMasterCleanupRequest.retryExitStatus { + scheduleCleanupRetry(for: key) + } else { + cleanupRequestsByControlMaster.removeValue(forKey: key) + } + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift new file mode 100644 index 000000000000..b8ff603a9ebc --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift @@ -0,0 +1,79 @@ +public import CmuxCore +public import Foundation + +@MainActor +extension NativeSSHConnectionBroker { + /// Begins an ownership handoff while foreground SSH authentication still + /// holds the resolved ControlPath authentication lock. + /// + /// The returned lease prevents another live cmux process from resetting + /// the newly authenticated master before the workspace configuration is + /// ready to adopt it. + /// + /// - Parameters: + /// - controlPath: Exact resolved cmux-owned ControlPath. + /// - ownerWorkspaceID: Workspace that is authenticating the master. + /// - Returns: A handoff lease, or `nil` when ownership cannot be acquired. + public func beginControlMasterAdoption( + controlPath: String, + ownerWorkspaceID: UUID + ) -> NativeSSHControlMasterAdoptionHandoff? { + let lease = NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: ownerWorkspaceID, + generation: UUID() + ) + guard controlMasterOwnershipRegistry.retain( + controlPath: controlPath, + lease: lease + ) else { + return nil + } + let ownershipRegistry = controlMasterOwnershipRegistry + return NativeSSHControlMasterAdoptionHandoff( + controlPath: controlPath, + lease: lease, + releaseHandler: { + ownershipRegistry.release(lease: lease) + } + ) + } + + /// Atomically transfers a foreground-authentication handoff to the + /// workspace's retained configuration lease. + /// + /// The durable lease is acquired before the temporary lease is released, + /// so another process never observes an unowned master between phases. + /// + /// - Parameters: + /// - handoff: Temporary lease returned by + /// ``beginControlMasterAdoption(controlPath:ownerWorkspaceID:)``. + /// - configuration: Retained workspace configuration receiving ownership. + /// - Returns: `true` when the durable lease acquired ownership. + public func completeControlMasterAdoption( + _ handoff: NativeSSHControlMasterAdoptionHandoff, + configuration: WorkspaceRemoteConfiguration + ) -> Bool { + guard configuration.ownerWorkspaceID == + handoff.lease.ownerWorkspaceID, + let lease = NativeSSHControlMasterLeaseIdentity( + configuration: configuration + ), + controlMasterOwnershipRegistry.retain( + controlPath: handoff.controlPath, + lease: lease + ) else { + return false + } + handoff.release() + return true + } + + /// Releases a foreground-authentication handoff that will not be adopted. + /// + /// - Parameter handoff: Temporary lease to release. + public func cancelControlMasterAdoption( + _ handoff: NativeSSHControlMasterAdoptionHandoff + ) { + handoff.release() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index ee74bfe135dd..fe967499eecb 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -1,15 +1,8 @@ public import CmuxCore public import CmuxRemoteWorkspace internal import CmuxFoundation -internal import Darwin internal import Foundation -private enum NativeSSHCleanupPolicy { - static let processTimeoutMilliseconds = 5_000 - static let forcedTerminationDelayMilliseconds = 1_000 - static let retryDelayMilliseconds = 31_000 -} - /// Owns cmux-native SSH master lifetimes and serializes reconnect attempts per endpoint. /// /// Workspace ownership is reference-counted by `ownerWorkspaceID`. Only the @@ -19,25 +12,30 @@ private enum NativeSSHCleanupPolicy { /// remain independent. @MainActor public final class NativeSSHConnectionBroker { - private nonisolated let sharingOptions: SSHConnectionSharingOptions - private let clock: any RemoteProxyRetryClock + nonisolated let sharingOptions: SSHConnectionSharingOptions + let clock: any RemoteProxyRetryClock private let jitterMilliseconds: @MainActor @Sendable () -> Int - private let cleanupLauncherOverride: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)? + let cleanupLauncherOverride: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)? private nonisolated let conflictedMasterResetEventHub: NativeSSHControlMasterResetEventHub + nonisolated let controlMasterOwnershipRegistry: + any NativeSSHControlMasterOwnershipTracking private let conflictedMasterResetCoordinator: NativeSSHControlMasterResetCoordinator - private var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] - private var ownersByControlMaster: [NativeSSHControlMasterKey: Set] = [:] + var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] + var ownersByControlMaster: [NativeSSHControlMasterKey: Set] = [:] var attemptStates: [NativeSSHConnectionKey: NativeSSHConnectionAttemptState] = [:] - private var cleanupRequestsByControlMaster: [ + var cleanupRequestsByControlMaster: [ NativeSSHControlMasterKey: NativeSSHControlMasterCleanupRequest ] = [:] - private var cleanupRetryTasks: [NativeSSHControlMasterKey: Task] = [:] - private var cleanupProcesses: [UUID: Process] = [:] - private var cleanupControlMasterKeysByProcessID: [UUID: NativeSSHControlMasterKey] = [:] - private var cleanupProcessIDByControlMaster: [NativeSSHControlMasterKey: UUID] = [:] - private var cleanupTimeoutTasks: [UUID: Task] = [:] - private var cleanupTerminationRequested: Set = [] + var cleanupRetryTasks: [NativeSSHControlMasterKey: Task] = [:] + var cleanupProcesses: [UUID: Process] = [:] + var cleanupControlMasterKeysByProcessID: [UUID: NativeSSHControlMasterKey] = [:] + var cleanupProcessIDByControlMaster: [NativeSSHControlMasterKey: UUID] = [:] + var cleanupTimeoutTasks: [UUID: Task] = [:] + var cleanupTerminationRequested: Set = [] + var cleanupAuthorizations: [ + UUID: NativeSSHControlMasterResetAuthorization + ] = [:] /// Creates the process-wide broker with continuous-clock jitter and local cleanup launching. /// @@ -48,12 +46,18 @@ public final class NativeSSHConnectionBroker { self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = nil let eventHub = NativeSSHControlMasterResetEventHub() + let ownershipRegistry = + NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) self.conflictedMasterResetEventHub = eventHub + self.controlMasterOwnershipRegistry = ownershipRegistry self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( sharingOptions: sharingOptions, processRunner: RemoteSessionProcessRunner(), clock: clock, - eventHub: eventHub + eventHub: eventHub, + ownershipRegistry: ownershipRegistry ) } @@ -74,12 +78,18 @@ public final class NativeSSHConnectionBroker { self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = cleanupLauncher let eventHub = NativeSSHControlMasterResetEventHub() + let ownershipRegistry = + NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) self.conflictedMasterResetEventHub = eventHub + self.controlMasterOwnershipRegistry = ownershipRegistry self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( sharingOptions: sharingOptions, processRunner: RemoteSessionProcessRunner(), clock: clock, - eventHub: eventHub + eventHub: eventHub, + ownershipRegistry: ownershipRegistry ) } @@ -88,7 +98,10 @@ public final class NativeSSHConnectionBroker { clock: any RemoteProxyRetryClock, jitterMilliseconds: @escaping @MainActor @Sendable () -> Int, cleanupLauncher: @escaping @MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void, - conflictedMasterResetRunner: any RemoteSessionProcessRunning = RemoteSessionProcessRunner() + conflictedMasterResetRunner: any RemoteSessionProcessRunning = + RemoteSessionProcessRunner(), + controlMasterOwnershipRegistry: + any NativeSSHControlMasterOwnershipTracking ) { self.sharingOptions = sharingOptions self.clock = clock @@ -96,11 +109,13 @@ public final class NativeSSHConnectionBroker { self.cleanupLauncherOverride = cleanupLauncher let eventHub = NativeSSHControlMasterResetEventHub() self.conflictedMasterResetEventHub = eventHub + self.controlMasterOwnershipRegistry = controlMasterOwnershipRegistry self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( sharingOptions: sharingOptions, processRunner: conflictedMasterResetRunner, clock: clock, - eventHub: eventHub + eventHub: eventHub, + ownershipRegistry: controlMasterOwnershipRegistry ) } @@ -154,6 +169,11 @@ public final class NativeSSHConnectionBroker { let generation = configuration.sshControlMasterLeaseGeneration else { return } + if let lease = NativeSSHControlMasterLeaseIdentity( + configuration: configuration + ) { + controlMasterOwnershipRegistry.release(lease: lease) + } if let resetKey = NativeSSHControlMasterResetKey( configuration: configuration, sharingOptions: sharingOptions @@ -174,6 +194,22 @@ public final class NativeSSHConnectionBroker { removeLease(ownerWorkspaceID: ownerWorkspaceID, key: key) } + /// Registers this process before a coordinator adopts an exact socket. + nonisolated func retainResolvedControlMasterLease( + for configuration: WorkspaceRemoteConfiguration, + controlPath: String + ) -> Bool { + guard let lease = NativeSSHControlMasterLeaseIdentity( + configuration: configuration + ) else { + return false + } + return controlMasterOwnershipRegistry.retain( + controlPath: controlPath, + lease: lease + ) + } + /// Coalesces an inherited-master reset after OpenSSH confirms a relay bind conflict. func resetConflictedControlMaster( for configuration: WorkspaceRemoteConfiguration @@ -231,64 +267,6 @@ public final class NativeSSHConnectionBroker { } } - private func removeLease(ownerWorkspaceID: UUID, key: NativeSSHControlMasterKey) { - guard var leases = ownerLeases[ownerWorkspaceID], - let previousConfiguration = leases.removeValue(forKey: key) else { - return - } - if leases.isEmpty { - ownerLeases.removeValue(forKey: ownerWorkspaceID) - } else { - ownerLeases[ownerWorkspaceID] = leases - } - var owners = ownersByControlMaster[key] ?? [] - owners.remove(ownerWorkspaceID) - guard owners.isEmpty else { - ownersByControlMaster[key] = owners - return - } - ownersByControlMaster.removeValue(forKey: key) - let arguments = RemoteControlMasterCleanup().cleanupArguments( - configuration: previousConfiguration - ) - let authenticationLockPath = sharingOptions.foregroundAuthenticationLockPath( - destination: previousConfiguration.destination, - port: previousConfiguration.port, - options: previousConfiguration.sshOptions - ) - let request = NativeSSHControlMasterCleanupRequest( - arguments: arguments, - environment: previousConfiguration.sshProcessEnvironment, - authenticationLockPath: authenticationLockPath - ) - beginCleanup(request, for: key) - } - - private func beginCleanup( - _ request: NativeSSHControlMasterCleanupRequest, - for key: NativeSSHControlMasterKey - ) { - if let cleanupLauncherOverride { - cleanupLauncherOverride(request) - cleanupRequestsByControlMaster.removeValue(forKey: key) - } else { - cleanupRequestsByControlMaster[key] = request - launchCleanup(request, for: key) - } - } - - private func cancelCleanup(for key: NativeSSHControlMasterKey) { - cleanupRequestsByControlMaster.removeValue(forKey: key) - cleanupRetryTasks.removeValue(forKey: key)?.cancel() - guard let cleanupID = cleanupProcessIDByControlMaster[key], - let process = cleanupProcesses[cleanupID], - process.isRunning else { - return - } - cleanupTerminationRequested.insert(cleanupID) - process.terminate() - } - private func acquireConnectionAttempt( for key: NativeSSHConnectionKey ) async throws -> NativeSSHConnectionPermit { @@ -381,113 +359,4 @@ public final class NativeSSHConnectionBroker { } } - private func launchCleanup( - _ request: NativeSSHControlMasterCleanupRequest, - for key: NativeSSHControlMasterKey - ) { - guard cleanupRequestsByControlMaster[key] != nil, - ownersByControlMaster[key]?.isEmpty != false, - cleanupProcessIDByControlMaster[key] == nil else { - return - } - let cleanupID = UUID() - let process = Process() - let invocation = request.processInvocation - process.executableURL = invocation.executableURL - process.arguments = invocation.arguments - process.environment = request.environment - process.standardInput = FileHandle.nullDevice - process.standardOutput = FileHandle.nullDevice - process.standardError = FileHandle.nullDevice - process.terminationHandler = { [weak self] _ in - Task { @MainActor in - self?.cleanupProcessDidTerminate(cleanupID) - } - } - do { - try process.run() - } catch { - scheduleCleanupRetry(for: key) - return - } - cleanupProcesses[cleanupID] = process - cleanupControlMasterKeysByProcessID[cleanupID] = key - cleanupProcessIDByControlMaster[key] = cleanupID - scheduleCleanupTimeout( - cleanupID, - afterMilliseconds: NativeSSHCleanupPolicy.processTimeoutMilliseconds - ) - } - - private func scheduleCleanupRetry(for key: NativeSSHControlMasterKey) { - guard cleanupRequestsByControlMaster[key] != nil, - ownersByControlMaster[key]?.isEmpty != false, - cleanupRetryTasks[key] == nil else { - return - } - let clock = self.clock - cleanupRetryTasks[key] = Task { @MainActor [weak self] in - guard (try? await clock.sleep( - forMilliseconds: NativeSSHCleanupPolicy.retryDelayMilliseconds - )) != nil, - !Task.isCancelled else { - return - } - self?.retryCleanup(for: key) - } - } - - private func retryCleanup(for key: NativeSSHControlMasterKey) { - cleanupRetryTasks.removeValue(forKey: key) - guard let request = cleanupRequestsByControlMaster[key], - ownersByControlMaster[key]?.isEmpty != false else { - cleanupRequestsByControlMaster.removeValue(forKey: key) - return - } - launchCleanup(request, for: key) - } - - private func scheduleCleanupTimeout(_ cleanupID: UUID, afterMilliseconds delay: Int) { - let clock = self.clock - cleanupTimeoutTasks[cleanupID] = Task { @MainActor [weak self] in - guard (try? await clock.sleep(forMilliseconds: delay)) != nil else { return } - self?.cleanupProcessTimedOut(cleanupID) - } - } - - private func cleanupProcessTimedOut(_ cleanupID: UUID) { - guard let process = cleanupProcesses[cleanupID], process.isRunning else { - cleanupProcessDidTerminate(cleanupID) - return - } - if cleanupTerminationRequested.insert(cleanupID).inserted { - process.terminate() - scheduleCleanupTimeout( - cleanupID, - afterMilliseconds: NativeSSHCleanupPolicy.forcedTerminationDelayMilliseconds - ) - } else { - _ = Darwin.kill(process.processIdentifier, SIGKILL) - } - } - - private func cleanupProcessDidTerminate(_ cleanupID: UUID) { - cleanupTimeoutTasks.removeValue(forKey: cleanupID)?.cancel() - let terminationWasRequested = cleanupTerminationRequested.remove(cleanupID) != nil - let process = cleanupProcesses.removeValue(forKey: cleanupID) - guard let key = cleanupControlMasterKeysByProcessID.removeValue(forKey: cleanupID) else { return } - if cleanupProcessIDByControlMaster[key] == cleanupID { - cleanupProcessIDByControlMaster.removeValue(forKey: key) - } - guard cleanupRequestsByControlMaster[key] != nil, - ownersByControlMaster[key]?.isEmpty != false else { - return - } - if terminationWasRequested || - process?.terminationStatus == NativeSSHControlMasterCleanupRequest.retryExitStatus { - scheduleCleanupRetry(for: key) - } else { - cleanupRequestsByControlMaster.removeValue(forKey: key) - } - } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift new file mode 100644 index 000000000000..280d27a11bef --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift @@ -0,0 +1,45 @@ +internal import Foundation + +/// A temporary ownership lease that bridges foreground SSH authentication to +/// installation of the workspace's durable ControlMaster lease. +// SAFETY: `lock` serializes every read and mutation of the release closure. +public final class NativeSSHControlMasterAdoptionHandoff: + @unchecked Sendable, + Equatable +{ + let controlPath: String + let lease: NativeSSHControlMasterLeaseIdentity + // lint:allow lock - transfer, cancellation, and deinit race to release once. + private let lock = NSLock() + private var releaseHandler: (@Sendable () -> Void)? + + init( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity, + releaseHandler: @escaping @Sendable () -> Void + ) { + self.controlPath = controlPath + self.lease = lease + self.releaseHandler = releaseHandler + } + + func release() { + let handler = lock.withLock { + defer { releaseHandler = nil } + return releaseHandler + } + handler?() + } + + /// Compares handoff identity. + public static func == ( + lhs: NativeSSHControlMasterAdoptionHandoff, + rhs: NativeSSHControlMasterAdoptionHandoff + ) -> Bool { + lhs === rhs + } + + deinit { + release() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterLeaseIdentity.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterLeaseIdentity.swift new file mode 100644 index 000000000000..037c707a97c8 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterLeaseIdentity.swift @@ -0,0 +1,23 @@ +internal import CmuxCore +internal import Foundation + +/// Identifies one retained workspace generation using a resolved SSH master. +struct NativeSSHControlMasterLeaseIdentity: Hashable, Sendable { + let ownerWorkspaceID: UUID + let generation: UUID + + init(ownerWorkspaceID: UUID, generation: UUID) { + self.ownerWorkspaceID = ownerWorkspaceID + self.generation = generation + } + + init?(configuration: WorkspaceRemoteConfiguration) { + guard let ownerWorkspaceID = configuration.ownerWorkspaceID, + let generation = + configuration.sshControlMasterLeaseGeneration else { + return nil + } + self.ownerWorkspaceID = ownerWorkspaceID + self.generation = generation + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift new file mode 100644 index 000000000000..9034c9aa0c81 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift @@ -0,0 +1,266 @@ +internal import CmuxFoundation +internal import Darwin +internal import Foundation + +/// Cross-process resolved-socket leases backed by advisory file locks. +/// +/// Every process adopting a socket holds a shared ownership lock. Recovery +/// first takes the alias-independent authentication lock, drops this process's +/// shared lease, and attempts a nonblocking exclusive ownership lock. A live +/// sibling process makes that attempt fail closed; a crashed process releases +/// its kernel-held lease automatically. +// SAFETY: `lock` protects both maps and every descriptor/lock-state transition. +final class NativeSSHControlMasterOwnershipRegistry: + NativeSSHControlMasterOwnershipTracking, + @unchecked Sendable +{ + private struct Entry { + let descriptor: Int32 + var leases: Set + var resetID: UUID? + } + + // lint:allow lock - registry operations are short nonblocking fd updates. + private let lock = NSLock() + private let sharingOptions: SSHConnectionSharingOptions + private var entries: [String: Entry] = [:] + private var controlPathByLease: [ + NativeSSHControlMasterLeaseIdentity: String + ] = [:] + + init(sharingOptions: SSHConnectionSharingOptions) { + self.sharingOptions = sharingOptions + try? FileManager.default.createDirectory( + atPath: sharingOptions.controlMasterLockDirectoryPath, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + } + + deinit { + let descriptors = lock.withLock { + let descriptors = entries.values.map(\.descriptor) + entries.removeAll() + controlPathByLease.removeAll() + return descriptors + } + for descriptor in descriptors { + _ = flock(descriptor, LOCK_UN) + _ = Darwin.close(descriptor) + } + } + + func retain( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity + ) -> Bool { + lock.withLock { + if controlPathByLease[lease] == controlPath { + return entries[controlPath]?.resetID == nil + } + removeLeaseLocked(lease) + if var entry = entries[controlPath] { + guard entry.resetID == nil else { return false } + entry.leases.insert(lease) + entries[controlPath] = entry + controlPathByLease[lease] = controlPath + return true + } + guard let lockPath = + sharingOptions.resolvedControlMasterOwnershipLockPath( + controlPath: controlPath + ), + let descriptor = openLockFile(lockPath) else { + return false + } + guard flock(descriptor, LOCK_SH | LOCK_NB) == 0 else { + _ = Darwin.close(descriptor) + return false + } + entries[controlPath] = Entry( + descriptor: descriptor, + leases: [lease], + resetID: nil + ) + controlPathByLease[lease] = controlPath + return true + } + } + + func release(lease: NativeSSHControlMasterLeaseIdentity) { + lock.withLock { + removeLeaseLocked(lease) + } + } + + func beginReset( + controlPath: String + ) -> NativeSSHControlMasterResetAuthorization? { + guard let authenticationPath = + sharingOptions.resolvedControlMasterAuthenticationLockPath( + controlPath: controlPath + ), + let authenticationDescriptor = openLockFile( + authenticationPath + ) else { + return nil + } + guard acquireAuthenticationLock(authenticationDescriptor) else { + _ = Darwin.close(authenticationDescriptor) + return nil + } + + let resetID = UUID() + let authorized = lock.withLock { + beginOwnershipResetLocked( + controlPath: controlPath, + resetID: resetID + ) + } + guard authorized else { + releaseAuthenticationLock(authenticationDescriptor) + _ = Darwin.close(authenticationDescriptor) + return nil + } + + return NativeSSHControlMasterResetAuthorization { [self] in + finishReset( + controlPath: controlPath, + resetID: resetID, + authenticationDescriptor: authenticationDescriptor + ) + } + } + + private func beginOwnershipResetLocked( + controlPath: String, + resetID: UUID + ) -> Bool { + guard var entry = entries[controlPath] else { + guard let lockPath = + sharingOptions.resolvedControlMasterOwnershipLockPath( + controlPath: controlPath + ), + let descriptor = openLockFile(lockPath) else { + return false + } + guard flock(descriptor, LOCK_EX | LOCK_NB) == 0 else { + _ = Darwin.close(descriptor) + return false + } + entries[controlPath] = Entry( + descriptor: descriptor, + leases: [], + resetID: resetID + ) + return true + } + guard entry.resetID == nil else { + return false + } + _ = flock(entry.descriptor, LOCK_UN) + guard flock(entry.descriptor, LOCK_EX | LOCK_NB) == 0 else { + if flock(entry.descriptor, LOCK_SH | LOCK_NB) != 0 { + removeEntryLocked(controlPath) + } + return false + } + entry.resetID = resetID + entries[controlPath] = entry + return true + } + + private func finishReset( + controlPath: String, + resetID: UUID, + authenticationDescriptor: Int32 + ) { + lock.withLock { + guard var entry = entries[controlPath], + entry.resetID == resetID else { + return + } + _ = flock(entry.descriptor, LOCK_UN) + entry.resetID = nil + if entry.leases.isEmpty { + _ = Darwin.close(entry.descriptor) + entries.removeValue(forKey: controlPath) + } else if flock(entry.descriptor, LOCK_SH | LOCK_NB) == 0 { + entries[controlPath] = entry + } else { + removeEntryLocked(controlPath) + } + } + releaseAuthenticationLock(authenticationDescriptor) + _ = Darwin.close(authenticationDescriptor) + } + + private func removeLeaseLocked( + _ lease: NativeSSHControlMasterLeaseIdentity + ) { + guard let controlPath = controlPathByLease.removeValue(forKey: lease), + var entry = entries[controlPath] else { + return + } + entry.leases.remove(lease) + guard entry.leases.isEmpty, entry.resetID == nil else { + entries[controlPath] = entry + return + } + _ = flock(entry.descriptor, LOCK_UN) + _ = Darwin.close(entry.descriptor) + entries.removeValue(forKey: controlPath) + } + + private func removeEntryLocked(_ controlPath: String) { + guard let entry = entries.removeValue(forKey: controlPath) else { + return + } + _ = flock(entry.descriptor, LOCK_UN) + _ = Darwin.close(entry.descriptor) + controlPathByLease = controlPathByLease.filter { + $0.value != controlPath + } + } + + private func openLockFile(_ path: String) -> Int32? { + let descriptor = Darwin.open( + path, + O_CREAT | O_RDWR | O_CLOEXEC | O_NOFOLLOW, + S_IRUSR | S_IWUSR + ) + guard descriptor >= 0 else { return nil } + var metadata = stat() + guard fstat(descriptor, &metadata) == 0, + metadata.st_uid == getuid(), + metadata.st_mode & S_IFMT == S_IFREG else { + _ = Darwin.close(descriptor) + return nil + } + _ = fchmod(descriptor, S_IRUSR | S_IWUSR) + return descriptor + } + + /// Matches the POSIX record locks used by zsh's `zsystem flock`. + private func acquireAuthenticationLock(_ descriptor: Int32) -> Bool { + var fileLock = Darwin.flock( + l_start: 0, + l_len: 0, + l_pid: 0, + l_type: Int16(F_WRLCK), + l_whence: Int16(SEEK_SET) + ) + return fcntl(descriptor, F_SETLK, &fileLock) == 0 + } + + private func releaseAuthenticationLock(_ descriptor: Int32) { + var fileLock = Darwin.flock( + l_start: 0, + l_len: 0, + l_pid: 0, + l_type: Int16(F_UNLCK), + l_whence: Int16(SEEK_SET) + ) + _ = fcntl(descriptor, F_SETLK, &fileLock) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift new file mode 100644 index 000000000000..1f784472715b --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift @@ -0,0 +1,11 @@ +/// Tracks process ownership of exact cmux-owned SSH master sockets. +protocol NativeSSHControlMasterOwnershipTracking: Sendable { + func retain( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity + ) -> Bool + func release(lease: NativeSSHControlMasterLeaseIdentity) + func beginReset( + controlPath: String + ) -> NativeSSHControlMasterResetAuthorization? +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift new file mode 100644 index 000000000000..33ab950ada43 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift @@ -0,0 +1,25 @@ +internal import Foundation + +/// Holds the resolved authentication and process-ownership locks for one reset. +// SAFETY: `lock` serializes every read and mutation of the release closure. +final class NativeSSHControlMasterResetAuthorization: @unchecked Sendable { + // lint:allow lock - release can arrive from task completion or deinit. + private let lock = NSLock() + private var releaseHandler: (@Sendable () -> Void)? + + init(releaseHandler: @escaping @Sendable () -> Void) { + self.releaseHandler = releaseHandler + } + + func release() { + let handler = lock.withLock { + defer { releaseHandler = nil } + return releaseHandler + } + handler?() + } + + deinit { + release() + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index 01d322afab29..18eb2be4f0ef 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -27,6 +27,8 @@ final class NativeSSHControlMasterResetCoordinator { private let processRunner: any RemoteSessionProcessRunning private let clock: any RemoteProxyRetryClock private let eventHub: NativeSSHControlMasterResetEventHub + private let ownershipRegistry: + any NativeSSHControlMasterOwnershipTracking private var leases: [ UUID: [NativeSSHControlMasterResetKey: WorkspaceRemoteConfiguration] ] = [:] @@ -38,12 +40,14 @@ final class NativeSSHControlMasterResetCoordinator { sharingOptions: SSHConnectionSharingOptions, processRunner: any RemoteSessionProcessRunning, clock: any RemoteProxyRetryClock, - eventHub: NativeSSHControlMasterResetEventHub + eventHub: NativeSSHControlMasterResetEventHub, + ownershipRegistry: any NativeSSHControlMasterOwnershipTracking ) { self.sharingOptions = sharingOptions self.processRunner = processRunner self.clock = clock self.eventHub = eventHub + self.ownershipRegistry = ownershipRegistry } func retainWorkspace( @@ -144,6 +148,25 @@ final class NativeSSHControlMasterResetCoordinator { inFlightResets[resolvedControlPath] = inFlight return await inFlight.task.value } + guard let lease = NativeSSHControlMasterLeaseIdentity( + configuration: configuration + ), + ownershipRegistry.retain( + controlPath: resolvedControlPath, + lease: lease + ) else { + return .deferred( + "resolved SSH master ownership is busy in another cmux process" + ) + } + guard let resetAuthorization = ownershipRegistry.beginReset( + controlPath: resolvedControlPath + ) else { + return .deferred( + "resolved SSH master is in use by another cmux process " + + "or foreground authentication" + ) + } let resolvedOptions = pathResolver.replacingControlPath( in: effectiveOptions, @@ -153,21 +176,17 @@ final class NativeSSHControlMasterResetCoordinator { configuration: configuration, sshOptionsOverride: resolvedOptions ) - let authenticationLockPath = sharingOptions.foregroundAuthenticationLockPath( - destination: configuration.destination, - port: configuration.port, - options: effectiveOptions - ) let request = NativeSSHControlMasterCleanupRequest( arguments: arguments, environment: configuration.sshProcessEnvironment, - authenticationLockPath: authenticationLockPath + authenticationLockPath: nil ) let resetID = UUID() let processRunner = self.processRunner let clock = self.clock let eventHub = self.eventHub let task = Task { + defer { resetAuthorization.release() } let outcome = await Self.runReset( request: request, processRunner: processRunner, diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift index 9df012fef8e1..0a7406174b90 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift @@ -10,10 +10,29 @@ public struct RemoteSessionStrings: Sendable, Equatable { /// Format for the suspended-auto-reconnect state detail /// (`remote.state.suspended.detail`); `%@` is the display target. public let suspendedDetailFormat: String + /// Retry detail when the reverse SSH relay is unavailable. + public let reverseRelayUnavailableRetrying: String + /// Retry detail when a live owner prevents relay-port recovery. + public let reverseRelayPortUnavailableRetrying: String - /// Creates the strings bundle with both formats app-resolved. - public init(connectedVMNoProxyFormat: String, suspendedDetailFormat: String) { + /// Creates the app-resolved strings bundle. + /// + /// - Parameters: + /// - connectedVMNoProxyFormat: Connected-without-proxy detail format. + /// - suspendedDetailFormat: Suspended reconnect detail format. + /// - reverseRelayUnavailableRetrying: Generic relay retry detail. + /// - reverseRelayPortUnavailableRetrying: Port-conflict retry detail. + public init( + connectedVMNoProxyFormat: String, + suspendedDetailFormat: String, + reverseRelayUnavailableRetrying: String, + reverseRelayPortUnavailableRetrying: String + ) { self.connectedVMNoProxyFormat = connectedVMNoProxyFormat self.suspendedDetailFormat = suspendedDetailFormat + self.reverseRelayUnavailableRetrying = + reverseRelayUnavailableRetrying + self.reverseRelayPortUnavailableRetrying = + reverseRelayPortUnavailableRetrying } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 4d383ba30a22..3137b62c2741 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -225,10 +225,7 @@ extension RemoteSessionCoordinator { let retryDelay = 2.0 publishDaemonStatus( .error, - detail: String( - localized: "remoteSession.reverseRelay.unavailableRetrying", - defaultValue: "Remote SSH relay unavailable; retrying in 2 seconds" - ) + detail: strings.reverseRelayUnavailableRetrying ) scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: retryDelay) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index f8d0d7c42aa4..eacd544a3b88 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -96,6 +96,16 @@ extension RemoteSessionCoordinator { /// master. private func resolvedReverseRelayControlMasterSSHOptionsLocked() -> [String]? { if let reverseRelayResolvedControlMasterSSHOptions { + let sharingOptions = SSHConnectionSharingOptions() + guard let resolvedPath = sharingOptions.cmuxOwnedControlPath( + in: reverseRelayResolvedControlMasterSSHOptions + ), + connectionBroker.retainResolvedControlMasterLease( + for: configuration, + controlPath: resolvedPath + ) else { + return nil + } return reverseRelayResolvedControlMasterSSHOptions } @@ -150,6 +160,16 @@ extension RemoteSessionCoordinator { in: effectiveOptions, with: resolvedPath ) + guard connectionBroker.retainResolvedControlMasterLease( + for: configuration, + controlPath: resolvedPath + ) else { + debugLog( + "remote.relay.controlmaster.ownershipBusy " + + "\(debugConfigSummary())" + ) + return nil + } guard let observation = connectionBroker.observeControlMasterResets( controlPath: resolvedPath, handler: { [weak self] in diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index c92d159fec02..5df34e6d9f8e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -104,10 +104,7 @@ extension RemoteSessionCoordinator { private func publishReverseRelayPortUnavailableLocked() { publishDaemonStatus( .error, - detail: String( - localized: "remoteSession.reverseRelay.portUnavailableRetrying", - defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" - ) + detail: strings.reverseRelayPortUnavailableRetrying ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index c216fe2d6721..3184a01caf69 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -42,7 +42,7 @@ struct NativeSSHConnectionBrokerTests { #expect(recorder.requests[0].arguments.contains(resolvedOwnedSSHOptions[2])) let request = recorder.requests[0] let lockPath = request.authenticationLockPath - #expect(lockPath?.contains("cmux-ssh-501-auth-") == true) + #expect(lockPath?.contains("cmux-ssh-501-resolved-auth-") == true) #expect(request.processInvocation.executableURL.path == "/bin/zsh") #expect(request.processInvocation.arguments.contains(lockPath.map { $0 + ".inflight" } ?? "") == true) #expect(request.processInvocation.arguments[1].contains("zsystem flock -t 4 -e")) @@ -174,7 +174,9 @@ struct NativeSSHConnectionBrokerTests { clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, - conflictedMasterResetRunner: runner + conflictedMasterResetRunner: runner, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() ) let firstLease = broker.retainWorkspace(configuration( owner: UUID(), @@ -204,7 +206,7 @@ struct NativeSSHConnectionBrokerTests { #expect(await secondReset.value == .reset) #expect(runner.requests.count == 1) let request = try #require(runner.requests.first) - #expect(request.executable == "/bin/zsh") + #expect(request.executable == "/usr/bin/ssh") #expect(request.arguments.contains("-O")) #expect(request.arguments.contains("exit")) #expect(request.arguments.contains(resolvedOwnedSSHOptions[2])) @@ -293,7 +295,9 @@ struct NativeSSHConnectionBrokerTests { sharingOptions: sharingOptions, clock: clock, jitterMilliseconds: { 900 }, - cleanupLauncher: { _ in } + cleanupLauncher: { _ in }, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() ) let leaderConfiguration = configuration( owner: UUID(), @@ -387,7 +391,9 @@ struct NativeSSHConnectionBrokerTests { sharingOptions: sharingOptions, clock: clock, jitterMilliseconds: { 200 }, - cleanupLauncher: { _ in } + cleanupLauncher: { _ in }, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() ) let configuration = configuration(owner: UUID()) @@ -429,7 +435,11 @@ struct NativeSSHConnectionBrokerTests { sharingOptions: sharingOptions, clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, - cleanupLauncher: { request in cleanupRecorder.requests.append(request) } + cleanupLauncher: { + request in cleanupRecorder.requests.append(request) + }, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift new file mode 100644 index 000000000000..c467b11b4e89 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift @@ -0,0 +1,142 @@ +import CmuxCore +import CmuxFoundation +import CmuxRemoteWorkspace +import Darwin +import Foundation +import Testing +@testable import CmuxRemoteSession + +@MainActor +@Suite("Native SSH ownership-gated recovery") +struct NativeSSHControlMasterOwnershipRecoveryTests { + @Test("A live foreign owner prevents destructive reset") + func foreignOwnerFailsClosed() async { + let runner = RecordingProcessRunner() + let broker = NativeSSHConnectionBroker( + sharingOptions: SSHConnectionSharingOptions(userID: 501), + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + conflictedMasterResetRunner: runner, + controlMasterOwnershipRegistry: + DenyingControlMasterOwnershipRegistry() + ) + let lease = broker.retainWorkspace(WorkspaceRemoteConfiguration( + destination: "alice@example.test", + port: nil, + identityFile: nil, + sshOptions: [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567", + ], + localProxyPort: nil, + relayPort: 64_001, + relayID: "relay-id", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-test.sock", + ownerWorkspaceID: UUID(), + terminalStartupCommand: nil, + preserveAfterTerminalExit: true, + persistentDaemonSlot: "ssh-test" + )) + + guard case .deferred = + await broker.resetConflictedControlMaster(for: lease) else { + Issue.record("Expected a live foreign owner to defer reset") + return + } + #expect(runner.requests.isEmpty) + } + + @Test("Foreground authentication hands ownership to the workspace without a gap") + func foregroundAuthenticationOwnershipHandoff() throws { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-control-handoff-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: scratchDirectory) } + let sharingOptions = SSHConnectionSharingOptions( + userID: Int(getuid()), + authenticationLockDirectoryPath: scratchDirectory.path + ) + let firstRegistry = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let secondRegistry = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let broker = NativeSSHConnectionBroker( + sharingOptions: sharingOptions, + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + controlMasterOwnershipRegistry: firstRegistry + ) + let ownerWorkspaceID = UUID() + let controlPath = + "/tmp/cmux-ssh-\(getuid())-" + + "0123456789abcdef0123456789abcdef01234567" + let handoff = try #require( + broker.beginControlMasterAdoption( + controlPath: controlPath, + ownerWorkspaceID: ownerWorkspaceID + ) + ) + #expect(secondRegistry.beginReset(controlPath: controlPath) == nil) + + let configuration = broker.retainWorkspace( + WorkspaceRemoteConfiguration( + destination: "alice@example.test", + port: nil, + identityFile: nil, + sshOptions: [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=\(controlPath)", + ], + localProxyPort: nil, + relayPort: 64_001, + relayID: "relay-id", + relayToken: String(repeating: "a", count: 64), + localSocketPath: "/tmp/cmux-test.sock", + ownerWorkspaceID: ownerWorkspaceID, + terminalStartupCommand: nil, + preserveAfterTerminalExit: true, + persistentDaemonSlot: "ssh-test" + ) + ) + #expect(broker.completeControlMasterAdoption( + handoff, + configuration: configuration + )) + #expect(secondRegistry.beginReset(controlPath: controlPath) == nil) + + broker.releaseWorkspace(configuration) + let authorization = try #require( + secondRegistry.beginReset(controlPath: controlPath) + ) + authorization.release() + } +} + +private final class DenyingControlMasterOwnershipRegistry: + NativeSSHControlMasterOwnershipTracking, + Sendable +{ + func retain( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity + ) -> Bool { + true + } + + func release(lease: NativeSSHControlMasterLeaseIdentity) {} + + func beginReset( + controlPath: String + ) -> NativeSSHControlMasterResetAuthorization? { + nil + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift new file mode 100644 index 000000000000..d2ffc6a2da15 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift @@ -0,0 +1,172 @@ +import CmuxFoundation +import Darwin +import Foundation +import Testing +@testable import CmuxRemoteSession + +@Suite("Native SSH cross-process master ownership") +struct NativeSSHControlMasterOwnershipRegistryTests { + @Test("A live sibling process blocks reset until it releases its lease") + func liveSiblingBlocksReset() throws { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-control-owner-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: scratchDirectory) } + let sharingOptions = SSHConnectionSharingOptions( + userID: Int(getuid()), + authenticationLockDirectoryPath: scratchDirectory.path + ) + let controlPath = resolvedControlPath(userID: Int(getuid())) + let first = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let second = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let firstLease = NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: UUID(), + generation: UUID() + ) + let secondLease = NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: UUID(), + generation: UUID() + ) + + #expect(first.retain( + controlPath: controlPath, + lease: firstLease + )) + #expect(second.retain( + controlPath: controlPath, + lease: secondLease + )) + #expect(first.beginReset(controlPath: controlPath) == nil) + + second.release(lease: secondLease) + let authorization = try #require( + first.beginReset(controlPath: controlPath) + ) + #expect(!second.retain( + controlPath: controlPath, + lease: secondLease + )) + authorization.release() + #expect(second.retain( + controlPath: controlPath, + lease: secondLease + )) + } + + @Test("Resolved authentication blocks reset across different aliases") + func resolvedAuthenticationBlocksReset() throws { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-control-auth-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: scratchDirectory) } + let sharingOptions = SSHConnectionSharingOptions( + userID: Int(getuid()), + authenticationLockDirectoryPath: scratchDirectory.path + ) + let controlPath = resolvedControlPath(userID: Int(getuid())) + let registry = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let lease = NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: UUID(), + generation: UUID() + ) + let authenticationPath = try #require( + sharingOptions.resolvedControlMasterAuthenticationLockPath( + controlPath: controlPath + ) + ) + #expect(FileManager.default.createFile( + atPath: authenticationPath, + contents: Data() + )) + let process = Process() + let stdin = Pipe() + let stdout = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/zsh") + process.arguments = [ + "-fc", + """ + zmodload zsh/system || exit 1 + zsystem flock -t 2 -e -f cmux_test_auth_fd "$1" || exit 2 + print 'ready' + IFS= read -r _ || true + exit 0 + """, + "cmux-auth-lock", + authenticationPath, + ] + process.standardInput = stdin + process.standardOutput = stdout + process.standardError = FileHandle.nullDevice + try process.run() + defer { + try? stdin.fileHandleForWriting.close() + if process.isRunning { + process.terminate() + } + process.waitUntilExit() + } + let ready = stdout.fileHandleForReading.readData(ofLength: 6) + #expect(String(decoding: ready, as: UTF8.self) == "ready\n") + #expect(registry.retain(controlPath: controlPath, lease: lease)) + #expect(registry.beginReset(controlPath: controlPath) == nil) + + try stdin.fileHandleForWriting.close() + process.waitUntilExit() + #expect(process.terminationStatus == 0) + let authorization = try #require( + registry.beginReset(controlPath: controlPath) + ) + authorization.release() + } + + @Test("Authorization deinit restores the process shared lease") + func authorizationDeinitRestoresSharedLease() throws { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-control-auth-deinit-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: scratchDirectory) } + let sharingOptions = SSHConnectionSharingOptions( + userID: Int(getuid()), + authenticationLockDirectoryPath: scratchDirectory.path + ) + let controlPath = resolvedControlPath(userID: Int(getuid())) + let first = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let second = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let lease = NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: UUID(), + generation: UUID() + ) + #expect(first.retain(controlPath: controlPath, lease: lease)) + + var authorization = first.beginReset(controlPath: controlPath) + #expect(authorization != nil) + authorization = nil + + #expect(second.beginReset(controlPath: controlPath) == nil) + first.release(lease: lease) + let secondAuthorization = try #require( + second.beginReset(controlPath: controlPath) + ) + secondAuthorization.release() + } + + private func resolvedControlPath(userID: Int) -> String { + "/tmp/cmux-ssh-\(userID)-0123456789abcdef0123456789abcdef01234567" + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift index bf91a5330a7d..07dcd8d3950d 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -141,23 +141,6 @@ struct NativeSSHControlMasterResetTests { #expect(!runner.exitRequests[0].arguments.contains( "ControlPath=/tmp/cmux-ssh-501-%C" )) - let unresolvedLock = try #require( - sharingOptions.foregroundAuthenticationLockPath( - destination: "first.example.test", - port: nil, - options: unresolvedOptions - ) - ) - let resolvedLock = try #require( - sharingOptions.foregroundAuthenticationLockPath( - destination: "first.example.test", - port: nil, - options: resolvedOptions - ) - ) - #expect(unresolvedLock != resolvedLock) - #expect(runner.exitRequests[0].arguments.contains(unresolvedLock)) - #expect(!runner.exitRequests[0].arguments.contains(resolvedLock)) _ = firstObservation _ = secondObservation } @@ -462,7 +445,9 @@ struct NativeSSHControlMasterResetTests { clock: clock, jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, - conflictedMasterResetRunner: processRunner + conflictedMasterResetRunner: processRunner, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift new file mode 100644 index 000000000000..1a65be7e06ad --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift @@ -0,0 +1,23 @@ +import Foundation +@testable import CmuxRemoteSession + +/// Test fake that authorizes ownership without opening process-global locks. +final class PermissiveNativeSSHControlMasterOwnershipRegistry: + NativeSSHControlMasterOwnershipTracking, + Sendable +{ + func retain( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity + ) -> Bool { + true + } + + func release(lease: NativeSSHControlMasterLeaseIdentity) {} + + func beginReset( + controlPath: String + ) -> NativeSSHControlMasterResetAuthorization? { + NativeSSHControlMasterResetAuthorization {} + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift index 82efbe70f468..0383ae927c43 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift @@ -370,7 +370,9 @@ struct RemoteDaemonUploadTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: "", + reverseRelayPortUnavailableRetrying: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift index 6f14552bce25..bef4deff5816 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift @@ -134,7 +134,9 @@ struct RemotePTYIntentionalCleanupTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: "", + reverseRelayPortUnavailableRetrying: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift index 16bdce028256..f72f5ea14901 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift @@ -341,7 +341,9 @@ struct RemotePortScanGatingTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: "", + reverseRelayPortUnavailableRetrying: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift index 50066b599916..3d50989fc643 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift @@ -312,7 +312,9 @@ struct RemoteReconnectPolicyTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: "", + reverseRelayPortUnavailableRetrying: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift index 05e526940434..440c03dbcc49 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift @@ -399,7 +399,9 @@ struct RemoteRelaySlotTeardownTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: "", + reverseRelayPortUnavailableRetrying: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 545098d2bdad..6e4cfcb0d4b4 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -70,13 +70,10 @@ struct RemoteSessionReverseRelayStartupTests { var statuses = host.daemonStatuses.makeAsyncIterator() let status = await statuses.next() - #expect(status?.detail == String( - localized: "remoteSession.reverseRelay.portUnavailableRetrying", - defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" - )) + #expect(status?.detail == "test relay port unavailable") let request = runner.requests.first - #expect(request?.executable == "/bin/zsh") + #expect(request?.executable == "/usr/bin/ssh") #expect(request?.arguments.contains("-O") == true) #expect(request?.arguments.contains("exit") == true) #expect(request?.arguments.contains("-R") == false) @@ -210,7 +207,9 @@ struct RemoteSessionReverseRelayStartupTests { clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, - conflictedMasterResetRunner: effectiveRunner + conflictedMasterResetRunner: effectiveRunner, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() ) let configuration = connectionBroker.retainWorkspace(rawConfiguration) let coordinator = RemoteSessionCoordinator( @@ -232,7 +231,11 @@ struct RemoteSessionReverseRelayStartupTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: + "test relay unavailable", + reverseRelayPortUnavailableRetrying: + "test relay port unavailable" ), clock: clock ) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index dbf8e5b71333..f1717ea3e1db 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -318,10 +318,7 @@ struct RemoteSessionReverseRelayTransportTests { launcher.emitTermination(detail: rawFailure) let status = try #require(await statuses.next()) - #expect(status.detail == String( - localized: "remoteSession.reverseRelay.unavailableRetrying", - defaultValue: "Remote SSH relay unavailable; retrying in 2 seconds" - )) + #expect(status.detail == "test relay unavailable") #expect(status.detail?.contains(rawFailure) == false) #expect(await clock.nextRequestedDelay() == 2_000) _ = await coordinator.stopAndWait(cleanupScope: .transport) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift index b0c053acf183..146c24691ff3 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift @@ -178,7 +178,9 @@ struct RemoteSessionSSHRemoteCommandOverrideTests { ), strings: RemoteSessionStrings( connectedVMNoProxyFormat: "%@", - suspendedDetailFormat: "%@" + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: "", + reverseRelayPortUnavailableRetrying: "" ) ) } diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index 3896bd75e0f6..dd4b86d07f99 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -139907,6 +139907,23 @@ } } }, + "remoteSession.controlMaster.ownershipUnavailable": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "SSH connection is busy in another cmux process." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "別の cmux プロセスが SSH 接続を使用しています。" + } + } + } + }, "remoteSession.reverseRelay.portUnavailableRetrying": { "extractionState": "manual", "localizations": { diff --git a/Sources/RemoteSessionStrings+App.swift b/Sources/RemoteSessionStrings+App.swift index 0985a08153d9..7a64d4a7d806 100644 --- a/Sources/RemoteSessionStrings+App.swift +++ b/Sources/RemoteSessionStrings+App.swift @@ -17,6 +17,16 @@ extension RemoteSessionStrings { suspendedDetailFormat: String( localized: "remote.state.suspended.detail", defaultValue: "Can't reach %@ — automatic reconnect is paused. Use Reconnect when your network is back." + ), + reverseRelayUnavailableRetrying: String( + localized: "remoteSession.reverseRelay.unavailableRetrying", + defaultValue: "Remote SSH relay unavailable; retrying in 2 seconds" + ), + reverseRelayPortUnavailableRetrying: String( + localized: + "remoteSession.reverseRelay.portUnavailableRetrying", + defaultValue: + "Remote SSH relay port unavailable; retrying in 2 seconds" ) ) } diff --git a/Sources/SSHPTYAttachStartupCommandBuilder.swift b/Sources/SSHPTYAttachStartupCommandBuilder.swift index 753998d2766d..f6144ef81ef2 100644 --- a/Sources/SSHPTYAttachStartupCommandBuilder.swift +++ b/Sources/SSHPTYAttachStartupCommandBuilder.swift @@ -103,25 +103,46 @@ enum SSHPTYAttachStartupCommandBuilder { } private static func foregroundAuthLines(_ auth: ForegroundAuth) -> [String] { - let sshCommand = sshForegroundAuthCommand(auth) - let quotedToken = shellQuote(auth.token) - return [ + let readinessInsideResolvedLock = + foregroundAuthenticationReadyShellLines( + auth, + includeResolvedControlPath: true, + requireSuccess: true, + cliVariable: "CMUX_SSH_ATTACH_CLI" + ) + let (sshCommand, reportsReadiness) = + sshForegroundAuthCommand( + auth, + successShellLines: readinessInsideResolvedLock + ) + var lines = [ "cmux_ssh_attach_foreground_auth() {", " \(sshCommand)", "cmux_ssh_auth_status=$?", " if [ \"$cmux_ssh_auth_status\" -ne 0 ]; then return \"$cmux_ssh_auth_status\"; fi", - "cmux_ssh_auth_token=\(quotedToken)", - "cmux_ssh_auth_payload=\"{\\\"workspace_id\\\":\\\"$CMUX_WORKSPACE_ID\\\",\\\"foreground_auth_token\\\":\\\"$cmux_ssh_auth_token\\\"}\"", - "\"$cmux_ssh_attach_cli\" --socket \"$CMUX_SOCKET_PATH\" rpc workspace.remote.foreground_auth_ready \"$cmux_ssh_auth_payload\" >/dev/null 2>&1 || true", - "unset cmux_ssh_auth_payload cmux_ssh_auth_status cmux_ssh_auth_token", + ] + if !reportsReadiness { + lines += foregroundAuthenticationReadyShellLines( + auth, + includeResolvedControlPath: false, + requireSuccess: false, + cliVariable: "cmux_ssh_attach_cli" + ) + } + lines += [ + "unset cmux_ssh_auth_status", "}", "cmux_ssh_attach_foreground_auth", "cmux_ssh_auth_status=$?", "if [ \"$cmux_ssh_auth_status\" -ne 0 ]; then exit \"$cmux_ssh_auth_status\"; fi", ] + return lines } - private static func sshForegroundAuthCommand(_ auth: ForegroundAuth) -> String { + private static func sshForegroundAuthCommand( + _ auth: ForegroundAuth, + successShellLines: [String] + ) -> (command: String, reportsReadiness: Bool) { let sharingOptions = SSHConnectionSharingOptions() var arguments = ["ssh"] let options = sharingOptions.mergingDefaults(into: auth.sshOptions) @@ -151,6 +172,13 @@ enum SSHPTYAttachStartupCommandBuilder { destination: auth.destination, options: options ) + let resolvedAuthenticationLockLines = + resolvedControlMasterAuthenticationLockLines( + sharingOptions: sharingOptions, + sshArguments: arguments, + destination: auth.destination, + options: options + ) arguments += ["-T", auth.destination, "true"] let command = arguments.map(shellQuote).joined(separator: " ") guard let lockPath = sharingOptions.foregroundAuthenticationLockPath( @@ -158,7 +186,7 @@ enum SSHPTYAttachStartupCommandBuilder { port: auth.port, options: options ) else { - return command + return (command, false) } let inFlightPath = lockPath + ".inflight" var lockedCommand = [ @@ -174,15 +202,89 @@ enum SSHPTYAttachStartupCommandBuilder { ": >> \"$cmux_ssh_auth_lock_path\" || exit 255", "zmodload zsh/system || exit 255", "zsystem flock -t 45 -e -f cmux_ssh_auth_lock_fd \"$cmux_ssh_auth_lock_path\" || exit 255", + ] + lockedCommand += resolvedAuthenticationLockLines + lockedCommand += [ preflight, preflight == nil ? nil : "cmux_ssh_preflight_control_path", "command \(command)", "cmux_ssh_auth_status=$?", "if [ \"$cmux_ssh_auth_status\" -ne 0 ]; then exit \"$cmux_ssh_auth_status\"; fi", ].compactMap { $0 } + lockedCommand += successShellLines lockedCommand += sharingOptions.successfulForegroundAuthenticationCleanupShellLines() lockedCommand.append("exit 0") - return "/bin/zsh -fc \(shellQuote(lockedCommand.joined(separator: "\n")))" + return ( + "CMUX_SSH_ATTACH_CLI=\"$cmux_ssh_attach_cli\" " + + "/bin/zsh -fc " + + shellQuote(lockedCommand.joined(separator: "\n")), + true + ) + } + + private static func foregroundAuthenticationReadyShellLines( + _ auth: ForegroundAuth, + includeResolvedControlPath: Bool, + requireSuccess: Bool, + cliVariable: String + ) -> [String] { + let payload: String + if includeResolvedControlPath { + payload = + "cmux_ssh_auth_payload=\"{\\\"workspace_id\\\":\\\"" + + "$CMUX_WORKSPACE_ID\\\",\\\"foreground_auth_token\\\":\\\"" + + "$cmux_ssh_auth_token\\\",\\\"control_path\\\":\\\"" + + "$cmux_ssh_resolved_control_path\\\"}\"" + } else { + payload = + "cmux_ssh_auth_payload=\"{\\\"workspace_id\\\":\\\"" + + "$CMUX_WORKSPACE_ID\\\",\\\"foreground_auth_token\\\":\\\"" + + "$cmux_ssh_auth_token\\\"}\"" + } + let failureHandling = requireSuccess ? " || exit 255" : " || true" + return [ + "cmux_ssh_auth_token=\(shellQuote(auth.token))", + payload, + "\"$\(cliVariable)\" --socket \"$CMUX_SOCKET_PATH\" rpc " + + "workspace.remote.foreground_auth_ready " + + "\"$cmux_ssh_auth_payload\" >/dev/null 2>&1" + + failureHandling, + "unset cmux_ssh_auth_payload cmux_ssh_auth_token", + ] + } + + private static func resolvedControlMasterAuthenticationLockLines( + sharingOptions: SSHConnectionSharingOptions, + sshArguments: [String], + destination: String, + options: [String] + ) -> [String] { + guard sharingOptions.cmuxOwnedControlPath(in: options) != nil else { + return [] + } + let sshPrefix = sshArguments.map(shellQuote).joined(separator: " ") + let quotedDestination = shellQuote(destination) + let lockPrefix = URL( + fileURLWithPath: sharingOptions.controlMasterLockDirectoryPath, + isDirectory: true + ) + .appendingPathComponent( + "cmux-ssh-\(sharingOptions.userID)-resolved-auth-", + isDirectory: false + ) + .path + return [ + #"cmux_ssh_resolved_control_path="$(command \#(sshPrefix) -G \#(quotedDestination) 2>/dev/null | awk 'tolower($1) == "controlpath" { $1 = ""; sub(/^[[:space:]]+/, ""); print; exit }')" "#, + "case \"$cmux_ssh_resolved_control_path\" in", + " /tmp/cmux-ssh-\(sharingOptions.userID)-*) ;;", + " *) exit 255 ;;", + "esac", + "cmux_ssh_resolved_control_basename=\"${cmux_ssh_resolved_control_path##*/}\"", + "case \"$cmux_ssh_resolved_control_basename\" in ''|*[!A-Za-z0-9._-]*) exit 255 ;; esac", + "cmux_ssh_resolved_auth_lock_path=\(shellQuote(lockPrefix))\"$cmux_ssh_resolved_control_basename.lock\"", + ": >> \"$cmux_ssh_resolved_auth_lock_path\" || exit 255", + "zsystem flock -t 45 -e -f cmux_ssh_resolved_auth_lock_fd \"$cmux_ssh_resolved_auth_lock_path\" || exit 255", + ] } static func sshOptionsWithRestoreControlDefaults(_ options: [String], relayPort: Int? = nil) -> [String] { diff --git a/Sources/TerminalController+ControlWorkspaceContext.swift b/Sources/TerminalController+ControlWorkspaceContext.swift index fa8343e02158..590512929f11 100644 --- a/Sources/TerminalController+ControlWorkspaceContext.swift +++ b/Sources/TerminalController+ControlWorkspaceContext.swift @@ -436,13 +436,27 @@ extension TerminalController: ControlWorkspaceContext { func controlWorkspaceRemoteForegroundAuthReady( workspaceID: UUID, - foregroundAuthToken: String? + foregroundAuthToken: String?, + resolvedControlPath: String? ) -> ControlWorkspaceRemoteResolution { guard let owner = AppDelegate.shared?.tabManagerFor(tabId: workspaceID), let workspace = owner.tabs.first(where: { $0.id == workspaceID }) else { return .notFound(workspaceID: workspaceID) } - workspace.notifyRemoteForegroundAuthenticationReady(token: foregroundAuthToken) + guard workspace.notifyRemoteForegroundAuthenticationReady( + token: foregroundAuthToken, + resolvedControlPath: resolvedControlPath + ) else { + return .unavailable( + workspaceID: workspaceID, + message: String( + localized: + "remoteSession.controlMaster.ownershipUnavailable", + defaultValue: + "SSH connection is busy in another cmux process." + ) + ) + } notifyRemotePTYControllerAvailabilityChanged() let windowId = AppDelegate.shared?.windowId(for: owner) return .resolved( diff --git a/Sources/Workspace+RemoteSessionLifecycle.swift b/Sources/Workspace+RemoteSessionLifecycle.swift index f93d8b687110..346b8e41b716 100644 --- a/Sources/Workspace+RemoteSessionLifecycle.swift +++ b/Sources/Workspace+RemoteSessionLifecycle.swift @@ -198,17 +198,67 @@ extension Workspace { return trimmed.isEmpty ? nil : trimmed } - func notifyRemoteForegroundAuthenticationReady(token: String? = nil) { - guard let foregroundAuthToken = Self.normalizedForegroundAuthToken(token) else { return } + @discardableResult + func notifyRemoteForegroundAuthenticationReady( + token: String? = nil, + resolvedControlPath: String? = nil + ) -> Bool { + guard let foregroundAuthToken = + Self.normalizedForegroundAuthToken(token) else { + return false + } + let controlMasterAdoption: + NativeSSHControlMasterAdoptionHandoff? + if let resolvedControlPath { + guard let adoption = + nativeSSHConnectionBroker.beginControlMasterAdoption( + controlPath: resolvedControlPath, + ownerWorkspaceID: id + ) else { + return false + } + controlMasterAdoption = adoption + } else { + controlMasterAdoption = nil + } guard let remoteConfiguration else { - remoteForegroundAuthenticationPhase = .readyBeforeConfiguration(token: foregroundAuthToken) - return + cancelPendingRemoteControlMasterAdoption() + remoteForegroundAuthenticationPhase = + .readyBeforeConfiguration( + token: foregroundAuthToken, + controlMasterAdoption: controlMasterAdoption + ) + return true } guard Self.normalizedForegroundAuthToken(remoteConfiguration.foregroundAuthToken) == foregroundAuthToken, remoteForegroundAuthenticationPhase == .authenticating(token: foregroundAuthToken) else { - return + if let controlMasterAdoption { + nativeSSHConnectionBroker.cancelControlMasterAdoption( + controlMasterAdoption + ) + } + return true + } + remoteForegroundAuthenticationPhase = + .readyBeforeConfiguration( + token: foregroundAuthToken, + controlMasterAdoption: controlMasterAdoption + ) + return configureRemoteConnection( + remoteConfiguration, + autoConnect: true + ) + } + + func cancelPendingRemoteControlMasterAdoption() { + if case .readyBeforeConfiguration( + _, + let controlMasterAdoption? + ) = remoteForegroundAuthenticationPhase { + nativeSSHConnectionBroker.cancelControlMasterAdoption( + controlMasterAdoption + ) } remoteForegroundAuthenticationPhase = nil - configureRemoteConnection(remoteConfiguration, autoConnect: true) } } diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index f679f58ffd9b..843a9cb666d4 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -2381,7 +2381,12 @@ final class Workspace: Identifiable, ObservableObject { var remoteSessionTransitionTask: Task? var remoteSessionTransitionID: UUID? enum RemoteForegroundAuthenticationPhase: Equatable { - case readyBeforeConfiguration(token: String), authenticating(token: String) + case readyBeforeConfiguration( + token: String, + controlMasterAdoption: + NativeSSHControlMasterAdoptionHandoff? + ) + case authenticating(token: String) } var remoteForegroundAuthenticationPhase: RemoteForegroundAuthenticationPhase? var activeRemoteSessionControllerID: UUID? @@ -5412,9 +5417,41 @@ final class Workspace: Identifiable, ObservableObject { return payload } - func configureRemoteConnection(_ configuration: WorkspaceRemoteConfiguration, autoConnect: Bool = true) { + @discardableResult + func configureRemoteConnection( + _ configuration: WorkspaceRemoteConfiguration, + autoConnect: Bool = true + ) -> Bool { var configuration = configuration.scopedToOwnerWorkspace(id) configuration = nativeSSHConnectionBroker.retainWorkspace(configuration) + let foregroundAuthToken = + Self.normalizedForegroundAuthToken( + configuration.foregroundAuthToken + ) + let pendingControlMasterAdoption: + NativeSSHControlMasterAdoptionHandoff? + if case .readyBeforeConfiguration( + let readyToken, + let controlMasterAdoption + ) = remoteForegroundAuthenticationPhase, + readyToken == foregroundAuthToken { + pendingControlMasterAdoption = controlMasterAdoption + } else { + cancelPendingRemoteControlMasterAdoption() + pendingControlMasterAdoption = nil + } + if let pendingControlMasterAdoption, + !nativeSSHConnectionBroker.completeControlMasterAdoption( + pendingControlMasterAdoption, + configuration: configuration + ) { + nativeSSHConnectionBroker.cancelControlMasterAdoption( + pendingControlMasterAdoption + ) + nativeSSHConnectionBroker.releaseWorkspace(configuration) + remoteForegroundAuthenticationPhase = nil + return false + } defer { TerminalController.shared.notifyRemotePTYControllerAvailabilityChanged() } let previousConfiguration = remoteConfiguration let previousPresentedDirectory = presentedCurrentDirectory @@ -5476,9 +5513,12 @@ final class Workspace: Identifiable, ObservableObject { } applyRemoteProxyEndpointUpdate(nil) applyBrowserRemoteWorkspaceStatusToPanels() - let foregroundAuthToken = Self.normalizedForegroundAuthToken(configuration.foregroundAuthToken) let foregroundAuthenticationWasReady = foregroundAuthToken.map { - remoteForegroundAuthenticationPhase == .readyBeforeConfiguration(token: $0) + guard case .readyBeforeConfiguration(let token, _) = + remoteForegroundAuthenticationPhase else { + return false + } + return token == $0 } ?? false let shouldAutoConnect = autoConnect || foregroundAuthenticationWasReady remoteForegroundAuthenticationPhase = nil @@ -5492,7 +5532,7 @@ final class Workspace: Identifiable, ObservableObject { shouldStartController: false, finalCleanup: false ) - return + return true } guard shouldAutoConnect else { remoteForegroundAuthenticationPhase = foregroundAuthToken.map { .authenticating(token: $0) } @@ -5504,7 +5544,7 @@ final class Workspace: Identifiable, ObservableObject { shouldStartController: false, finalCleanup: false ) - return + return true } remoteConnectionState = .connecting applyBrowserRemoteWorkspaceStatusToPanels() @@ -5514,6 +5554,7 @@ final class Workspace: Identifiable, ObservableObject { shouldStartController: true, finalCleanup: false ) + return true } func disconnectRemoteConnection(clearConfiguration: Bool = false, disconnectedDetail: String? = nil) { @@ -5537,7 +5578,7 @@ final class Workspace: Identifiable, ObservableObject { shouldStartController: false, finalCleanup: clearConfiguration ) - remoteForegroundAuthenticationPhase = nil + cancelPendingRemoteControlMasterAdoption() remoteDisconnectPlaceholderPanelIds.formUnion(activeRemoteTerminalSurfaceIds) activeRemoteTerminalSurfaceIds.removeAll() let remoteDirectoryPanelIdsToClear = clearConfiguration ? remoteDirectoryTrustRequiredPanelIds.union(remoteDirectoryReportPanelIds) : [] diff --git a/cmuxTests/SSHForegroundAuthenticationMarkerCleanupTests.swift b/cmuxTests/SSHForegroundAuthenticationMarkerCleanupTests.swift index f04b62385678..c2a017ae4529 100644 --- a/cmuxTests/SSHForegroundAuthenticationMarkerCleanupTests.swift +++ b/cmuxTests/SSHForegroundAuthenticationMarkerCleanupTests.swift @@ -16,6 +16,8 @@ struct SSHForegroundAuthenticationMarkerCleanupTests { .appendingPathComponent("cmux-ssh-restored-auth-\(UUID().uuidString)", isDirectory: true) let fakeCLI = root.appendingPathComponent("cmux") let fakeSSH = root.appendingPathComponent("ssh") + let foregroundAuthPayloadLog = + root.appendingPathComponent("foreground-auth-payload.json") let socketHash = UUID().uuidString .replacingOccurrences(of: "-", with: "") .lowercased() + "01234567" @@ -32,17 +34,33 @@ struct SSHForegroundAuthenticationMarkerCleanupTests { options: sshOptions )) let inFlightPath = lockPath + ".inflight" + let resolvedAuthenticationLockPath = try #require( + SSHConnectionSharingOptions() + .resolvedControlMasterAuthenticationLockPath( + controlPath: controlPath + ) + ) try fileManager.createDirectory(at: root, withIntermediateDirectories: true) defer { try? fileManager.removeItem(at: root) unlink(lockPath) unlink(inFlightPath) + unlink(resolvedAuthenticationLockPath) } try Self.writeShellFile(at: fakeCLI, lines: [ "#!/bin/sh", - "case \" $* \" in *\" ssh-pty-attach \"*) exit 253 ;; *) exit 0 ;; esac", + "case \" $* \" in", + " *\" workspace.remote.foreground_auth_ready \"*)", + " for argument in \"$@\"; do cmux_test_last_argument=\"$argument\"; done", + " printf '%s\\n' \"$cmux_test_last_argument\" > \"$CMUX_TEST_AUTH_PAYLOAD_LOG\"", + " /bin/zsh -fc 'zmodload zsh/system || exit 2; : >> \"$CMUX_TEST_RESOLVED_AUTH_LOCK\" || exit 2; if zsystem flock -t 0 -e -f cmux_test_lock_fd \"$CMUX_TEST_RESOLVED_AUTH_LOCK\"; then exit 1; fi; exit 0'", + " exit $?", + " ;;", + " *\" ssh-pty-attach \"*) exit 253 ;;", + " *) exit 0 ;;", + "esac", ]) try Self.writeShellFile(at: fakeSSH, lines: [ "#!/bin/sh", @@ -65,6 +83,10 @@ struct SSHForegroundAuthenticationMarkerCleanupTests { environment["CMUX_WORKSPACE_ID"] = "11111111-1111-1111-1111-111111111111" environment["CMUX_SURFACE_ID"] = "22222222-2222-2222-2222-222222222222" environment["CMUX_TEST_CONTROL_PATH"] = controlPath + environment["CMUX_TEST_AUTH_PAYLOAD_LOG"] = + foregroundAuthPayloadLog.path + environment["CMUX_TEST_RESOLVED_AUTH_LOCK"] = + resolvedAuthenticationLockPath let command = SSHPTYAttachStartupCommandBuilder.command( sessionID: "ssh-test-session", @@ -83,6 +105,11 @@ struct SSHForegroundAuthenticationMarkerCleanupTests { !fileManager.fileExists(atPath: inFlightPath), "Successful restored authentication must remove its owned in-flight marker before releasing the lock" ) + let payloadData = try Data(contentsOf: foregroundAuthPayloadLog) + let payload = try #require( + JSONSerialization.jsonObject(with: payloadData) as? [String: String] + ) + #expect(payload["control_path"] == controlPath) } private static func writeShellFile(at url: URL, lines: [String]) throws { diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 5fe6c18d3f70..f1fa942eb424 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -983,10 +983,19 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { terminalStartupCommand: "ssh cmux-macmini", foregroundAuthToken: "token-a" ) - workspace.notifyRemoteForegroundAuthenticationReady(token: "token-a") + let resolvedControlPath = + "/tmp/cmux-ssh-\(getuid())-" + + "0123456789abcdef0123456789abcdef01234567" + XCTAssertTrue(workspace.notifyRemoteForegroundAuthenticationReady( + token: "token-a", + resolvedControlPath: resolvedControlPath + )) XCTAssertEqual(workspace.remoteConnectionState, .disconnected) XCTAssertNil(workspace.activeRemoteSessionControllerID) - workspace.configureRemoteConnection(config, autoConnect: false) + XCTAssertTrue(workspace.configureRemoteConnection( + config, + autoConnect: false + )) XCTAssertEqual(workspace.remoteConnectionState, .connecting) XCTAssertNotNil(workspace.activeRemoteSessionControllerID) workspace.disconnectRemoteConnection(clearConfiguration: true) From 74d63237a90c0940b9aa39b016256fbbd4fb7df8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 03:31:16 -0700 Subject: [PATCH 24/39] fix: preserve SSH master lifecycle invariants --- .../NativeSSHConnectionBroker+Cleanup.swift | 2 +- .../NativeSSHConnectionBroker.swift | 2 +- ...trolMasterExclusiveUseAuthorization.swift} | 6 +- ...iveSSHControlMasterOwnershipRegistry.swift | 129 +++++++++++++----- ...iveSSHControlMasterOwnershipTracking.swift | 5 +- ...tiveSSHControlMasterResetCoordinator.swift | 15 +- ...oordinator+ReverseRelayControlMaster.swift | 12 +- .../BlockingControlMasterResetRunner.swift | 1 + ...HControlMasterOwnershipRecoveryTests.swift | 8 +- ...HControlMasterOwnershipRegistryTests.swift | 36 +++++ ...iveSSHControlMasterOwnershipRegistry.swift | 10 +- ...oteSessionReverseRelayTransportTests.swift | 50 +++++++ 12 files changed, 212 insertions(+), 64 deletions(-) rename Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/{NativeSSHControlMasterResetAuthorization.swift => NativeSSHControlMasterExclusiveUseAuthorization.swift} (77%) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift index 00712848e669..10bd0ad0aab5 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift @@ -77,7 +77,7 @@ extension NativeSSHConnectionBroker { cleanupProcessIDByControlMaster[key] == nil else { return } - guard let authorization = controlMasterOwnershipRegistry.beginReset( + guard let authorization = controlMasterOwnershipRegistry.beginCleanup( controlPath: key.controlPath ) else { scheduleCleanupRetry(for: key) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index fe967499eecb..276d35d1cba7 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -34,7 +34,7 @@ public final class NativeSSHConnectionBroker { var cleanupTimeoutTasks: [UUID: Task] = [:] var cleanupTerminationRequested: Set = [] var cleanupAuthorizations: [ - UUID: NativeSSHControlMasterResetAuthorization + UUID: NativeSSHControlMasterExclusiveUseAuthorization ] = [:] /// Creates the process-wide broker with continuous-clock jitter and local cleanup launching. diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUseAuthorization.swift similarity index 77% rename from Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift rename to Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUseAuthorization.swift index 33ab950ada43..7672c880ac4c 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetAuthorization.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUseAuthorization.swift @@ -1,8 +1,10 @@ internal import Foundation -/// Holds the resolved authentication and process-ownership locks for one reset. +/// Holds authentication and process-ownership locks for one exclusive operation. // SAFETY: `lock` serializes every read and mutation of the release closure. -final class NativeSSHControlMasterResetAuthorization: @unchecked Sendable { +final class NativeSSHControlMasterExclusiveUseAuthorization: + @unchecked Sendable +{ // lint:allow lock - release can arrive from task completion or deinit. private let lock = NSLock() private var releaseHandler: (@Sendable () -> Void)? diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift index 9034c9aa0c81..558deb346d9e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift @@ -17,7 +17,12 @@ final class NativeSSHControlMasterOwnershipRegistry: private struct Entry { let descriptor: Int32 var leases: Set - var resetID: UUID? + var exclusiveUseID: UUID? + } + + private enum ExclusiveUsePurpose { + case conflictedMasterReset + case ordinaryCleanup } // lint:allow lock - registry operations are short nonblocking fd updates. @@ -56,11 +61,11 @@ final class NativeSSHControlMasterOwnershipRegistry: ) -> Bool { lock.withLock { if controlPathByLease[lease] == controlPath { - return entries[controlPath]?.resetID == nil + return entries[controlPath]?.exclusiveUseID == nil } removeLeaseLocked(lease) if var entry = entries[controlPath] { - guard entry.resetID == nil else { return false } + guard entry.exclusiveUseID == nil else { return false } entry.leases.insert(lease) entries[controlPath] = entry controlPathByLease[lease] = controlPath @@ -80,7 +85,7 @@ final class NativeSSHControlMasterOwnershipRegistry: entries[controlPath] = Entry( descriptor: descriptor, leases: [lease], - resetID: nil + exclusiveUseID: nil ) controlPathByLease[lease] = controlPath return true @@ -95,7 +100,26 @@ final class NativeSSHControlMasterOwnershipRegistry: func beginReset( controlPath: String - ) -> NativeSSHControlMasterResetAuthorization? { + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + beginExclusiveUse( + controlPath: controlPath, + purpose: .conflictedMasterReset + ) + } + + func beginCleanup( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + beginExclusiveUse( + controlPath: controlPath, + purpose: .ordinaryCleanup + ) + } + + private func beginExclusiveUse( + controlPath: String, + purpose: ExclusiveUsePurpose + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { guard let authenticationPath = sharingOptions.resolvedControlMasterAuthenticationLockPath( controlPath: controlPath @@ -110,12 +134,20 @@ final class NativeSSHControlMasterOwnershipRegistry: return nil } - let resetID = UUID() + let exclusiveUseID = UUID() let authorized = lock.withLock { - beginOwnershipResetLocked( - controlPath: controlPath, - resetID: resetID - ) + switch purpose { + case .conflictedMasterReset: + beginOwnershipResetLocked( + controlPath: controlPath, + exclusiveUseID: exclusiveUseID + ) + case .ordinaryCleanup: + beginOwnershipCleanupLocked( + controlPath: controlPath, + exclusiveUseID: exclusiveUseID + ) + } } guard authorized else { releaseAuthenticationLock(authenticationDescriptor) @@ -123,39 +155,39 @@ final class NativeSSHControlMasterOwnershipRegistry: return nil } - return NativeSSHControlMasterResetAuthorization { [self] in - finishReset( + return NativeSSHControlMasterExclusiveUseAuthorization { [self] in + finishExclusiveUse( controlPath: controlPath, - resetID: resetID, + exclusiveUseID: exclusiveUseID, authenticationDescriptor: authenticationDescriptor ) } } + private func beginOwnershipCleanupLocked( + controlPath: String, + exclusiveUseID: UUID + ) -> Bool { + guard entries[controlPath] == nil else { + return false + } + return beginUnownedExclusiveUseLocked( + controlPath: controlPath, + exclusiveUseID: exclusiveUseID + ) + } + private func beginOwnershipResetLocked( controlPath: String, - resetID: UUID + exclusiveUseID: UUID ) -> Bool { guard var entry = entries[controlPath] else { - guard let lockPath = - sharingOptions.resolvedControlMasterOwnershipLockPath( - controlPath: controlPath - ), - let descriptor = openLockFile(lockPath) else { - return false - } - guard flock(descriptor, LOCK_EX | LOCK_NB) == 0 else { - _ = Darwin.close(descriptor) - return false - } - entries[controlPath] = Entry( - descriptor: descriptor, - leases: [], - resetID: resetID + return beginUnownedExclusiveUseLocked( + controlPath: controlPath, + exclusiveUseID: exclusiveUseID ) - return true } - guard entry.resetID == nil else { + guard entry.exclusiveUseID == nil else { return false } _ = flock(entry.descriptor, LOCK_UN) @@ -165,23 +197,46 @@ final class NativeSSHControlMasterOwnershipRegistry: } return false } - entry.resetID = resetID + entry.exclusiveUseID = exclusiveUseID entries[controlPath] = entry return true } - private func finishReset( + private func beginUnownedExclusiveUseLocked( + controlPath: String, + exclusiveUseID: UUID + ) -> Bool { + guard let lockPath = + sharingOptions.resolvedControlMasterOwnershipLockPath( + controlPath: controlPath + ), + let descriptor = openLockFile(lockPath) else { + return false + } + guard flock(descriptor, LOCK_EX | LOCK_NB) == 0 else { + _ = Darwin.close(descriptor) + return false + } + entries[controlPath] = Entry( + descriptor: descriptor, + leases: [], + exclusiveUseID: exclusiveUseID + ) + return true + } + + private func finishExclusiveUse( controlPath: String, - resetID: UUID, + exclusiveUseID: UUID, authenticationDescriptor: Int32 ) { lock.withLock { guard var entry = entries[controlPath], - entry.resetID == resetID else { + entry.exclusiveUseID == exclusiveUseID else { return } _ = flock(entry.descriptor, LOCK_UN) - entry.resetID = nil + entry.exclusiveUseID = nil if entry.leases.isEmpty { _ = Darwin.close(entry.descriptor) entries.removeValue(forKey: controlPath) @@ -203,7 +258,7 @@ final class NativeSSHControlMasterOwnershipRegistry: return } entry.leases.remove(lease) - guard entry.leases.isEmpty, entry.resetID == nil else { + guard entry.leases.isEmpty, entry.exclusiveUseID == nil else { entries[controlPath] = entry return } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift index 1f784472715b..0c40d08177c8 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift @@ -7,5 +7,8 @@ protocol NativeSSHControlMasterOwnershipTracking: Sendable { func release(lease: NativeSSHControlMasterLeaseIdentity) func beginReset( controlPath: String - ) -> NativeSSHControlMasterResetAuthorization? + ) -> NativeSSHControlMasterExclusiveUseAuthorization? + func beginCleanup( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index 18eb2be4f0ef..95843d8b3e3e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -19,7 +19,6 @@ enum NativeSSHControlMasterResetOutcome: Sendable, Equatable { final class NativeSSHControlMasterResetCoordinator { private struct InFlightReset { let id: UUID - var authorizationKeys: Set let task: Task } @@ -75,15 +74,6 @@ final class NativeSSHControlMasterResetCoordinator { } else { leases[ownerWorkspaceID] = ownerLeases } - for reset in inFlightResets.values { - let remainsOwned = reset.authorizationKeys.contains { - authorizationKey in - leases.values.contains { $0[authorizationKey] != nil } - } - if !remainsOwned { - reset.task.cancel() - } - } } func reset( @@ -143,9 +133,7 @@ final class NativeSSHControlMasterResetCoordinator { ) else { return .ignored("workspace no longer owns this cmux SSH master") } - if var inFlight = inFlightResets[resolvedControlPath] { - inFlight.authorizationKeys.insert(key) - inFlightResets[resolvedControlPath] = inFlight + if let inFlight = inFlightResets[resolvedControlPath] { return await inFlight.task.value } guard let lease = NativeSSHControlMasterLeaseIdentity( @@ -199,7 +187,6 @@ final class NativeSSHControlMasterResetCoordinator { } inFlightResets[resolvedControlPath] = InFlightReset( id: resetID, - authorizationKeys: [key], task: task ) let outcome = await task.value diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index eacd544a3b88..2b3c25c37f42 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -99,11 +99,13 @@ extension RemoteSessionCoordinator { let sharingOptions = SSHConnectionSharingOptions() guard let resolvedPath = sharingOptions.cmuxOwnedControlPath( in: reverseRelayResolvedControlMasterSSHOptions - ), - connectionBroker.retainResolvedControlMasterLease( - for: configuration, - controlPath: resolvedPath - ) else { + ) else { + return reverseRelayResolvedControlMasterSSHOptions + } + guard connectionBroker.retainResolvedControlMasterLease( + for: configuration, + controlPath: resolvedPath + ) else { return nil } return reverseRelayResolvedControlMasterSSHOptions diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift index 7a1afebf3620..4247b117cb89 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift @@ -26,6 +26,7 @@ final class BlockingControlMasterResetRunner: lock.withLock { _requests.append(request) } startsContinuation.yield() release.wait() + try operation?.throwIfCancelled() return RemoteCommandResult(status: 0, stdout: "", stderr: "") } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift index c467b11b4e89..131c23689c72 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift @@ -136,7 +136,13 @@ private final class DenyingControlMasterOwnershipRegistry: func beginReset( controlPath: String - ) -> NativeSSHControlMasterResetAuthorization? { + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + nil + } + + func beginCleanup( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { nil } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift index d2ffc6a2da15..0e2ad8a1c7a3 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift @@ -129,6 +129,42 @@ struct NativeSSHControlMasterOwnershipRegistryTests { authorization.release() } + @Test("Ordinary cleanup never overrides a live local lease") + func cleanupRespectsLocalLease() throws { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-control-cleanup-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: scratchDirectory) } + let registry = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: SSHConnectionSharingOptions( + userID: Int(getuid()), + authenticationLockDirectoryPath: scratchDirectory.path + ) + ) + let controlPath = resolvedControlPath(userID: Int(getuid())) + let lease = NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: UUID(), + generation: UUID() + ) + + #expect(registry.retain(controlPath: controlPath, lease: lease)) + #expect(registry.beginCleanup(controlPath: controlPath) == nil) + + let reset = try #require( + registry.beginReset(controlPath: controlPath) + ) + reset.release() + #expect(registry.beginCleanup(controlPath: controlPath) == nil) + + registry.release(lease: lease) + let cleanup = try #require( + registry.beginCleanup(controlPath: controlPath) + ) + cleanup.release() + } + @Test("Authorization deinit restores the process shared lease") func authorizationDeinitRestoresSharedLease() throws { let scratchDirectory = FileManager.default.temporaryDirectory diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift index 1a65be7e06ad..0ee1fd6227d6 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift @@ -17,7 +17,13 @@ final class PermissiveNativeSSHControlMasterOwnershipRegistry: func beginReset( controlPath: String - ) -> NativeSSHControlMasterResetAuthorization? { - NativeSSHControlMasterResetAuthorization {} + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + NativeSSHControlMasterExclusiveUseAuthorization {} + } + + func beginCleanup( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + NativeSSHControlMasterExclusiveUseAuthorization {} } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index f1717ea3e1db..0380bf08fe04 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -282,6 +282,56 @@ struct RemoteSessionReverseRelayTransportTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } + @Test("A cached custom ControlPath remains reusable") + func cachedCustomControlPathRemainsReusable() async throws { + let runner = RecordingProcessRunner() + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator( + runner: runner, + sshOptions: [ + "StrictHostKeyChecking=accept-new", + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=~/.ssh/custom-%C", + ] + ) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + + let first = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: "127.0.0.1:64044:127.0.0.1:55001", + relayPort: 64_044 + ) + } + coordinator.queue.sync { + coordinator.stopReverseRelayViaControlMasterLocked() + } + let second = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: "127.0.0.1:64044:127.0.0.1:55002", + relayPort: 64_044 + ) + } + + guard case .started = first else { + Issue.record("Expected the initial custom-master relay to start") + return + } + guard case .started = second else { + Issue.record("Expected the cached custom master to remain usable") + return + } + #expect(runner.requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 2) + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test("A standalone non-bind failure publishes only a generic retry status") func standaloneFailurePublishesSanitizedStatus() async throws { let rawFailure = "Permission denied: secret diagnostic" From b1c1349a0f114896b6df3679551990866f4ad1e3 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 04:00:38 -0700 Subject: [PATCH 25/39] fix: close SSH ownership coordination gaps --- .../NativeSSHConnectionBroker.swift | 8 +++ ...iveSSHControlMasterOwnershipRegistry.swift | 58 ++++++++-------- .../Hosting/RemoteSessionStrings.swift | 8 ++- ...emoteSessionCoordinator+ReverseRelay.swift | 8 --- ...oordinator+ReverseRelayControlMaster.swift | 49 ++++++++----- .../Session/RemoteSessionCoordinator.swift | 4 +- .../NativeSSHConnectionBrokerTests.swift | 24 +++++++ ...HControlMasterOwnershipRegistryTests.swift | 69 +++++++++++++++++++ ...tiveSSHControlMasterResetTestSupport.swift | 16 +++-- ...iveSSHControlMasterOwnershipRegistry.swift | 15 +++- .../RemoteDaemonUploadTests.swift | 3 +- .../RemotePTYIntentionalCleanupTests.swift | 3 +- .../RemotePortScanGatingTests.swift | 3 +- .../RemoteReconnectPolicyTests.swift | 3 +- .../RemoteRelaySlotTeardownTests.swift | 3 +- ...emoteSessionReverseRelayStartupTests.swift | 11 +-- ...oteSessionReverseRelayTransportTests.swift | 47 ++++++++++++- ...SessionSSHRemoteCommandOverrideTests.swift | 3 +- Sources/RemoteSessionStrings+App.swift | 6 ++ 19 files changed, 267 insertions(+), 74 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 276d35d1cba7..6526da2488a5 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -147,6 +147,14 @@ public final class NativeSSHConnectionBroker { ) } guard let nextKey else { return leasedConfiguration } + if let lease = NativeSSHControlMasterLeaseIdentity( + configuration: leasedConfiguration + ) { + _ = controlMasterOwnershipRegistry.retain( + controlPath: nextKey.controlPath, + lease: lease + ) + } cancelCleanup(for: nextKey) var leases = ownerLeases[ownerWorkspaceID] ?? [:] let isNewMaster = leases[nextKey] == nil diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift index 558deb346d9e..b1dfb083a9df 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift @@ -120,40 +120,43 @@ final class NativeSSHControlMasterOwnershipRegistry: controlPath: String, purpose: ExclusiveUsePurpose ) -> NativeSSHControlMasterExclusiveUseAuthorization? { - guard let authenticationPath = - sharingOptions.resolvedControlMasterAuthenticationLockPath( - controlPath: controlPath - ), - let authenticationDescriptor = openLockFile( - authenticationPath - ) else { - return nil - } - guard acquireAuthenticationLock(authenticationDescriptor) else { - _ = Darwin.close(authenticationDescriptor) - return nil - } - let exclusiveUseID = UUID() - let authorized = lock.withLock { + let authenticationDescriptor = lock.withLock { () -> Int32? in + guard entries[controlPath]?.exclusiveUseID == nil, + let authenticationPath = + sharingOptions + .resolvedControlMasterAuthenticationLockPath( + controlPath: controlPath + ), + let descriptor = openLockFile(authenticationPath) else { + return nil + } + guard acquireAuthenticationLock(descriptor) else { + _ = Darwin.close(descriptor) + return nil + } + + let authorized: Bool switch purpose { case .conflictedMasterReset: - beginOwnershipResetLocked( + authorized = beginOwnershipResetLocked( controlPath: controlPath, exclusiveUseID: exclusiveUseID ) case .ordinaryCleanup: - beginOwnershipCleanupLocked( + authorized = beginOwnershipCleanupLocked( controlPath: controlPath, exclusiveUseID: exclusiveUseID ) } + guard authorized else { + releaseAuthenticationLock(descriptor) + _ = Darwin.close(descriptor) + return nil + } + return descriptor } - guard authorized else { - releaseAuthenticationLock(authenticationDescriptor) - _ = Darwin.close(authenticationDescriptor) - return nil - } + guard let authenticationDescriptor else { return nil } return NativeSSHControlMasterExclusiveUseAuthorization { [self] in finishExclusiveUse( @@ -245,9 +248,9 @@ final class NativeSSHControlMasterOwnershipRegistry: } else { removeEntryLocked(controlPath) } + releaseAuthenticationLock(authenticationDescriptor) + _ = Darwin.close(authenticationDescriptor) } - releaseAuthenticationLock(authenticationDescriptor) - _ = Darwin.close(authenticationDescriptor) } private func removeLeaseLocked( @@ -296,7 +299,8 @@ final class NativeSSHControlMasterOwnershipRegistry: return descriptor } - /// Matches the POSIX record locks used by zsh's `zsystem flock`. + /// Conflicts with zsh's POSIX `zsystem flock`, but stays bound to this + /// descriptor so another registry in the same process cannot release it. private func acquireAuthenticationLock(_ descriptor: Int32) -> Bool { var fileLock = Darwin.flock( l_start: 0, @@ -305,7 +309,7 @@ final class NativeSSHControlMasterOwnershipRegistry: l_type: Int16(F_WRLCK), l_whence: Int16(SEEK_SET) ) - return fcntl(descriptor, F_SETLK, &fileLock) == 0 + return fcntl(descriptor, F_OFD_SETLK, &fileLock) == 0 } private func releaseAuthenticationLock(_ descriptor: Int32) { @@ -316,6 +320,6 @@ final class NativeSSHControlMasterOwnershipRegistry: l_type: Int16(F_UNLCK), l_whence: Int16(SEEK_SET) ) - _ = fcntl(descriptor, F_SETLK, &fileLock) + _ = fcntl(descriptor, F_OFD_SETLK, &fileLock) } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift index 0a7406174b90..29bc7e2e1703 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Hosting/RemoteSessionStrings.swift @@ -14,6 +14,8 @@ public struct RemoteSessionStrings: Sendable, Equatable { public let reverseRelayUnavailableRetrying: String /// Retry detail when a live owner prevents relay-port recovery. public let reverseRelayPortUnavailableRetrying: String + /// Detail when another cmux process temporarily owns the SSH master. + public let controlMasterOwnershipUnavailable: String /// Creates the app-resolved strings bundle. /// @@ -22,11 +24,13 @@ public struct RemoteSessionStrings: Sendable, Equatable { /// - suspendedDetailFormat: Suspended reconnect detail format. /// - reverseRelayUnavailableRetrying: Generic relay retry detail. /// - reverseRelayPortUnavailableRetrying: Port-conflict retry detail. + /// - controlMasterOwnershipUnavailable: Cross-process ownership detail. public init( connectedVMNoProxyFormat: String, suspendedDetailFormat: String, reverseRelayUnavailableRetrying: String, - reverseRelayPortUnavailableRetrying: String + reverseRelayPortUnavailableRetrying: String, + controlMasterOwnershipUnavailable: String ) { self.connectedVMNoProxyFormat = connectedVMNoProxyFormat self.suspendedDetailFormat = suspendedDetailFormat @@ -34,5 +38,7 @@ public struct RemoteSessionStrings: Sendable, Equatable { reverseRelayUnavailableRetrying self.reverseRelayPortUnavailableRetrying = reverseRelayPortUnavailableRetrying + self.controlMasterOwnershipUnavailable = + controlMasterOwnershipUnavailable } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 3137b62c2741..6f8bcd2e3aaa 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -208,14 +208,6 @@ extension RemoteSessionCoordinator { let detail = stderrDetail ?? "status=\(process.terminationStatus)" debugLog("remote.relay.exit \(detail)") - if let relayPort = configuration.relayPort, - beginConflictedControlMasterExitIfNeededLocked( - startupFailure: detail, - remotePath: remotePath, - relayPort: relayPort - ) { - return - } publishReverseRelayFailureLocked(remotePath: remotePath) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index 2b3c25c37f42..e1b267860fdd 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -12,18 +12,33 @@ enum ReverseRelayControlMasterStartOutcome: Sendable { extension RemoteSessionCoordinator { /// Matches the connection-sharing defaults used by foreground authentication. var reverseRelayControlMasterSSHOptions: [String] { - SSHConnectionSharingOptions().mergingDefaults( + connectionBroker.sharingOptions.mergingDefaults( into: configuration.sshOptions ) } + /// Claims the exact cmux-owned socket before background SSH can reuse it. + func prepareControlMasterOwnershipLocked() throws { + guard configuration.transport != .ssh || + resolvedControlMasterSSHOptionsLocked() != nil else { + throw NSError( + domain: "cmux.remote.control-master", + code: 1, + userInfo: [ + NSLocalizedDescriptionKey: + strings.controlMasterOwnershipUnavailable, + ] + ) + } + } + /// Prefers the already-authenticated shared transport without creating one. func startReverseRelayViaControlMasterLocked( forwardSpec: String, relayPort: Int ) -> ReverseRelayControlMasterStartOutcome { guard let effectiveSSHOptions = - resolvedReverseRelayControlMasterSSHOptionsLocked() else { + resolvedControlMasterSSHOptionsLocked() else { return .unavailable } guard let arguments = configuration.reverseRelayControlMasterArguments( @@ -75,7 +90,7 @@ extension RemoteSessionCoordinator { guard let forwardSpec = reverseRelayControlMasterForwardSpec else { return } reverseRelayControlMasterForwardSpec = nil guard let effectiveSSHOptions = - reverseRelayResolvedControlMasterSSHOptions else { + resolvedControlMasterSSHOptions else { return } guard let arguments = configuration.reverseRelayControlMasterArguments( @@ -88,19 +103,19 @@ extension RemoteSessionCoordinator { _ = try? sshExec(arguments: arguments, timeout: 4) } - /// Resolves cmux's `%C` template before adopting a shared master. + /// Resolves cmux's `%C` template before any background SSH command can + /// adopt the shared master. /// - /// The exact socket path is both the command target and the reset-event - /// identity. If OpenSSH cannot produce that identity, relay startup falls - /// back to its standalone transport without touching the unresolved - /// master. - private func resolvedReverseRelayControlMasterSSHOptionsLocked() -> [String]? { - if let reverseRelayResolvedControlMasterSSHOptions { - let sharingOptions = SSHConnectionSharingOptions() + /// The exact socket path is both the process-ownership lease and reset + /// identity. Custom paths remain user-managed. If OpenSSH cannot resolve a + /// cmux-owned path, the connection attempt fails before daemon bootstrap. + func resolvedControlMasterSSHOptionsLocked() -> [String]? { + if let resolvedControlMasterSSHOptions { + let sharingOptions = connectionBroker.sharingOptions guard let resolvedPath = sharingOptions.cmuxOwnedControlPath( - in: reverseRelayResolvedControlMasterSSHOptions + in: resolvedControlMasterSSHOptions ) else { - return reverseRelayResolvedControlMasterSSHOptions + return resolvedControlMasterSSHOptions } guard connectionBroker.retainResolvedControlMasterLease( for: configuration, @@ -108,18 +123,18 @@ extension RemoteSessionCoordinator { ) else { return nil } - return reverseRelayResolvedControlMasterSSHOptions + return resolvedControlMasterSSHOptions } let effectiveOptions = reverseRelayControlMasterSSHOptions - let sharingOptions = SSHConnectionSharingOptions() + let sharingOptions = connectionBroker.sharingOptions let resolver = NativeSSHControlPathResolver( sharingOptions: sharingOptions ) guard let ownedPath = sharingOptions.cmuxOwnedControlPath( in: effectiveOptions ) else { - reverseRelayResolvedControlMasterSSHOptions = effectiveOptions + resolvedControlMasterSSHOptions = effectiveOptions return effectiveOptions } @@ -183,7 +198,7 @@ extension RemoteSessionCoordinator { return nil } conflictedControlMasterResetObservation = observation - reverseRelayResolvedControlMasterSSHOptions = resolvedOptions + resolvedControlMasterSSHOptions = resolvedOptions return resolvedOptions } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index 5b94196818a7..a9f925ef1c66 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -83,7 +83,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? - var reverseRelayResolvedControlMasterSSHOptions: [String]? + var resolvedControlMasterSSHOptions: [String]? var conflictedControlMasterResetObservation: NativeSSHControlMasterResetObservation? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] @@ -245,7 +245,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { func beginConnectionAttemptLocked() { guard !isStopping else { return } - Self.killOrphanedRemoteSSHProcesses( destination: configuration.destination, relayPort: configuration.relayPort, @@ -279,6 +278,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { publishState(connectionState, detail: connectDetail) publishDaemonStatus(.bootstrapping, detail: bootstrapDetail) do { + try prepareControlMasterOwnershipLocked() let requiredCapabilities = requiredDaemonCapabilities let hello: DaemonHello if configuration.skipDaemonBootstrap { diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index 3184a01caf69..7c079e9d0904 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -68,6 +68,30 @@ struct NativeSSHConnectionBrokerTests { #expect(recorder.requests.isEmpty) } + @Test("An exact cmux ControlPath is owned as soon as the workspace is retained") + func exactPathRetainsProcessOwnershipImmediately() { + let registry = + PermissiveNativeSSHControlMasterOwnershipRegistry() + let broker = NativeSSHConnectionBroker( + sharingOptions: sharingOptions, + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + controlMasterOwnershipRegistry: registry + ) + let expectedPath = + "/tmp/cmux-ssh-501-" + + "0123456789abcdef0123456789abcdef01234567" + + let lease = broker.retainWorkspace(configuration( + owner: UUID(), + sshOptions: resolvedOwnedSSHOptions + )) + + #expect(registry.retainedControlPaths == [expectedPath]) + broker.releaseWorkspace(lease) + } + @Test("Unresolved templates authorize reset but not last-owner cleanup") func unresolvedTemplatesOnlyAuthorizeReset() { let recorder = CleanupRequestRecorder() diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift index 0e2ad8a1c7a3..2c7fe113d610 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift @@ -165,6 +165,53 @@ struct NativeSSHControlMasterOwnershipRegistryTests { cleanup.release() } + @Test("A rejected local operation preserves the active authentication lock") + func rejectedOperationPreservesAuthenticationLock() throws { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-control-exclusive-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: scratchDirectory) } + let sharingOptions = SSHConnectionSharingOptions( + userID: Int(getuid()), + authenticationLockDirectoryPath: scratchDirectory.path + ) + let first = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let second = NativeSSHControlMasterOwnershipRegistry( + sharingOptions: sharingOptions + ) + let controlPath = resolvedControlPath(userID: Int(getuid())) + let authenticationPath = try #require( + sharingOptions.resolvedControlMasterAuthenticationLockPath( + controlPath: controlPath + ) + ) + + let reset = try #require( + first.beginReset(controlPath: controlPath) + ) + #expect(second.beginCleanup(controlPath: controlPath) == nil) + #expect( + try childCanAcquireAuthenticationLock( + at: authenticationPath + ) == false + ) + + reset.release() + let cleanup = try #require( + second.beginCleanup(controlPath: controlPath) + ) + cleanup.release() + #expect( + try childCanAcquireAuthenticationLock( + at: authenticationPath + ) + ) + } + @Test("Authorization deinit restores the process shared lease") func authorizationDeinitRestoresSharedLease() throws { let scratchDirectory = FileManager.default.temporaryDirectory @@ -205,4 +252,26 @@ struct NativeSSHControlMasterOwnershipRegistryTests { private func resolvedControlPath(userID: Int) -> String { "/tmp/cmux-ssh-\(userID)-0123456789abcdef0123456789abcdef01234567" } + + private func childCanAcquireAuthenticationLock( + at path: String + ) throws -> Bool { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/zsh") + process.arguments = [ + "-fc", + """ + zmodload zsh/system || exit 1 + zsystem flock -t 0 -e "$1" + """, + "cmux-auth-lock-probe", + path, + ] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + return process.terminationStatus == 0 + } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift index 66669bac02f3..0fcf3e1be256 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift @@ -93,7 +93,7 @@ final class FixedStatusResetRunner: RemoteSessionProcessRunning, @unchecked Sendable { - // lint:allow lock - process calls increment one test counter. + // lint:allow lock - guards one counter and the broadcast test gate. private let lock = NSLock() private let status: Int32 private var count = 0 @@ -174,7 +174,8 @@ final class BlockingResolvingResetRunner: private let resolvedPath: String private let resolutionContinuation: AsyncStream.Continuation private let exitContinuation: AsyncStream.Continuation - private let exitRelease = DispatchSemaphore(value: 0) + private let exitCondition = NSCondition() + private var exitFinished = false private var _exitCount = 0 init(resolvedPath: String) { @@ -205,12 +206,19 @@ final class BlockingResolvingResetRunner: _exitCount += 1 } exitContinuation.yield() - exitRelease.wait() + exitCondition.lock() + while !exitFinished { + exitCondition.wait() + } + exitCondition.unlock() } return RemoteCommandResult(status: 0, stdout: "", stderr: "") } func finishExit() { - exitRelease.signal() + exitCondition.lock() + exitFinished = true + exitCondition.broadcast() + exitCondition.unlock() } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift index 0ee1fd6227d6..2dc6c0b7ef9f 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift @@ -4,13 +4,24 @@ import Foundation /// Test fake that authorizes ownership without opening process-global locks. final class PermissiveNativeSSHControlMasterOwnershipRegistry: NativeSSHControlMasterOwnershipTracking, - Sendable + @unchecked Sendable { + // lint:allow lock - test assertions read the retained-path snapshot. + private let lock = NSLock() + private var _retainedControlPaths: [String] = [] + + var retainedControlPaths: [String] { + lock.withLock { _retainedControlPaths } + } + func retain( controlPath: String, lease: NativeSSHControlMasterLeaseIdentity ) -> Bool { - true + lock.withLock { + _retainedControlPaths.append(controlPath) + } + return true } func release(lease: NativeSSHControlMasterLeaseIdentity) {} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift index 0383ae927c43..778155deed36 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteDaemonUploadTests.swift @@ -372,7 +372,8 @@ struct RemoteDaemonUploadTests { connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@", reverseRelayUnavailableRetrying: "", - reverseRelayPortUnavailableRetrying: "" + reverseRelayPortUnavailableRetrying: "", + controlMasterOwnershipUnavailable: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift index bef4deff5816..3a33c51f2032 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePTYIntentionalCleanupTests.swift @@ -136,7 +136,8 @@ struct RemotePTYIntentionalCleanupTests { connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@", reverseRelayUnavailableRetrying: "", - reverseRelayPortUnavailableRetrying: "" + reverseRelayPortUnavailableRetrying: "", + controlMasterOwnershipUnavailable: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift index f72f5ea14901..73d6d0eac02b 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemotePortScanGatingTests.swift @@ -343,7 +343,8 @@ struct RemotePortScanGatingTests { connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@", reverseRelayUnavailableRetrying: "", - reverseRelayPortUnavailableRetrying: "" + reverseRelayPortUnavailableRetrying: "", + controlMasterOwnershipUnavailable: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift index 3d50989fc643..2d1083b41b9a 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteReconnectPolicyTests.swift @@ -314,7 +314,8 @@ struct RemoteReconnectPolicyTests { connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@", reverseRelayUnavailableRetrying: "", - reverseRelayPortUnavailableRetrying: "" + reverseRelayPortUnavailableRetrying: "", + controlMasterOwnershipUnavailable: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift index 440c03dbcc49..9728b6b89386 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift @@ -401,7 +401,8 @@ struct RemoteRelaySlotTeardownTests { connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@", reverseRelayUnavailableRetrying: "", - reverseRelayPortUnavailableRetrying: "" + reverseRelayPortUnavailableRetrying: "", + controlMasterOwnershipUnavailable: "" ) ) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 6e4cfcb0d4b4..9dc09f2cbf28 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -170,7 +170,9 @@ struct RemoteSessionReverseRelayStartupTests { relayPort: Int = 64_044, sshOptions: [String]? = nil, clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock(), - providesResolvedControlPath: Bool = true + providesResolvedControlPath: Bool = true, + ownershipRegistry: any NativeSSHControlMasterOwnershipTracking = + PermissiveNativeSSHControlMasterOwnershipRegistry() ) throws -> (coordinator: RemoteSessionCoordinator, scratchDirectory: URL) { let scratchDirectory = FileManager.default.temporaryDirectory .appendingPathComponent( @@ -208,8 +210,7 @@ struct RemoteSessionReverseRelayStartupTests { jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, conflictedMasterResetRunner: effectiveRunner, - controlMasterOwnershipRegistry: - PermissiveNativeSSHControlMasterOwnershipRegistry() + controlMasterOwnershipRegistry: ownershipRegistry ) let configuration = connectionBroker.retainWorkspace(rawConfiguration) let coordinator = RemoteSessionCoordinator( @@ -235,7 +236,9 @@ struct RemoteSessionReverseRelayStartupTests { reverseRelayUnavailableRetrying: "test relay unavailable", reverseRelayPortUnavailableRetrying: - "test relay port unavailable" + "test relay port unavailable", + controlMasterOwnershipUnavailable: + "test control master unavailable" ), clock: clock ) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index 0380bf08fe04..b795ab485f59 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -53,6 +53,47 @@ struct RemoteSessionReverseRelayTransportTests { ) } + @Test("Connection preparation owns the resolved path before shared SSH use") + func connectionPreparationRetainsResolvedPath() async throws { + let runner = RecordingProcessRunner { request in + if request.arguments.first == "-G" { + return RemoteCommandResult( + status: 0, + stdout: "controlpath \(ResolvedControlPathFixture.path)\n", + stderr: "" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + let registry = PermissiveNativeSSHControlMasterOwnershipRegistry() + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator( + runner: runner, + providesResolvedControlPath: false, + ownershipRegistry: registry + ) + let coordinator = fixture.coordinator + defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } + #expect(registry.retainedControlPaths.isEmpty) + + try coordinator.queue.sync { + try coordinator.prepareControlMasterOwnershipLocked() + } + let options = coordinator.queue.sync { + coordinator.resolvedControlMasterSSHOptions + } + + #expect(options?.contains( + "ControlPath=\(ResolvedControlPathFixture.path)" + ) == true) + #expect( + registry.retainedControlPaths == + [ResolvedControlPathFixture.path] + ) + #expect(runner.requests.first?.arguments.first == "-G") + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test("An explicitly disabled ControlMaster uses the standalone fallback") func disabledControlMasterUsesStandaloneFallback() async throws { let runner = RecordingProcessRunner() @@ -374,8 +415,8 @@ struct RemoteSessionReverseRelayTransportTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test("A late standalone bind failure triggers owned-master recovery") - func lateStandaloneConflictTriggersRecovery() async throws { + @Test("A standalone bind failure never exits a shared master") + func standaloneConflictFailsClosed() async throws { let relayPort = 64_047 let clock = ManualBrokerClock() let runner = RecordingProcessRunner { request in @@ -422,7 +463,7 @@ struct RemoteSessionReverseRelayTransportTests { #expect(await clock.nextRequestedDelay() == 2_000) coordinator.queue.sync {} #expect(!runner.requests.contains(where: Self.isMetadataInstallRequest)) - #expect(runner.requests.contains(where: { + #expect(!runner.requests.contains(where: { Self.isControlCommand("exit", in: $0.arguments) })) #expect(coordinator.queue.sync { diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift index 146c24691ff3..f1c5d59fdc99 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionSSHRemoteCommandOverrideTests.swift @@ -180,7 +180,8 @@ struct RemoteSessionSSHRemoteCommandOverrideTests { connectedVMNoProxyFormat: "%@", suspendedDetailFormat: "%@", reverseRelayUnavailableRetrying: "", - reverseRelayPortUnavailableRetrying: "" + reverseRelayPortUnavailableRetrying: "", + controlMasterOwnershipUnavailable: "" ) ) } diff --git a/Sources/RemoteSessionStrings+App.swift b/Sources/RemoteSessionStrings+App.swift index 7a64d4a7d806..0a10c62e9cb6 100644 --- a/Sources/RemoteSessionStrings+App.swift +++ b/Sources/RemoteSessionStrings+App.swift @@ -27,6 +27,12 @@ extension RemoteSessionStrings { "remoteSession.reverseRelay.portUnavailableRetrying", defaultValue: "Remote SSH relay port unavailable; retrying in 2 seconds" + ), + controlMasterOwnershipUnavailable: String( + localized: + "remoteSession.controlMaster.ownershipUnavailable", + defaultValue: + "SSH connection is busy in another cmux process." ) ) } From 531388f5ed8e72e94e38016fbdcfe8c699e87046 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 04:24:48 -0700 Subject: [PATCH 26/39] fix: bound reverse relay termination --- .../FoundationRemoteReverseRelayProcess.swift | 18 ++++++- ...dationRemoteReverseRelayProcessTests.swift | 52 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift index dd5865b96009..81e0527799a7 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -1,3 +1,4 @@ +internal import Darwin internal import Foundation /// Foundation-backed handle for one dedicated SSH reverse-relay process. @@ -12,15 +13,18 @@ final class FoundationRemoteReverseRelayProcess: private let process: Process private let stderrPipe: Pipe private let stderrDrainGracePeriod: TimeInterval + private let terminationGracePeriod: TimeInterval init( process: Process, stderrPipe: Pipe, - stderrDrainGracePeriod: TimeInterval = 0.5 + stderrDrainGracePeriod: TimeInterval = 0.5, + terminationGracePeriod: TimeInterval = 2 ) { self.process = process self.stderrPipe = stderrPipe self.stderrDrainGracePeriod = stderrDrainGracePeriod + self.terminationGracePeriod = terminationGracePeriod } var isRunning: Bool { @@ -93,7 +97,19 @@ final class FoundationRemoteReverseRelayProcess: } func terminate() { + guard process.isRunning else { return } + let process = process + let processID = process.processIdentifier process.terminate() + DispatchQueue.global(qos: .utility).asyncAfter( + deadline: .now() + max(0, terminationGracePeriod) + ) { + guard process.isRunning, + process.processIdentifier == processID else { + return + } + _ = Darwin.kill(processID, SIGKILL) + } } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift index 94152a763131..e815e76adfba 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift @@ -1,4 +1,5 @@ import CmuxFoundation +import Darwin import Foundation import Testing @testable import CmuxRemoteSession @@ -168,4 +169,55 @@ struct FoundationRemoteReverseRelayProcessTests { #expect(Date().timeIntervalSince(startedAt) < 1) #expect(!process.isRunning) } + + @Test("Startup deadline force-kills a process that ignores SIGTERM") + func startupDeadlineForceKillsAfterGracePeriod() async throws { + let process = Process() + let stdoutPipe = Pipe() + let stderrPipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + """ + trap '' TERM + printf 'ready\\n' + while :; do :; done + """, + ] + process.standardInput = FileHandle.nullDevice + process.standardOutput = stdoutPipe + process.standardError = stderrPipe + let relayProcess = FoundationRemoteReverseRelayProcess( + process: process, + stderrPipe: stderrPipe, + stderrDrainGracePeriod: 0.05, + terminationGracePeriod: 0.05 + ) + let startupRecorder = ResetEventRecorder() + let (terminations, continuation) = AsyncStream.makeStream() + + try process.run() + let readiness = stdoutPipe.fileHandleForReading.readData(ofLength: 6) + #expect(String(data: readiness, encoding: .utf8) == "ready\n") + relayProcess.captureLifecycle( + startupMarker: "marker-that-never-arrives", + startupTimeout: 0.05, + startupHandler: { + startupRecorder.record() + }, + terminationHandler: { detail in + continuation.yield(detail) + continuation.finish() + } + ) + let startedAt = Date() + + var iterator = terminations.makeAsyncIterator() + #expect(await iterator.next() != nil) + #expect(startupRecorder.count == 0) + #expect(Date().timeIntervalSince(startedAt) < 1) + #expect(!process.isRunning) + #expect(process.terminationReason == .uncaughtSignal) + #expect(process.terminationStatus == SIGKILL) + } } From bff27d0e65cf3478b04a85d6db21232c92ceb30a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 05:00:39 -0700 Subject: [PATCH 27/39] fix: preserve SSH master recovery identity --- .../SSHAgentSocketResolver.swift | 12 +- .../SSHConnectionSharingOptions.swift | 45 ++---- .../SSHConnectionSharingOptionsTests.swift | 20 +++ ...nectionBroker+ControlMasterOwnership.swift | 1 + .../NativeSSHConnectionBroker.swift | 8 +- ...ativeSSHControlMasterAdoptionHandoff.swift | 28 +++- ...tiveSSHControlMasterResetCoordinator.swift | 82 ++-------- ...emoteSessionCoordinator+ReverseRelay.swift | 5 +- ...oordinator+ReverseRelayControlMaster.swift | 15 +- ...ssionCoordinator+ReverseRelayStartup.swift | 9 +- .../BlockingControlMasterResetRunner.swift | 14 +- .../NativeSSHConnectionBrokerTests.swift | 46 ------ ...SSHControlMasterAdoptionHandoffTests.swift | 32 ++++ ...HControlMasterOwnershipRecoveryTests.swift | 11 +- ...tiveSSHControlMasterResetTestSupport.swift | 102 ------------ .../NativeSSHControlMasterResetTests.swift | 151 +++++++----------- ...emoteSessionReverseRelayStartupTests.swift | 15 +- 17 files changed, 225 insertions(+), 371 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift diff --git a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHAgentSocketResolver.swift b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHAgentSocketResolver.swift index 180ecf0f7535..29619a3359a6 100644 --- a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHAgentSocketResolver.swift +++ b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHAgentSocketResolver.swift @@ -35,18 +35,18 @@ public struct SSHAgentSocketResolver: Sendable { .lowercased() } - /// Reads the last non-empty value for an OpenSSH-style option. + /// Reads the first non-empty value for an OpenSSH-style option. /// - /// OpenSSH applies the last repeated `-o` value, so this method scans in - /// reverse order. + /// OpenSSH keeps the first value obtained for command-line configuration, + /// so later duplicate `-o` values do not override the first. /// /// - Parameters: /// - key: The option key to read. /// - options: Option strings such as `ForwardAgent=yes`. - /// - Returns: The last non-empty matching option value, or `nil`. + /// - Returns: The first non-empty matching option value, or `nil`. public func optionValue(named key: String, in options: [String]) -> String? { let loweredKey = key.lowercased() - for option in options.reversed() { + for option in options { let trimmed = option.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { continue } let parts = trimmed.split( @@ -100,7 +100,7 @@ public struct SSHAgentSocketResolver: Sendable { return normalizedOptional((trimmed as NSString).expandingTildeInPath) ?? trimmed } - /// Resolves the last `ForwardAgent` option into an agent socket path candidate. + /// Resolves the effective `ForwardAgent` option into an agent socket path candidate. /// /// - Parameter options: OpenSSH-style option strings. /// - Returns: A socket path candidate, or `nil` when no usable `ForwardAgent` value exists. diff --git a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift index a76d564b2e89..f64b5c37bf6e 100644 --- a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift +++ b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SSHConnectionSharingOptions.swift @@ -94,10 +94,16 @@ public struct SSHConnectionSharingOptions: Sendable { } } } - let controlMaster = firstOptionValue(named: "ControlMaster", in: merged) + let controlMaster = resolver.optionValue( + named: "ControlMaster", + in: merged + ) let controlMasterDisabled = isDisabled(controlMaster) if !controlMasterDisabled, - let controlPath = firstOptionValue(named: "ControlPath", in: merged), + let controlPath = resolver.optionValue( + named: "ControlPath", + in: merged + ), isLegacyRelayScopedControlPath(controlPath) { merged = merged.map { option in guard resolver.optionKey(option) == "controlpath" else { return option } @@ -163,10 +169,17 @@ public struct SSHConnectionSharingOptions: Sendable { /// - Parameter options: OpenSSH `-o` values to inspect. /// - Returns: The cmux-owned path, or `nil` for user-managed paths. public func cmuxOwnedControlPath(in options: [String]) -> String? { - guard !isDisabled(firstOptionValue(named: "ControlMaster", in: options)) else { + let resolver = SSHAgentSocketResolver() + guard !isDisabled(resolver.optionValue( + named: "ControlMaster", + in: options + )) else { return nil } - guard let rawPath = firstOptionValue(named: "ControlPath", in: options) else { + guard let rawPath = resolver.optionValue( + named: "ControlPath", + in: options + ) else { return nil } let path = rawPath.trimmingCharacters(in: .whitespacesAndNewlines) @@ -354,30 +367,6 @@ public struct SSHConnectionSharingOptions: Sendable { return ["no", "false", "off", "0"].contains(value) } - /// OpenSSH keeps the first value obtained for command-line configuration - /// options. Ownership decisions must inspect the same value that `ssh` - /// will use, especially when callers supplied a duplicate option. - private func firstOptionValue(named key: String, in options: [String]) -> String? { - let loweredKey = key.lowercased() - for option in options { - let trimmed = option.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } - let parts = trimmed.split( - maxSplits: 1, - omittingEmptySubsequences: true, - whereSeparator: { $0 == "=" || $0.isWhitespace } - ) - guard parts.count == 2, parts[0].lowercased() == loweredKey else { - continue - } - let value = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) - if !value.isEmpty { - return value - } - } - return nil - } - private func shellQuote(_ value: String) -> String { let safePattern = "^[A-Za-z0-9_@%+=:,./-]+$" if value.range(of: safePattern, options: .regularExpression) != nil { diff --git a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift index 945048ad432c..2326b100a16c 100644 --- a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift +++ b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/SSHConnectionSharingOptionsTests.swift @@ -85,6 +85,26 @@ struct SSHConnectionSharingOptionsTests { ) } + @Test("Shared option parsing follows OpenSSH's first-value rule") + func optionResolverUsesFirstValue() { + let resolver = SSHAgentSocketResolver() + + #expect(resolver.optionValue( + named: "ControlMaster", + in: [ + "ControlMaster=no", + "ControlMaster=auto", + ] + ) == "no") + #expect(resolver.optionValue( + named: "ControlPersist", + in: [ + "ControlPersist=600", + "ControlPersist=no", + ] + ) == "600") + } + @Test("Effective custom ssh_config control settings replace cmux defaults") func preservesResolvedSSHConfigSettings() { let output = """ diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift index b8ff603a9ebc..fe831e6ebab4 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift @@ -32,6 +32,7 @@ extension NativeSSHConnectionBroker { return NativeSSHControlMasterAdoptionHandoff( controlPath: controlPath, lease: lease, + clock: clock, releaseHandler: { ownershipRegistry.release(lease: lease) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 6526da2488a5..9bf65f85b8b0 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -220,9 +220,13 @@ public final class NativeSSHConnectionBroker { /// Coalesces an inherited-master reset after OpenSSH confirms a relay bind conflict. func resetConflictedControlMaster( - for configuration: WorkspaceRemoteConfiguration + for configuration: WorkspaceRemoteConfiguration, + resolvedControlPath: String ) async -> NativeSSHControlMasterResetOutcome { - await conflictedMasterResetCoordinator.reset(for: configuration) + await conflictedMasterResetCoordinator.reset( + for: configuration, + resolvedControlPath: resolvedControlPath + ) } /// Observes resets that invalidate an exact cmux-owned control socket. diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift index 280d27a11bef..31123189e348 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift @@ -1,8 +1,10 @@ +internal import CmuxRemoteWorkspace internal import Foundation /// A temporary ownership lease that bridges foreground SSH authentication to -/// installation of the workspace's durable ControlMaster lease. -// SAFETY: `lock` serializes every read and mutation of the release closure. +/// installation of the workspace's durable ControlMaster lease. Unconsumed +/// handoffs expire so an interrupted restore cannot retain ownership forever. +// SAFETY: `lock` serializes the release closure and expiration task. public final class NativeSSHControlMasterAdoptionHandoff: @unchecked Sendable, Equatable @@ -12,22 +14,38 @@ public final class NativeSSHControlMasterAdoptionHandoff: // lint:allow lock - transfer, cancellation, and deinit race to release once. private let lock = NSLock() private var releaseHandler: (@Sendable () -> Void)? + private var expirationTask: Task? = nil init( controlPath: String, lease: NativeSSHControlMasterLeaseIdentity, + clock: any RemoteProxyRetryClock, + expirationMilliseconds: Int = 30_000, releaseHandler: @escaping @Sendable () -> Void ) { self.controlPath = controlPath self.lease = lease self.releaseHandler = releaseHandler + self.expirationTask = Task { [weak self, clock] in + do { + try await clock.sleep( + forMilliseconds: expirationMilliseconds + ) + } catch { + return + } + self?.release() + } } func release() { - let handler = lock.withLock { - defer { releaseHandler = nil } - return releaseHandler + let (handler, expirationTask) = lock.withLock { + let result = (releaseHandler, self.expirationTask) + releaseHandler = nil + self.expirationTask = nil + return result } + expirationTask?.cancel() handler?() } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift index 95843d8b3e3e..c4b958e7ee1d 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift @@ -77,7 +77,8 @@ final class NativeSSHControlMasterResetCoordinator { } func reset( - for configuration: WorkspaceRemoteConfiguration + for configuration: WorkspaceRemoteConfiguration, + resolvedControlPath: String ) async -> NativeSSHControlMasterResetOutcome { guard let ownerWorkspaceID = configuration.ownerWorkspaceID, let generation = configuration.sshControlMasterLeaseGeneration, @@ -93,39 +94,22 @@ final class NativeSSHControlMasterResetCoordinator { return .ignored("workspace no longer owns this cmux SSH master") } + guard !resolvedControlPath.contains("%"), + sharingOptions.cmuxOwnedControlPath(in: [ + "ControlMaster=auto", + "ControlPath=\(resolvedControlPath)", + ]) == resolvedControlPath else { + return .ignored("could not identify the cmux SSH master socket") + } let effectiveOptions = sharingOptions.mergingDefaults( into: configuration.sshOptions ) let pathResolver = NativeSSHControlPathResolver( sharingOptions: sharingOptions ) - let resolvedControlPath: String? - if let exactPath = pathResolver.resolvedControlPath( - effectiveOptions: effectiveOptions - ) { - resolvedControlPath = exactPath - } else { - let resolution = await Self.resolveControlPath( - configuration: configuration, - effectiveOptions: effectiveOptions, - resolver: pathResolver, - processRunner: processRunner - ) - switch resolution { - case .resolved(let path): - resolvedControlPath = path - case .unavailable: - return .ignored("could not resolve the cmux SSH master socket") - case .retry(let detail): - return .deferred(detail) - } - } guard !Task.isCancelled else { return .deferred("control-master reset cancelled") } - guard let resolvedControlPath else { - return .ignored("could not resolve the cmux SSH master socket") - } guard ownsLease( ownerWorkspaceID: ownerWorkspaceID, generation: generation, @@ -204,54 +188,6 @@ final class NativeSSHControlMasterResetCoordinator { leases[ownerWorkspaceID]?[key]?.sshControlMasterLeaseGeneration == generation } - private nonisolated static func resolveControlPath( - configuration: WorkspaceRemoteConfiguration, - effectiveOptions: [String], - resolver: NativeSSHControlPathResolver, - processRunner: any RemoteSessionProcessRunning - ) async -> ControlPathResolutionOutcome { - let cancellation = RemoteProcessCancellationOperation() - return await withTaskCancellationHandler { - await withCheckedContinuation { continuation in - DispatchQueue.global(qos: .utility).async { - let request = RemoteProcessRequest( - executable: "/usr/bin/ssh", - arguments: resolver.resolutionArguments( - configuration: configuration, - effectiveOptions: effectiveOptions - ), - environment: configuration.sshProcessEnvironment, - timeout: 5 - ) - do { - let result = try processRunner.run( - request, - operation: cancellation - ) - guard result.status == 0 else { - continuation.resume(returning: .unavailable) - return - } - if let path = resolver.resolvedControlPath( - effectiveOptions: effectiveOptions, - sshConfigOutput: result.stdout - ) { - continuation.resume(returning: .resolved(path)) - } else { - continuation.resume(returning: .unavailable) - } - } catch { - continuation.resume(returning: .retry( - error.localizedDescription - )) - } - } - } - } onCancel: { - cancellation.cancel() - } - } - private nonisolated static func runReset( request: NativeSSHControlMasterCleanupRequest, processRunner: any RemoteSessionProcessRunning, diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 6f8bcd2e3aaa..50013fdff470 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -91,14 +91,15 @@ extension RemoteSessionCoordinator { "target=\(configuration.displayTarget) controlMaster=1" ) return - case .bindingConflict(let detail): + case .bindingConflict(let detail, let controlPath): debugLog( "remote.relay.startFailed relayPort=\(relayPort) error=\(detail)" ) if beginConflictedControlMasterExitIfNeededLocked( startupFailure: detail, remotePath: remotePath, - relayPort: relayPort + relayPort: relayPort, + resolvedControlPath: controlPath ) { return } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index e1b267860fdd..c89e08c550cf 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -6,7 +6,7 @@ internal import Foundation enum ReverseRelayControlMasterStartOutcome: Sendable { case started case unavailable - case bindingConflict(String) + case bindingConflict(String, controlPath: String?) } extension RemoteSessionCoordinator { @@ -70,7 +70,18 @@ extension RemoteSessionCoordinator { debugConfigSummary() ) if let bindingConflict { - return .bindingConflict(bindingConflict) + let ownedControlPath = + connectionBroker.sharingOptions.cmuxOwnedControlPath( + in: effectiveSSHOptions + ) + let resolvedControlPath = + ownedControlPath?.contains("%") == false + ? ownedControlPath + : nil + return .bindingConflict( + bindingConflict, + controlPath: resolvedControlPath + ) } return .unavailable } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift index 5df34e6d9f8e..189409f179f8 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -12,7 +12,8 @@ extension RemoteSessionCoordinator { func beginConflictedControlMasterExitIfNeededLocked( startupFailure: String, remotePath: String, - relayPort: Int + relayPort: Int, + resolvedControlPath: String? ) -> Bool { guard Self.isReverseRelayPortBindingFailure( startupFailure, @@ -31,6 +32,9 @@ extension RemoteSessionCoordinator { ) return false } + guard let resolvedControlPath else { + return false + } let token = UUID() let configuration = self.configuration @@ -38,7 +42,8 @@ extension RemoteSessionCoordinator { let task = Task { [weak self] in let outcome = await connectionBroker.resetConflictedControlMaster( - for: configuration + for: configuration, + resolvedControlPath: resolvedControlPath ) guard !Task.isCancelled else { return } self?.queue.async { [weak self] in diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift index 4247b117cb89..0095a49e8395 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift @@ -9,7 +9,8 @@ final class BlockingControlMasterResetRunner: private let startsContinuation: AsyncStream.Continuation private let lock = NSLock() private var _requests: [RemoteProcessRequest] = [] - private let release = DispatchSemaphore(value: 0) + private let releaseCondition = NSCondition() + private var finished = false init() { (starts, startsContinuation) = AsyncStream.makeStream() @@ -25,12 +26,19 @@ final class BlockingControlMasterResetRunner: ) throws -> RemoteCommandResult { lock.withLock { _requests.append(request) } startsContinuation.yield() - release.wait() + releaseCondition.lock() + while !finished { + releaseCondition.wait() + } + releaseCondition.unlock() try operation?.throwIfCancelled() return RemoteCommandResult(status: 0, stdout: "", stderr: "") } func finish() { - release.signal() + releaseCondition.lock() + finished = true + releaseCondition.broadcast() + releaseCondition.unlock() } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index 7c079e9d0904..8c0b555413e2 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -190,52 +190,6 @@ struct NativeSSHConnectionBrokerTests { #expect(recorder.requests[1].arguments.contains(replacement.sshOptions[2])) } - @Test("Concurrent owners coalesce one conflicted-master reset") - func concurrentOwnersCoalesceConflictReset() async throws { - let runner = BlockingControlMasterResetRunner() - let broker = NativeSSHConnectionBroker( - sharingOptions: sharingOptions, - clock: RecordingImmediateClock(), - jitterMilliseconds: { 200 }, - cleanupLauncher: { _ in }, - conflictedMasterResetRunner: runner, - controlMasterOwnershipRegistry: - PermissiveNativeSSHControlMasterOwnershipRegistry() - ) - let firstLease = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "first-alias", - sshOptions: resolvedOwnedSSHOptions - )) - let secondLease = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "second-alias", - sshOptions: resolvedOwnedSSHOptions - )) - - let firstReset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: firstLease) - } - var starts = runner.starts.makeAsyncIterator() - _ = try #require(await starts.next()) - - let secondReset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: secondLease) - } - await Task.yield() - #expect(runner.requests.count == 1) - - runner.finish() - #expect(await firstReset.value == .reset) - #expect(await secondReset.value == .reset) - #expect(runner.requests.count == 1) - let request = try #require(runner.requests.first) - #expect(request.executable == "/usr/bin/ssh") - #expect(request.arguments.contains("-O")) - #expect(request.arguments.contains("exit")) - #expect(request.arguments.contains(resolvedOwnedSSHOptions[2])) - } - @Test("Cleanup reuses the shared path without negotiating a replacement master") func cleanupArgumentsAreReuseOnly() { let configuration = configuration( diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift new file mode 100644 index 000000000000..0856b5fc718e --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift @@ -0,0 +1,32 @@ +import Foundation +import Testing +@testable import CmuxRemoteSession + +@Suite("Native SSH ControlMaster adoption handoff") +struct NativeSSHControlMasterAdoptionHandoffTests { + @Test("An unconsumed handoff expires and releases ownership once") + func unconsumedHandoffExpires() async { + let clock = ManualBrokerClock() + let recorder = ResetEventRecorder() + let handoff = NativeSSHControlMasterAdoptionHandoff( + controlPath: "/tmp/cmux-ssh-501-test", + lease: NativeSSHControlMasterLeaseIdentity( + ownerWorkspaceID: UUID(), + generation: UUID() + ), + clock: clock, + expirationMilliseconds: 10, + releaseHandler: { + recorder.record() + } + ) + + #expect(await clock.nextRequestedDelay() == 10) + await clock.resumeNextSleep() + await Task.yield() + #expect(recorder.count == 1) + + handoff.release() + #expect(recorder.count == 1) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift index 131c23689c72..85a71a59ad9f 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift @@ -11,6 +11,8 @@ import Testing struct NativeSSHControlMasterOwnershipRecoveryTests { @Test("A live foreign owner prevents destructive reset") func foreignOwnerFailsClosed() async { + let controlPath = + "/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567" let runner = RecordingProcessRunner() let broker = NativeSSHConnectionBroker( sharingOptions: SSHConnectionSharingOptions(userID: 501), @@ -28,7 +30,7 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { sshOptions: [ "ControlMaster=auto", "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567", + "ControlPath=\(controlPath)", ], localProxyPort: nil, relayPort: 64_001, @@ -42,7 +44,10 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { )) guard case .deferred = - await broker.resetConflictedControlMaster(for: lease) else { + await broker.resetConflictedControlMaster( + for: lease, + resolvedControlPath: controlPath + ) else { Issue.record("Expected a live foreign owner to defer reset") return } @@ -69,7 +74,7 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { ) let broker = NativeSSHConnectionBroker( sharingOptions: sharingOptions, - clock: RecordingImmediateClock(), + clock: SystemRemoteProxyRetryClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, controlMasterOwnershipRegistry: firstRegistry diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift index 0fcf3e1be256..f9e195209f6d 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift @@ -120,105 +120,3 @@ final class FixedStatusResetRunner: ) } } - -final class ResolvingResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - // lint:allow lock - process calls append test request snapshots. - private let lock = NSLock() - private let pathsByDestination: [String: String] - private var requests: [RemoteProcessRequest] = [] - - init(pathsByDestination: [String: String]) { - self.pathsByDestination = pathsByDestination - } - - var exitRequests: [RemoteProcessRequest] { - lock.withLock { - requests.filter { - $0.arguments.contains("-O") && $0.arguments.contains("exit") - } - } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - lock.withLock { - requests.append(request) - } - if request.arguments.contains("-G"), - let destination = request.arguments.last, - let path = pathsByDestination[destination] { - return RemoteCommandResult( - status: 0, - stdout: "controlpath \(path)\n", - stderr: "" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } -} - -final class BlockingResolvingResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - let resolutions: AsyncStream - let exits: AsyncStream - - // lint:allow lock - process calls increment one test counter. - private let lock = NSLock() - private let resolvedPath: String - private let resolutionContinuation: AsyncStream.Continuation - private let exitContinuation: AsyncStream.Continuation - private let exitCondition = NSCondition() - private var exitFinished = false - private var _exitCount = 0 - - init(resolvedPath: String) { - self.resolvedPath = resolvedPath - (resolutions, resolutionContinuation) = AsyncStream.makeStream() - (exits, exitContinuation) = AsyncStream.makeStream() - } - - var exitCount: Int { - lock.withLock { _exitCount } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - if request.arguments.contains("-G") { - resolutionContinuation.yield() - return RemoteCommandResult( - status: 0, - stdout: "controlpath \(resolvedPath)\n", - stderr: "" - ) - } - if request.arguments.contains("-O"), - request.arguments.contains("exit") { - lock.withLock { - _exitCount += 1 - } - exitContinuation.yield() - exitCondition.lock() - while !exitFinished { - exitCondition.wait() - } - exitCondition.unlock() - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - - func finishExit() { - exitCondition.lock() - exitFinished = true - exitCondition.broadcast() - exitCondition.unlock() - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift index 07dcd8d3950d..6a20fc22fe0a 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift @@ -49,7 +49,10 @@ struct NativeSSHControlMasterResetTests { } ) - #expect(await broker.resetConflictedControlMaster(for: first) == .reset) + #expect(await broker.resetConflictedControlMaster( + for: first, + resolvedControlPath: firstResolvedPath + ) == .reset) #expect(recorder.count == 2) _ = firstObservation _ = secondObservation @@ -66,7 +69,10 @@ struct NativeSSHControlMasterResetTests { )) let reset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: lease) + await broker.resetConflictedControlMaster( + for: lease, + resolvedControlPath: firstResolvedPath + ) } #expect(await clock.nextRequestedDelay() == 2_000) await clock.resumeNextSleep() @@ -90,7 +96,10 @@ struct NativeSSHControlMasterResetTests { )) let reset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: lease) + await broker.resetConflictedControlMaster( + for: lease, + resolvedControlPath: firstResolvedPath + ) } #expect(await clock.nextRequestedDelay() == 2_000) await clock.resumeNextSleep() @@ -105,10 +114,7 @@ struct NativeSSHControlMasterResetTests { func expandedPathsDoNotNotifyDifferentHost() async throws { let firstRecorder = ResetEventRecorder() let secondRecorder = ResetEventRecorder() - let runner = ResolvingResetRunner(pathsByDestination: [ - "first.example.test": firstResolvedPath, - "second.example.test": secondResolvedPath, - ]) + let runner = RecordingProcessRunner() let broker = makeBroker(processRunner: runner) let first = broker.retainWorkspace(configuration( owner: UUID(), @@ -131,14 +137,17 @@ struct NativeSSHControlMasterResetTests { } ) - #expect(await broker.resetConflictedControlMaster(for: first) == .reset) + #expect(await broker.resetConflictedControlMaster( + for: first, + resolvedControlPath: firstResolvedPath + ) == .reset) #expect(firstRecorder.count == 1) #expect(secondRecorder.count == 0) - #expect(runner.exitRequests.count == 1) - #expect(runner.exitRequests[0].arguments.contains( + #expect(runner.requests.count == 1) + #expect(runner.requests[0].arguments.contains( "ControlPath=\(firstResolvedPath)" )) - #expect(!runner.exitRequests[0].arguments.contains( + #expect(!runner.requests[0].arguments.contains( "ControlPath=/tmp/cmux-ssh-501-%C" )) _ = firstObservation @@ -148,10 +157,7 @@ struct NativeSSHControlMasterResetTests { @Test("Different aliases resolving to one socket share reset fanout") func aliasesResolvingToSamePathShareFanout() async throws { let recorder = ResetEventRecorder() - let runner = ResolvingResetRunner(pathsByDestination: [ - "first-alias": firstResolvedPath, - "second-alias": firstResolvedPath, - ]) + let runner = RecordingProcessRunner() let broker = makeBroker(processRunner: runner) let first = broker.retainWorkspace(configuration( owner: UUID(), @@ -174,18 +180,19 @@ struct NativeSSHControlMasterResetTests { } ) - #expect(await broker.resetConflictedControlMaster(for: first) == .reset) + #expect(await broker.resetConflictedControlMaster( + for: first, + resolvedControlPath: firstResolvedPath + ) == .reset) #expect(recorder.count == 2) - #expect(runner.exitRequests.count == 1) + #expect(runner.requests.count == 1) _ = firstObservation _ = secondObservation } - @Test("Concurrent aliases coalesce only after exact path resolution") + @Test("Concurrent aliases coalesce by their authoritative exact path") func concurrentAliasesCoalesceByResolvedPath() async { - let runner = BlockingResolvingResetRunner( - resolvedPath: firstResolvedPath - ) + let runner = BlockingControlMasterResetRunner() let broker = makeBroker(processRunner: runner) let first = broker.retainWorkspace(configuration( owner: UUID(), @@ -199,65 +206,27 @@ struct NativeSSHControlMasterResetTests { )) let firstReset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: first) + await broker.resetConflictedControlMaster( + for: first, + resolvedControlPath: firstResolvedPath + ) } let secondReset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: second) + await broker.resetConflictedControlMaster( + for: second, + resolvedControlPath: firstResolvedPath + ) } - var resolutions = runner.resolutions.makeAsyncIterator() - #expect(await resolutions.next() != nil) - #expect(await resolutions.next() != nil) - var exits = runner.exits.makeAsyncIterator() - #expect(await exits.next() != nil) - runner.finishExit() - - #expect(await firstReset.value == .reset) - #expect(await secondReset.value == .reset) - #expect(runner.exitCount == 1) - } - - @Test("A coalesced alias keeps its resolved-socket reset alive") - func coalescedAliasKeepsResetAlive() async throws { - let runner = BlockingResolvingResetRunner( - resolvedPath: firstResolvedPath - ) - let recorder = ResetEventRecorder() - let broker = makeBroker(processRunner: runner) - let first = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "first-alias", - options: unresolvedOptions - )) - let second = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "second-alias", - options: unresolvedOptions - )) - let observation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - - let firstReset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: first) - } - var resolutions = runner.resolutions.makeAsyncIterator() - #expect(await resolutions.next() != nil) - var exits = runner.exits.makeAsyncIterator() - #expect(await exits.next() != nil) - let secondReset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: second) + var starts = runner.starts.makeAsyncIterator() + #expect(await starts.next() != nil) + for _ in 0..<3 { + await Task.yield() } - #expect(await resolutions.next() != nil) - await Task.yield() - broker.releaseWorkspace(first) - runner.finishExit() + runner.finish() #expect(await firstReset.value == .reset) #expect(await secondReset.value == .reset) - #expect(recorder.count == 1) - _ = observation + #expect(runner.requests.count == 1) } @Test("A successful exit still invalidates after its lease is released") @@ -276,7 +245,10 @@ struct NativeSSHControlMasterResetTests { ) let reset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: lease) + await broker.resetConflictedControlMaster( + for: lease, + resolvedControlPath: firstResolvedPath + ) } var starts = runner.starts.makeAsyncIterator() #expect(await starts.next() != nil) @@ -288,33 +260,25 @@ struct NativeSSHControlMasterResetTests { _ = observation } - @Test("ControlPath resolution failure never exits a master") - func resolutionFailureFailsClosed() async { - let runner = RecordingProcessRunner { request in - if request.arguments.contains("-G") { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: "configuration resolution failed" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } + @Test("An unresolved reset identity never exits a master") + func unresolvedResetIdentityFailsClosed() async { + let runner = RecordingProcessRunner() let broker = makeBroker(processRunner: runner) let lease = broker.retainWorkspace(configuration( owner: UUID(), options: unresolvedOptions )) - let outcome = await broker.resetConflictedControlMaster(for: lease) + let outcome = await broker.resetConflictedControlMaster( + for: lease, + resolvedControlPath: "/tmp/cmux-ssh-501-%C" + ) guard case .ignored = outcome else { - Issue.record("Expected resolution failure to be ignored") + Issue.record("Expected unresolved identity to be ignored") return } - #expect(!runner.requests.contains(where: { - $0.arguments.contains("-O") && $0.arguments.contains("exit") - })) + #expect(runner.requests.isEmpty) } @Test("A reset-wrapper no-op remains deferred and emits no reset") @@ -336,7 +300,10 @@ struct NativeSSHControlMasterResetTests { ) let reset = Task { @MainActor in - await broker.resetConflictedControlMaster(for: lease) + await broker.resetConflictedControlMaster( + for: lease, + resolvedControlPath: firstResolvedPath + ) } #expect(await clock.nextRequestedDelay() == 2_000) await clock.resumeNextSleep() diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 9dc09f2cbf28..e569ad2d2964 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -53,7 +53,8 @@ struct RemoteSessionReverseRelayStartupTests { coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "Connection refused", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044 + relayPort: 64_044, + resolvedControlPath: ResolvedControlPathFixture.path ) } #expect(!ignoredUnrelatedFailure) @@ -63,7 +64,8 @@ struct RemoteSessionReverseRelayStartupTests { coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "Error: remote port forwarding failed for listen port 64044", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044 + relayPort: 64_044, + resolvedControlPath: ResolvedControlPathFixture.path ) } #expect(beganRecovery) @@ -89,7 +91,8 @@ struct RemoteSessionReverseRelayStartupTests { coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "remote port forwarding failed for listen port 64044", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044 + relayPort: 64_044, + resolvedControlPath: ResolvedControlPathFixture.path ) } #expect(!beganSecondRecovery) @@ -117,7 +120,8 @@ struct RemoteSessionReverseRelayStartupTests { _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "Error: remote port forwarding failed for listen port \(relayPort)", remotePath: "/tmp/cmuxd-remote", - relayPort: relayPort + relayPort: relayPort, + resolvedControlPath: ResolvedControlPathFixture.path ) } @@ -144,7 +148,8 @@ struct RemoteSessionReverseRelayStartupTests { _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( startupFailure: "remote port forwarding failed for listen port 64044", remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044 + relayPort: 64_044, + resolvedControlPath: ResolvedControlPathFixture.path ) } From 14e5ab1dd10841c11e6191533099cbcd97122daa Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 05:43:24 -0700 Subject: [PATCH 28/39] fix: cancel only inherited relay forwards --- ...RemoteConfiguration+SSHBatchCommands.swift | 16 + ...ceRemoteConfiguration+SSHControlPath.swift | 43 ++ ...teConfigurationSSHBatchCommandsTests.swift | 48 ++ ...nectionBroker+ControlMasterOwnership.swift | 6 +- .../NativeSSHConnectionBroker.swift | 99 +--- ...iveSSHControlMasterOwnershipRegistry.swift | 12 +- ...iveSSHControlMasterOwnershipTracking.swift | 2 +- ...tiveSSHControlMasterResetCoordinator.swift | 308 ------------ .../NativeSSHControlMasterResetEventHub.swift | 63 --- .../NativeSSHControlMasterResetKey.swift | 55 --- ...SessionCoordinator+RelayProvisioning.swift | 34 ++ ...emoteSessionCoordinator+ReverseRelay.swift | 13 +- ...oordinator+ReverseRelayControlMaster.swift | 167 +++++-- ...ssionCoordinator+ReverseRelayStartup.swift | 127 ----- .../Session/RemoteSessionCoordinator.swift | 1 - .../Values/ReverseRelayStartupPhase.swift | 29 +- .../BlockingControlMasterResetRunner.swift | 44 -- ...dationRemoteReverseRelayProcessTests.swift | 4 +- .../NativeSSHConnectionBrokerTests.swift | 4 +- ...SSHControlMasterAdoptionHandoffTests.swift | 2 +- ...HControlMasterOwnershipRecoveryTests.swift | 30 +- ...HControlMasterOwnershipRegistryTests.swift | 34 +- ...tiveSSHControlMasterResetTestSupport.swift | 122 ----- .../NativeSSHControlMasterResetTests.swift | 442 ------------------ ...iveSSHControlMasterOwnershipRegistry.swift | 2 +- ...SessionInheritedForwardRecoveryTests.swift | 382 +++++++++++++++ ...emoteSessionReverseRelayStartupTests.swift | 164 ------- ...oteSessionReverseRelayTransportTests.swift | 98 ---- .../ResolvedControlPathProcessRunner.swift | 2 +- .../SynchronousEventRecorder.swift | 18 + ...alController+ControlWorkspaceContext.swift | 16 +- Sources/Workspace.swift | 7 +- .../WorkspaceRemoteConnectionTests.swift | 4 + 33 files changed, 762 insertions(+), 1636 deletions(-) create mode 100644 Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHControlPath.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift delete mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/SynchronousEventRecorder.swift diff --git a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift index 9ed953d3d926..73f585ee41bb 100644 --- a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift +++ b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHBatchCommands.swift @@ -90,6 +90,22 @@ extension WorkspaceRemoteConfiguration { return arguments } + /// Builds a non-interactive command that reuses the supplied exact ControlPath. + /// + /// - Parameters: + /// - command: Remote shell command to execute. + /// - effectiveSSHOptions: Options carrying the authenticated ControlPath. + /// - Returns: Arguments for `/usr/bin/ssh`. + public func batchSSHCommandArguments( + command: String, + effectiveSSHOptions: [String] + ) -> [String] { + ["-T"] + + SSHHostConfiguredRemoteCommand().overrideArguments + + batchSSHArguments(sshOptions: effectiveSSHOptions) + + ["-o", "RequestTTY=no", destination, command] + } + // Shared batch-mode `ssh` options: keepalives, BatchMode, no new // ControlMaster (existing ControlPath sockets may be reused), port, // identity, then the configuration's options minus diff --git a/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHControlPath.swift b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHControlPath.swift new file mode 100644 index 000000000000..586a30e7102a --- /dev/null +++ b/Packages/macOS/CmuxCore/Sources/CmuxCore/Remote/WorkspaceRemoteConfiguration+SSHControlPath.swift @@ -0,0 +1,43 @@ +internal import CmuxFoundation + +extension WorkspaceRemoteConfiguration { + /// Returns a copy whose first `ControlPath` option is the exact authenticated socket. + /// + /// - Parameter controlPath: Resolved path reported while foreground + /// authentication still owns the ControlMaster. + /// - Returns: A configuration that reuses that exact socket identity. + public func withResolvedSSHControlPath( + _ controlPath: String + ) -> WorkspaceRemoteConfiguration { + let resolver = SSHAgentSocketResolver() + let resolvedOptions = ["ControlPath=\(controlPath)"] + + sshOptions.filter { + resolver.optionKey($0) != "controlpath" + } + return WorkspaceRemoteConfiguration( + transport: transport, + terminalTransport: terminalTransport, + terminalProfile: terminalProfile, + destination: destination, + port: port, + identityFile: identityFile, + sshOptions: resolvedOptions, + localProxyPort: localProxyPort, + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + localSocketPath: localSocketPath, + ownerWorkspaceID: ownerWorkspaceID, + managedCloudVMID: managedCloudVMID, + terminalStartupCommand: terminalStartupCommand, + foregroundAuthToken: foregroundAuthToken, + agentSocketPath: agentSocketPath, + daemonWebSocketEndpoint: daemonWebSocketEndpoint, + preserveAfterTerminalExit: preserveAfterTerminalExit, + persistentDaemonSlot: persistentDaemonSlot, + skipDaemonBootstrap: skipDaemonBootstrap, + sshControlMasterLeaseGeneration: + sshControlMasterLeaseGeneration + ) + } +} diff --git a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift index ae9848b40edc..8a60c9ad2af6 100644 --- a/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift +++ b/Packages/macOS/CmuxCore/Tests/CmuxCoreTests/WorkspaceRemoteConfigurationSSHBatchCommandsTests.swift @@ -167,6 +167,54 @@ struct WorkspaceRemoteConfigurationSSHBatchCommandsTests { ) } + @Test("batch command reuses the supplied authenticated ControlPath") + func batchCommandUsesEffectiveControlPath() { + let effectiveOptions = [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=/tmp/cmux-ssh-resolved", + "StrictHostKeyChecking=accept-new", + ] + + #expect( + configuration().batchSSHCommandArguments( + command: "printf relay-metadata", + effectiveSSHOptions: effectiveOptions + ) == [ + "-T", + "-o", "RemoteCommand=none", + "-o", "ConnectTimeout=6", + "-o", "ServerAliveInterval=20", + "-o", "ServerAliveCountMax=2", + "-o", "BatchMode=yes", + "-o", "ControlMaster=no", + "-p", "2222", + "-i", "/Users/test/.ssh/id_ed25519", + "-o", "ControlPath=/tmp/cmux-ssh-resolved", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "RequestTTY=no", + "cmux-macmini", + "printf relay-metadata", + ] + ) + } + + @Test("resolved ControlPath replaces every unresolved option") + func resolvedControlPathReplacesTemplates() { + let resolved = configuration( + sshOptions: [ + "StrictHostKeyChecking=accept-new", + "ControlPath=/tmp/cmux-ssh-%C", + "ControlPath=~/.ssh/ignored-%C", + ] + ).withResolvedSSHControlPath("/tmp/cmux-ssh-resolved") + + #expect(resolved.sshOptions == [ + "ControlPath=/tmp/cmux-ssh-resolved", + "StrictHostKeyChecking=accept-new", + ]) + } + @Test("reverse relay ControlMaster commands require a usable ControlPath") func reverseRelayRequiresControlPath() { #expect( diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift index fe831e6ebab4..0dee5661470d 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift @@ -6,9 +6,9 @@ extension NativeSSHConnectionBroker { /// Begins an ownership handoff while foreground SSH authentication still /// holds the resolved ControlPath authentication lock. /// - /// The returned lease prevents another live cmux process from resetting - /// the newly authenticated master before the workspace configuration is - /// ready to adopt it. + /// The returned lease prevents another live cmux process from recovering + /// an inherited forward on the newly authenticated master before the + /// workspace configuration is ready to adopt it. /// /// - Parameters: /// - controlPath: Exact resolved cmux-owned ControlPath. diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 9bf65f85b8b0..c9cbbc2a1ff2 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -16,10 +16,8 @@ public final class NativeSSHConnectionBroker { let clock: any RemoteProxyRetryClock private let jitterMilliseconds: @MainActor @Sendable () -> Int let cleanupLauncherOverride: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)? - private nonisolated let conflictedMasterResetEventHub: NativeSSHControlMasterResetEventHub nonisolated let controlMasterOwnershipRegistry: any NativeSSHControlMasterOwnershipTracking - private let conflictedMasterResetCoordinator: NativeSSHControlMasterResetCoordinator var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] var ownersByControlMaster: [NativeSSHControlMasterKey: Set] = [:] @@ -45,20 +43,11 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = nil - let eventHub = NativeSSHControlMasterResetEventHub() let ownershipRegistry = NativeSSHControlMasterOwnershipRegistry( sharingOptions: sharingOptions ) - self.conflictedMasterResetEventHub = eventHub self.controlMasterOwnershipRegistry = ownershipRegistry - self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( - sharingOptions: sharingOptions, - processRunner: RemoteSessionProcessRunner(), - clock: clock, - eventHub: eventHub, - ownershipRegistry: ownershipRegistry - ) } /// Creates a broker with an injected cleanup launcher. @@ -77,20 +66,11 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = cleanupLauncher - let eventHub = NativeSSHControlMasterResetEventHub() let ownershipRegistry = NativeSSHControlMasterOwnershipRegistry( sharingOptions: sharingOptions ) - self.conflictedMasterResetEventHub = eventHub self.controlMasterOwnershipRegistry = ownershipRegistry - self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( - sharingOptions: sharingOptions, - processRunner: RemoteSessionProcessRunner(), - clock: clock, - eventHub: eventHub, - ownershipRegistry: ownershipRegistry - ) } nonisolated init( @@ -98,8 +78,6 @@ public final class NativeSSHConnectionBroker { clock: any RemoteProxyRetryClock, jitterMilliseconds: @escaping @MainActor @Sendable () -> Int, cleanupLauncher: @escaping @MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void, - conflictedMasterResetRunner: any RemoteSessionProcessRunning = - RemoteSessionProcessRunner(), controlMasterOwnershipRegistry: any NativeSSHControlMasterOwnershipTracking ) { @@ -107,16 +85,7 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = jitterMilliseconds self.cleanupLauncherOverride = cleanupLauncher - let eventHub = NativeSSHControlMasterResetEventHub() - self.conflictedMasterResetEventHub = eventHub self.controlMasterOwnershipRegistry = controlMasterOwnershipRegistry - self.conflictedMasterResetCoordinator = NativeSSHControlMasterResetCoordinator( - sharingOptions: sharingOptions, - processRunner: conflictedMasterResetRunner, - clock: clock, - eventHub: eventHub, - ownershipRegistry: controlMasterOwnershipRegistry - ) } /// Retains the cmux-owned master used by a configured workspace. @@ -129,23 +98,24 @@ public final class NativeSSHConnectionBroker { @discardableResult public func retainWorkspace(_ configuration: WorkspaceRemoteConfiguration) -> WorkspaceRemoteConfiguration { guard let ownerWorkspaceID = configuration.ownerWorkspaceID else { return configuration } - let nextKey = NativeSSHControlMasterKey( - configuration: configuration, - sharingOptions: sharingOptions + guard configuration.transport == .ssh else { return configuration } + let effectiveOptions = sharingOptions.mergingDefaults( + into: configuration.sshOptions ) - let resetKey = NativeSSHControlMasterResetKey( - configuration: configuration, + guard sharingOptions.cmuxOwnedControlPath( + in: effectiveOptions + ) != nil else { + return configuration + } + let leasedConfiguration = + configuration.withSSHControlMasterLeaseGeneration(UUID()) + let nextKey = NativeSSHControlMasterKey( + configuration: leasedConfiguration, sharingOptions: sharingOptions ) - guard nextKey != nil || resetKey != nil else { return configuration } - let leasedConfiguration = configuration.withSSHControlMasterLeaseGeneration(UUID()) - if let resetKey { - conflictedMasterResetCoordinator.retainWorkspace( - leasedConfiguration, - ownerWorkspaceID: ownerWorkspaceID, - key: resetKey - ) - } + // An unresolved `%C` template still needs a generation so the + // coordinator can retain its exact resolved socket before reuse. + // Lifecycle ownership remains exact-path-only. guard let nextKey else { return leasedConfiguration } if let lease = NativeSSHControlMasterLeaseIdentity( configuration: leasedConfiguration @@ -182,16 +152,6 @@ public final class NativeSSHConnectionBroker { ) { controlMasterOwnershipRegistry.release(lease: lease) } - if let resetKey = NativeSSHControlMasterResetKey( - configuration: configuration, - sharingOptions: sharingOptions - ) { - conflictedMasterResetCoordinator.releaseWorkspace( - ownerWorkspaceID: ownerWorkspaceID, - generation: generation, - key: resetKey - ) - } guard let key = NativeSSHControlMasterKey( configuration: configuration, sharingOptions: sharingOptions @@ -218,32 +178,19 @@ public final class NativeSSHConnectionBroker { ) } - /// Coalesces an inherited-master reset after OpenSSH confirms a relay bind conflict. - func resetConflictedControlMaster( - for configuration: WorkspaceRemoteConfiguration, - resolvedControlPath: String - ) async -> NativeSSHControlMasterResetOutcome { - await conflictedMasterResetCoordinator.reset( - for: configuration, - resolvedControlPath: resolvedControlPath - ) - } - - /// Observes resets that invalidate an exact cmux-owned control socket. - nonisolated func observeControlMasterResets( - controlPath: String, - handler: @escaping @Sendable () -> Void - ) -> NativeSSHControlMasterResetObservation? { + /// Serializes a narrow inherited-forward cancellation against other cmux processes. + nonisolated func beginReverseForwardRecovery( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { guard !controlPath.contains("%"), sharingOptions.cmuxOwnedControlPath(in: [ - "ControlMaster=auto", - "ControlPath=\(controlPath)", + "ControlMaster=auto", + "ControlPath=\(controlPath)", ]) == controlPath else { return nil } - return conflictedMasterResetEventHub.observe( - controlPath: controlPath, - handler: handler + return controlMasterOwnershipRegistry.beginRecovery( + controlPath: controlPath ) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift index b1dfb083a9df..3d62cbbd9de5 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift @@ -21,7 +21,7 @@ final class NativeSSHControlMasterOwnershipRegistry: } private enum ExclusiveUsePurpose { - case conflictedMasterReset + case reverseForwardRecovery case ordinaryCleanup } @@ -98,12 +98,12 @@ final class NativeSSHControlMasterOwnershipRegistry: } } - func beginReset( + func beginRecovery( controlPath: String ) -> NativeSSHControlMasterExclusiveUseAuthorization? { beginExclusiveUse( controlPath: controlPath, - purpose: .conflictedMasterReset + purpose: .reverseForwardRecovery ) } @@ -138,8 +138,8 @@ final class NativeSSHControlMasterOwnershipRegistry: let authorized: Bool switch purpose { - case .conflictedMasterReset: - authorized = beginOwnershipResetLocked( + case .reverseForwardRecovery: + authorized = beginRecoveryLocked( controlPath: controlPath, exclusiveUseID: exclusiveUseID ) @@ -180,7 +180,7 @@ final class NativeSSHControlMasterOwnershipRegistry: ) } - private func beginOwnershipResetLocked( + private func beginRecoveryLocked( controlPath: String, exclusiveUseID: UUID ) -> Bool { diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift index 0c40d08177c8..5bd719fa6b59 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipTracking.swift @@ -5,7 +5,7 @@ protocol NativeSSHControlMasterOwnershipTracking: Sendable { lease: NativeSSHControlMasterLeaseIdentity ) -> Bool func release(lease: NativeSSHControlMasterLeaseIdentity) - func beginReset( + func beginRecovery( controlPath: String ) -> NativeSSHControlMasterExclusiveUseAuthorization? func beginCleanup( diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift deleted file mode 100644 index c4b958e7ee1d..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetCoordinator.swift +++ /dev/null @@ -1,308 +0,0 @@ -internal import CmuxCore -internal import CmuxFoundation -internal import CmuxRemoteWorkspace -internal import Foundation - -/// Outcome of one broker-authorized conflicted-master migration. -enum NativeSSHControlMasterResetOutcome: Sendable, Equatable { - case reset - case deferred(String) - case ignored(String) -} - -/// Broker-owned state for disruptive ControlMaster resets. -/// -/// The parent ``NativeSSHConnectionBroker`` calls every mutating method on the -/// main actor. Keeping this state in one collaborator makes reset authorization -/// and coalescing independent from the normal last-owner cleanup lifecycle. -@MainActor -final class NativeSSHControlMasterResetCoordinator { - private struct InFlightReset { - let id: UUID - let task: Task - } - - private let sharingOptions: SSHConnectionSharingOptions - private let processRunner: any RemoteSessionProcessRunning - private let clock: any RemoteProxyRetryClock - private let eventHub: NativeSSHControlMasterResetEventHub - private let ownershipRegistry: - any NativeSSHControlMasterOwnershipTracking - private var leases: [ - UUID: [NativeSSHControlMasterResetKey: WorkspaceRemoteConfiguration] - ] = [:] - private var inFlightResets: [ - String: InFlightReset - ] = [:] - - nonisolated init( - sharingOptions: SSHConnectionSharingOptions, - processRunner: any RemoteSessionProcessRunning, - clock: any RemoteProxyRetryClock, - eventHub: NativeSSHControlMasterResetEventHub, - ownershipRegistry: any NativeSSHControlMasterOwnershipTracking - ) { - self.sharingOptions = sharingOptions - self.processRunner = processRunner - self.clock = clock - self.eventHub = eventHub - self.ownershipRegistry = ownershipRegistry - } - - func retainWorkspace( - _ configuration: WorkspaceRemoteConfiguration, - ownerWorkspaceID: UUID, - key: NativeSSHControlMasterResetKey - ) { - var ownerLeases = leases[ownerWorkspaceID] ?? [:] - ownerLeases[key] = configuration - leases[ownerWorkspaceID] = ownerLeases - } - - func releaseWorkspace( - ownerWorkspaceID: UUID, - generation: UUID, - key: NativeSSHControlMasterResetKey - ) { - guard var ownerLeases = leases[ownerWorkspaceID], - ownerLeases[key]?.sshControlMasterLeaseGeneration == generation else { - return - } - ownerLeases.removeValue(forKey: key) - if ownerLeases.isEmpty { - leases.removeValue(forKey: ownerWorkspaceID) - } else { - leases[ownerWorkspaceID] = ownerLeases - } - } - - func reset( - for configuration: WorkspaceRemoteConfiguration, - resolvedControlPath: String - ) async -> NativeSSHControlMasterResetOutcome { - guard let ownerWorkspaceID = configuration.ownerWorkspaceID, - let generation = configuration.sshControlMasterLeaseGeneration, - let key = NativeSSHControlMasterResetKey( - configuration: configuration, - sharingOptions: sharingOptions - ), - ownsLease( - ownerWorkspaceID: ownerWorkspaceID, - generation: generation, - key: key - ) else { - return .ignored("workspace no longer owns this cmux SSH master") - } - - guard !resolvedControlPath.contains("%"), - sharingOptions.cmuxOwnedControlPath(in: [ - "ControlMaster=auto", - "ControlPath=\(resolvedControlPath)", - ]) == resolvedControlPath else { - return .ignored("could not identify the cmux SSH master socket") - } - let effectiveOptions = sharingOptions.mergingDefaults( - into: configuration.sshOptions - ) - let pathResolver = NativeSSHControlPathResolver( - sharingOptions: sharingOptions - ) - guard !Task.isCancelled else { - return .deferred("control-master reset cancelled") - } - guard ownsLease( - ownerWorkspaceID: ownerWorkspaceID, - generation: generation, - key: key - ) else { - return .ignored("workspace no longer owns this cmux SSH master") - } - if let inFlight = inFlightResets[resolvedControlPath] { - return await inFlight.task.value - } - guard let lease = NativeSSHControlMasterLeaseIdentity( - configuration: configuration - ), - ownershipRegistry.retain( - controlPath: resolvedControlPath, - lease: lease - ) else { - return .deferred( - "resolved SSH master ownership is busy in another cmux process" - ) - } - guard let resetAuthorization = ownershipRegistry.beginReset( - controlPath: resolvedControlPath - ) else { - return .deferred( - "resolved SSH master is in use by another cmux process " + - "or foreground authentication" - ) - } - - let resolvedOptions = pathResolver.replacingControlPath( - in: effectiveOptions, - with: resolvedControlPath - ) - let arguments = RemoteControlMasterCleanup().cleanupArguments( - configuration: configuration, - sshOptionsOverride: resolvedOptions - ) - let request = NativeSSHControlMasterCleanupRequest( - arguments: arguments, - environment: configuration.sshProcessEnvironment, - authenticationLockPath: nil - ) - let resetID = UUID() - let processRunner = self.processRunner - let clock = self.clock - let eventHub = self.eventHub - let task = Task { - defer { resetAuthorization.release() } - let outcome = await Self.runReset( - request: request, - processRunner: processRunner, - clock: clock - ) - if case .reset = outcome { - eventHub.emit(controlPath: resolvedControlPath) - } - return outcome - } - inFlightResets[resolvedControlPath] = InFlightReset( - id: resetID, - task: task - ) - let outcome = await task.value - if inFlightResets[resolvedControlPath]?.id == resetID { - inFlightResets.removeValue(forKey: resolvedControlPath) - } - return outcome - } - - private func ownsLease( - ownerWorkspaceID: UUID, - generation: UUID, - key: NativeSSHControlMasterResetKey - ) -> Bool { - leases[ownerWorkspaceID]?[key]?.sshControlMasterLeaseGeneration == generation - } - - private nonisolated static func runReset( - request: NativeSSHControlMasterCleanupRequest, - processRunner: any RemoteSessionProcessRunning, - clock: any RemoteProxyRetryClock - ) async -> NativeSSHControlMasterResetOutcome { - let maximumAttempts = 3 - for attemptIndex in 0.. ResetAttemptOutcome { - let cancellation = RemoteProcessCancellationOperation() - return await withTaskCancellationHandler { - await withCheckedContinuation { continuation in - DispatchQueue.global(qos: .utility).async { - let invocation = request.processInvocation( - noOpExitStatus: - NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus - ) - let processRequest = RemoteProcessRequest( - executable: invocation.executableURL.path, - arguments: invocation.arguments, - environment: request.environment, - timeout: 5 - ) - do { - let result = try processRunner.run( - processRequest, - operation: cancellation - ) - let detail = bestErrorLine( - stderr: result.stderr, - stdout: result.stdout - ) ?? "ssh exited \(result.status)" - if result.status == 0 { - continuation.resume(returning: .reset) - } else if [ - NativeSSHControlMasterCleanupRequest.retryExitStatus, - NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus, - ].contains(result.status) { - continuation.resume(returning: .retry(detail)) - } else { - continuation.resume(returning: .ignored(detail)) - } - } catch { - continuation.resume(returning: .retry(error.localizedDescription)) - } - } - } - } onCancel: { - cancellation.cancel() - } - } - - private nonisolated static func bestErrorLine( - stderr: String, - stdout: String - ) -> String? { - for text in [stderr, stdout] { - if let line = text - .split(whereSeparator: \.isNewline) - .map(String.init) - .last(where: { - !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - }) { - return line.trimmingCharacters(in: .whitespacesAndNewlines) - } - } - return nil - } -} - -private enum ResetAttemptOutcome: Sendable { - case reset - case retry(String) - case ignored(String) -} - -private enum ControlPathResolutionOutcome: Sendable { - case resolved(String) - case unavailable - case retry(String) -} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift deleted file mode 100644 index bdff7806e6db..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetEventHub.swift +++ /dev/null @@ -1,63 +0,0 @@ -internal import Foundation - -/// Process-local fanout for disruptive shared-ControlMaster resets. -final class NativeSSHControlMasterResetEventHub: @unchecked Sendable { - // lint:allow lock - subscription and synchronous event snapshots are tiny. - private let lock = NSLock() - private var observers: [ - UUID: ( - controlPath: String, - handler: @Sendable () -> Void - ) - ] = [:] - - func observe( - controlPath: String, - handler: @escaping @Sendable () -> Void - ) -> NativeSSHControlMasterResetObservation { - let id = UUID() - lock.withLock { - observers[id] = (controlPath: controlPath, handler: handler) - } - return NativeSSHControlMasterResetObservation { [weak self] in - self?.removeObserver(id) - } - } - - func emit(controlPath: String) { - let handlers = lock.withLock { - observers.values.compactMap { observer in - observer.controlPath == controlPath ? observer.handler : nil - } - } - for handler in handlers { - handler() - } - } - - private func removeObserver(_ id: UUID) { - lock.withLock { - _ = observers.removeValue(forKey: id) - } - } -} - -/// Lifetime token for one ControlMaster-reset observer. -final class NativeSSHControlMasterResetObservation: @unchecked Sendable { - // lint:allow lock - cancellation exchanges one closure exactly once. - private let lock = NSLock() - private var cancellation: (@Sendable () -> Void)? - - init(cancellation: @escaping @Sendable () -> Void) { - self.cancellation = cancellation - } - - deinit { - let cancellation = lock.withLock { - let value = self.cancellation - self.cancellation = nil - return value - } - cancellation?() - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift deleted file mode 100644 index 7a9938bf22d3..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterResetKey.swift +++ /dev/null @@ -1,55 +0,0 @@ -internal import CmuxCore -internal import CmuxFoundation -internal import Foundation - -/// Identifies the effective cmux-owned master that may be reset after an -/// OpenSSH-confirmed reverse-forward bind conflict. -/// -/// Resolved socket paths are globally stable. An unresolved `%` template is -/// intentionally scoped to one owner and its full explicit SSH identity: -/// without `ssh -G` expansion, coalescing separate owners could falsely claim -/// that an exit for one effective socket reset another. -struct NativeSSHControlMasterResetKey: Hashable, Sendable { - let controlPath: String - let destination: String? - let port: Int? - let identityFile: String? - let effectiveOptions: [String] - let ownerWorkspaceID: UUID? - - init?( - configuration: WorkspaceRemoteConfiguration, - sharingOptions: SSHConnectionSharingOptions - ) { - guard configuration.transport == .ssh else { return nil } - let effectiveOptions = sharingOptions.mergingDefaults( - into: configuration.sshOptions - ) - guard let controlPath = sharingOptions.cmuxOwnedControlPath( - in: effectiveOptions - ) else { - return nil - } - self.controlPath = controlPath - if controlPath.contains("%") { - let destination = configuration.destination - .trimmingCharacters(in: .whitespacesAndNewlines) - guard !destination.isEmpty, - let ownerWorkspaceID = configuration.ownerWorkspaceID else { - return nil - } - self.destination = destination - self.port = configuration.port - self.identityFile = configuration.identityFile? - .trimmingCharacters(in: .whitespacesAndNewlines) - self.effectiveOptions = effectiveOptions - self.ownerWorkspaceID = ownerWorkspaceID - } else { - self.destination = nil - self.port = nil - self.identityFile = nil - self.effectiveOptions = [] - self.ownerWorkspaceID = nil - } - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift index e45e50190a5e..d256abf70ec0 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift @@ -5,6 +5,40 @@ internal import Foundation // (the CmuxCore SSH-option-normalization precedent); the script text is // wire/process behavior pinned by tests — do not alter. extension RemoteSessionCoordinator { + /// Proves that stale relay metadata belongs to this exact relay identity. + static func remoteRelayMetadataOwnershipProbeScript( + relayPort: Int, + relayID: String, + relayToken: String, + persistentDaemonSlot: String? + ) -> String { + let authPayload = + "{\"relay_id\":\"\(relayID)\",\"relay_token\":\"\(relayToken)\"}" + let normalizedSlot = normalizedPersistentDaemonSlotForRemoteCleanup( + persistentDaemonSlot + ) + guard persistentDaemonSlot == nil || normalizedSlot != nil else { + return "exit 64" + } + let slotCheck: String + if let normalizedSlot { + slotCheck = """ + [ -r "$slot_file" ] || exit 64 + [ "$(tr -d '\\r\\n' < "$slot_file")" = \(normalizedSlot.shellSingleQuoted) ] || exit 64 + """ + } else { + slotCheck = "[ ! -e \"$slot_file\" ] || exit 64" + } + return """ + relay_directory="$HOME/.cmux/relay" + auth_file="$relay_directory/\(relayPort).auth" + slot_file="$relay_directory/\(relayPort).slot" + [ -r "$auth_file" ] || exit 64 + [ "$(tr -d '\\r\\n' < "$auth_file")" = \(authPayload.shellSingleQuoted) ] || exit 64 + \(slotCheck) + """ + } + /// Builds a direct persistent-slot shutdown script when no relay metadata exists. static func remotePersistentDaemonStopScript( daemonRemotePath: String, diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 50013fdff470..ddb805793567 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -26,7 +26,6 @@ extension RemoteSessionCoordinator { } guard reverseRelayProcess == nil else { return } guard reverseRelayControlMasterForwardSpec == nil else { return } - guard reverseRelayStartupPhase.allowsRelayLaunch else { return } cancelReverseRelayRestartLocked() launchReverseRelayLocked( @@ -48,7 +47,6 @@ extension RemoteSessionCoordinator { ) { guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } guard reverseRelayControlMasterForwardSpec == nil else { return } - guard reverseRelayStartupPhase.allowsRelayLaunch else { return } var relayServer: RemoteCLIRelayServer? do { @@ -91,18 +89,10 @@ extension RemoteSessionCoordinator { "target=\(configuration.displayTarget) controlMaster=1" ) return - case .bindingConflict(let detail, let controlPath): + case .bindingConflict(let detail): debugLog( "remote.relay.startFailed relayPort=\(relayPort) error=\(detail)" ) - if beginConflictedControlMasterExitIfNeededLocked( - startupFailure: detail, - remotePath: remotePath, - relayPort: relayPort, - resolvedControlPath: controlPath - ) { - return - } publishReverseRelayFailureLocked( remotePath: remotePath ) @@ -263,7 +253,6 @@ extension RemoteSessionCoordinator { @discardableResult func stopReverseRelayLocked(cleanupScope: RemoteRelayCleanupScope = .transport) -> Bool { - cancelReverseRelayStartupLocked() if let reverseRelayProcess, reverseRelayProcess.isRunning { reverseRelayProcess.terminate() } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index c89e08c550cf..c9fe86f09228 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -6,7 +6,7 @@ internal import Foundation enum ReverseRelayControlMasterStartOutcome: Sendable { case started case unavailable - case bindingConflict(String, controlPath: String?) + case bindingConflict(String) } extension RemoteSessionCoordinator { @@ -70,18 +70,16 @@ extension RemoteSessionCoordinator { debugConfigSummary() ) if let bindingConflict { - let ownedControlPath = - connectionBroker.sharingOptions.cmuxOwnedControlPath( - in: effectiveSSHOptions - ) - let resolvedControlPath = - ownedControlPath?.contains("%") == false - ? ownedControlPath - : nil - return .bindingConflict( - bindingConflict, - controlPath: resolvedControlPath - ) + if recoverInheritedReverseForwardLocked( + forwardSpec: forwardSpec, + relayPort: relayPort, + effectiveSSHOptions: effectiveSSHOptions + ) { + reverseRelayControlMasterForwardSpec = + forwardSpec + return .started + } + return .bindingConflict(bindingConflict) } return .unavailable } @@ -96,6 +94,119 @@ extension RemoteSessionCoordinator { } } + /// Cancels only a forward whose persisted relay identity matches this workspace. + /// + /// A bind diagnostic alone is ambiguous: an unrelated remote process may + /// own the port. Recovery therefore requires the exact cmux-owned + /// ControlPath, cross-process exclusive ownership, matching relay metadata, + /// and a successful OpenSSH `cancel` for the listen address before retrying. + private func recoverInheritedReverseForwardLocked( + forwardSpec: String, + relayPort: Int, + effectiveSSHOptions: [String] + ) -> Bool { + guard reverseRelayStartupPhase.canAttemptRecovery, + reverseRelayControlMasterForwardSpec == nil, + let relayID = configuration.relayID? + .trimmingCharacters(in: .whitespacesAndNewlines), + !relayID.isEmpty, + let relayToken = configuration.relayToken? + .trimmingCharacters(in: .whitespacesAndNewlines), + !relayToken.isEmpty, + let controlPath = + connectionBroker.sharingOptions.cmuxOwnedControlPath( + in: effectiveSSHOptions + ), + !controlPath.contains("%"), + let authorization = + connectionBroker.beginReverseForwardRecovery( + controlPath: controlPath + ) else { + return false + } + reverseRelayStartupPhase = .recoveryAttempted + defer { authorization.release() } + + let probeScript = Self.remoteRelayMetadataOwnershipProbeScript( + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + persistentDaemonSlot: configuration.persistentDaemonSlot + ) + let probeCommand = "sh -c \(probeScript.shellSingleQuoted)" + do { + let probe = try sshExec( + arguments: configuration.batchSSHCommandArguments( + command: probeCommand, + effectiveSSHOptions: effectiveSSHOptions + ), + timeout: 6 + ) + guard probe.status == 0 else { + debugLog( + "remote.relay.inheritedForward.recoveryIgnored " + + "reason=metadata-mismatch relayPort=\(relayPort) " + + debugConfigSummary() + ) + return false + } + + let listenSpec = "127.0.0.1:\(relayPort)" + guard let cancelArguments = + configuration.reverseRelayControlMasterArguments( + controlCommand: "cancel", + forwardSpec: listenSpec, + effectiveSSHOptions: effectiveSSHOptions + ) else { + return false + } + let cancellation = try sshExec( + arguments: cancelArguments, + timeout: 4 + ) + guard cancellation.status == 0 else { + debugLog( + "remote.relay.inheritedForward.recoveryIgnored " + + "reason=forward-not-owned relayPort=\(relayPort) " + + debugConfigSummary() + ) + return false + } + + guard let forwardArguments = + configuration.reverseRelayControlMasterArguments( + controlCommand: "forward", + forwardSpec: forwardSpec, + effectiveSSHOptions: effectiveSSHOptions + ) else { + return false + } + let retry = try sshExec( + arguments: forwardArguments, + timeout: 6 + ) + guard retry.status == 0 else { + debugLog( + "remote.relay.inheritedForward.retryFailed " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + return false + } + debugLog( + "remote.relay.inheritedForward.recovered " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + return true + } catch { + debugLog( + "remote.relay.inheritedForward.recoveryIgnored " + + "relayPort=\(relayPort) \(error.localizedDescription) " + + debugConfigSummary() + ) + return false + } + } + /// Cancels only the exact forward this coordinator successfully installed. func stopReverseRelayViaControlMasterLocked() { guard let forwardSpec = reverseRelayControlMasterForwardSpec else { return } @@ -117,7 +228,7 @@ extension RemoteSessionCoordinator { /// Resolves cmux's `%C` template before any background SSH command can /// adopt the shared master. /// - /// The exact socket path is both the process-ownership lease and reset + /// The exact socket path is both the process-ownership lease and recovery /// identity. Custom paths remain user-managed. If OpenSSH cannot resolve a /// cmux-owned path, the connection attempt fails before daemon bootstrap. func resolvedControlMasterSSHOptionsLocked() -> [String]? { @@ -198,35 +309,7 @@ extension RemoteSessionCoordinator { ) return nil } - guard let observation = connectionBroker.observeControlMasterResets( - controlPath: resolvedPath, - handler: { [weak self] in - self?.queue.async { [weak self] in - self?.sharedControlMasterDidResetLocked() - } - } - ) else { - return nil - } - conflictedControlMasterResetObservation = observation resolvedControlMasterSSHOptions = resolvedOptions return resolvedOptions } - - /// Invalidates a relay installed on a shared master that another owner - /// exited during conflict recovery. - func sharedControlMasterDidResetLocked() { - guard reverseRelayControlMasterForwardSpec != nil else { return } - reverseRelayControlMasterForwardSpec = nil - debugLog( - "remote.relay.controlmaster.resetObserved \(debugConfigSummary())" - ) - guard !isStopping, - daemonReady, - let remotePath = daemonRemotePath, - !remotePath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - return - } - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) - } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift deleted file mode 100644 index 189409f179f8..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift +++ /dev/null @@ -1,127 +0,0 @@ -internal import CmuxFoundation -internal import Foundation - -extension RemoteSessionCoordinator { - /// Exits a cmux-owned ControlPersist master only after OpenSSH proves that - /// the configured relay port is already bound. - /// - /// OpenSSH cannot cancel an inherited reverse forward without its original - /// full target specification. Custom ControlPaths fail closed here: cmux - /// never terminates a master it did not create. - @discardableResult - func beginConflictedControlMasterExitIfNeededLocked( - startupFailure: String, - remotePath: String, - relayPort: Int, - resolvedControlPath: String? - ) -> Bool { - guard Self.isReverseRelayPortBindingFailure( - startupFailure, - relayPort: relayPort - ) else { - return false - } - guard reverseRelayStartupPhase.canAttemptRecovery else { - return false - } - guard reverseRelayControlMasterForwardSpec == nil else { - debugLog( - "remote.relay.conflictedMaster.exitSkipped " + - "reason=current-forward-owned relayPort=\(relayPort) " + - debugConfigSummary() - ) - return false - } - guard let resolvedControlPath else { - return false - } - - let token = UUID() - let configuration = self.configuration - let connectionBroker = self.connectionBroker - - let task = Task { [weak self] in - let outcome = await connectionBroker.resetConflictedControlMaster( - for: configuration, - resolvedControlPath: resolvedControlPath - ) - guard !Task.isCancelled else { return } - self?.queue.async { [weak self] in - self?.finishConflictedControlMasterExitLocked( - token: token, - outcome: outcome, - remotePath: remotePath, - relayPort: relayPort - ) - } - } - reverseRelayStartupPhase = .exitingConflictedControlMaster( - token: token, - task: task - ) - debugLog( - "remote.relay.conflictedMaster.exitBegin " + - "relayPort=\(relayPort) \(debugConfigSummary())" - ) - return true - } - - /// Re-enters queue confinement and retries only after `ssh -O exit` - /// completes and this recovery phase still owns the token. - private func finishConflictedControlMasterExitLocked( - token: UUID, - outcome: NativeSSHControlMasterResetOutcome, - remotePath: String, - relayPort: Int - ) { - guard reverseRelayStartupPhase.token == token else { return } - switch outcome { - case .reset: - reverseRelayStartupPhase = .recoveryAttempted - debugLog( - "remote.relay.conflictedMaster.exited " + - "relayPort=\(relayPort) \(debugConfigSummary())" - ) - case .deferred(let detail): - reverseRelayStartupPhase = .recoveryAvailable - debugLog( - "remote.relay.conflictedMaster.exitDeferred " + - "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" - ) - publishReverseRelayPortUnavailableLocked() - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) - return - case .ignored(let detail): - reverseRelayStartupPhase = .recoveryAttempted - debugLog( - "remote.relay.conflictedMaster.exitIgnored " + - "relayPort=\(relayPort) \(detail) \(debugConfigSummary())" - ) - publishReverseRelayPortUnavailableLocked() - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) - return - } - - guard !isStopping else { return } - scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) - } - - private func publishReverseRelayPortUnavailableLocked() { - publishDaemonStatus( - .error, - detail: strings.reverseRelayPortUnavailableRetrying - ) - } - - /// Cancels the in-flight OpenSSH recovery and invalidates its continuation. - func cancelReverseRelayStartupLocked() { - guard case .exitingConflictedControlMaster( - _, - let task - ) = reverseRelayStartupPhase else { - return - } - reverseRelayStartupPhase = .recoveryAttempted - task.cancel() - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index a9f925ef1c66..ab6fc2d839e7 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -84,7 +84,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? var resolvedControlMasterSSHOptions: [String]? - var conflictedControlMasterResetObservation: NativeSSHControlMasterResetObservation? var cliRelayServer: RemoteCLIRelayServer? var remotePortScanTTYNames: [UUID: String] = [:] /// Stable publication state for best-effort remote TTY attribution scans. diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift index d1f9aa3a0adf..3473608854ef 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift @@ -1,39 +1,12 @@ -internal import Foundation - -/// Queue-confined phase for one conflict-triggered legacy-master recovery. +/// Queue-confined phase for one inherited-forward cancellation attempt. enum ReverseRelayStartupPhase: Sendable { case recoveryAvailable - case exitingConflictedControlMaster( - token: UUID, - task: Task - ) case recoveryAttempted - var allowsRelayLaunch: Bool { - if case .exitingConflictedControlMaster = self { - return false - } - return true - } - var canAttemptRecovery: Bool { if case .recoveryAvailable = self { return true } return false } - - var isRecovering: Bool { - if case .exitingConflictedControlMaster = self { - return true - } - return false - } - - var token: UUID? { - guard case .exitingConflictedControlMaster(let token, _) = self else { - return nil - } - return token - } } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift deleted file mode 100644 index 0095a49e8395..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingControlMasterResetRunner.swift +++ /dev/null @@ -1,44 +0,0 @@ -import Foundation -@testable import CmuxRemoteSession - -final class BlockingControlMasterResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - let starts: AsyncStream - private let startsContinuation: AsyncStream.Continuation - private let lock = NSLock() - private var _requests: [RemoteProcessRequest] = [] - private let releaseCondition = NSCondition() - private var finished = false - - init() { - (starts, startsContinuation) = AsyncStream.makeStream() - } - - var requests: [RemoteProcessRequest] { - lock.withLock { _requests } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - lock.withLock { _requests.append(request) } - startsContinuation.yield() - releaseCondition.lock() - while !finished { - releaseCondition.wait() - } - releaseCondition.unlock() - try operation?.throwIfCancelled() - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - - func finish() { - releaseCondition.lock() - finished = true - releaseCondition.broadcast() - releaseCondition.unlock() - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift index e815e76adfba..aea3234f115f 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift @@ -146,7 +146,7 @@ struct FoundationRemoteReverseRelayProcessTests { stderrPipe: stderrPipe, stderrDrainGracePeriod: 0.05 ) - let startupRecorder = ResetEventRecorder() + let startupRecorder = SynchronousEventRecorder() let (terminations, continuation) = AsyncStream.makeStream() let startedAt = Date() @@ -193,7 +193,7 @@ struct FoundationRemoteReverseRelayProcessTests { stderrDrainGracePeriod: 0.05, terminationGracePeriod: 0.05 ) - let startupRecorder = ResetEventRecorder() + let startupRecorder = SynchronousEventRecorder() let (terminations, continuation) = AsyncStream.makeStream() try process.run() diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index 8c0b555413e2..4c83ca7f3ea3 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -92,8 +92,8 @@ struct NativeSSHConnectionBrokerTests { broker.releaseWorkspace(lease) } - @Test("Unresolved templates authorize reset but not last-owner cleanup") - func unresolvedTemplatesOnlyAuthorizeReset() { + @Test("Unresolved templates carry a generation but not last-owner cleanup") + func unresolvedTemplatesCarryGenerationOnly() { let recorder = CleanupRequestRecorder() let broker = makeBroker(cleanupRecorder: recorder) let templateOptions = sharingOptions.mergingDefaults(into: []) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift index 0856b5fc718e..75302346fa98 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift @@ -7,7 +7,7 @@ struct NativeSSHControlMasterAdoptionHandoffTests { @Test("An unconsumed handoff expires and releases ownership once") func unconsumedHandoffExpires() async { let clock = ManualBrokerClock() - let recorder = ResetEventRecorder() + let recorder = SynchronousEventRecorder() let handoff = NativeSSHControlMasterAdoptionHandoff( controlPath: "/tmp/cmux-ssh-501-test", lease: NativeSSHControlMasterLeaseIdentity( diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift index 85a71a59ad9f..9c3366d49b58 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift @@ -9,21 +9,19 @@ import Testing @MainActor @Suite("Native SSH ownership-gated recovery") struct NativeSSHControlMasterOwnershipRecoveryTests { - @Test("A live foreign owner prevents destructive reset") - func foreignOwnerFailsClosed() async { + @Test("A live foreign owner prevents inherited-forward recovery") + func foreignOwnerFailsClosed() { let controlPath = "/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567" - let runner = RecordingProcessRunner() let broker = NativeSSHConnectionBroker( sharingOptions: SSHConnectionSharingOptions(userID: 501), clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, - conflictedMasterResetRunner: runner, controlMasterOwnershipRegistry: DenyingControlMasterOwnershipRegistry() ) - let lease = broker.retainWorkspace(WorkspaceRemoteConfiguration( + _ = broker.retainWorkspace(WorkspaceRemoteConfiguration( destination: "alice@example.test", port: nil, identityFile: nil, @@ -43,15 +41,11 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { persistentDaemonSlot: "ssh-test" )) - guard case .deferred = - await broker.resetConflictedControlMaster( - for: lease, - resolvedControlPath: controlPath - ) else { - Issue.record("Expected a live foreign owner to defer reset") - return - } - #expect(runner.requests.isEmpty) + #expect( + broker.beginReverseForwardRecovery( + controlPath: controlPath + ) == nil + ) } @Test("Foreground authentication hands ownership to the workspace without a gap") @@ -89,7 +83,7 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { ownerWorkspaceID: ownerWorkspaceID ) ) - #expect(secondRegistry.beginReset(controlPath: controlPath) == nil) + #expect(secondRegistry.beginRecovery(controlPath: controlPath) == nil) let configuration = broker.retainWorkspace( WorkspaceRemoteConfiguration( @@ -116,11 +110,11 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { handoff, configuration: configuration )) - #expect(secondRegistry.beginReset(controlPath: controlPath) == nil) + #expect(secondRegistry.beginRecovery(controlPath: controlPath) == nil) broker.releaseWorkspace(configuration) let authorization = try #require( - secondRegistry.beginReset(controlPath: controlPath) + secondRegistry.beginRecovery(controlPath: controlPath) ) authorization.release() } @@ -139,7 +133,7 @@ private final class DenyingControlMasterOwnershipRegistry: func release(lease: NativeSSHControlMasterLeaseIdentity) {} - func beginReset( + func beginRecovery( controlPath: String ) -> NativeSSHControlMasterExclusiveUseAuthorization? { nil diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift index 2c7fe113d610..4376ccbd7e15 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRegistryTests.swift @@ -6,8 +6,8 @@ import Testing @Suite("Native SSH cross-process master ownership") struct NativeSSHControlMasterOwnershipRegistryTests { - @Test("A live sibling process blocks reset until it releases its lease") - func liveSiblingBlocksReset() throws { + @Test("A live sibling process blocks recovery until it releases its lease") + func liveSiblingBlocksRecovery() throws { let scratchDirectory = FileManager.default.temporaryDirectory .appendingPathComponent( "cmux-control-owner-\(UUID().uuidString)", @@ -42,11 +42,11 @@ struct NativeSSHControlMasterOwnershipRegistryTests { controlPath: controlPath, lease: secondLease )) - #expect(first.beginReset(controlPath: controlPath) == nil) + #expect(first.beginRecovery(controlPath: controlPath) == nil) second.release(lease: secondLease) let authorization = try #require( - first.beginReset(controlPath: controlPath) + first.beginRecovery(controlPath: controlPath) ) #expect(!second.retain( controlPath: controlPath, @@ -59,8 +59,8 @@ struct NativeSSHControlMasterOwnershipRegistryTests { )) } - @Test("Resolved authentication blocks reset across different aliases") - func resolvedAuthenticationBlocksReset() throws { + @Test("Resolved authentication blocks recovery across different aliases") + func resolvedAuthenticationBlocksRecovery() throws { let scratchDirectory = FileManager.default.temporaryDirectory .appendingPathComponent( "cmux-control-auth-\(UUID().uuidString)", @@ -118,13 +118,13 @@ struct NativeSSHControlMasterOwnershipRegistryTests { let ready = stdout.fileHandleForReading.readData(ofLength: 6) #expect(String(decoding: ready, as: UTF8.self) == "ready\n") #expect(registry.retain(controlPath: controlPath, lease: lease)) - #expect(registry.beginReset(controlPath: controlPath) == nil) + #expect(registry.beginRecovery(controlPath: controlPath) == nil) try stdin.fileHandleForWriting.close() process.waitUntilExit() #expect(process.terminationStatus == 0) let authorization = try #require( - registry.beginReset(controlPath: controlPath) + registry.beginRecovery(controlPath: controlPath) ) authorization.release() } @@ -152,10 +152,10 @@ struct NativeSSHControlMasterOwnershipRegistryTests { #expect(registry.retain(controlPath: controlPath, lease: lease)) #expect(registry.beginCleanup(controlPath: controlPath) == nil) - let reset = try #require( - registry.beginReset(controlPath: controlPath) + let recovery = try #require( + registry.beginRecovery(controlPath: controlPath) ) - reset.release() + recovery.release() #expect(registry.beginCleanup(controlPath: controlPath) == nil) registry.release(lease: lease) @@ -190,8 +190,8 @@ struct NativeSSHControlMasterOwnershipRegistryTests { ) ) - let reset = try #require( - first.beginReset(controlPath: controlPath) + let recovery = try #require( + first.beginRecovery(controlPath: controlPath) ) #expect(second.beginCleanup(controlPath: controlPath) == nil) #expect( @@ -200,7 +200,7 @@ struct NativeSSHControlMasterOwnershipRegistryTests { ) == false ) - reset.release() + recovery.release() let cleanup = try #require( second.beginCleanup(controlPath: controlPath) ) @@ -237,14 +237,14 @@ struct NativeSSHControlMasterOwnershipRegistryTests { ) #expect(first.retain(controlPath: controlPath, lease: lease)) - var authorization = first.beginReset(controlPath: controlPath) + var authorization = first.beginRecovery(controlPath: controlPath) #expect(authorization != nil) authorization = nil - #expect(second.beginReset(controlPath: controlPath) == nil) + #expect(second.beginRecovery(controlPath: controlPath) == nil) first.release(lease: lease) let secondAuthorization = try #require( - second.beginReset(controlPath: controlPath) + second.beginRecovery(controlPath: controlPath) ) secondAuthorization.release() } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift deleted file mode 100644 index f9e195209f6d..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTestSupport.swift +++ /dev/null @@ -1,122 +0,0 @@ -import CmuxFoundation -import Foundation -@testable import CmuxRemoteSession - -final class ResetEventRecorder: @unchecked Sendable { - // lint:allow lock - event callbacks increment one test counter. - private let lock = NSLock() - private var value = 0 - - var count: Int { - lock.withLock { value } - } - - func record() { - lock.withLock { - value += 1 - } - } -} - -final class RetryThenSuccessResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - // lint:allow lock - process calls consume one scripted test counter. - private let lock = NSLock() - private let retryCount: Int - private var count = 0 - - init(retryCount: Int) { - self.retryCount = retryCount - } - - var requestCount: Int { - lock.withLock { count } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - let attempt = lock.withLock { - count += 1 - return count - } - if attempt <= retryCount { - return RemoteCommandResult( - status: NativeSSHControlMasterCleanupRequest.retryExitStatus, - stdout: "", - stderr: "foreground authentication still active" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } -} - -final class ThrowThenSuccessResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - // lint:allow lock - process calls consume one scripted test counter. - private let lock = NSLock() - private let throwCount: Int - private var count = 0 - - init(throwCount: Int) { - self.throwCount = throwCount - } - - var requestCount: Int { - lock.withLock { count } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - let attempt = lock.withLock { - count += 1 - return count - } - if attempt <= throwCount { - throw NSError( - domain: "NativeSSHControlMasterResetTests", - code: attempt - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } -} - -final class FixedStatusResetRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - // lint:allow lock - guards one counter and the broadcast test gate. - private let lock = NSLock() - private let status: Int32 - private var count = 0 - - init(status: Int32) { - self.status = status - } - - var requestCount: Int { - lock.withLock { count } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - lock.withLock { - count += 1 - } - return RemoteCommandResult( - status: status, - stdout: "", - stderr: "cleanup wrapper skipped ssh" - ) - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift deleted file mode 100644 index 6a20fc22fe0a..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterResetTests.swift +++ /dev/null @@ -1,442 +0,0 @@ -import CmuxCore -import CmuxFoundation -import CmuxRemoteWorkspace -import Foundation -import Testing -@testable import CmuxRemoteSession - -@MainActor -@Suite("Native SSH conflicted-master reset") -struct NativeSSHControlMasterResetTests { - private let sharingOptions = SSHConnectionSharingOptions(userID: 501) - private let firstResolvedPath = - "/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567" - private let secondResolvedPath = - "/tmp/cmux-ssh-501-89abcdef0123456789abcdef0123456789abcdef" - private let resolvedOptions = [ - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567", - ] - private let unresolvedOptions = [ - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=/tmp/cmux-ssh-501-%C", - ] - - @Test("A successful global exit notifies every shared-master owner") - func successfulExitNotifiesSiblingOwners() async throws { - let recorder = ResetEventRecorder() - let broker = makeBroker(processRunner: RecordingProcessRunner()) - let first = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "first-alias", - options: resolvedOptions - )) - _ = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "second-alias", - options: resolvedOptions - )) - let firstObservation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - let secondObservation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - - #expect(await broker.resetConflictedControlMaster( - for: first, - resolvedControlPath: firstResolvedPath - ) == .reset) - #expect(recorder.count == 2) - _ = firstObservation - _ = secondObservation - } - - @Test("Authentication-lock deferrals retry before resetting") - func authenticationDeferralRetries() async { - let clock = ManualBrokerClock() - let runner = RetryThenSuccessResetRunner(retryCount: 2) - let broker = makeBroker(clock: clock, processRunner: runner) - let lease = broker.retainWorkspace(configuration( - owner: UUID(), - options: resolvedOptions - )) - - let reset = Task { @MainActor in - await broker.resetConflictedControlMaster( - for: lease, - resolvedControlPath: firstResolvedPath - ) - } - #expect(await clock.nextRequestedDelay() == 2_000) - await clock.resumeNextSleep() - #expect(await clock.nextRequestedDelay() == 2_000) - await clock.resumeNextSleep() - - #expect( - await reset.value == NativeSSHControlMasterResetOutcome.reset - ) - #expect(runner.requestCount == 3) - } - - @Test("Transient reset runner errors retry before resetting") - func transientRunnerErrorsRetry() async { - let clock = ManualBrokerClock() - let runner = ThrowThenSuccessResetRunner(throwCount: 2) - let broker = makeBroker(clock: clock, processRunner: runner) - let lease = broker.retainWorkspace(configuration( - owner: UUID(), - options: resolvedOptions - )) - - let reset = Task { @MainActor in - await broker.resetConflictedControlMaster( - for: lease, - resolvedControlPath: firstResolvedPath - ) - } - #expect(await clock.nextRequestedDelay() == 2_000) - await clock.resumeNextSleep() - #expect(await clock.nextRequestedDelay() == 2_000) - await clock.resumeNextSleep() - - #expect(await reset.value == .reset) - #expect(runner.requestCount == 3) - } - - @Test("Expanded paths keep unrelated hosts isolated") - func expandedPathsDoNotNotifyDifferentHost() async throws { - let firstRecorder = ResetEventRecorder() - let secondRecorder = ResetEventRecorder() - let runner = RecordingProcessRunner() - let broker = makeBroker(processRunner: runner) - let first = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "first.example.test", - options: unresolvedOptions - )) - _ = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "second.example.test", - options: unresolvedOptions - )) - let firstObservation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - firstRecorder.record() - } - ) - let secondObservation = try #require( - broker.observeControlMasterResets(controlPath: secondResolvedPath) { - secondRecorder.record() - } - ) - - #expect(await broker.resetConflictedControlMaster( - for: first, - resolvedControlPath: firstResolvedPath - ) == .reset) - #expect(firstRecorder.count == 1) - #expect(secondRecorder.count == 0) - #expect(runner.requests.count == 1) - #expect(runner.requests[0].arguments.contains( - "ControlPath=\(firstResolvedPath)" - )) - #expect(!runner.requests[0].arguments.contains( - "ControlPath=/tmp/cmux-ssh-501-%C" - )) - _ = firstObservation - _ = secondObservation - } - - @Test("Different aliases resolving to one socket share reset fanout") - func aliasesResolvingToSamePathShareFanout() async throws { - let recorder = ResetEventRecorder() - let runner = RecordingProcessRunner() - let broker = makeBroker(processRunner: runner) - let first = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "first-alias", - options: unresolvedOptions - )) - _ = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "second-alias", - options: unresolvedOptions - )) - let firstObservation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - let secondObservation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - - #expect(await broker.resetConflictedControlMaster( - for: first, - resolvedControlPath: firstResolvedPath - ) == .reset) - #expect(recorder.count == 2) - #expect(runner.requests.count == 1) - _ = firstObservation - _ = secondObservation - } - - @Test("Concurrent aliases coalesce by their authoritative exact path") - func concurrentAliasesCoalesceByResolvedPath() async { - let runner = BlockingControlMasterResetRunner() - let broker = makeBroker(processRunner: runner) - let first = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "first-alias", - options: unresolvedOptions - )) - let second = broker.retainWorkspace(configuration( - owner: UUID(), - destination: "second-alias", - options: unresolvedOptions - )) - - let firstReset = Task { @MainActor in - await broker.resetConflictedControlMaster( - for: first, - resolvedControlPath: firstResolvedPath - ) - } - let secondReset = Task { @MainActor in - await broker.resetConflictedControlMaster( - for: second, - resolvedControlPath: firstResolvedPath - ) - } - var starts = runner.starts.makeAsyncIterator() - #expect(await starts.next() != nil) - for _ in 0..<3 { - await Task.yield() - } - runner.finish() - - #expect(await firstReset.value == .reset) - #expect(await secondReset.value == .reset) - #expect(runner.requests.count == 1) - } - - @Test("A successful exit still invalidates after its lease is released") - func successfulExitAfterReleaseStillInvalidates() async throws { - let runner = BlockingControlMasterResetRunner() - let recorder = ResetEventRecorder() - let broker = makeBroker(processRunner: runner) - let lease = broker.retainWorkspace(configuration( - owner: UUID(), - options: resolvedOptions - )) - let observation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - - let reset = Task { @MainActor in - await broker.resetConflictedControlMaster( - for: lease, - resolvedControlPath: firstResolvedPath - ) - } - var starts = runner.starts.makeAsyncIterator() - #expect(await starts.next() != nil) - broker.releaseWorkspace(lease) - runner.finish() - - #expect(await reset.value == .reset) - #expect(recorder.count == 1) - _ = observation - } - - @Test("An unresolved reset identity never exits a master") - func unresolvedResetIdentityFailsClosed() async { - let runner = RecordingProcessRunner() - let broker = makeBroker(processRunner: runner) - let lease = broker.retainWorkspace(configuration( - owner: UUID(), - options: unresolvedOptions - )) - - let outcome = await broker.resetConflictedControlMaster( - for: lease, - resolvedControlPath: "/tmp/cmux-ssh-501-%C" - ) - - guard case .ignored = outcome else { - Issue.record("Expected unresolved identity to be ignored") - return - } - #expect(runner.requests.isEmpty) - } - - @Test("A reset-wrapper no-op remains deferred and emits no reset") - func resetWrapperNoOpDoesNotCountAsReset() async throws { - let clock = ManualBrokerClock() - let runner = FixedStatusResetRunner( - status: NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus - ) - let recorder = ResetEventRecorder() - let broker = makeBroker(clock: clock, processRunner: runner) - let lease = broker.retainWorkspace(configuration( - owner: UUID(), - options: resolvedOptions - )) - let observation = try #require( - broker.observeControlMasterResets(controlPath: firstResolvedPath) { - recorder.record() - } - ) - - let reset = Task { @MainActor in - await broker.resetConflictedControlMaster( - for: lease, - resolvedControlPath: firstResolvedPath - ) - } - #expect(await clock.nextRequestedDelay() == 2_000) - await clock.resumeNextSleep() - #expect(await clock.nextRequestedDelay() == 2_000) - await clock.resumeNextSleep() - - guard case .deferred = await reset.value else { - Issue.record("Expected skipped reset attempts to remain deferred") - return - } - #expect(runner.requestCount == 3) - #expect(recorder.count == 0) - _ = observation - } - - @Test("Cleanup wrapper distinguishes ordinary and reset-only no-ops") - func cleanupWrapperUsesResetOnlySkippedStatus() throws { - let request = NativeSSHControlMasterCleanupRequest( - arguments: ["-V"], - environment: nil, - authenticationLockPath: "/dev/null/cmux-test.lock" - ) - let normalInvocation = request.processInvocation - let resetInvocation = request.processInvocation( - noOpExitStatus: - NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus - ) - let runner = RemoteSessionProcessRunner() - - let normal = try runner.run( - RemoteProcessRequest( - executable: normalInvocation.executableURL.path, - arguments: normalInvocation.arguments, - timeout: 2 - ), - operation: nil - ) - let reset = try runner.run( - RemoteProcessRequest( - executable: resetInvocation.executableURL.path, - arguments: resetInvocation.arguments, - timeout: 2 - ), - operation: nil - ) - - #expect(normal.status == 0) - #expect( - reset.status == - NativeSSHControlMasterCleanupRequest.resetSkippedExitStatus - ) - } - - @Test("Unresolved authorization keys remain owner-scoped") - func unresolvedTemplatesUseConservativeAuthorizationKeys() throws { - let first = configuration( - owner: UUID(), - destination: "shared-alias", - options: [ - "ControlMaster=auto", - "ControlPath=/tmp/cmux-ssh-501-%C", - "User=alice", - ] - ) - let second = configuration( - owner: UUID(), - destination: "shared-alias", - options: [ - "ControlMaster=auto", - "ControlPath=/tmp/cmux-ssh-501-%C", - "User=bob", - ] - ) - let matchingSibling = configuration( - owner: UUID(), - destination: "shared-alias", - options: [ - "ControlMaster=auto", - "ControlPath=/tmp/cmux-ssh-501-%C", - "User=alice", - ] - ) - let firstKey = try #require(NativeSSHControlMasterResetKey( - configuration: first, - sharingOptions: sharingOptions - )) - let secondKey = try #require(NativeSSHControlMasterResetKey( - configuration: second, - sharingOptions: sharingOptions - )) - let matchingSiblingKey = try #require(NativeSSHControlMasterResetKey( - configuration: matchingSibling, - sharingOptions: sharingOptions - )) - - #expect(firstKey != secondKey) - #expect(firstKey != matchingSiblingKey) - } - - private func makeBroker( - clock: any RemoteProxyRetryClock = RecordingImmediateClock(), - processRunner: any RemoteSessionProcessRunning - ) -> NativeSSHConnectionBroker { - NativeSSHConnectionBroker( - sharingOptions: sharingOptions, - clock: clock, - jitterMilliseconds: { 200 }, - cleanupLauncher: { _ in }, - conflictedMasterResetRunner: processRunner, - controlMasterOwnershipRegistry: - PermissiveNativeSSHControlMasterOwnershipRegistry() - ) - } - - private func configuration( - owner: UUID, - destination: String = "alice@example.test", - options: [String] - ) -> WorkspaceRemoteConfiguration { - WorkspaceRemoteConfiguration( - destination: destination, - port: nil, - identityFile: nil, - sshOptions: options, - localProxyPort: nil, - relayPort: 64_001, - relayID: "relay-id", - relayToken: "token", - localSocketPath: "/tmp/cmux-test.sock", - ownerWorkspaceID: owner, - terminalStartupCommand: nil, - preserveAfterTerminalExit: true, - persistentDaemonSlot: "ssh-test" - ) - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift index 2dc6c0b7ef9f..44eb912db56b 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/PermissiveNativeSSHControlMasterOwnershipRegistry.swift @@ -26,7 +26,7 @@ final class PermissiveNativeSSHControlMasterOwnershipRegistry: func release(lease: NativeSSHControlMasterLeaseIdentity) {} - func beginReset( + func beginRecovery( controlPath: String ) -> NativeSSHControlMasterExclusiveUseAuthorization? { NativeSSHControlMasterExclusiveUseAuthorization {} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift new file mode 100644 index 000000000000..6e0f4fa8aab3 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift @@ -0,0 +1,382 @@ +import CmuxCore +import CmuxFoundation +import Foundation +import Testing +@testable import CmuxRemoteSession + +@Suite("Inherited reverse-forward recovery") +struct RemoteSessionInheritedForwardRecoveryTests { + @Test("Metadata probe requires exact relay identity and slot") + func metadataProbeMatchesExactLeaseIdentity() throws { + let home = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-relay-probe-\(UUID().uuidString)", + isDirectory: true + ) + let relayDirectory = home + .appendingPathComponent(".cmux", isDirectory: true) + .appendingPathComponent("relay", isDirectory: true) + try FileManager.default.createDirectory( + at: relayDirectory, + withIntermediateDirectories: true + ) + defer { try? FileManager.default.removeItem(at: home) } + let authFile = relayDirectory.appendingPathComponent("64044.auth") + let slotFile = relayDirectory.appendingPathComponent("64044.slot") + let token = String(repeating: "a", count: 64) + try """ + {"relay_id":"relay-startup-cancellation","relay_token":"\(token)"} + """.write(to: authFile, atomically: true, encoding: .utf8) + try "ssh-test\n".write( + to: slotFile, + atomically: true, + encoding: .utf8 + ) + let script = + RemoteSessionCoordinator.remoteRelayMetadataOwnershipProbeScript( + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: token, + persistentDaemonSlot: "ssh-test" + ) + + #expect(try Self.runShellScript(script, home: home) == 0) + + try "other-slot".write( + to: slotFile, + atomically: true, + encoding: .utf8 + ) + #expect(try Self.runShellScript(script, home: home) == 64) + + try """ + {"relay_id":"another-relay","relay_token":"\(token)"} + """.write(to: authFile, atomically: true, encoding: .utf8) + try "ssh-test".write( + to: slotFile, + atomically: true, + encoding: .utf8 + ) + #expect(try Self.runShellScript(script, home: home) == 64) + } + + @Test("Matching metadata cancels only the stale forward and retries it") + func matchingMetadataRecoversStaleForward() async throws { + let runner = InheritedForwardRecoveryProcessRunner(mode: .success) + let launcher = RecordingReverseRelayLauncher() + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher + ) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" + ) + } + + let requests = runner.requests + let forwards = requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + } + let cancellations = requests.filter { + Self.isControlCommand("cancel", in: $0.arguments) + } + let probe = try #require( + requests.first(where: Self.isMetadataOwnershipProbe) + ) + #expect(forwards.count == 2) + #expect(cancellations.count == 1) + #expect( + Self.reverseForward(in: cancellations[0].arguments) + == "127.0.0.1:64044" + ) + #expect( + probe.arguments.contains( + "ControlPath=\(ResolvedControlPathFixture.path)" + ) + ) + #expect(probe.arguments.contains("BatchMode=yes")) + #expect(!requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + #expect(launcher.launchCount == 0) + #expect(coordinator.queue.sync { + coordinator.reverseRelayControlMasterForwardSpec != nil && + coordinator.reverseRelayProcess == nil + }) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("Mismatched metadata leaves an ambiguous listener untouched") + func metadataMismatchFailsClosed() async throws { + let runner = InheritedForwardRecoveryProcessRunner( + mode: .metadataMismatch + ) + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator(runner: runner) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + + let outcome = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: "127.0.0.1:64044:127.0.0.1:55001", + relayPort: 64_044 + ) + } + + guard case .bindingConflict = outcome else { + Issue.record("Expected the collision to remain unresolved") + return + } + let requests = runner.requests + #expect( + requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 1 + ) + #expect(requests.contains(where: Self.isMetadataOwnershipProbe)) + #expect(!requests.contains(where: { + Self.isControlCommand("cancel", in: $0.arguments) + })) + #expect(!requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("A rejected cancel does not retry or exit the master") + func rejectedCancellationFailsClosed() async throws { + let runner = InheritedForwardRecoveryProcessRunner( + mode: .cancellationFailure + ) + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator(runner: runner) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + + let outcome = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: "127.0.0.1:64044:127.0.0.1:55001", + relayPort: 64_044 + ) + } + + guard case .bindingConflict = outcome else { + Issue.record("Expected the rejected cancel to fail closed") + return + } + let requests = runner.requests + #expect( + requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 1 + ) + #expect( + requests.filter { + Self.isControlCommand("cancel", in: $0.arguments) + }.count == 1 + ) + #expect(!requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + @Test("A custom ControlPath never authorizes stale-forward recovery") + func customControlPathFailsClosed() async throws { + let runner = InheritedForwardRecoveryProcessRunner(mode: .success) + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator( + runner: runner, + sshOptions: [ + "StrictHostKeyChecking=accept-new", + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=~/.ssh/custom-%C", + ] + ) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + + let outcome = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: "127.0.0.1:64044:127.0.0.1:55001", + relayPort: 64_044 + ) + } + + guard case .bindingConflict = outcome else { + Issue.record("Expected the custom master collision to fail closed") + return + } + let requests = runner.requests + #expect( + requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 1 + ) + #expect(!requests.contains(where: Self.isMetadataOwnershipProbe)) + #expect(!requests.contains(where: { + Self.isControlCommand("cancel", in: $0.arguments) + })) + #expect(!requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } + + private static func reverseForward( + in arguments: [String] + ) -> String? { + guard let reverseIndex = arguments.firstIndex(of: "-R") else { + return nil + } + let valueIndex = arguments.index(after: reverseIndex) + return arguments.indices.contains(valueIndex) + ? arguments[valueIndex] + : nil + } + + private static func isMetadataOwnershipProbe( + _ request: RemoteProcessRequest + ) -> Bool { + request.arguments.last?.contains("tr -d") == true && + request.arguments.last?.contains( + "relay-startup-cancellation" + ) == true + } + + private static func runShellScript( + _ script: String, + home: URL + ) throws -> Int32 { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = ["-c", script] + process.environment = [ + "HOME": home.path, + "PATH": "/usr/bin:/bin", + ] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + return process.terminationStatus + } +} + +private final class InheritedForwardRecoveryProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + enum Mode: Equatable, Sendable { + case success + case metadataMismatch + case cancellationFailure + } + + // lint:allow lock - synchronous test requests consume one scripted counter. + private let lock = NSLock() + private let mode: Mode + private var _requests: [RemoteProcessRequest] = [] + private var forwardAttempts = 0 + + init(mode: Mode) { + self.mode = mode + } + + var requests: [RemoteProcessRequest] { + lock.withLock { _requests } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { + _requests.append(request) + if Self.isControlCommand("forward", in: request.arguments) { + forwardAttempts += 1 + if forwardAttempts == 1 { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: + "remote port forwarding failed for listen port 64044" + ) + } + } + if Self.isMetadataOwnershipProbe(request), + mode == .metadataMismatch { + return RemoteCommandResult( + status: 64, + stdout: "", + stderr: "" + ) + } + if Self.isControlCommand("cancel", in: request.arguments), + mode == .cancellationFailure { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "cancel failed" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } + + private static func isMetadataOwnershipProbe( + _ request: RemoteProcessRequest + ) -> Bool { + request.arguments.last?.contains("tr -d") == true && + request.arguments.last?.contains( + "relay-startup-cancellation" + ) == true + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index e569ad2d2964..435efa62ca4a 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -35,138 +35,6 @@ struct RemoteSessionReverseRelayStartupTests { )) } - @Test("Confirmed bind conflict exits the configured master once") - func confirmedConflictExitsConfiguredMaster() async throws { - let host = ReverseRelayRecoveryHost() - let runner = RecordingProcessRunner { _ in - RemoteCommandResult( - status: 255, - stdout: "", - stderr: "Control socket connect: No such file or directory" - ) - } - let fixture = try await Self.makeCoordinator(host: host, runner: runner) - let coordinator = fixture.coordinator - defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } - - let ignoredUnrelatedFailure = coordinator.queue.sync { - coordinator.beginConflictedControlMasterExitIfNeededLocked( - startupFailure: "Connection refused", - remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - resolvedControlPath: ResolvedControlPathFixture.path - ) - } - #expect(!ignoredUnrelatedFailure) - #expect(runner.requests.isEmpty) - - let beganRecovery = coordinator.queue.sync { - coordinator.beginConflictedControlMasterExitIfNeededLocked( - startupFailure: "Error: remote port forwarding failed for listen port 64044", - remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - resolvedControlPath: ResolvedControlPathFixture.path - ) - } - #expect(beganRecovery) - - var statuses = host.daemonStatuses.makeAsyncIterator() - let status = await statuses.next() - #expect(status?.detail == "test relay port unavailable") - - let request = runner.requests.first - #expect(request?.executable == "/usr/bin/ssh") - #expect(request?.arguments.contains("-O") == true) - #expect(request?.arguments.contains("exit") == true) - #expect(request?.arguments.contains("-R") == false) - #expect(request?.arguments.contains("StrictHostKeyChecking=accept-new") == true) - #expect(request?.arguments.last == "user@example.test") - - let recoveryAttempted = coordinator.queue.sync { - !coordinator.reverseRelayStartupPhase.isRecovering && - coordinator.reverseRelayRestartTask != nil - } - #expect(recoveryAttempted) - let beganSecondRecovery = coordinator.queue.sync { - coordinator.beginConflictedControlMasterExitIfNeededLocked( - startupFailure: "remote port forwarding failed for listen port 64044", - remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - resolvedControlPath: ResolvedControlPathFixture.path - ) - } - #expect(!beganSecondRecovery) - #expect(runner.requests.count == 1) - _ = await coordinator.stopAndWait(cleanupScope: .transport) - } - - @Test("Successful master exit schedules relay retry after reconnect can recreate it") - func successfulMasterExitSchedulesRetry() async throws { - let runner = RecordingProcessRunner() - let launcher = RecordingReverseRelayLauncher() - let clock = ManualBrokerClock() - let relayPort = 64_046 - let fixture = try await Self.makeCoordinator( - runner: runner, - reverseRelayLauncher: launcher, - relayPort: relayPort, - clock: clock - ) - let coordinator = fixture.coordinator - defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } - - coordinator.queue.async { - coordinator.daemonReady = true - _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( - startupFailure: "Error: remote port forwarding failed for listen port \(relayPort)", - remotePath: "/tmp/cmuxd-remote", - relayPort: relayPort, - resolvedControlPath: ResolvedControlPathFixture.path - ) - } - - #expect(await clock.nextRequestedDelay() == 2_000) - let recoveryRequest = runner.requests.first - #expect(recoveryRequest?.arguments.contains("-O") == true) - #expect(recoveryRequest?.arguments.contains("exit") == true) - #expect(launcher.launchCount == 0) - #expect(coordinator.queue.sync { - coordinator.reverseRelayStartupPhase.allowsRelayLaunch && - coordinator.reverseRelayProcess == nil - }) - _ = await coordinator.stopAndWait(cleanupScope: .transport) - } - - @Test("Stop detaches from a broker-owned reset without waiting") - func stopDetachesFromConflictedMasterReset() async throws { - let runner = BlockingConflictedMasterExitRunner() - let fixture = try await Self.makeCoordinator(runner: runner) - let coordinator = fixture.coordinator - defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } - - coordinator.queue.async { - _ = coordinator.beginConflictedControlMasterExitIfNeededLocked( - startupFailure: "remote port forwarding failed for listen port 64044", - remotePath: "/tmp/cmuxd-remote", - relayPort: 64_044, - resolvedControlPath: ResolvedControlPathFixture.path - ) - } - - var started = runner.started.makeAsyncIterator() - #expect(await started.next() != nil) - - _ = await coordinator.stopAndWait(cleanupScope: .transport) - - let startupCleared = coordinator.queue.sync { - !coordinator.reverseRelayStartupPhase.isRecovering && - coordinator.reverseRelayStartupPhase.allowsRelayLaunch && - coordinator.reverseRelayProcess == nil - } - #expect(startupCleared) - runner.finish() - } - @MainActor static func makeCoordinator( host: any RemoteSessionHosting = NoopRemoteSessionHost(), @@ -214,7 +82,6 @@ struct RemoteSessionReverseRelayStartupTests { clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, - conflictedMasterResetRunner: effectiveRunner, controlMasterOwnershipRegistry: ownershipRegistry ) let configuration = connectionBroker.retainWorkspace(rawConfiguration) @@ -335,37 +202,6 @@ final class StubReverseRelayProcess: func terminate() {} } -private final class BlockingConflictedMasterExitRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - let started: AsyncStream - - private let startedContinuation: AsyncStream.Continuation - private let release = DispatchSemaphore(value: 0) - - init() { - (started, startedContinuation) = AsyncStream.makeStream() - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - guard request.arguments.contains("-O"), - request.arguments.contains("exit") else { - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - startedContinuation.yield() - release.wait() - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - - func finish() { - release.signal() - } -} - final class ReverseRelayRecoveryHost: RemoteSessionHosting, @unchecked Sendable { let daemonStatuses: AsyncStream private let daemonStatusContinuation: AsyncStream.Continuation diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift index b795ab485f59..a29200193f79 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayTransportTests.swift @@ -225,104 +225,6 @@ struct RemoteSessionReverseRelayTransportTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test("A bind conflict exits a cmux-owned master") - func ownedConflictExitsMaster() async throws { - let clock = ManualBrokerClock() - let runner = RecordingProcessRunner { request in - if Self.isControlCommand("forward", in: request.arguments) { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: """ - mux_client_forward: forwarding request failed: remote port forwarding failed for listen port 64044 - muxclient: master forward request failed - """ - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - let launcher = RecordingReverseRelayLauncher() - let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( - runner: runner, - reverseRelayLauncher: launcher, - clock: clock - ) - let coordinator = fixture.coordinator - defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } - - coordinator.queue.sync { - coordinator.daemonReady = true - coordinator.daemonRemotePath = "/tmp/cmuxd-remote" - coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") - } - - #expect(await clock.nextRequestedDelay() == 2_000) - #expect(runner.requests.contains(where: { - Self.isControlCommand("exit", in: $0.arguments) - })) - let exitRequest = try #require(runner.requests.first(where: { - Self.isControlCommand("exit", in: $0.arguments) - })) - #expect( - exitRequest.arguments.contains( - "ControlPath=\(ResolvedControlPathFixture.path)" - ) - ) - #expect(launcher.launchCount == 0) - _ = await coordinator.stopAndWait(cleanupScope: .transport) - } - - @Test("A custom ControlPath is never exited") - func customControlPathFailsClosed() async throws { - let clock = ManualBrokerClock() - let runner = RecordingProcessRunner { request in - if Self.isControlCommand("forward", in: request.arguments) { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: "remote port forwarding failed for listen port 64044" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - let launcher = RecordingReverseRelayLauncher() - let fixture = try await RemoteSessionReverseRelayStartupTests.makeCoordinator( - runner: runner, - reverseRelayLauncher: launcher, - sshOptions: [ - "StrictHostKeyChecking=accept-new", - "ControlMaster=auto", - "ControlPersist=600", - "ControlPath=~/.ssh/custom-%C", - "ControlPath=\(SSHConnectionSharingOptions().defaultControlPath)", - ], - clock: clock - ) - let coordinator = fixture.coordinator - defer { try? FileManager.default.removeItem(at: fixture.scratchDirectory) } - - coordinator.queue.sync { - coordinator.daemonReady = true - coordinator.daemonRemotePath = "/tmp/cmuxd-remote" - coordinator.startReverseRelayLocked(remotePath: "/tmp/cmuxd-remote") - } - - #expect(await clock.nextRequestedDelay() == 2_000) - #expect(!runner.requests.contains(where: { - Self.isControlCommand("exit", in: $0.arguments) - })) - let forwardRequest = try #require(runner.requests.first(where: { - Self.isControlCommand("forward", in: $0.arguments) - })) - #expect(forwardRequest.arguments.contains("ControlPath=~/.ssh/custom-%C")) - #expect(coordinator.queue.sync { - !coordinator.reverseRelayStartupPhase.isRecovering && - coordinator.reverseRelayStartupPhase.allowsRelayLaunch - }) - #expect(launcher.launchCount == 0) - _ = await coordinator.stopAndWait(cleanupScope: .transport) - } - @Test("A cached custom ControlPath remains reusable") func cachedCustomControlPathRemainsReusable() async throws { let runner = RecordingProcessRunner() diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift index 80ec537066d4..a6f6440d0bd0 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift @@ -9,7 +9,7 @@ enum ResolvedControlPathFixture { } /// Gives relay tests deterministic `ssh -G` expansion while preserving their -/// existing process-runner scripts for forward, cancel, and reset commands. +/// existing process-runner scripts for forward and cancel commands. final class ResolvedControlPathProcessRunner: RemoteSessionProcessRunning, @unchecked Sendable diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/SynchronousEventRecorder.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/SynchronousEventRecorder.swift new file mode 100644 index 000000000000..9b2058319f7b --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/SynchronousEventRecorder.swift @@ -0,0 +1,18 @@ +import Foundation + +/// Counts synchronous test callbacks behind a short, nonblocking lock. +final class SynchronousEventRecorder: @unchecked Sendable { + // lint:allow lock - event callbacks increment one test counter. + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func record() { + lock.withLock { + value += 1 + } + } +} diff --git a/Sources/TerminalController+ControlWorkspaceContext.swift b/Sources/TerminalController+ControlWorkspaceContext.swift index 590512929f11..8bf6168f6a31 100644 --- a/Sources/TerminalController+ControlWorkspaceContext.swift +++ b/Sources/TerminalController+ControlWorkspaceContext.swift @@ -668,7 +668,21 @@ extension TerminalController: ControlWorkspaceContext { persistentDaemonSlot: persistentDaemonSlot?.isEmpty == true ? nil : persistentDaemonSlot, skipDaemonBootstrap: skipDaemonBootstrap ) - workspace.configureRemoteConnection(config, autoConnect: autoConnect) + guard workspace.configureRemoteConnection( + config, + autoConnect: autoConnect + ) else { + return .err( + code: "unavailable", + message: String( + localized: + "remoteSession.controlMaster.ownershipUnavailable", + defaultValue: + "SSH connection is busy in another cmux process." + ), + data: nil + ) + } notifyRemotePTYControllerAvailabilityChanged() let windowId = AppDelegate.shared?.windowId(for: owner) diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index 843a9cb666d4..7394277427b4 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -5423,7 +5423,6 @@ final class Workspace: Identifiable, ObservableObject { autoConnect: Bool = true ) -> Bool { var configuration = configuration.scopedToOwnerWorkspace(id) - configuration = nativeSSHConnectionBroker.retainWorkspace(configuration) let foregroundAuthToken = Self.normalizedForegroundAuthToken( configuration.foregroundAuthToken @@ -5440,6 +5439,12 @@ final class Workspace: Identifiable, ObservableObject { cancelPendingRemoteControlMasterAdoption() pendingControlMasterAdoption = nil } + if let pendingControlMasterAdoption { + configuration = configuration.withResolvedSSHControlPath( + pendingControlMasterAdoption.controlPath + ) + } + configuration = nativeSSHConnectionBroker.retainWorkspace(configuration) if let pendingControlMasterAdoption, !nativeSSHConnectionBroker.completeControlMasterAdoption( pendingControlMasterAdoption, diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index f1fa942eb424..3709c2132607 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -996,6 +996,10 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { config, autoConnect: false )) + XCTAssertEqual( + workspace.remoteConfiguration?.sshOptions.first, + "ControlPath=\(resolvedControlPath)" + ) XCTAssertEqual(workspace.remoteConnectionState, .connecting) XCTAssertNotNil(workspace.activeRemoteSessionControllerID) workspace.disconnectRemoteConnection(clearConfiguration: true) From 009bd2a1cabdab3c92ba3c455a846816ec286bd6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 05:51:55 -0700 Subject: [PATCH 29/39] fix: retry inherited forward recovery --- ...oordinator+ReverseRelayControlMaster.swift | 4 +- .../Session/RemoteSessionCoordinator.swift | 1 - .../Values/ReverseRelayStartupPhase.swift | 12 --- ...SessionInheritedForwardRecoveryTests.swift | 82 +++++++++++++++++-- 4 files changed, 75 insertions(+), 24 deletions(-) delete mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index c9fe86f09228..52857e43d94f 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -105,8 +105,7 @@ extension RemoteSessionCoordinator { relayPort: Int, effectiveSSHOptions: [String] ) -> Bool { - guard reverseRelayStartupPhase.canAttemptRecovery, - reverseRelayControlMasterForwardSpec == nil, + guard reverseRelayControlMasterForwardSpec == nil, let relayID = configuration.relayID? .trimmingCharacters(in: .whitespacesAndNewlines), !relayID.isEmpty, @@ -124,7 +123,6 @@ extension RemoteSessionCoordinator { ) else { return false } - reverseRelayStartupPhase = .recoveryAttempted defer { authorization.release() } let probeScript = Self.remoteRelayMetadataOwnershipProbeScript( diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index ab6fc2d839e7..34b453241d09 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -80,7 +80,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonReady = false var daemonBootstrapVersion: String? var daemonRemotePath: String? - var reverseRelayStartupPhase = ReverseRelayStartupPhase.recoveryAvailable var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? var resolvedControlMasterSSHOptions: [String]? diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift deleted file mode 100644 index 3473608854ef..000000000000 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift +++ /dev/null @@ -1,12 +0,0 @@ -/// Queue-confined phase for one inherited-forward cancellation attempt. -enum ReverseRelayStartupPhase: Sendable { - case recoveryAvailable - case recoveryAttempted - - var canAttemptRecovery: Bool { - if case .recoveryAvailable = self { - return true - } - return false - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift index 6e0f4fa8aab3..ac53adbfe94e 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift @@ -203,6 +203,64 @@ struct RemoteSessionInheritedForwardRecoveryTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } + @Test("A transient metadata failure can recover on the next relay attempt") + func transientMetadataFailureRetriesRecovery() async throws { + let runner = InheritedForwardRecoveryProcessRunner( + mode: .transientMetadataFailure + ) + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator(runner: runner) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + let forwardSpec = "127.0.0.1:64044:127.0.0.1:55001" + + let firstOutcome = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: forwardSpec, + relayPort: 64_044 + ) + } + guard case .bindingConflict = firstOutcome else { + Issue.record("Expected the transient probe failure to fail closed") + return + } + + let secondOutcome = coordinator.queue.sync { + coordinator.startReverseRelayViaControlMasterLocked( + forwardSpec: forwardSpec, + relayPort: 64_044 + ) + } + guard case .started = secondOutcome else { + Issue.record("Expected the next relay attempt to recover") + return + } + + let requests = runner.requests + #expect( + requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 3 + ) + #expect( + requests.filter(Self.isMetadataOwnershipProbe).count == 2 + ) + #expect( + requests.filter { + Self.isControlCommand("cancel", in: $0.arguments) + }.count == 1 + ) + #expect(!requests.contains(where: { + Self.isControlCommand("exit", in: $0.arguments) + })) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + @Test("A custom ControlPath never authorizes stale-forward recovery") func customControlPathFailsClosed() async throws { let runner = InheritedForwardRecoveryProcessRunner(mode: .success) @@ -309,6 +367,7 @@ private final class InheritedForwardRecoveryProcessRunner: case success case metadataMismatch case cancellationFailure + case transientMetadataFailure } // lint:allow lock - synchronous test requests consume one scripted counter. @@ -316,6 +375,7 @@ private final class InheritedForwardRecoveryProcessRunner: private let mode: Mode private var _requests: [RemoteProcessRequest] = [] private var forwardAttempts = 0 + private var metadataProbeAttempts = 0 init(mode: Mode) { self.mode = mode @@ -333,7 +393,9 @@ private final class InheritedForwardRecoveryProcessRunner: _requests.append(request) if Self.isControlCommand("forward", in: request.arguments) { forwardAttempts += 1 - if forwardAttempts == 1 { + let failingForwardAttempts = + mode == .transientMetadataFailure ? 2 : 1 + if forwardAttempts <= failingForwardAttempts { return RemoteCommandResult( status: 255, stdout: "", @@ -342,13 +404,17 @@ private final class InheritedForwardRecoveryProcessRunner: ) } } - if Self.isMetadataOwnershipProbe(request), - mode == .metadataMismatch { - return RemoteCommandResult( - status: 64, - stdout: "", - stderr: "" - ) + if Self.isMetadataOwnershipProbe(request) { + metadataProbeAttempts += 1 + if mode == .metadataMismatch || + (mode == .transientMetadataFailure && + metadataProbeAttempts == 1) { + return RemoteCommandResult( + status: 64, + stdout: "", + stderr: "" + ) + } } if Self.isControlCommand("cancel", in: request.arguments), mode == .cancellationFailure { From ecdc7dd460770c2851a4a41021b6f4e803c9934b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:12:43 -0700 Subject: [PATCH 30/39] fix: bound SSH relay recovery lifecycle --- .../NativeSSHConnectionBroker+Cleanup.swift | 30 +++++++---- .../NativeSSHConnectionBroker.swift | 7 +-- ...NativeSSHControlMasterPendingCleanup.swift | 11 ++++ ...emoteSessionCoordinator+ReverseRelay.swift | 2 + .../Session/RemoteSessionCoordinator.swift | 10 ++++ ...iveSSHControlMasterOwnershipRegistry.swift | 44 ++++++++++++++++ .../NativeSSHConnectionBrokerTests.swift | 30 +++++++++++ ...SessionInheritedForwardRecoveryTests.swift | 52 ++++++++++++------- 8 files changed, 154 insertions(+), 32 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterPendingCleanup.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/CleanupBlockingNativeSSHControlMasterOwnershipRegistry.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift index 10bd0ad0aab5..4d4a549a950a 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift @@ -7,6 +7,7 @@ extension NativeSSHConnectionBroker { private static let cleanupProcessTimeoutMilliseconds = 5_000 private static let cleanupForcedTerminationDelayMilliseconds = 1_000 private static let cleanupRetryDelayMilliseconds = 31_000 + private static let cleanupMaximumRetryCount = 3 func removeLease( ownerWorkspaceID: UUID, @@ -49,15 +50,19 @@ extension NativeSSHConnectionBroker { ) { if let cleanupLauncherOverride { cleanupLauncherOverride(request) - cleanupRequestsByControlMaster.removeValue(forKey: key) + pendingCleanupsByControlMaster.removeValue(forKey: key) } else { - cleanupRequestsByControlMaster[key] = request + pendingCleanupsByControlMaster[key] = + NativeSSHControlMasterPendingCleanup( + request: request, + retriesRemaining: Self.cleanupMaximumRetryCount + ) launchCleanup(request, for: key) } } func cancelCleanup(for key: NativeSSHControlMasterKey) { - cleanupRequestsByControlMaster.removeValue(forKey: key) + pendingCleanupsByControlMaster.removeValue(forKey: key) cleanupRetryTasks.removeValue(forKey: key)?.cancel() guard let cleanupID = cleanupProcessIDByControlMaster[key], let process = cleanupProcesses[cleanupID], @@ -72,7 +77,7 @@ extension NativeSSHConnectionBroker { _ request: NativeSSHControlMasterCleanupRequest, for key: NativeSSHControlMasterKey ) { - guard cleanupRequestsByControlMaster[key] != nil, + guard pendingCleanupsByControlMaster[key] != nil, ownersByControlMaster[key]?.isEmpty != false, cleanupProcessIDByControlMaster[key] == nil else { return @@ -119,11 +124,16 @@ extension NativeSSHConnectionBroker { } private func scheduleCleanupRetry(for key: NativeSSHControlMasterKey) { - guard cleanupRequestsByControlMaster[key] != nil, + guard var pendingCleanup = pendingCleanupsByControlMaster[key], ownersByControlMaster[key]?.isEmpty != false, cleanupRetryTasks[key] == nil else { return } + guard pendingCleanup.consumeRetry() else { + pendingCleanupsByControlMaster.removeValue(forKey: key) + return + } + pendingCleanupsByControlMaster[key] = pendingCleanup let clock = self.clock cleanupRetryTasks[key] = Task { @MainActor [weak self] in guard (try? await clock.sleep( @@ -138,12 +148,12 @@ extension NativeSSHConnectionBroker { private func retryCleanup(for key: NativeSSHControlMasterKey) { cleanupRetryTasks.removeValue(forKey: key) - guard let request = cleanupRequestsByControlMaster[key], + guard let pendingCleanup = pendingCleanupsByControlMaster[key], ownersByControlMaster[key]?.isEmpty != false else { - cleanupRequestsByControlMaster.removeValue(forKey: key) + pendingCleanupsByControlMaster.removeValue(forKey: key) return } - launchCleanup(request, for: key) + launchCleanup(pendingCleanup.request, for: key) } private func scheduleCleanupTimeout( @@ -190,7 +200,7 @@ extension NativeSSHConnectionBroker { if cleanupProcessIDByControlMaster[key] == cleanupID { cleanupProcessIDByControlMaster.removeValue(forKey: key) } - guard cleanupRequestsByControlMaster[key] != nil, + guard pendingCleanupsByControlMaster[key] != nil, ownersByControlMaster[key]?.isEmpty != false else { return } @@ -199,7 +209,7 @@ extension NativeSSHConnectionBroker { NativeSSHControlMasterCleanupRequest.retryExitStatus { scheduleCleanupRetry(for: key) } else { - cleanupRequestsByControlMaster.removeValue(forKey: key) + pendingCleanupsByControlMaster.removeValue(forKey: key) } } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index c9cbbc2a1ff2..179d901f0ded 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -22,8 +22,8 @@ public final class NativeSSHConnectionBroker { var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] var ownersByControlMaster: [NativeSSHControlMasterKey: Set] = [:] var attemptStates: [NativeSSHConnectionKey: NativeSSHConnectionAttemptState] = [:] - var cleanupRequestsByControlMaster: [ - NativeSSHControlMasterKey: NativeSSHControlMasterCleanupRequest + var pendingCleanupsByControlMaster: [ + NativeSSHControlMasterKey: NativeSSHControlMasterPendingCleanup ] = [:] var cleanupRetryTasks: [NativeSSHControlMasterKey: Task] = [:] var cleanupProcesses: [UUID: Process] = [:] @@ -77,7 +77,8 @@ public final class NativeSSHConnectionBroker { sharingOptions: SSHConnectionSharingOptions, clock: any RemoteProxyRetryClock, jitterMilliseconds: @escaping @MainActor @Sendable () -> Int, - cleanupLauncher: @escaping @MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void, + cleanupLauncher: + (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)?, controlMasterOwnershipRegistry: any NativeSSHControlMasterOwnershipTracking ) { diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterPendingCleanup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterPendingCleanup.swift new file mode 100644 index 000000000000..ca0df7698e81 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterPendingCleanup.swift @@ -0,0 +1,11 @@ +/// A last-owner cleanup request with a finite retry budget. +struct NativeSSHControlMasterPendingCleanup { + let request: NativeSSHControlMasterCleanupRequest + private(set) var retriesRemaining: Int + + mutating func consumeRetry() -> Bool { + guard retriesRemaining > 0 else { return false } + retriesRemaining -= 1 + return true + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index ddb805793567..2e026a2ad1a5 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -83,6 +83,7 @@ extension RemoteSessionCoordinator { scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) return } + restoreReadyDaemonStatusLocked() recordHeartbeatActivityLocked() debugLog( "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + @@ -179,6 +180,7 @@ extension RemoteSessionCoordinator { scheduleReverseRelayRestartLocked(remotePath: remotePath, delay: 2.0) return } + restoreReadyDaemonStatusLocked() recordHeartbeatActivityLocked() debugLog( "remote.relay.start relayPort=\(relayPort) localRelayPort=\(localRelayPort) " + diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index 34b453241d09..669001a4ac4e 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -80,6 +80,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonReady = false var daemonBootstrapVersion: String? var daemonRemotePath: String? + var readyDaemonStatus: WorkspaceRemoteDaemonStatus? var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? var resolvedControlMasterSSHOptions: [String]? @@ -471,9 +472,18 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { capabilities: capabilities, remotePath: remotePath ) + if state == .ready { + readyDaemonStatus = status + } host.publishDaemonStatus(status) } + /// Re-publishes the daemon hello snapshot after relay recovery succeeds. + func restoreReadyDaemonStatusLocked() { + guard daemonReady, let readyDaemonStatus else { return } + host.publishDaemonStatus(readyDaemonStatus) + } + func publishProxyEndpoint(_ endpoint: BrowserProxyEndpoint?) { host.publishProxyEndpoint(endpoint) } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/CleanupBlockingNativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/CleanupBlockingNativeSSHControlMasterOwnershipRegistry.swift new file mode 100644 index 000000000000..05782ef4a56f --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/CleanupBlockingNativeSSHControlMasterOwnershipRegistry.swift @@ -0,0 +1,44 @@ +import Foundation +@testable import CmuxRemoteSession + +/// Rejects cleanup ownership and records every bounded retry attempt. +final class CleanupBlockingNativeSSHControlMasterOwnershipRegistry: + NativeSSHControlMasterOwnershipTracking, + @unchecked Sendable +{ + let cleanupAttempts: AsyncStream + private let continuation: AsyncStream.Continuation + // lint:allow lock - synchronous callbacks increment one test counter. + private let lock = NSLock() + private var cleanupAttemptCount = 0 + + init() { + (cleanupAttempts, continuation) = AsyncStream.makeStream() + } + + func retain( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity + ) -> Bool { + true + } + + func release(lease: NativeSSHControlMasterLeaseIdentity) {} + + func beginRecovery( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + nil + } + + func beginCleanup( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + let attempt = lock.withLock { + cleanupAttemptCount += 1 + return cleanupAttemptCount + } + continuation.yield(attempt) + return nil + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index 4c83ca7f3ea3..8d520d137748 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -264,6 +264,36 @@ struct NativeSSHConnectionBrokerTests { #expect(FileManager.default.fileExists(atPath: markerPath)) } + @Test("Ownership-blocked cleanup exhausts its bounded retry budget") + func ownershipBlockedCleanupIsBounded() async { + let clock = ManualBrokerClock() + let ownershipRegistry = + CleanupBlockingNativeSSHControlMasterOwnershipRegistry() + let broker = NativeSSHConnectionBroker( + sharingOptions: sharingOptions, + clock: clock, + jitterMilliseconds: { 200 }, + cleanupLauncher: nil, + controlMasterOwnershipRegistry: ownershipRegistry + ) + let configuration = broker.retainWorkspace(configuration( + owner: UUID() + )) + var attempts = ownershipRegistry.cleanupAttempts.makeAsyncIterator() + + broker.releaseWorkspace(configuration) + #expect(await attempts.next() == 1) + + for expectedAttempt in 2...4 { + #expect(await clock.nextRequestedDelay() == 31_000) + await clock.resumeNextSleep() + #expect(await attempts.next() == expectedAttempt) + } + + #expect(broker.pendingCleanupsByControlMaster.isEmpty) + #expect(broker.cleanupRetryTasks.isEmpty) + } + @Test("Same-host attempts are FIFO and separated by bounded jitter") func sameHostAttemptsAreSerialized() async throws { let clock = ManualBrokerClock() diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift index ac53adbfe94e..be026fbf622e 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift @@ -208,38 +208,50 @@ struct RemoteSessionInheritedForwardRecoveryTests { let runner = InheritedForwardRecoveryProcessRunner( mode: .transientMetadataFailure ) + let host = ReverseRelayRecoveryHost() let fixture = try await RemoteSessionReverseRelayStartupTests - .makeCoordinator(runner: runner) + .makeCoordinator(host: host, runner: runner) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem( at: fixture.scratchDirectory ) } - let forwardSpec = "127.0.0.1:64044:127.0.0.1:55001" + let readyStatus = WorkspaceRemoteDaemonStatus( + state: .ready, + detail: "Remote daemon ready", + version: "0.64.20", + name: "cmuxd-remote", + capabilities: ["reverse-relay"], + remotePath: "/tmp/cmuxd-remote" + ) - let firstOutcome = coordinator.queue.sync { - coordinator.startReverseRelayViaControlMasterLocked( - forwardSpec: forwardSpec, - relayPort: 64_044 + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.publishDaemonStatus( + readyStatus.state, + detail: readyStatus.detail, + version: readyStatus.version, + name: readyStatus.name, + capabilities: readyStatus.capabilities, + remotePath: readyStatus.remotePath ) - } - guard case .bindingConflict = firstOutcome else { - Issue.record("Expected the transient probe failure to fail closed") - return - } - - let secondOutcome = coordinator.queue.sync { - coordinator.startReverseRelayViaControlMasterLocked( - forwardSpec: forwardSpec, - relayPort: 64_044 + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" ) } - guard case .started = secondOutcome else { - Issue.record("Expected the next relay attempt to recover") - return + coordinator.queue.sync { + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" + ) } + var statuses = host.daemonStatuses.makeAsyncIterator() + #expect(await statuses.next() == readyStatus) + #expect(await statuses.next()?.state == .error) + #expect(await statuses.next() == readyStatus) + let requests = runner.requests #expect( requests.filter { @@ -334,6 +346,7 @@ struct RemoteSessionInheritedForwardRecoveryTests { _ request: RemoteProcessRequest ) -> Bool { request.arguments.last?.contains("tr -d") == true && + request.arguments.last?.contains("auth_file=") == true && request.arguments.last?.contains( "relay-startup-cancellation" ) == true @@ -441,6 +454,7 @@ private final class InheritedForwardRecoveryProcessRunner: _ request: RemoteProcessRequest ) -> Bool { request.arguments.last?.contains("tr -d") == true && + request.arguments.last?.contains("auth_file=") == true && request.arguments.last?.contains( "relay-startup-cancellation" ) == true From 69cd31515fc0874704f21d0f8a39b5cc0a24f610 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:27:23 -0700 Subject: [PATCH 31/39] refactor: isolate SSH recovery lifecycle state --- ...ativeSSHControlMasterAdoptionHandoff.swift | 3 +- ...eSSHControlMasterExclusiveUsePurpose.swift | 5 + ...NativeSSHControlMasterOwnershipEntry.swift | 8 + ...iveSSHControlMasterOwnershipRegistry.swift | 32 ++- .../FoundationRemoteReverseRelayProcess.swift | 194 ++------------- .../Process/ReverseRelayStderrCapture.swift | 235 ++++++++++++++++++ ...enyingControlMasterOwnershipRegistry.swift | 27 ++ ...lakyResolvedControlPathProcessRunner.swift | 51 ++++ .../InheritedForwardRecoveryMode.swift | 6 + ...nheritedForwardRecoveryProcessRunner.swift | 84 +++++++ ...HControlMasterOwnershipRecoveryTests.swift | 26 -- .../RecordedReverseRelayLaunch.swift | 5 + .../RecordingReverseRelayLauncher.swift | 70 ++++++ ...SessionInheritedForwardRecoveryTests.swift | 89 ------- ...emoteSessionReverseRelayStartupTests.swift | 102 -------- .../ResolvedControlPathFixture.swift | 7 + .../ResolvedControlPathProcessRunner.swift | 56 ----- .../ReverseRelayRecoveryHost.swift | 35 +++ .../StubReverseRelayProcess.swift | 12 + 19 files changed, 577 insertions(+), 470 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUsePurpose.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipEntry.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/ReverseRelayStderrCapture.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/DenyingControlMasterOwnershipRegistry.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FlakyResolvedControlPathProcessRunner.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordedReverseRelayLaunch.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordingReverseRelayLauncher.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathFixture.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ReverseRelayRecoveryHost.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/StubReverseRelayProcess.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift index 31123189e348..820ca9921acd 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift @@ -4,7 +4,8 @@ internal import Foundation /// A temporary ownership lease that bridges foreground SSH authentication to /// installation of the workspace's durable ControlMaster lease. Unconsumed /// handoffs expire so an interrupted restore cannot retain ownership forever. -// SAFETY: `lock` serializes the release closure and expiration task. +/// +/// The synchronous lock serializes the release closure and expiration task. public final class NativeSSHControlMasterAdoptionHandoff: @unchecked Sendable, Equatable diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUsePurpose.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUsePurpose.swift new file mode 100644 index 000000000000..f1d56ae8e52e --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterExclusiveUsePurpose.swift @@ -0,0 +1,5 @@ +/// The operation requesting exclusive use of a resolved ControlMaster socket. +enum NativeSSHControlMasterExclusiveUsePurpose { + case reverseForwardRecovery + case ordinaryCleanup +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipEntry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipEntry.swift new file mode 100644 index 000000000000..004d1b763798 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipEntry.swift @@ -0,0 +1,8 @@ +internal import Foundation + +/// One resolved ControlMaster socket's in-process ownership state. +struct NativeSSHControlMasterOwnershipEntry { + let descriptor: Int32 + var leases: Set + var exclusiveUseID: UUID? +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift index 3d62cbbd9de5..3469d8537805 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift @@ -14,28 +14,22 @@ final class NativeSSHControlMasterOwnershipRegistry: NativeSSHControlMasterOwnershipTracking, @unchecked Sendable { - private struct Entry { - let descriptor: Int32 - var leases: Set - var exclusiveUseID: UUID? - } - - private enum ExclusiveUsePurpose { - case reverseForwardRecovery - case ordinaryCleanup - } - + private let sharingOptions: SSHConnectionSharingOptions // lint:allow lock - registry operations are short nonblocking fd updates. private let lock = NSLock() - private let sharingOptions: SSHConnectionSharingOptions - private var entries: [String: Entry] = [:] + private var entries: [ + String: NativeSSHControlMasterOwnershipEntry + ] = [:] private var controlPathByLease: [ NativeSSHControlMasterLeaseIdentity: String ] = [:] - init(sharingOptions: SSHConnectionSharingOptions) { + init( + sharingOptions: SSHConnectionSharingOptions, + fileManager: FileManager = .default + ) { self.sharingOptions = sharingOptions - try? FileManager.default.createDirectory( + try? fileManager.createDirectory( atPath: sharingOptions.controlMasterLockDirectoryPath, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700] @@ -82,7 +76,8 @@ final class NativeSSHControlMasterOwnershipRegistry: _ = Darwin.close(descriptor) return false } - entries[controlPath] = Entry( + entries[controlPath] = + NativeSSHControlMasterOwnershipEntry( descriptor: descriptor, leases: [lease], exclusiveUseID: nil @@ -118,7 +113,7 @@ final class NativeSSHControlMasterOwnershipRegistry: private func beginExclusiveUse( controlPath: String, - purpose: ExclusiveUsePurpose + purpose: NativeSSHControlMasterExclusiveUsePurpose ) -> NativeSSHControlMasterExclusiveUseAuthorization? { let exclusiveUseID = UUID() let authenticationDescriptor = lock.withLock { () -> Int32? in @@ -220,7 +215,8 @@ final class NativeSSHControlMasterOwnershipRegistry: _ = Darwin.close(descriptor) return false } - entries[controlPath] = Entry( + entries[controlPath] = + NativeSSHControlMasterOwnershipEntry( descriptor: descriptor, leases: [], exclusiveUseID: exclusiveUseID diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift index 81e0527799a7..004f1479bf7b 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/FoundationRemoteReverseRelayProcess.swift @@ -1,5 +1,6 @@ internal import Darwin internal import Foundation +internal import CmuxRemoteWorkspace /// Foundation-backed handle for one dedicated SSH reverse-relay process. /// @@ -14,17 +15,20 @@ final class FoundationRemoteReverseRelayProcess: private let stderrPipe: Pipe private let stderrDrainGracePeriod: TimeInterval private let terminationGracePeriod: TimeInterval + private let clock: any RemoteProxyRetryClock init( process: Process, stderrPipe: Pipe, stderrDrainGracePeriod: TimeInterval = 0.5, - terminationGracePeriod: TimeInterval = 2 + terminationGracePeriod: TimeInterval = 2, + clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock() ) { self.process = process self.stderrPipe = stderrPipe self.stderrDrainGracePeriod = stderrDrainGracePeriod self.terminationGracePeriod = terminationGracePeriod + self.clock = clock } var isRunning: Bool { @@ -82,7 +86,8 @@ final class FoundationRemoteReverseRelayProcess: startupTimeout: startupTimeout, startupTimeoutHandler: startupTimeoutHandler, startupHandler: startupHandler, - terminationHandler: terminationHandler + terminationHandler: terminationHandler, + clock: clock ) readHandle.readabilityHandler = { handle in capture.receive(handle.availableData) @@ -101,9 +106,15 @@ final class FoundationRemoteReverseRelayProcess: let process = process let processID = process.processIdentifier process.terminate() - DispatchQueue.global(qos: .utility).asyncAfter( - deadline: .now() + max(0, terminationGracePeriod) - ) { + let delayMilliseconds = Int( + (max(0, terminationGracePeriod) * 1_000).rounded(.up) + ) + Task { [clock] in + guard (try? await clock.sleep( + forMilliseconds: delayMilliseconds + )) != nil else { + return + } guard process.isRunning, process.processIdentifier == processID else { return @@ -112,176 +123,3 @@ final class FoundationRemoteReverseRelayProcess: } } } - -/// Event-driven stderr tail with a bounded post-termination drain. -private final class ReverseRelayStderrCapture: @unchecked Sendable { - private struct Completion { - let stderr: String - let status: Int32 - } - - // lint:allow lock - FileHandle and Process callbacks are synchronous; the - // critical sections only append bounded data and update lifecycle bits. - private let lock = NSLock() - private let readHandle: FileHandle - private let startupMarker: Data? - private let startupTimeout: TimeInterval? - private let startupTimeoutHandler: (@Sendable () -> Void)? - private let startupHandler: (@Sendable () -> Void)? - private let terminationHandler: @Sendable (String?) -> Void - private let byteLimit: Int - private let drainGracePeriod: TimeInterval - private var tail = Data() - private var startupReported = false - private var startupExpired = false - private var sawEOF = false - private var terminationStatus: Int32? - private var drainDeadlineScheduled = false - private var completed = false - - init( - readHandle: FileHandle, - byteLimit: Int = 8192, - drainGracePeriod: TimeInterval, - startupMarker: String?, - startupTimeout: TimeInterval?, - startupTimeoutHandler: (@Sendable () -> Void)?, - startupHandler: (@Sendable () -> Void)?, - terminationHandler: @escaping @Sendable (String?) -> Void - ) { - self.readHandle = readHandle - self.byteLimit = byteLimit - self.drainGracePeriod = drainGracePeriod - self.startupMarker = startupMarker?.data(using: .utf8) - self.startupTimeout = startupTimeout - self.startupTimeoutHandler = startupTimeoutHandler - self.startupHandler = startupHandler - self.terminationHandler = terminationHandler - } - - func startStartupDeadline() { - guard let startupTimeout else { return } - DispatchQueue.global(qos: .utility).asyncAfter( - deadline: .now() + max(0, startupTimeout) - ) { [weak self] in - self?.startupDeadlineElapsed() - } - } - - func receive(_ data: Data) { - let result = lock.withLock { () -> ( - completion: Completion?, - reportStartup: Bool - ) in - if data.isEmpty { - sawEOF = true - } else { - tail.append(data) - let reportStartup = - !completed && - !startupReported && - !startupExpired && - startupMarker.map { tail.range(of: $0) != nil } == true - if reportStartup { - startupReported = true - } - if tail.count > byteLimit { - tail.removeFirst(tail.count - byteLimit) - } - return (takeCompletionIfReady(), reportStartup) - } - return (takeCompletionIfReady(), false) - } - if result.reportStartup { - startupHandler?() - } - finish(result.completion) - } - - func processDidTerminate(status: Int32) { - let result = lock.withLock { () -> ( - completion: Completion?, - scheduleDeadline: Bool - ) in - terminationStatus = status - let completion = takeCompletionIfReady() - if let completion { - return (completion, false) - } - guard !drainDeadlineScheduled else { - return (nil, false) - } - drainDeadlineScheduled = true - return (nil, true) - } - finish(result.completion) - if result.scheduleDeadline { - DispatchQueue.global(qos: .utility).asyncAfter( - deadline: .now() + max(0, drainGracePeriod) - ) { [self] in - drainDeadlineElapsed() - } - } - } - - private func drainDeadlineElapsed() { - let completion = lock.withLock { - takeCompletionIfReady(force: true) - } - finish(completion) - } - - private func startupDeadlineElapsed() { - let shouldTerminate = lock.withLock { - guard !startupReported, - !startupExpired, - !completed, - terminationStatus == nil else { - return false - } - startupExpired = true - return true - } - if shouldTerminate { - startupTimeoutHandler?() - } - } - - private func takeCompletionIfReady(force: Bool = false) -> Completion? { - guard !completed, - (sawEOF || force), - let terminationStatus else { - return nil - } - completed = true - return Completion( - stderr: String(data: tail, encoding: .utf8) ?? "", - status: terminationStatus - ) - } - - private func finish(_ completion: Completion?) { - guard let completion else { return } - readHandle.readabilityHandler = nil - try? readHandle.close() - terminationHandler( - Self.preferredTerminationDetail(stderr: completion.stderr) - ?? "status=\(completion.status)" - ) - } - - private static func preferredTerminationDetail(stderr: String) -> String? { - let lines = stderr - .split(separator: "\n") - .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } - .filter { !$0.isEmpty } - if let forwardFailure = lines.last(where: { - $0.localizedCaseInsensitiveContains( - "remote port forwarding failed for listen" - ) - }) { - return forwardFailure - } - return RemoteSessionCoordinator.bestErrorLine(stderr: stderr) - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/ReverseRelayStderrCapture.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/ReverseRelayStderrCapture.swift new file mode 100644 index 000000000000..77d2187e755f --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Process/ReverseRelayStderrCapture.swift @@ -0,0 +1,235 @@ +internal import Foundation +internal import CmuxRemoteWorkspace + +/// Event-driven stderr tail with bounded startup and drain deadlines. +// SAFETY: synchronous FileHandle and Process callbacks serialize all mutable +// lifecycle state through `lock`; deadline tasks call the same guarded methods. +final class ReverseRelayStderrCapture: @unchecked Sendable { + // lint:allow lock - callback critical sections only update bounded state. + private let lock = NSLock() + private let readHandle: FileHandle + private let startupMarker: Data? + private let startupTimeout: TimeInterval? + private let startupTimeoutHandler: (@Sendable () -> Void)? + private let startupHandler: (@Sendable () -> Void)? + private let terminationHandler: @Sendable (String?) -> Void + private let byteLimit: Int + private let drainGracePeriod: TimeInterval + private let clock: any RemoteProxyRetryClock + private var tail = Data() + private var startupReported = false + private var startupExpired = false + private var sawEOF = false + private var terminationStatus: Int32? + private var drainDeadlineScheduled = false + private var completed = false + private var startupDeadlineTask: Task? + private var drainDeadlineTask: Task? + + init( + readHandle: FileHandle, + byteLimit: Int = 8192, + drainGracePeriod: TimeInterval, + startupMarker: String?, + startupTimeout: TimeInterval?, + startupTimeoutHandler: (@Sendable () -> Void)?, + startupHandler: (@Sendable () -> Void)?, + terminationHandler: @escaping @Sendable (String?) -> Void, + clock: any RemoteProxyRetryClock + ) { + self.readHandle = readHandle + self.byteLimit = byteLimit + self.drainGracePeriod = drainGracePeriod + self.startupMarker = startupMarker?.data(using: .utf8) + self.startupTimeout = startupTimeout + self.startupTimeoutHandler = startupTimeoutHandler + self.startupHandler = startupHandler + self.terminationHandler = terminationHandler + self.clock = clock + } + + func startStartupDeadline() { + guard let startupTimeout else { return } + let delayMilliseconds = Int( + (max(0, startupTimeout) * 1_000).rounded(.up) + ) + let task = Task { [weak self, clock] in + guard (try? await clock.sleep( + forMilliseconds: delayMilliseconds + )) != nil else { + return + } + self?.startupDeadlineElapsed() + } + let retained = lock.withLock { + guard !startupReported, !startupExpired, !completed else { + return false + } + startupDeadlineTask?.cancel() + startupDeadlineTask = task + return true + } + if !retained { + task.cancel() + } + } + + func receive(_ data: Data) { + let result = lock.withLock { () -> ( + completion: (stderr: String, status: Int32)?, + reportStartup: Bool + ) in + if data.isEmpty { + sawEOF = true + } else { + tail.append(data) + let reportStartup = + !completed && + !startupReported && + !startupExpired && + startupMarker.map { tail.range(of: $0) != nil } == true + if reportStartup { + startupReported = true + } + if tail.count > byteLimit { + tail.removeFirst(tail.count - byteLimit) + } + return (takeCompletionIfReady(), reportStartup) + } + return (takeCompletionIfReady(), false) + } + if result.reportStartup { + cancelStartupDeadline() + startupHandler?() + } + finish(result.completion) + } + + func processDidTerminate(status: Int32) { + let result = lock.withLock { () -> ( + completion: (stderr: String, status: Int32)?, + scheduleDeadline: Bool + ) in + terminationStatus = status + let completion = takeCompletionIfReady() + if let completion { + return (completion, false) + } + guard !drainDeadlineScheduled else { + return (nil, false) + } + drainDeadlineScheduled = true + return (nil, true) + } + finish(result.completion) + if result.scheduleDeadline { + scheduleDrainDeadline() + } + } + + private func scheduleDrainDeadline() { + let delayMilliseconds = Int( + (max(0, drainGracePeriod) * 1_000).rounded(.up) + ) + let task = Task { [weak self, clock] in + guard (try? await clock.sleep( + forMilliseconds: delayMilliseconds + )) != nil else { + return + } + self?.drainDeadlineElapsed() + } + let retained = lock.withLock { + guard !completed else { return false } + drainDeadlineTask?.cancel() + drainDeadlineTask = task + return true + } + if !retained { + task.cancel() + } + } + + private func drainDeadlineElapsed() { + let completion = lock.withLock { + drainDeadlineTask = nil + return takeCompletionIfReady(force: true) + } + finish(completion) + } + + private func startupDeadlineElapsed() { + let shouldTerminate = lock.withLock { + startupDeadlineTask = nil + guard !startupReported, + !startupExpired, + !completed, + terminationStatus == nil else { + return false + } + startupExpired = true + return true + } + if shouldTerminate { + startupTimeoutHandler?() + } + } + + private func cancelStartupDeadline() { + let task = lock.withLock { + defer { startupDeadlineTask = nil } + return startupDeadlineTask + } + task?.cancel() + } + + private func takeCompletionIfReady( + force: Bool = false + ) -> (stderr: String, status: Int32)? { + guard !completed, + (sawEOF || force), + let terminationStatus else { + return nil + } + completed = true + return ( + stderr: String(data: tail, encoding: .utf8) ?? "", + status: terminationStatus + ) + } + + private func finish( + _ completion: (stderr: String, status: Int32)? + ) { + guard let completion else { return } + let tasks = lock.withLock { + let tasks = (startupDeadlineTask, drainDeadlineTask) + startupDeadlineTask = nil + drainDeadlineTask = nil + return tasks + } + tasks.0?.cancel() + tasks.1?.cancel() + readHandle.readabilityHandler = nil + try? readHandle.close() + terminationHandler( + preferredTerminationDetail(stderr: completion.stderr) + ?? "status=\(completion.status)" + ) + } + + private func preferredTerminationDetail(stderr: String) -> String? { + let lines = stderr + .split(separator: "\n") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + if let forwardFailure = lines.last(where: { + $0.localizedCaseInsensitiveContains( + "remote port forwarding failed for listen" + ) + }) { + return forwardFailure + } + return RemoteSessionCoordinator.bestErrorLine(stderr: stderr) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/DenyingControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/DenyingControlMasterOwnershipRegistry.swift new file mode 100644 index 000000000000..7eda36ea7409 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/DenyingControlMasterOwnershipRegistry.swift @@ -0,0 +1,27 @@ +@testable import CmuxRemoteSession + +final class DenyingControlMasterOwnershipRegistry: + NativeSSHControlMasterOwnershipTracking, + Sendable +{ + func retain( + controlPath: String, + lease: NativeSSHControlMasterLeaseIdentity + ) -> Bool { + true + } + + func release(lease: NativeSSHControlMasterLeaseIdentity) {} + + func beginRecovery( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + nil + } + + func beginCleanup( + controlPath: String + ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + nil + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FlakyResolvedControlPathProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FlakyResolvedControlPathProcessRunner.swift new file mode 100644 index 000000000000..ec2e48ff104f --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FlakyResolvedControlPathProcessRunner.swift @@ -0,0 +1,51 @@ +import Foundation +@testable import CmuxRemoteSession + +final class FlakyResolvedControlPathProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - resolution calls consume one scripted test counter. + private let lock = NSLock() + private let base: any RemoteSessionProcessRunning + private let failureCount: Int + private var attempts = 0 + + init( + base: any RemoteSessionProcessRunning, + failureCount: Int + ) { + self.base = base + self.failureCount = failureCount + } + + var resolutionAttempts: Int { + lock.withLock { attempts } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + if request.executable == "/usr/bin/ssh", + request.arguments.first == "-G" { + let attempt = lock.withLock { + attempts += 1 + return attempts + } + guard attempt > failureCount else { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "temporary configuration failure" + ) + } + return RemoteCommandResult( + status: 0, + stdout: "controlpath \(ResolvedControlPathFixture.path)\n", + stderr: "" + ) + } + return try base.run(request, operation: operation) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift new file mode 100644 index 000000000000..3f3701739716 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift @@ -0,0 +1,6 @@ +enum InheritedForwardRecoveryMode: Equatable, Sendable { + case success + case metadataMismatch + case cancellationFailure + case transientMetadataFailure +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift new file mode 100644 index 000000000000..0fa74dbcc30c --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift @@ -0,0 +1,84 @@ +import Foundation +@testable import CmuxRemoteSession + +final class InheritedForwardRecoveryProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + // lint:allow lock - synchronous test requests consume one scripted counter. + private let lock = NSLock() + private let mode: InheritedForwardRecoveryMode + private var _requests: [RemoteProcessRequest] = [] + private var forwardAttempts = 0 + private var metadataProbeAttempts = 0 + + init(mode: InheritedForwardRecoveryMode) { + self.mode = mode + } + + var requests: [RemoteProcessRequest] { + lock.withLock { _requests } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { + _requests.append(request) + if Self.isControlCommand("forward", in: request.arguments) { + forwardAttempts += 1 + let failingForwardAttempts = + mode == .transientMetadataFailure ? 2 : 1 + if forwardAttempts <= failingForwardAttempts { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: + "remote port forwarding failed for listen port 64044" + ) + } + } + if Self.isMetadataOwnershipProbe(request) { + metadataProbeAttempts += 1 + if mode == .metadataMismatch || + (mode == .transientMetadataFailure && + metadataProbeAttempts == 1) { + return RemoteCommandResult( + status: 64, + stdout: "", + stderr: "" + ) + } + } + if Self.isControlCommand("cancel", in: request.arguments), + mode == .cancellationFailure { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "cancel failed" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } + + private static func isMetadataOwnershipProbe( + _ request: RemoteProcessRequest + ) -> Bool { + request.arguments.last?.contains("tr -d") == true && + request.arguments.last?.contains("auth_file=") == true && + request.arguments.last?.contains( + "relay-startup-cancellation" + ) == true + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift index 9c3366d49b58..1e634497176c 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift @@ -119,29 +119,3 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { authorization.release() } } - -private final class DenyingControlMasterOwnershipRegistry: - NativeSSHControlMasterOwnershipTracking, - Sendable -{ - func retain( - controlPath: String, - lease: NativeSSHControlMasterLeaseIdentity - ) -> Bool { - true - } - - func release(lease: NativeSSHControlMasterLeaseIdentity) {} - - func beginRecovery( - controlPath: String - ) -> NativeSSHControlMasterExclusiveUseAuthorization? { - nil - } - - func beginCleanup( - controlPath: String - ) -> NativeSSHControlMasterExclusiveUseAuthorization? { - nil - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordedReverseRelayLaunch.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordedReverseRelayLaunch.swift new file mode 100644 index 000000000000..4e4abb963e8c --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordedReverseRelayLaunch.swift @@ -0,0 +1,5 @@ +struct RecordedReverseRelayLaunch: Sendable { + let arguments: [String] + let localRelayPort: Int + let startupMarker: String +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordingReverseRelayLauncher.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordingReverseRelayLauncher.swift new file mode 100644 index 000000000000..9e851a47e956 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RecordingReverseRelayLauncher.swift @@ -0,0 +1,70 @@ +import Foundation +import Testing +@testable import CmuxRemoteSession + +/// Synchronous launcher callbacks cannot await; the lock protects only callback snapshots and a counter. +final class RecordingReverseRelayLauncher: + RemoteReverseRelayLaunching, + @unchecked Sendable +{ + let launches: AsyncStream + let process = StubReverseRelayProcess() + + private let lock = NSLock() + private var _launchCount = 0 + private var startupHandler: (@Sendable (any RemoteReverseRelayProcess) -> Void)? + private var terminationHandler: (@Sendable (any RemoteReverseRelayProcess, String?) -> Void)? + private let launchContinuation: AsyncStream.Continuation + + init() { + (launches, launchContinuation) = AsyncStream.makeStream() + } + + func launch( + arguments: [String], + environment: [String: String]?, + startupMarker: String, + startupHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess + ) -> Void, + terminationHandler: @escaping @Sendable ( + any RemoteReverseRelayProcess, + String? + ) -> Void + ) throws -> any RemoteReverseRelayProcess { + let reverseArgumentIndex = try #require(arguments.firstIndex(of: "-R")) + let reverseArgument = try #require( + arguments.indices.contains(arguments.index(after: reverseArgumentIndex)) + ? arguments[arguments.index(after: reverseArgumentIndex)] + : nil + ) + let localRelayPort = try #require( + Int(reverseArgument.split(separator: ":").last ?? "") + ) + launchContinuation.yield(RecordedReverseRelayLaunch( + arguments: arguments, + localRelayPort: localRelayPort, + startupMarker: startupMarker + )) + lock.withLock { + _launchCount += 1 + self.startupHandler = startupHandler + self.terminationHandler = terminationHandler + } + return process + } + + var launchCount: Int { + lock.withLock { _launchCount } + } + + func emitStartupReady() { + let handler = lock.withLock { startupHandler } + handler?(process) + } + + func emitTermination(detail: String?) { + let handler = lock.withLock { terminationHandler } + handler?(process, detail) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift index be026fbf622e..c7719f283ba8 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift @@ -371,92 +371,3 @@ struct RemoteSessionInheritedForwardRecoveryTests { return process.terminationStatus } } - -private final class InheritedForwardRecoveryProcessRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - enum Mode: Equatable, Sendable { - case success - case metadataMismatch - case cancellationFailure - case transientMetadataFailure - } - - // lint:allow lock - synchronous test requests consume one scripted counter. - private let lock = NSLock() - private let mode: Mode - private var _requests: [RemoteProcessRequest] = [] - private var forwardAttempts = 0 - private var metadataProbeAttempts = 0 - - init(mode: Mode) { - self.mode = mode - } - - var requests: [RemoteProcessRequest] { - lock.withLock { _requests } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - lock.withLock { - _requests.append(request) - if Self.isControlCommand("forward", in: request.arguments) { - forwardAttempts += 1 - let failingForwardAttempts = - mode == .transientMetadataFailure ? 2 : 1 - if forwardAttempts <= failingForwardAttempts { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: - "remote port forwarding failed for listen port 64044" - ) - } - } - if Self.isMetadataOwnershipProbe(request) { - metadataProbeAttempts += 1 - if mode == .metadataMismatch || - (mode == .transientMetadataFailure && - metadataProbeAttempts == 1) { - return RemoteCommandResult( - status: 64, - stdout: "", - stderr: "" - ) - } - } - if Self.isControlCommand("cancel", in: request.arguments), - mode == .cancellationFailure { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: "cancel failed" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - } - - private static func isControlCommand( - _ command: String, - in arguments: [String] - ) -> Bool { - arguments.indices.dropLast().contains(where: { - arguments[$0] == "-O" && arguments[$0 + 1] == command - }) - } - - private static func isMetadataOwnershipProbe( - _ request: RemoteProcessRequest - ) -> Bool { - request.arguments.last?.contains("tr -d") == true && - request.arguments.last?.contains("auth_file=") == true && - request.arguments.last?.contains( - "relay-startup-cancellation" - ) == true - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index 435efa62ca4a..d5ef757b3bb4 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -117,105 +117,3 @@ struct RemoteSessionReverseRelayStartupTests { return (coordinator, scratchDirectory) } } - -struct RecordedReverseRelayLaunch: Sendable { - let arguments: [String] - let localRelayPort: Int - let startupMarker: String -} - -/// Synchronous launcher callbacks cannot await; the lock protects only callback snapshots and a counter. -final class RecordingReverseRelayLauncher: - RemoteReverseRelayLaunching, - @unchecked Sendable -{ - let launches: AsyncStream - let process = StubReverseRelayProcess() - - private let lock = NSLock() - private var _launchCount = 0 - private var startupHandler: (@Sendable (any RemoteReverseRelayProcess) -> Void)? - private var terminationHandler: (@Sendable (any RemoteReverseRelayProcess, String?) -> Void)? - private let launchContinuation: AsyncStream.Continuation - - init() { - (launches, launchContinuation) = AsyncStream.makeStream() - } - - func launch( - arguments: [String], - environment: [String: String]?, - startupMarker: String, - startupHandler: @escaping @Sendable ( - any RemoteReverseRelayProcess - ) -> Void, - terminationHandler: @escaping @Sendable ( - any RemoteReverseRelayProcess, - String? - ) -> Void - ) throws -> any RemoteReverseRelayProcess { - let reverseArgumentIndex = try #require(arguments.firstIndex(of: "-R")) - let reverseArgument = try #require( - arguments.indices.contains(arguments.index(after: reverseArgumentIndex)) - ? arguments[arguments.index(after: reverseArgumentIndex)] - : nil - ) - let localRelayPort = try #require( - Int(reverseArgument.split(separator: ":").last ?? "") - ) - launchContinuation.yield(RecordedReverseRelayLaunch( - arguments: arguments, - localRelayPort: localRelayPort, - startupMarker: startupMarker - )) - lock.withLock { - _launchCount += 1 - self.startupHandler = startupHandler - self.terminationHandler = terminationHandler - } - return process - } - - var launchCount: Int { - lock.withLock { _launchCount } - } - - func emitStartupReady() { - let handler = lock.withLock { startupHandler } - handler?(process) - } - - func emitTermination(detail: String?) { - let handler = lock.withLock { terminationHandler } - handler?(process, detail) - } -} - -/// Immutable fake process used by the injected launcher. -final class StubReverseRelayProcess: - RemoteReverseRelayProcess, - @unchecked Sendable -{ - let isRunning = true - let terminationStatus: Int32 = 0 - - func terminate() {} -} - -final class ReverseRelayRecoveryHost: RemoteSessionHosting, @unchecked Sendable { - let daemonStatuses: AsyncStream - private let daemonStatusContinuation: AsyncStream.Continuation - - init() { - (daemonStatuses, daemonStatusContinuation) = AsyncStream.makeStream() - } - - func publishConnectionState(_ state: WorkspaceRemoteConnectionState, detail: String?) {} - func publishDaemonStatus(_ status: WorkspaceRemoteDaemonStatus) { - daemonStatusContinuation.yield(status) - } - func publishProxyEndpoint(_ endpoint: BrowserProxyEndpoint?) {} - func publishPortsSnapshot(detectedByPanel: [UUID: [Int]], detected: [Int]) {} - func publishHeartbeat(count: Int, lastSeenAt: Date?) {} - func publishBootstrapRemoteTTY(_ ttyName: String) {} -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathFixture.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathFixture.swift new file mode 100644 index 000000000000..6ef96788493d --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathFixture.swift @@ -0,0 +1,7 @@ +import CmuxFoundation + +enum ResolvedControlPathFixture { + static let path = + "/tmp/cmux-ssh-\(SSHConnectionSharingOptions().userID)-" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift index a6f6440d0bd0..01bccf1bf869 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ResolvedControlPathProcessRunner.swift @@ -1,13 +1,6 @@ -import CmuxFoundation import Foundation @testable import CmuxRemoteSession -enum ResolvedControlPathFixture { - static let path = - "/tmp/cmux-ssh-\(SSHConnectionSharingOptions().userID)-" + - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -} - /// Gives relay tests deterministic `ssh -G` expansion while preserving their /// existing process-runner scripts for forward and cancel commands. final class ResolvedControlPathProcessRunner: @@ -35,52 +28,3 @@ final class ResolvedControlPathProcessRunner: return try base.run(request, operation: operation) } } - -final class FlakyResolvedControlPathProcessRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - // lint:allow lock - resolution calls consume one scripted test counter. - private let lock = NSLock() - private let base: any RemoteSessionProcessRunning - private let failureCount: Int - private var attempts = 0 - - init( - base: any RemoteSessionProcessRunning, - failureCount: Int - ) { - self.base = base - self.failureCount = failureCount - } - - var resolutionAttempts: Int { - lock.withLock { attempts } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - if request.executable == "/usr/bin/ssh", - request.arguments.first == "-G" { - let attempt = lock.withLock { - attempts += 1 - return attempts - } - guard attempt > failureCount else { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: "temporary configuration failure" - ) - } - return RemoteCommandResult( - status: 0, - stdout: "controlpath \(ResolvedControlPathFixture.path)\n", - stderr: "" - ) - } - return try base.run(request, operation: operation) - } -} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ReverseRelayRecoveryHost.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ReverseRelayRecoveryHost.swift new file mode 100644 index 000000000000..32366a06b767 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/ReverseRelayRecoveryHost.swift @@ -0,0 +1,35 @@ +import CmuxCore +import Foundation +@testable import CmuxRemoteSession + +final class ReverseRelayRecoveryHost: + RemoteSessionHosting, + @unchecked Sendable +{ + let daemonStatuses: AsyncStream + private let daemonStatusContinuation: + AsyncStream.Continuation + + init() { + (daemonStatuses, daemonStatusContinuation) = AsyncStream.makeStream() + } + + func publishConnectionState( + _ state: WorkspaceRemoteConnectionState, + detail: String? + ) {} + + func publishDaemonStatus(_ status: WorkspaceRemoteDaemonStatus) { + daemonStatusContinuation.yield(status) + } + + func publishProxyEndpoint(_ endpoint: BrowserProxyEndpoint?) {} + + func publishPortsSnapshot( + detectedByPanel: [UUID: [Int]], + detected: [Int] + ) {} + + func publishHeartbeat(count: Int, lastSeenAt: Date?) {} + func publishBootstrapRemoteTTY(_ ttyName: String) {} +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/StubReverseRelayProcess.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/StubReverseRelayProcess.swift new file mode 100644 index 000000000000..59e5e6ff2e20 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/StubReverseRelayProcess.swift @@ -0,0 +1,12 @@ +@testable import CmuxRemoteSession + +/// Immutable fake process used by the injected launcher. +final class StubReverseRelayProcess: + RemoteReverseRelayProcess, + @unchecked Sendable +{ + let isRunning = true + let terminationStatus: Int32 = 0 + + func terminate() {} +} From 7c47bdc2ccae7e984a622d2bf8bebca5bfbf638b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:45:03 -0700 Subject: [PATCH 32/39] Address SSH relay ownership review findings --- .../ControlCommandCoordinator+Workspace.swift | 18 +++-- ...trolCommandCoordinatorWorkspaceTests.swift | 47 +++++++++++++ .../NativeSSHConnectionBroker+Cleanup.swift | 23 +++---- ...nectionBroker+ControlMasterOwnership.swift | 8 +++ ...NativeSSHControlMasterCleanupRequest.swift | 54 +-------------- ...iveSSHControlMasterOwnershipRegistry.swift | 3 + ...dationRemoteReverseRelayProcessTests.swift | 6 -- .../NativeSSHConnectionBrokerTests.swift | 66 +------------------ ...SSHControlMasterAdoptionHandoffTests.swift | 37 ++++++++++- ...alController+ControlWorkspaceContext.swift | 18 ++--- .../Workspace+RemoteSessionLifecycle.swift | 7 +- Sources/Workspace.swift | 8 +++ 12 files changed, 136 insertions(+), 159 deletions(-) diff --git a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift index 4c0a150011a2..6f04f085bac7 100644 --- a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift +++ b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift @@ -798,13 +798,17 @@ extension ControlCommandCoordinator { guard let workspaceID = resolution.workspaceID else { return .err(code: "invalid_params", message: "Missing workspace_id", data: nil) } - // Legacy `v2RawString(...)?.trimmingCharacters(...)`: trimmed, but an - // empty string stays "" (NOT nil), so use the raw-trim, not the - // empty-to-nil variant. - let token = rawString(params, "foreground_auth_token")? - .trimmingCharacters(in: .whitespacesAndNewlines) - let controlPath = rawString(params, "control_path")? - .trimmingCharacters(in: .whitespacesAndNewlines) + guard let token = optionalTrimmedRawString( + params, + "foreground_auth_token" + ) else { + return .err( + code: "invalid_params", + message: "Missing foreground_auth_token", + data: nil + ) + } + let controlPath = optionalTrimmedRawString(params, "control_path") return workspaceRemoteResult(context?.controlWorkspaceRemoteForegroundAuthReady( workspaceID: workspaceID, foregroundAuthToken: token, diff --git a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift index 656e4e67ede3..e873f448484a 100644 --- a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift +++ b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandCoordinatorWorkspaceTests.swift @@ -266,4 +266,51 @@ struct ControlCommandCoordinatorWorkspaceTests { "/tmp/cmux-ssh-501-0123456789abcdef" ) } + + @Test func foregroundAuthenticationRequiresNonemptyToken() { + let (coordinator, context) = coordinator() + let workspaceID = UUID() + let requests: [[String: JSONValue]] = [ + ["workspace_id": .string(workspaceID.uuidString)], + [ + "workspace_id": .string(workspaceID.uuidString), + "foreground_auth_token": .string(" \n "), + ], + ] + + for params in requests { + guard case .err(let code, let message, _) = + coordinator.handle(request( + "workspace.remote.foreground_auth_ready", + params + )) else { + Issue.record("missing foreground-auth token was accepted") + continue + } + #expect(code == "invalid_params") + #expect(message == "Missing foreground_auth_token") + } + #expect(context.foregroundAuthCall == nil) + } + + @Test func foregroundAuthenticationNormalizesBlankControlPath() { + let (coordinator, context) = coordinator() + let workspaceID = UUID() + context.foregroundAuthResolution = .unavailable( + workspaceID: workspaceID, + message: "localized ownership unavailable" + ) + + _ = coordinator.handle(request( + "workspace.remote.foreground_auth_ready", + [ + "workspace_id": .string(workspaceID.uuidString), + "foreground_auth_token": .string("auth-token"), + "control_path": .string(" \n "), + ] + )) + + #expect(context.foregroundAuthCall?.token == "auth-token") + #expect(context.foregroundAuthCall?.controlPath == nil) + } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift index 4d4a549a950a..104821b39173 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+Cleanup.swift @@ -4,6 +4,8 @@ internal import Foundation @MainActor extension NativeSSHConnectionBroker { + // A cross-process lock holder cannot signal this process when it exits, so + // cleanup retries use injected, capped deadlines instead of open-ended polling. private static let cleanupProcessTimeoutMilliseconds = 5_000 private static let cleanupForcedTerminationDelayMilliseconds = 1_000 private static let cleanupRetryDelayMilliseconds = 31_000 @@ -32,14 +34,9 @@ extension NativeSSHConnectionBroker { let arguments = RemoteControlMasterCleanup().cleanupArguments( configuration: previousConfiguration ) - let authenticationLockPath = - sharingOptions.resolvedControlMasterAuthenticationLockPath( - controlPath: key.controlPath - ) let request = NativeSSHControlMasterCleanupRequest( arguments: arguments, - environment: previousConfiguration.sshProcessEnvironment, - authenticationLockPath: authenticationLockPath + environment: previousConfiguration.sshProcessEnvironment ) beginCleanup(request, for: key) } @@ -90,11 +87,9 @@ extension NativeSSHConnectionBroker { } let cleanupID = UUID() let process = Process() - let invocation = NativeSSHControlMasterCleanupRequest( - arguments: request.arguments, - environment: request.environment, - authenticationLockPath: nil - ).processInvocation + // The registry authorization already holds the exact authentication + // and ownership locks; a second shell lock would deadlock itself. + let invocation = request.processInvocation process.executableURL = invocation.executableURL process.arguments = invocation.arguments process.environment = request.environment @@ -190,7 +185,7 @@ extension NativeSSHConnectionBroker { cleanupAuthorizations.removeValue(forKey: cleanupID)?.release() let terminationWasRequested = cleanupTerminationRequested.remove(cleanupID) != nil - let process = cleanupProcesses.removeValue(forKey: cleanupID) + cleanupProcesses.removeValue(forKey: cleanupID) guard let key = cleanupControlMasterKeysByProcessID.removeValue( forKey: cleanupID @@ -204,9 +199,7 @@ extension NativeSSHConnectionBroker { ownersByControlMaster[key]?.isEmpty != false else { return } - if terminationWasRequested || - process?.terminationStatus == - NativeSSHControlMasterCleanupRequest.retryExitStatus { + if terminationWasRequested { scheduleCleanupRetry(for: key) } else { pendingCleanupsByControlMaster.removeValue(forKey: key) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift index 0dee5661470d..046e5e5e2f43 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker+ControlMasterOwnership.swift @@ -18,6 +18,14 @@ extension NativeSSHConnectionBroker { controlPath: String, ownerWorkspaceID: UUID ) -> NativeSSHControlMasterAdoptionHandoff? { + let normalizedControlPath = + controlPath.trimmingCharacters(in: .whitespacesAndNewlines) + guard normalizedControlPath == controlPath, + sharingOptions.resolvedControlMasterOwnershipLockPath( + controlPath: controlPath + ) != nil else { + return nil + } let lease = NativeSSHControlMasterLeaseIdentity( ownerWorkspaceID: ownerWorkspaceID, generation: UUID() diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift index 17bc0b21d932..b1d7d8b050fa 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterCleanupRequest.swift @@ -8,72 +8,22 @@ public struct NativeSSHControlMasterCleanupRequest: Sendable { /// Environment passed to the cleanup process, including an injected agent socket when present. public let environment: [String: String]? - /// Advisory lock shared with foreground authentication for this master. - public let authenticationLockPath: String? - /// Creates a cleanup process request. /// /// - Parameters: /// - arguments: Arguments passed to `/usr/bin/ssh`. /// - environment: Optional process environment. - /// - authenticationLockPath: User-private lock path for this master. public init( arguments: [String], - environment: [String: String]?, - authenticationLockPath: String? + environment: [String: String]? ) { self.arguments = arguments self.environment = environment - self.authenticationLockPath = authenticationLockPath } } extension NativeSSHControlMasterCleanupRequest { - static let retryExitStatus: Int32 = 75 - static let resetSkippedExitStatus: Int32 = 76 - var processInvocation: (executableURL: URL, arguments: [String]) { - processInvocation(noOpExitStatus: 0) - } - - func processInvocation( - noOpExitStatus: Int32 - ) -> (executableURL: URL, arguments: [String]) { - guard let authenticationLockPath else { - return (URL(fileURLWithPath: "/usr/bin/ssh"), arguments) - } - let inFlightPath = authenticationLockPath + ".inflight" - let script = """ - umask 077 - : >> "$1" || exit \(noOpExitStatus) - zmodload zsh/system || exit \(noOpExitStatus) - zsystem flock -t 4 -e -f cmux_ssh_auth_lock_fd "$1" || exit \(Self.retryExitStatus) - cmux_auth_pid="$(/bin/cat -- "$2" 2>/dev/null || true)" - case "$cmux_auth_pid" in - ''|*[!0-9]*) ;; - *) - if /bin/kill -0 "$cmux_auth_pid" 2>/dev/null; then exit \(Self.retryExitStatus); fi - cmux_auth_mtime="$(/usr/bin/stat -f %m -- "$2" 2>/dev/null || true)" - cmux_now="$(/bin/date +%s)" - case "$cmux_auth_mtime:$cmux_now" in - *[!0-9:]*|:*) ;; - *) if [ $((cmux_now - cmux_auth_mtime)) -le 30 ]; then exit \(Self.retryExitStatus); fi ;; - esac - ;; - esac - /bin/rm -f -- "$2" 2>/dev/null || true - shift 2 - exec /usr/bin/ssh "$@" - """ - return ( - URL(fileURLWithPath: "/bin/zsh"), - [ - "-fc", - script, - "cmux-ssh-cleanup", - authenticationLockPath, - inFlightPath, - ] + arguments - ) + (URL(fileURLWithPath: "/usr/bin/ssh"), arguments) } } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift index 3469d8537805..c50e9ec76b87 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterOwnershipRegistry.swift @@ -188,6 +188,9 @@ final class NativeSSHControlMasterOwnershipRegistry: guard entry.exclusiveUseID == nil else { return false } + // The exact authentication lock remains exclusive across this + // non-atomic flock conversion, so no other cmux process can race an + // exclusive conversion into the unlock/reacquire window. _ = flock(entry.descriptor, LOCK_UN) guard flock(entry.descriptor, LOCK_EX | LOCK_NB) == 0 else { if flock(entry.descriptor, LOCK_SH | LOCK_NB) != 0 { diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift index aea3234f115f..b8a11386b1e4 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/FoundationRemoteReverseRelayProcessTests.swift @@ -69,7 +69,6 @@ struct FoundationRemoteReverseRelayProcessTests { stderrDrainGracePeriod: 0.05 ) let (details, continuation) = AsyncStream.makeStream() - let startedAt = Date() try process.run() relayProcess.captureTermination { detail in @@ -79,7 +78,6 @@ struct FoundationRemoteReverseRelayProcessTests { var iterator = details.makeAsyncIterator() #expect(await iterator.next() == "proxy diagnostic") - #expect(Date().timeIntervalSince(startedAt) < 1) #expect(process.terminationStatus == 23) } @@ -148,7 +146,6 @@ struct FoundationRemoteReverseRelayProcessTests { ) let startupRecorder = SynchronousEventRecorder() let (terminations, continuation) = AsyncStream.makeStream() - let startedAt = Date() try process.run() relayProcess.captureLifecycle( @@ -166,7 +163,6 @@ struct FoundationRemoteReverseRelayProcessTests { var iterator = terminations.makeAsyncIterator() #expect(await iterator.next() != nil) #expect(startupRecorder.count == 0) - #expect(Date().timeIntervalSince(startedAt) < 1) #expect(!process.isRunning) } @@ -210,12 +206,10 @@ struct FoundationRemoteReverseRelayProcessTests { continuation.finish() } ) - let startedAt = Date() var iterator = terminations.makeAsyncIterator() #expect(await iterator.next() != nil) #expect(startupRecorder.count == 0) - #expect(Date().timeIntervalSince(startedAt) < 1) #expect(!process.isRunning) #expect(process.terminationReason == .uncaughtSignal) #expect(process.terminationStatus == SIGKILL) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift index 8d520d137748..01ac79098e88 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHConnectionBrokerTests.swift @@ -41,12 +41,8 @@ struct NativeSSHConnectionBrokerTests { #expect(recorder.requests.count == 1) #expect(recorder.requests[0].arguments.contains(resolvedOwnedSSHOptions[2])) let request = recorder.requests[0] - let lockPath = request.authenticationLockPath - #expect(lockPath?.contains("cmux-ssh-501-resolved-auth-") == true) - #expect(request.processInvocation.executableURL.path == "/bin/zsh") - #expect(request.processInvocation.arguments.contains(lockPath.map { $0 + ".inflight" } ?? "") == true) - #expect(request.processInvocation.arguments[1].contains("zsystem flock -t 4 -e")) - #expect(request.processInvocation.arguments[1].contains("/bin/kill -0")) + #expect(request.processInvocation.executableURL.path == "/usr/bin/ssh") + #expect(request.processInvocation.arguments == request.arguments) } @Test("A custom user-managed control path is never closed") @@ -206,64 +202,6 @@ struct NativeSSHConnectionBrokerTests { #expect(arguments.suffix(3) == ["-O", "exit", "alice@example.test"]) } - @Test("Cleanup yields to a live foreground authentication marker") - func cleanupYieldsToLiveAuthentication() throws { - let root = FileManager.default.temporaryDirectory - .appendingPathComponent("cmux-ssh-cleanup-test-\(UUID().uuidString)", isDirectory: true) - let lockPath = root.appendingPathComponent("auth.lock").path - let markerPath = lockPath + ".inflight" - try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) - defer { try? FileManager.default.removeItem(at: root) } - try "\(getpid())\n".write(toFile: markerPath, atomically: true, encoding: .utf8) - - let request = NativeSSHControlMasterCleanupRequest( - arguments: ["-Z"], - environment: nil, - authenticationLockPath: lockPath - ) - let invocation = request.processInvocation - let process = Process() - process.executableURL = invocation.executableURL - process.arguments = invocation.arguments - process.standardInput = FileHandle.nullDevice - process.standardOutput = FileHandle.nullDevice - process.standardError = FileHandle.nullDevice - try process.run() - process.waitUntilExit() - - #expect(process.terminationStatus == 75) - #expect(FileManager.default.fileExists(atPath: markerPath)) - } - - @Test("Cleanup requests a retry for a recent dead authentication marker") - func cleanupRequestsRetryForRecentDeadAuthentication() throws { - let root = FileManager.default.temporaryDirectory - .appendingPathComponent("cmux-ssh-cleanup-test-\(UUID().uuidString)", isDirectory: true) - let lockPath = root.appendingPathComponent("auth.lock").path - let markerPath = lockPath + ".inflight" - try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) - defer { try? FileManager.default.removeItem(at: root) } - try "2147483647\n".write(toFile: markerPath, atomically: true, encoding: .utf8) - - let request = NativeSSHControlMasterCleanupRequest( - arguments: ["-Z"], - environment: nil, - authenticationLockPath: lockPath - ) - let invocation = request.processInvocation - let process = Process() - process.executableURL = invocation.executableURL - process.arguments = invocation.arguments - process.standardInput = FileHandle.nullDevice - process.standardOutput = FileHandle.nullDevice - process.standardError = FileHandle.nullDevice - try process.run() - process.waitUntilExit() - - #expect(process.terminationStatus == 75) - #expect(FileManager.default.fileExists(atPath: markerPath)) - } - @Test("Ownership-blocked cleanup exhausts its bounded retry budget") func ownershipBlockedCleanupIsBounded() async { let clock = ManualBrokerClock() diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift index 75302346fa98..c7a6ec50a5b0 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterAdoptionHandoffTests.swift @@ -1,13 +1,45 @@ +import CmuxFoundation import Foundation import Testing @testable import CmuxRemoteSession @Suite("Native SSH ControlMaster adoption handoff") struct NativeSSHControlMasterAdoptionHandoffTests { + @MainActor + @Test( + "Adoption rejects paths that are not exact resolved cmux sockets", + arguments: [ + "", + " /tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567", + "/tmp/cmux-ssh-501-%C", + "/tmp/cmux-ssh-502-0123456789abcdef0123456789abcdef01234567", + "~/.ssh/custom-0123456789abcdef0123456789abcdef01234567", + ] + ) + func adoptionRejectsUnownedPath(controlPath: String) { + let registry = + PermissiveNativeSSHControlMasterOwnershipRegistry() + let broker = NativeSSHConnectionBroker( + sharingOptions: SSHConnectionSharingOptions(userID: 501), + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + controlMasterOwnershipRegistry: registry + ) + + #expect(broker.beginControlMasterAdoption( + controlPath: controlPath, + ownerWorkspaceID: UUID() + ) == nil) + #expect(registry.retainedControlPaths.isEmpty) + } + @Test("An unconsumed handoff expires and releases ownership once") func unconsumedHandoffExpires() async { let clock = ManualBrokerClock() let recorder = SynchronousEventRecorder() + let (releases, releaseContinuation) = + AsyncStream.makeStream() let handoff = NativeSSHControlMasterAdoptionHandoff( controlPath: "/tmp/cmux-ssh-501-test", lease: NativeSSHControlMasterLeaseIdentity( @@ -18,12 +50,15 @@ struct NativeSSHControlMasterAdoptionHandoffTests { expirationMilliseconds: 10, releaseHandler: { recorder.record() + releaseContinuation.yield() + releaseContinuation.finish() } ) #expect(await clock.nextRequestedDelay() == 10) + var releaseIterator = releases.makeAsyncIterator() await clock.resumeNextSleep() - await Task.yield() + #expect(await releaseIterator.next() != nil) #expect(recorder.count == 1) handoff.release() diff --git a/Sources/TerminalController+ControlWorkspaceContext.swift b/Sources/TerminalController+ControlWorkspaceContext.swift index 8bf6168f6a31..b13d6cd3c18e 100644 --- a/Sources/TerminalController+ControlWorkspaceContext.swift +++ b/Sources/TerminalController+ControlWorkspaceContext.swift @@ -449,12 +449,9 @@ extension TerminalController: ControlWorkspaceContext { ) else { return .unavailable( workspaceID: workspaceID, - message: String( - localized: - "remoteSession.controlMaster.ownershipUnavailable", - defaultValue: - "SSH connection is busy in another cmux process." - ) + message: + RemoteSessionStrings.appLocalized + .controlMasterOwnershipUnavailable ) } notifyRemotePTYControllerAvailabilityChanged() @@ -674,12 +671,9 @@ extension TerminalController: ControlWorkspaceContext { ) else { return .err( code: "unavailable", - message: String( - localized: - "remoteSession.controlMaster.ownershipUnavailable", - defaultValue: - "SSH connection is busy in another cmux process." - ), + message: + RemoteSessionStrings.appLocalized + .controlMasterOwnershipUnavailable, data: nil ) } diff --git a/Sources/Workspace+RemoteSessionLifecycle.swift b/Sources/Workspace+RemoteSessionLifecycle.swift index 346b8e41b716..bb3506c11c56 100644 --- a/Sources/Workspace+RemoteSessionLifecycle.swift +++ b/Sources/Workspace+RemoteSessionLifecycle.swift @@ -207,12 +207,15 @@ extension Workspace { Self.normalizedForegroundAuthToken(token) else { return false } + let normalizedControlPath = resolvedControlPath? + .trimmingCharacters(in: .whitespacesAndNewlines) let controlMasterAdoption: NativeSSHControlMasterAdoptionHandoff? - if let resolvedControlPath { + if let normalizedControlPath, + !normalizedControlPath.isEmpty { guard let adoption = nativeSSHConnectionBroker.beginControlMasterAdoption( - controlPath: resolvedControlPath, + controlPath: normalizedControlPath, ownerWorkspaceID: id ) else { return false diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index 7d6316f28ad2..b1fec3bd6d8c 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -5565,6 +5565,14 @@ final class Workspace: Identifiable, ObservableObject { ) nativeSSHConnectionBroker.releaseWorkspace(configuration) remoteForegroundAuthenticationPhase = nil + remoteConnectionState = .error + remoteConnectionDetail = + RemoteSessionStrings.appLocalized + .controlMasterOwnershipUnavailable + applyBrowserRemoteWorkspaceStatusToPanels() + postRemoteConnectionPresentationDidChange() + TerminalController.shared + .notifyRemotePTYControllerAvailabilityChanged() return false } defer { TerminalController.shared.notifyRemotePTYControllerAvailabilityChanged() } From f2a69daaf2a3933ce08cfef499a283895d254aa8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:59:21 -0700 Subject: [PATCH 33/39] Fix app target remote session import --- Sources/TerminalController+ControlWorkspaceContext.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/TerminalController+ControlWorkspaceContext.swift b/Sources/TerminalController+ControlWorkspaceContext.swift index b13d6cd3c18e..5d523e40a717 100644 --- a/Sources/TerminalController+ControlWorkspaceContext.swift +++ b/Sources/TerminalController+ControlWorkspaceContext.swift @@ -1,6 +1,7 @@ import CmuxControlSocket import CmuxCore import CmuxPanes +import CmuxRemoteSession import CmuxWorkspaces import Foundation From 1f69281bc28dabefffeabfc0995c15d7c594a056 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 07:20:06 -0700 Subject: [PATCH 34/39] fix: expose adopted SSH control path --- .../Connection/NativeSSHControlMasterAdoptionHandoff.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift index 820ca9921acd..24632e21c1d1 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterAdoptionHandoff.swift @@ -10,7 +10,8 @@ public final class NativeSSHControlMasterAdoptionHandoff: @unchecked Sendable, Equatable { - let controlPath: String + /// Exact cmux-owned ControlPath held by this adoption lease. + public let controlPath: String let lease: NativeSSHControlMasterLeaseIdentity // lint:allow lock - transfer, cancellation, and deinit race to release once. private let lock = NSLock() From 5f4cbed3c0a2006b909751a6f5b73fcc1354b927 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 07:34:38 -0700 Subject: [PATCH 35/39] test: cover rotated relay auth recovery --- ...SessionInheritedForwardRecoveryTests.swift | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift index c7719f283ba8..3d240572c846 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift @@ -6,8 +6,8 @@ import Testing @Suite("Inherited reverse-forward recovery") struct RemoteSessionInheritedForwardRecoveryTests { - @Test("Metadata probe requires exact relay identity and slot") - func metadataProbeMatchesExactLeaseIdentity() throws { + @Test("Persistent metadata probe accepts rotated auth for the exact slot") + func persistentMetadataProbeUsesDurableSlotIdentity() throws { let home = FileManager.default.temporaryDirectory .appendingPathComponent( "cmux-relay-probe-\(UUID().uuidString)", @@ -57,6 +57,54 @@ struct RemoteSessionInheritedForwardRecoveryTests { atomically: true, encoding: .utf8 ) + #expect(try Self.runShellScript(script, home: home) == 0) + + try FileManager.default.removeItem(at: authFile) + #expect(try Self.runShellScript(script, home: home) == 64) + } + + @Test("Nonpersistent metadata probe requires exact relay credentials") + func nonpersistentMetadataProbeUsesTransientCredentials() throws { + let home = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-relay-probe-\(UUID().uuidString)", + isDirectory: true + ) + let relayDirectory = home + .appendingPathComponent(".cmux", isDirectory: true) + .appendingPathComponent("relay", isDirectory: true) + try FileManager.default.createDirectory( + at: relayDirectory, + withIntermediateDirectories: true + ) + defer { try? FileManager.default.removeItem(at: home) } + let authFile = relayDirectory.appendingPathComponent("64044.auth") + let slotFile = relayDirectory.appendingPathComponent("64044.slot") + let token = String(repeating: "a", count: 64) + try """ + {"relay_id":"relay-startup-cancellation","relay_token":"\(token)"} + """.write(to: authFile, atomically: true, encoding: .utf8) + let script = + RemoteSessionCoordinator.remoteRelayMetadataOwnershipProbeScript( + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: token, + persistentDaemonSlot: nil + ) + + #expect(try Self.runShellScript(script, home: home) == 0) + + try """ + {"relay_id":"another-relay","relay_token":"\(token)"} + """.write(to: authFile, atomically: true, encoding: .utf8) + #expect(try Self.runShellScript(script, home: home) == 64) + + try FileManager.default.removeItem(at: authFile) + try "unexpected-slot".write( + to: slotFile, + atomically: true, + encoding: .utf8 + ) #expect(try Self.runShellScript(script, home: home) == 64) } From fbc1586323f6d9a34c40044b8cfaa4bad1ee1b7f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 07:35:13 -0700 Subject: [PATCH 36/39] fix: recover rotated persistent relay leases --- ...SessionCoordinator+RelayProvisioning.swift | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift index d256abf70ec0..624e3ab2dd6a 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+RelayProvisioning.swift @@ -5,7 +5,11 @@ internal import Foundation // (the CmuxCore SSH-option-normalization precedent); the script text is // wire/process behavior pinned by tests — do not alter. extension RemoteSessionCoordinator { - /// Proves that stale relay metadata belongs to this exact relay identity. + /// Proves that stale relay metadata belongs to this relay namespace. + /// + /// Persistent restores deliberately rotate relay credentials, so their + /// durable daemon slot is the cross-launch ownership identity. Relays + /// without a persistent slot remain scoped to their exact credentials. static func remoteRelayMetadataOwnershipProbeScript( relayPort: Int, relayID: String, @@ -20,22 +24,24 @@ extension RemoteSessionCoordinator { guard persistentDaemonSlot == nil || normalizedSlot != nil else { return "exit 64" } - let slotCheck: String + let ownershipCheck: String if let normalizedSlot { - slotCheck = """ + ownershipCheck = """ + [ -r "$auth_file" ] || exit 64 [ -r "$slot_file" ] || exit 64 [ "$(tr -d '\\r\\n' < "$slot_file")" = \(normalizedSlot.shellSingleQuoted) ] || exit 64 """ } else { - slotCheck = "[ ! -e \"$slot_file\" ] || exit 64" + ownershipCheck = """ + [ "$(tr -d '\\r\\n' < "$auth_file")" = \(authPayload.shellSingleQuoted) ] || exit 64 + [ ! -e "$slot_file" ] || exit 64 + """ } return """ relay_directory="$HOME/.cmux/relay" auth_file="$relay_directory/\(relayPort).auth" slot_file="$relay_directory/\(relayPort).slot" - [ -r "$auth_file" ] || exit 64 - [ "$(tr -d '\\r\\n' < "$auth_file")" = \(authPayload.shellSingleQuoted) ] || exit 64 - \(slotCheck) + \(ownershipCheck) """ } From f9f58d3f9783b6ae9a5c632ebfaf3cce74b9263e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 08:08:12 -0700 Subject: [PATCH 37/39] test: cover inherited SSH master reap --- ...emoteSessionInheritedMasterReapTests.swift | 123 ++++++++++++++++++ ...emoteSessionReverseRelayStartupTests.swift | 5 +- 2 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift new file mode 100644 index 000000000000..e6d5b7a95b6f --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift @@ -0,0 +1,123 @@ +import Foundation +import Testing +@testable import CmuxRemoteSession + +@Suite("Inherited ControlMaster reap") +struct RemoteSessionInheritedMasterReapTests { + @Test("Owned persistent relay exits its inherited master before retrying") + func ownedPersistentRelayExitsInheritedMaster() async throws { + let runner = InheritedMasterReapProcessRunner() + let launcher = RecordingReverseRelayLauncher() + let clock = ManualBrokerClock() + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator( + runner: runner, + reverseRelayLauncher: launcher, + persistentDaemonSlot: "ssh-persistent-slot", + clock: clock + ) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + var requests = runner.requestStream.makeAsyncIterator() + + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" + ) + } + + let initialRequests = runner.requests + try #require(!initialRequests.contains(where: { + Self.isControlCommand("cancel", in: $0.arguments) + })) + #expect(initialRequests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 1) + + var exitRequest: RemoteProcessRequest? + while let request = await requests.next() { + if Self.isControlCommand("exit", in: request.arguments) { + exitRequest = request + break + } + } + let reapingRequest = try #require(exitRequest) + #expect( + reapingRequest.arguments.contains( + "ControlPath=\(ResolvedControlPathFixture.path)" + ) + ) + #expect(!reapingRequest.arguments.contains("-R")) + #expect(launcher.launchCount == 0) + #expect(await clock.nextRequestedDelay() == 2_000) + #expect(runner.requests.filter { + Self.isControlCommand("forward", in: $0.arguments) + }.count == 1) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } +} + +private final class InheritedMasterReapProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let requestStream: AsyncStream + + // lint:allow lock - synchronous test requests append and snapshot only. + private let lock = NSLock() + private var recordedRequests: [RemoteProcessRequest] = [] + private let requestContinuation: + AsyncStream.Continuation + + init() { + (requestStream, requestContinuation) = AsyncStream.makeStream() + } + + var requests: [RemoteProcessRequest] { + lock.withLock { recordedRequests } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { + recordedRequests.append(request) + } + requestContinuation.yield(request) + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: + "remote port forwarding failed for listen port 64044" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index d5ef757b3bb4..b90c31f0000b 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -42,6 +42,7 @@ struct RemoteSessionReverseRelayStartupTests { reverseRelayLauncher: any RemoteReverseRelayLaunching = RemoteReverseRelayLauncher(), relayPort: Int = 64_044, sshOptions: [String]? = nil, + persistentDaemonSlot: String? = nil, clock: any RemoteProxyRetryClock = SystemRemoteProxyRetryClock(), providesResolvedControlPath: Bool = true, ownershipRegistry: any NativeSSHControlMasterOwnershipTracking = @@ -68,8 +69,8 @@ struct RemoteSessionReverseRelayStartupTests { localSocketPath: scratchDirectory.appendingPathComponent("relay.sock").path, ownerWorkspaceID: UUID(), terminalStartupCommand: nil, - preserveAfterTerminalExit: false, - persistentDaemonSlot: nil + preserveAfterTerminalExit: persistentDaemonSlot != nil, + persistentDaemonSlot: persistentDaemonSlot ) let effectiveRunner: any RemoteSessionProcessRunning if providesResolvedControlPath { From fbe642f918b26cf78bf842c70a4b5225c3ca93ef Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 08:23:23 -0700 Subject: [PATCH 38/39] fix: reap inherited SSH masters after relay conflicts --- .../NativeSSHConnectionBroker.swift | 85 +++++- ...ativeSSHControlMasterReapCoordinator.swift | 280 ++++++++++++++++++ .../NativeSSHControlMasterReapEventHub.swift | 45 +++ .../NativeSSHControlMasterReapLeaseKey.swift | 54 ++++ .../RemoteSessionCoordinator+Lifecycle.swift | 1 + ...emoteSessionCoordinator+ReverseRelay.swift | 30 +- ...oordinator+ReverseRelayControlMaster.swift | 140 ++------- ...ssionCoordinator+ReverseRelayStartup.swift | 202 +++++++++++++ ...RemoteSessionCoordinator+SystemPower.swift | 10 +- .../Session/RemoteSessionCoordinator.swift | 2 +- .../Values/ControlMasterReapState.swift | 9 + .../Values/ReverseRelayStartupPhase.swift | 35 +++ .../InheritedForwardRecoveryMode.swift | 2 +- ...nheritedForwardRecoveryProcessRunner.swift | 18 +- ...HControlMasterOwnershipRecoveryTests.swift | 25 +- ...SessionInheritedForwardRecoveryTests.swift | 107 +++---- ...emoteSessionInheritedMasterReapTests.swift | 255 ++++++++++++++++ ...emoteSessionReverseRelayStartupTests.swift | 1 + 18 files changed, 1091 insertions(+), 210 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapEventHub.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapLeaseKey.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ControlMasterReapState.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift index 179d901f0ded..f57bd79bda9d 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHConnectionBroker.swift @@ -5,19 +5,24 @@ internal import Foundation /// Owns cmux-native SSH master lifetimes and serializes reconnect attempts per endpoint. /// -/// Workspace ownership is reference-counted by `ownerWorkspaceID`. Only the -/// last workspace using a cmux-owned `ControlPath` may request `ssh -O exit`; -/// custom control paths remain entirely user-managed. Connection attempts for -/// the same `(destination, port)` run one at a time, while different endpoints -/// remain independent. +/// Workspace ownership is reference-counted by `ownerWorkspaceID`. Ordinary +/// cleanup exits a cmux-owned master only for its last workspace; authenticated +/// inherited-forward recovery may reap an exclusively owned master and +/// invalidates every sharing workspace. Custom control paths remain entirely +/// user-managed. Connection attempts for the same `(destination, port)` run +/// one at a time, while different endpoints remain independent. @MainActor public final class NativeSSHConnectionBroker { nonisolated let sharingOptions: SSHConnectionSharingOptions let clock: any RemoteProxyRetryClock private let jitterMilliseconds: @MainActor @Sendable () -> Int let cleanupLauncherOverride: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)? + private nonisolated let inheritedMasterReapEventHub: + NativeSSHControlMasterReapEventHub nonisolated let controlMasterOwnershipRegistry: any NativeSSHControlMasterOwnershipTracking + private let inheritedMasterReapCoordinator: + NativeSSHControlMasterReapCoordinator var ownerLeases: [UUID: [NativeSSHControlMasterKey: WorkspaceRemoteConfiguration]] = [:] var ownersByControlMaster: [NativeSSHControlMasterKey: Set] = [:] @@ -43,11 +48,20 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = nil + let eventHub = NativeSSHControlMasterReapEventHub() let ownershipRegistry = NativeSSHControlMasterOwnershipRegistry( sharingOptions: sharingOptions ) + self.inheritedMasterReapEventHub = eventHub self.controlMasterOwnershipRegistry = ownershipRegistry + self.inheritedMasterReapCoordinator = + NativeSSHControlMasterReapCoordinator( + sharingOptions: sharingOptions, + processRunner: RemoteSessionProcessRunner(), + eventHub: eventHub, + ownershipRegistry: ownershipRegistry + ) } /// Creates a broker with an injected cleanup launcher. @@ -66,11 +80,20 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = { Int.random(in: 100...350) } self.cleanupLauncherOverride = cleanupLauncher + let eventHub = NativeSSHControlMasterReapEventHub() let ownershipRegistry = NativeSSHControlMasterOwnershipRegistry( sharingOptions: sharingOptions ) + self.inheritedMasterReapEventHub = eventHub self.controlMasterOwnershipRegistry = ownershipRegistry + self.inheritedMasterReapCoordinator = + NativeSSHControlMasterReapCoordinator( + sharingOptions: sharingOptions, + processRunner: RemoteSessionProcessRunner(), + eventHub: eventHub, + ownershipRegistry: ownershipRegistry + ) } nonisolated init( @@ -79,6 +102,8 @@ public final class NativeSSHConnectionBroker { jitterMilliseconds: @escaping @MainActor @Sendable () -> Int, cleanupLauncher: (@MainActor @Sendable (NativeSSHControlMasterCleanupRequest) -> Void)?, + inheritedMasterReapRunner: any RemoteSessionProcessRunning = + RemoteSessionProcessRunner(), controlMasterOwnershipRegistry: any NativeSSHControlMasterOwnershipTracking ) { @@ -86,7 +111,16 @@ public final class NativeSSHConnectionBroker { self.clock = clock self.jitterMilliseconds = jitterMilliseconds self.cleanupLauncherOverride = cleanupLauncher + let eventHub = NativeSSHControlMasterReapEventHub() + self.inheritedMasterReapEventHub = eventHub self.controlMasterOwnershipRegistry = controlMasterOwnershipRegistry + self.inheritedMasterReapCoordinator = + NativeSSHControlMasterReapCoordinator( + sharingOptions: sharingOptions, + processRunner: inheritedMasterReapRunner, + eventHub: eventHub, + ownershipRegistry: controlMasterOwnershipRegistry + ) } /// Retains the cmux-owned master used by a configured workspace. @@ -110,6 +144,16 @@ public final class NativeSSHConnectionBroker { } let leasedConfiguration = configuration.withSSHControlMasterLeaseGeneration(UUID()) + if let reapKey = NativeSSHControlMasterReapLeaseKey( + configuration: leasedConfiguration, + sharingOptions: sharingOptions + ) { + inheritedMasterReapCoordinator.retainWorkspace( + leasedConfiguration, + ownerWorkspaceID: ownerWorkspaceID, + key: reapKey + ) + } let nextKey = NativeSSHControlMasterKey( configuration: leasedConfiguration, sharingOptions: sharingOptions @@ -153,6 +197,16 @@ public final class NativeSSHConnectionBroker { ) { controlMasterOwnershipRegistry.release(lease: lease) } + if let reapKey = NativeSSHControlMasterReapLeaseKey( + configuration: configuration, + sharingOptions: sharingOptions + ) { + inheritedMasterReapCoordinator.releaseWorkspace( + ownerWorkspaceID: ownerWorkspaceID, + generation: generation, + key: reapKey + ) + } guard let key = NativeSSHControlMasterKey( configuration: configuration, sharingOptions: sharingOptions @@ -179,10 +233,23 @@ public final class NativeSSHConnectionBroker { ) } - /// Serializes a narrow inherited-forward cancellation against other cmux processes. - nonisolated func beginReverseForwardRecovery( + /// Reaps an exclusively owned inherited master after remote metadata proof. + func reapInheritedControlMaster( + for configuration: WorkspaceRemoteConfiguration, + resolvedControlPath: String, + metadataProbeCommand: String + ) async -> NativeSSHControlMasterReapOutcome { + await inheritedMasterReapCoordinator.reap( + for: configuration, + resolvedControlPath: resolvedControlPath, + metadataProbeCommand: metadataProbeCommand + ) + } + + /// Observes successful reaps for one exact cmux-owned control socket. + nonisolated func controlMasterReapEvents( controlPath: String - ) -> NativeSSHControlMasterExclusiveUseAuthorization? { + ) async -> AsyncStream? { guard !controlPath.contains("%"), sharingOptions.cmuxOwnedControlPath(in: [ "ControlMaster=auto", @@ -190,7 +257,7 @@ public final class NativeSSHConnectionBroker { ]) == controlPath else { return nil } - return controlMasterOwnershipRegistry.beginRecovery( + return await inheritedMasterReapEventHub.events( controlPath: controlPath ) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift new file mode 100644 index 000000000000..d93089c41d2c --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift @@ -0,0 +1,280 @@ +internal import CmuxCore +internal import CmuxFoundation +internal import CmuxRemoteWorkspace +internal import Foundation + +/// Outcome of one broker-authorized inherited-ControlMaster reap. +enum NativeSSHControlMasterReapOutcome: Sendable, Equatable { + case reaped(eventID: UUID) + case deferred(String) + case ignored(String) +} + +/// Broker-owned state for disruptive inherited-ControlMaster reaps. +/// +/// Authorization, metadata proof, process execution, and sibling invalidation +/// form one operation so no caller can prove one socket and exit another. +@MainActor +final class NativeSSHControlMasterReapCoordinator { + private struct InFlightReap { + let id: UUID + let task: Task + } + + private let sharingOptions: SSHConnectionSharingOptions + private let processRunner: any RemoteSessionProcessRunning + private let eventHub: NativeSSHControlMasterReapEventHub + private let ownershipRegistry: + any NativeSSHControlMasterOwnershipTracking + private var leases: [ + UUID: [ + NativeSSHControlMasterReapLeaseKey: + WorkspaceRemoteConfiguration + ] + ] = [:] + private var inFlightReaps: [String: InFlightReap] = [:] + + nonisolated init( + sharingOptions: SSHConnectionSharingOptions, + processRunner: any RemoteSessionProcessRunning, + eventHub: NativeSSHControlMasterReapEventHub, + ownershipRegistry: any NativeSSHControlMasterOwnershipTracking + ) { + self.sharingOptions = sharingOptions + self.processRunner = processRunner + self.eventHub = eventHub + self.ownershipRegistry = ownershipRegistry + } + + func retainWorkspace( + _ configuration: WorkspaceRemoteConfiguration, + ownerWorkspaceID: UUID, + key: NativeSSHControlMasterReapLeaseKey + ) { + var ownerLeases = leases[ownerWorkspaceID] ?? [:] + ownerLeases[key] = configuration + leases[ownerWorkspaceID] = ownerLeases + } + + func releaseWorkspace( + ownerWorkspaceID: UUID, + generation: UUID, + key: NativeSSHControlMasterReapLeaseKey + ) { + guard var ownerLeases = leases[ownerWorkspaceID], + ownerLeases[key]?.sshControlMasterLeaseGeneration == + generation else { + return + } + ownerLeases.removeValue(forKey: key) + if ownerLeases.isEmpty { + leases.removeValue(forKey: ownerWorkspaceID) + } else { + leases[ownerWorkspaceID] = ownerLeases + } + } + + func reap( + for configuration: WorkspaceRemoteConfiguration, + resolvedControlPath: String, + metadataProbeCommand: String + ) async -> NativeSSHControlMasterReapOutcome { + guard let ownerWorkspaceID = configuration.ownerWorkspaceID, + let generation = + configuration.sshControlMasterLeaseGeneration, + let key = NativeSSHControlMasterReapLeaseKey( + configuration: configuration, + sharingOptions: sharingOptions + ), + ownsLease( + ownerWorkspaceID: ownerWorkspaceID, + generation: generation, + key: key + ) else { + return .ignored( + "workspace no longer owns this cmux SSH master" + ) + } + guard !resolvedControlPath.contains("%"), + sharingOptions.cmuxOwnedControlPath(in: [ + "ControlMaster=auto", + "ControlPath=\(resolvedControlPath)", + ]) == resolvedControlPath else { + return .ignored( + "could not identify the cmux SSH master socket" + ) + } + if let inFlight = inFlightReaps[resolvedControlPath] { + return await inFlight.task.value + } + guard let lease = NativeSSHControlMasterLeaseIdentity( + configuration: configuration + ), + ownershipRegistry.retain( + controlPath: resolvedControlPath, + lease: lease + ) else { + return .deferred( + "resolved SSH master ownership is busy in another cmux process" + ) + } + guard let authorization = ownershipRegistry.beginRecovery( + controlPath: resolvedControlPath + ) else { + return .deferred( + "resolved SSH master is in use by another cmux process " + + "or foreground authentication" + ) + } + + let resolver = NativeSSHControlPathResolver( + sharingOptions: sharingOptions + ) + let resolvedOptions = resolver.replacingControlPath( + in: sharingOptions.mergingDefaults( + into: configuration.sshOptions + ), + with: resolvedControlPath + ) + let probeRequest = RemoteProcessRequest( + executable: "/usr/bin/ssh", + arguments: configuration.batchSSHCommandArguments( + command: metadataProbeCommand, + effectiveSSHOptions: resolvedOptions + ), + environment: configuration.sshProcessEnvironment, + timeout: 6 + ) + let exitRequest = RemoteProcessRequest( + executable: "/usr/bin/ssh", + arguments: RemoteControlMasterCleanup().cleanupArguments( + configuration: configuration, + sshOptionsOverride: resolvedOptions + ), + environment: configuration.sshProcessEnvironment, + timeout: 5 + ) + let reapID = UUID() + let processRunner = self.processRunner + let eventHub = self.eventHub + let task: Task< + NativeSSHControlMasterReapOutcome, + Never + > = Task { + defer { authorization.release() } + let attempt = await Self.runReap( + metadataProbeRequest: probeRequest, + exitRequest: exitRequest, + processRunner: processRunner + ) + switch attempt { + case .reaped: + let eventID = await eventHub.emit( + controlPath: resolvedControlPath + ) + return .reaped(eventID: eventID) + case .deferred(let detail): + return .deferred(detail) + case .ignored(let detail): + return .ignored(detail) + } + } + inFlightReaps[resolvedControlPath] = InFlightReap( + id: reapID, + task: task + ) + let outcome = await task.value + if inFlightReaps[resolvedControlPath]?.id == reapID { + inFlightReaps.removeValue(forKey: resolvedControlPath) + } + return outcome + } + + private func ownsLease( + ownerWorkspaceID: UUID, + generation: UUID, + key: NativeSSHControlMasterReapLeaseKey + ) -> Bool { + leases[ownerWorkspaceID]?[key]? + .sshControlMasterLeaseGeneration == generation + } + + @concurrent + private static func runReap( + metadataProbeRequest: RemoteProcessRequest, + exitRequest: RemoteProcessRequest, + processRunner: any RemoteSessionProcessRunning + ) async -> ReapAttemptOutcome { + let cancellation = RemoteProcessCancellationOperation() + return await withTaskCancellationHandler { + guard !Task.isCancelled else { + return .deferred("control-master reap cancelled") + } + do { + let probe = try processRunner.run( + metadataProbeRequest, + operation: cancellation + ) + guard probe.status == 0 else { + let detail = bestErrorLine( + stderr: probe.stderr, + stdout: probe.stdout + ) ?? "ssh exited \(probe.status)" + return probe.status == 64 + ? .ignored("relay metadata did not match") + : .deferred( + "could not verify relay metadata: \(detail)" + ) + } + guard !Task.isCancelled else { + return .deferred("control-master reap cancelled") + } + let exit = try processRunner.run( + exitRequest, + operation: cancellation + ) + guard exit.status == 0 else { + let detail = bestErrorLine( + stderr: exit.stderr, + stdout: exit.stdout + ) ?? "ssh exited \(exit.status)" + return .ignored( + "control-master exit failed: \(detail)" + ) + } + return .reaped + } catch { + return .deferred(error.localizedDescription) + } + } onCancel: { + cancellation.cancel() + } + } + + private nonisolated static func bestErrorLine( + stderr: String, + stdout: String + ) -> String? { + for text in [stderr, stdout] { + if let line = text + .split(whereSeparator: \.isNewline) + .map(String.init) + .last(where: { + !$0.trimmingCharacters( + in: .whitespacesAndNewlines + ).isEmpty + }) { + return line.trimmingCharacters( + in: .whitespacesAndNewlines + ) + } + } + return nil + } +} + +private enum ReapAttemptOutcome: Sendable { + case reaped + case deferred(String) + case ignored(String) +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapEventHub.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapEventHub.swift new file mode 100644 index 000000000000..abb53a5417b1 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapEventHub.swift @@ -0,0 +1,45 @@ +internal import Foundation + +/// Process-local async fanout for disruptive shared-ControlMaster reaps. +actor NativeSSHControlMasterReapEventHub { + private var observers: [ + String: [UUID: AsyncStream.Continuation] + ] = [:] + + func events(controlPath: String) -> AsyncStream { + let observerID = UUID() + return AsyncStream(bufferingPolicy: .bufferingNewest(1)) { + continuation in + observers[controlPath, default: [:]][observerID] = continuation + continuation.onTermination = { [weak self] _ in + Task { + await self?.removeObserver( + observerID, + controlPath: controlPath + ) + } + } + } + } + + func emit(controlPath: String) -> UUID { + let eventID = UUID() + guard let continuations = observers[controlPath]?.values else { + return eventID + } + for continuation in continuations { + continuation.yield(eventID) + } + return eventID + } + + private func removeObserver( + _ observerID: UUID, + controlPath: String + ) { + observers[controlPath]?.removeValue(forKey: observerID) + if observers[controlPath]?.isEmpty == true { + observers.removeValue(forKey: controlPath) + } + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapLeaseKey.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapLeaseKey.swift new file mode 100644 index 000000000000..c1fcbecbab5b --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapLeaseKey.swift @@ -0,0 +1,54 @@ +internal import CmuxCore +internal import CmuxFoundation +internal import Foundation + +/// Identifies a retained workspace generation that may reap an inherited +/// cmux-owned ControlMaster after an authenticated relay conflict. +/// +/// Exact paths are globally stable. An unresolved `%` template remains scoped +/// to one owner and its complete explicit SSH identity until `ssh -G` resolves +/// the authoritative socket path. +struct NativeSSHControlMasterReapLeaseKey: Hashable, Sendable { + let controlPath: String + let destination: String? + let port: Int? + let identityFile: String? + let effectiveOptions: [String] + let ownerWorkspaceID: UUID? + + init?( + configuration: WorkspaceRemoteConfiguration, + sharingOptions: SSHConnectionSharingOptions + ) { + guard configuration.transport == .ssh else { return nil } + let effectiveOptions = sharingOptions.mergingDefaults( + into: configuration.sshOptions + ) + guard let controlPath = sharingOptions.cmuxOwnedControlPath( + in: effectiveOptions + ) else { + return nil + } + self.controlPath = controlPath + if controlPath.contains("%") { + let destination = configuration.destination + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !destination.isEmpty, + let ownerWorkspaceID = configuration.ownerWorkspaceID else { + return nil + } + self.destination = destination + self.port = configuration.port + self.identityFile = configuration.identityFile? + .trimmingCharacters(in: .whitespacesAndNewlines) + self.effectiveOptions = effectiveOptions + self.ownerWorkspaceID = ownerWorkspaceID + } else { + self.destination = nil + self.port = nil + self.identityFile = nil + self.effectiveOptions = [] + self.ownerWorkspaceID = nil + } + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+Lifecycle.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+Lifecycle.swift index a224b11c5f7c..a5561e4516eb 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+Lifecycle.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+Lifecycle.swift @@ -21,6 +21,7 @@ extension RemoteSessionCoordinator { consecutiveUnreachableProbeCount = 0 reconnectSuspended = false reachabilityProbeGeneration &+= 1 + cancelControlMasterReapObservationLocked() cancelReverseRelayRestartLocked() cancelRemotePortScanCoalesceLocked() let cleanupSucceeded = stopReverseRelayLocked(cleanupScope: cleanupScope) diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift index 2e026a2ad1a5..d12314b11f38 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelay.swift @@ -26,6 +26,8 @@ extension RemoteSessionCoordinator { } guard reverseRelayProcess == nil else { return } guard reverseRelayControlMasterForwardSpec == nil else { return } + guard controlMasterReapState.startupPhase + .allowsRelayLaunch else { return } cancelReverseRelayRestartLocked() launchReverseRelayLocked( @@ -47,6 +49,8 @@ extension RemoteSessionCoordinator { ) { guard !isStopping, daemonReady, reverseRelayProcess == nil else { return } guard reverseRelayControlMasterForwardSpec == nil else { return } + guard controlMasterReapState.startupPhase + .allowsRelayLaunch else { return } var relayServer: RemoteCLIRelayServer? do { @@ -90,10 +94,18 @@ extension RemoteSessionCoordinator { "target=\(configuration.displayTarget) controlMaster=1" ) return - case .bindingConflict(let detail): + case .bindingConflict(let detail, let controlPath): debugLog( "remote.relay.startFailed relayPort=\(relayPort) error=\(detail)" ) + if beginInheritedControlMasterReapIfNeededLocked( + startupFailure: detail, + remotePath: remotePath, + relayPort: relayPort, + resolvedControlPath: controlPath + ) { + return + } publishReverseRelayFailureLocked( remotePath: remotePath ) @@ -255,6 +267,7 @@ extension RemoteSessionCoordinator { @discardableResult func stopReverseRelayLocked(cleanupScope: RemoteRelayCleanupScope = .transport) -> Bool { + cancelReverseRelayStartupLocked() if let reverseRelayProcess, reverseRelayProcess.isRunning { reverseRelayProcess.terminate() } @@ -265,6 +278,21 @@ extension RemoteSessionCoordinator { return removeRemoteRelayMetadataLocked(cleanupScope: cleanupScope) } + /// Drops only local state after the shared master has already exited. + /// + /// Remote metadata intentionally survives so a persistent daemon and its + /// pinned lease remain available to the reconnecting transport. + func invalidateReverseRelayAfterControlMasterReapLocked() { + cancelReverseRelayRestartLocked() + if let reverseRelayProcess, reverseRelayProcess.isRunning { + reverseRelayProcess.terminate() + } + reverseRelayProcess = nil + reverseRelayControlMasterForwardSpec = nil + cliRelayServer?.stop() + cliRelayServer = nil + } + func reverseRelayArguments(relayPort: Int, localRelayPort: Int) -> [String] { // Fallback only: `-S none` prevents accidental adoption of a shared // transport after `-O forward` proved unavailable. diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift index 52857e43d94f..44e2e798f63f 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayControlMaster.swift @@ -6,7 +6,7 @@ internal import Foundation enum ReverseRelayControlMasterStartOutcome: Sendable { case started case unavailable - case bindingConflict(String) + case bindingConflict(String, controlPath: String?) } extension RemoteSessionCoordinator { @@ -70,16 +70,19 @@ extension RemoteSessionCoordinator { debugConfigSummary() ) if let bindingConflict { - if recoverInheritedReverseForwardLocked( - forwardSpec: forwardSpec, - relayPort: relayPort, - effectiveSSHOptions: effectiveSSHOptions - ) { - reverseRelayControlMasterForwardSpec = - forwardSpec - return .started - } - return .bindingConflict(bindingConflict) + let ownedControlPath = + connectionBroker.sharingOptions + .cmuxOwnedControlPath( + in: effectiveSSHOptions + ) + let resolvedControlPath = + ownedControlPath?.contains("%") == false + ? ownedControlPath + : nil + return .bindingConflict( + bindingConflict, + controlPath: resolvedControlPath + ) } return .unavailable } @@ -94,117 +97,6 @@ extension RemoteSessionCoordinator { } } - /// Cancels only a forward whose persisted relay identity matches this workspace. - /// - /// A bind diagnostic alone is ambiguous: an unrelated remote process may - /// own the port. Recovery therefore requires the exact cmux-owned - /// ControlPath, cross-process exclusive ownership, matching relay metadata, - /// and a successful OpenSSH `cancel` for the listen address before retrying. - private func recoverInheritedReverseForwardLocked( - forwardSpec: String, - relayPort: Int, - effectiveSSHOptions: [String] - ) -> Bool { - guard reverseRelayControlMasterForwardSpec == nil, - let relayID = configuration.relayID? - .trimmingCharacters(in: .whitespacesAndNewlines), - !relayID.isEmpty, - let relayToken = configuration.relayToken? - .trimmingCharacters(in: .whitespacesAndNewlines), - !relayToken.isEmpty, - let controlPath = - connectionBroker.sharingOptions.cmuxOwnedControlPath( - in: effectiveSSHOptions - ), - !controlPath.contains("%"), - let authorization = - connectionBroker.beginReverseForwardRecovery( - controlPath: controlPath - ) else { - return false - } - defer { authorization.release() } - - let probeScript = Self.remoteRelayMetadataOwnershipProbeScript( - relayPort: relayPort, - relayID: relayID, - relayToken: relayToken, - persistentDaemonSlot: configuration.persistentDaemonSlot - ) - let probeCommand = "sh -c \(probeScript.shellSingleQuoted)" - do { - let probe = try sshExec( - arguments: configuration.batchSSHCommandArguments( - command: probeCommand, - effectiveSSHOptions: effectiveSSHOptions - ), - timeout: 6 - ) - guard probe.status == 0 else { - debugLog( - "remote.relay.inheritedForward.recoveryIgnored " + - "reason=metadata-mismatch relayPort=\(relayPort) " + - debugConfigSummary() - ) - return false - } - - let listenSpec = "127.0.0.1:\(relayPort)" - guard let cancelArguments = - configuration.reverseRelayControlMasterArguments( - controlCommand: "cancel", - forwardSpec: listenSpec, - effectiveSSHOptions: effectiveSSHOptions - ) else { - return false - } - let cancellation = try sshExec( - arguments: cancelArguments, - timeout: 4 - ) - guard cancellation.status == 0 else { - debugLog( - "remote.relay.inheritedForward.recoveryIgnored " + - "reason=forward-not-owned relayPort=\(relayPort) " + - debugConfigSummary() - ) - return false - } - - guard let forwardArguments = - configuration.reverseRelayControlMasterArguments( - controlCommand: "forward", - forwardSpec: forwardSpec, - effectiveSSHOptions: effectiveSSHOptions - ) else { - return false - } - let retry = try sshExec( - arguments: forwardArguments, - timeout: 6 - ) - guard retry.status == 0 else { - debugLog( - "remote.relay.inheritedForward.retryFailed " + - "relayPort=\(relayPort) \(debugConfigSummary())" - ) - return false - } - debugLog( - "remote.relay.inheritedForward.recovered " + - "relayPort=\(relayPort) \(debugConfigSummary())" - ) - return true - } catch { - debugLog( - "remote.relay.inheritedForward.recoveryIgnored " + - "relayPort=\(relayPort) \(error.localizedDescription) " + - debugConfigSummary() - ) - return false - } - } - /// Cancels only the exact forward this coordinator successfully installed. func stopReverseRelayViaControlMasterLocked() { guard let forwardSpec = reverseRelayControlMasterForwardSpec else { return } @@ -243,6 +135,9 @@ extension RemoteSessionCoordinator { ) else { return nil } + observeControlMasterReapsLocked( + controlPath: resolvedPath + ) return resolvedControlMasterSSHOptions } @@ -307,6 +202,7 @@ extension RemoteSessionCoordinator { ) return nil } + observeControlMasterReapsLocked(controlPath: resolvedPath) resolvedControlMasterSSHOptions = resolvedOptions return resolvedOptions } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift new file mode 100644 index 000000000000..7b4e24b0d137 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+ReverseRelayStartup.swift @@ -0,0 +1,202 @@ +internal import CmuxFoundation +internal import Foundation + +extension RemoteSessionCoordinator { + /// Reaps a cmux-owned ControlPersist master only after OpenSSH proves that + /// this relay's port is bound and remote metadata proves cmux ownership. + /// + /// OpenSSH cannot cancel an inherited reverse forward without its original + /// local target. Reaping the exclusively owned master is therefore the only + /// backward-compatible operation that preserves the remote lease port. + @discardableResult + func beginInheritedControlMasterReapIfNeededLocked( + startupFailure: String, + remotePath: String, + relayPort: Int, + resolvedControlPath: String? + ) -> Bool { + guard Self.isReverseRelayPortBindingFailure( + startupFailure, + relayPort: relayPort + ) else { + return false + } + guard controlMasterReapState.startupPhase + .canAttemptRecovery else { + return false + } + guard reverseRelayControlMasterForwardSpec == nil else { + debugLog( + "remote.relay.inheritedMaster.reapSkipped " + + "reason=current-forward-owned relayPort=\(relayPort) " + + debugConfigSummary() + ) + return false + } + guard let resolvedControlPath, + let relayID = configuration.relayID? + .trimmingCharacters(in: .whitespacesAndNewlines), + !relayID.isEmpty, + let relayToken = configuration.relayToken? + .trimmingCharacters(in: .whitespacesAndNewlines), + !relayToken.isEmpty else { + return false + } + + let probeScript = Self.remoteRelayMetadataOwnershipProbeScript( + relayPort: relayPort, + relayID: relayID, + relayToken: relayToken, + persistentDaemonSlot: configuration.persistentDaemonSlot + ) + let metadataProbeCommand = + "sh -c \(probeScript.shellSingleQuoted)" + let token = UUID() + let configuration = self.configuration + let connectionBroker = self.connectionBroker + let task = Task { [weak self] in + let outcome = + await connectionBroker.reapInheritedControlMaster( + for: configuration, + resolvedControlPath: resolvedControlPath, + metadataProbeCommand: metadataProbeCommand + ) + guard !Task.isCancelled else { return } + self?.queue.async { [weak self] in + self?.finishInheritedControlMasterReapLocked( + token: token, + outcome: outcome, + remotePath: remotePath, + relayPort: relayPort + ) + } + } + controlMasterReapState.startupPhase = + .reapingInheritedControlMaster( + token: token, + task: task + ) + debugLog( + "remote.relay.inheritedMaster.reapBegin " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + return true + } + + private func finishInheritedControlMasterReapLocked( + token: UUID, + outcome: NativeSSHControlMasterReapOutcome, + remotePath: String, + relayPort: Int + ) { + guard controlMasterReapState.startupPhase.token == token else { + return + } + switch outcome { + case .reaped(let eventID): + controlMasterReapState.startupPhase = .recoveryAttempted + debugLog( + "remote.relay.inheritedMaster.reaped " + + "relayPort=\(relayPort) \(debugConfigSummary())" + ) + handleSharedControlMasterReapLocked(eventID: eventID) + case .deferred(let detail): + controlMasterReapState.startupPhase = .recoveryAvailable + debugLog( + "remote.relay.inheritedMaster.reapDeferred " + + "relayPort=\(relayPort) \(detail) " + + debugConfigSummary() + ) + publishReverseRelayPortUnavailableLocked() + scheduleReverseRelayRestartLocked( + remotePath: remotePath, + delay: 2.0 + ) + case .ignored(let detail): + controlMasterReapState.startupPhase = .recoveryAttempted + debugLog( + "remote.relay.inheritedMaster.reapIgnored " + + "relayPort=\(relayPort) \(detail) " + + debugConfigSummary() + ) + publishReverseRelayPortUnavailableLocked() + scheduleReverseRelayRestartLocked( + remotePath: remotePath, + delay: 2.0 + ) + } + } + + /// Invalidates every local transport that shared the reaped master and + /// enters the normal reconnect state machine. + func handleSharedControlMasterReapLocked(eventID: UUID) { + guard controlMasterReapState.lastHandledEventID != eventID else { + return + } + controlMasterReapState.lastHandledEventID = eventID + controlMasterReapState.startupPhase = .recoveryAttempted + debugLog( + "remote.relay.inheritedMaster.reapObserved " + + debugConfigSummary() + ) + guard !isStopping else { return } + resetTransportForReconnectLocked( + preservePersistentRelayMetadata: true + ) + publishDaemonStatus( + .error, + detail: strings.reverseRelayUnavailableRetrying + ) + _ = scheduleReconnectLocked(baseDelay: 2.0) + } + + func observeControlMasterReapsLocked(controlPath: String) { + guard controlMasterReapState.observedControlPath != + controlPath else { + return + } + controlMasterReapState.observationTask?.cancel() + controlMasterReapState.observedControlPath = controlPath + let connectionBroker = self.connectionBroker + controlMasterReapState.observationTask = Task { [weak self] in + guard let events = + await connectionBroker.controlMasterReapEvents( + controlPath: controlPath + ) else { + return + } + for await eventID in events { + guard !Task.isCancelled else { return } + self?.queue.async { [weak self] in + self?.handleSharedControlMasterReapLocked( + eventID: eventID + ) + } + } + } + } + + func cancelReverseRelayStartupLocked() { + guard case .reapingInheritedControlMaster( + _, + let task + ) = controlMasterReapState.startupPhase else { + return + } + controlMasterReapState.startupPhase = .recoveryAttempted + task.cancel() + } + + func cancelControlMasterReapObservationLocked() { + controlMasterReapState.observationTask?.cancel() + controlMasterReapState.observationTask = nil + controlMasterReapState.observedControlPath = nil + } + + private func publishReverseRelayPortUnavailableLocked() { + publishDaemonStatus( + .error, + detail: strings.reverseRelayPortUnavailableRetrying + ) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+SystemPower.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+SystemPower.swift index 945a2c66dc78..57c75e6c390f 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+SystemPower.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator+SystemPower.swift @@ -53,10 +53,16 @@ extension RemoteSessionCoordinator { return shouldReconnect } - private func resetTransportForReconnectLocked() { + func resetTransportForReconnectLocked( + preservePersistentRelayMetadata: Bool = false + ) { cancelTransportDependentWorkLocked() cancelReverseRelayRestartLocked() - stopReverseRelayLocked() + if preservePersistentRelayMetadata { + invalidateReverseRelayAfterControlMasterReapLocked() + } else { + stopReverseRelayLocked() + } failPendingPTYBridgeStartsLocked("remote daemon is not ready") releaseProxyLeaseLocked() proxyEndpoint = nil diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift index e0e3fea82b41..b27650426027 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Session/RemoteSessionCoordinator.swift @@ -72,7 +72,6 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { // // Every var below is confined to `queue` (see the isolation essay). // Internal so the coordinator's same-module extension files can reach them. - var isStopping = false var proxyLease: RemoteProxyLease? var proxyLeaseGeneration: UInt64 = 0 @@ -81,6 +80,7 @@ public final class RemoteSessionCoordinator: @unchecked Sendable { var daemonBootstrapVersion: String? var daemonRemotePath: String? var readyDaemonStatus: WorkspaceRemoteDaemonStatus? + var controlMasterReapState = ControlMasterReapState() var reverseRelayProcess: (any RemoteReverseRelayProcess)? var reverseRelayControlMasterForwardSpec: String? var resolvedControlMasterSSHOptions: [String]? diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ControlMasterReapState.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ControlMasterReapState.swift new file mode 100644 index 000000000000..dffb56b75839 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ControlMasterReapState.swift @@ -0,0 +1,9 @@ +internal import Foundation + +/// Queue-confined lifecycle state for inherited-ControlMaster recovery. +struct ControlMasterReapState { + var startupPhase = ReverseRelayStartupPhase.recoveryAvailable + var observationTask: Task? + var observedControlPath: String? + var lastHandledEventID: UUID? +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift new file mode 100644 index 000000000000..f8f9b769ed31 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Values/ReverseRelayStartupPhase.swift @@ -0,0 +1,35 @@ +internal import Foundation + +/// Queue-confined phase for one conflict-triggered inherited-master recovery. +enum ReverseRelayStartupPhase: Sendable { + case recoveryAvailable + case reapingInheritedControlMaster( + token: UUID, + task: Task + ) + case recoveryAttempted + + var allowsRelayLaunch: Bool { + if case .reapingInheritedControlMaster = self { + return false + } + return true + } + + var canAttemptRecovery: Bool { + if case .recoveryAvailable = self { + return true + } + return false + } + + var token: UUID? { + guard case .reapingInheritedControlMaster( + let token, + _ + ) = self else { + return nil + } + return token + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift index 3f3701739716..f42b9039bb19 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryMode.swift @@ -1,6 +1,6 @@ enum InheritedForwardRecoveryMode: Equatable, Sendable { case success case metadataMismatch - case cancellationFailure + case exitFailure case transientMetadataFailure } diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift index 0fa74dbcc30c..948a9fe65bed 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedForwardRecoveryProcessRunner.swift @@ -41,22 +41,28 @@ final class InheritedForwardRecoveryProcessRunner: } if Self.isMetadataOwnershipProbe(request) { metadataProbeAttempts += 1 - if mode == .metadataMismatch || - (mode == .transientMetadataFailure && - metadataProbeAttempts == 1) { + if mode == .metadataMismatch { return RemoteCommandResult( status: 64, stdout: "", stderr: "" ) } + if mode == .transientMetadataFailure && + metadataProbeAttempts == 1 { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: "temporary probe failure" + ) + } } - if Self.isControlCommand("cancel", in: request.arguments), - mode == .cancellationFailure { + if Self.isControlCommand("exit", in: request.arguments), + mode == .exitFailure { return RemoteCommandResult( status: 255, stdout: "", - stderr: "cancel failed" + stderr: "exit failed" ) } return RemoteCommandResult(status: 0, stdout: "", stderr: "") diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift index 1e634497176c..f6ac5966e194 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/NativeSSHControlMasterOwnershipRecoveryTests.swift @@ -10,18 +10,21 @@ import Testing @Suite("Native SSH ownership-gated recovery") struct NativeSSHControlMasterOwnershipRecoveryTests { @Test("A live foreign owner prevents inherited-forward recovery") - func foreignOwnerFailsClosed() { + func foreignOwnerFailsClosed() async { let controlPath = "/tmp/cmux-ssh-501-0123456789abcdef0123456789abcdef01234567" + let runner = RecordingProcessRunner() let broker = NativeSSHConnectionBroker( sharingOptions: SSHConnectionSharingOptions(userID: 501), clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, + inheritedMasterReapRunner: runner, controlMasterOwnershipRegistry: DenyingControlMasterOwnershipRegistry() ) - _ = broker.retainWorkspace(WorkspaceRemoteConfiguration( + let configuration = broker.retainWorkspace( + WorkspaceRemoteConfiguration( destination: "alice@example.test", port: nil, identityFile: nil, @@ -39,13 +42,19 @@ struct NativeSSHControlMasterOwnershipRecoveryTests { terminalStartupCommand: nil, preserveAfterTerminalExit: true, persistentDaemonSlot: "ssh-test" - )) - - #expect( - broker.beginReverseForwardRecovery( - controlPath: controlPath - ) == nil + ) ) + + guard case .deferred = await broker.reapInheritedControlMaster( + for: configuration, + resolvedControlPath: controlPath, + metadataProbeCommand: "true" + ) else { + Issue.record("Expected a foreign owner to defer the reap") + return + } + #expect(runner.requests.isEmpty) + broker.releaseWorkspace(configuration) } @Test("Foreground authentication hands ownership to the workspace without a gap") diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift index 3d240572c846..fdf2065e9196 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedForwardRecoveryTests.swift @@ -108,14 +108,16 @@ struct RemoteSessionInheritedForwardRecoveryTests { #expect(try Self.runShellScript(script, home: home) == 64) } - @Test("Matching metadata cancels only the stale forward and retries it") - func matchingMetadataRecoversStaleForward() async throws { + @Test("Matching transient metadata authorizes an inherited-master reap") + func matchingMetadataAuthorizesMasterReap() async throws { let runner = InheritedForwardRecoveryProcessRunner(mode: .success) let launcher = RecordingReverseRelayLauncher() + let clock = ManualBrokerClock() let fixture = try await RemoteSessionReverseRelayStartupTests .makeCoordinator( runner: runner, - reverseRelayLauncher: launcher + reverseRelayLauncher: launcher, + clock: clock ) let coordinator = fixture.coordinator defer { @@ -132,34 +134,31 @@ struct RemoteSessionInheritedForwardRecoveryTests { ) } + #expect(await clock.nextRequestedDelay() == 2_000) let requests = runner.requests let forwards = requests.filter { Self.isControlCommand("forward", in: $0.arguments) } - let cancellations = requests.filter { - Self.isControlCommand("cancel", in: $0.arguments) - } let probe = try #require( requests.first(where: Self.isMetadataOwnershipProbe) ) - #expect(forwards.count == 2) - #expect(cancellations.count == 1) - #expect( - Self.reverseForward(in: cancellations[0].arguments) - == "127.0.0.1:64044" - ) + #expect(forwards.count == 1) #expect( probe.arguments.contains( "ControlPath=\(ResolvedControlPathFixture.path)" ) ) #expect(probe.arguments.contains("BatchMode=yes")) - #expect(!requests.contains(where: { + #expect(requests.filter { Self.isControlCommand("exit", in: $0.arguments) + }.count == 1) + #expect(!requests.contains(where: { + Self.isControlCommand("cancel", in: $0.arguments) })) #expect(launcher.launchCount == 0) #expect(coordinator.queue.sync { - coordinator.reverseRelayControlMasterForwardSpec != nil && + !coordinator.daemonReady && + coordinator.reverseRelayControlMasterForwardSpec == nil && coordinator.reverseRelayProcess == nil }) @@ -171,8 +170,9 @@ struct RemoteSessionInheritedForwardRecoveryTests { let runner = InheritedForwardRecoveryProcessRunner( mode: .metadataMismatch ) + let clock = ManualBrokerClock() let fixture = try await RemoteSessionReverseRelayStartupTests - .makeCoordinator(runner: runner) + .makeCoordinator(runner: runner, clock: clock) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem( @@ -180,17 +180,15 @@ struct RemoteSessionInheritedForwardRecoveryTests { ) } - let outcome = coordinator.queue.sync { - coordinator.startReverseRelayViaControlMasterLocked( - forwardSpec: "127.0.0.1:64044:127.0.0.1:55001", - relayPort: 64_044 + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" ) } - guard case .bindingConflict = outcome else { - Issue.record("Expected the collision to remain unresolved") - return - } + #expect(await clock.nextRequestedDelay() == 2_000) let requests = runner.requests #expect( requests.filter { @@ -208,13 +206,14 @@ struct RemoteSessionInheritedForwardRecoveryTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } - @Test("A rejected cancel does not retry or exit the master") - func rejectedCancellationFailsClosed() async throws { + @Test("A rejected master exit does not cancel or retry the forward") + func rejectedMasterExitFailsClosed() async throws { let runner = InheritedForwardRecoveryProcessRunner( - mode: .cancellationFailure + mode: .exitFailure ) + let clock = ManualBrokerClock() let fixture = try await RemoteSessionReverseRelayStartupTests - .makeCoordinator(runner: runner) + .makeCoordinator(runner: runner, clock: clock) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem( @@ -222,17 +221,15 @@ struct RemoteSessionInheritedForwardRecoveryTests { ) } - let outcome = coordinator.queue.sync { - coordinator.startReverseRelayViaControlMasterLocked( - forwardSpec: "127.0.0.1:64044:127.0.0.1:55001", - relayPort: 64_044 + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" ) } - guard case .bindingConflict = outcome else { - Issue.record("Expected the rejected cancel to fail closed") - return - } + #expect(await clock.nextRequestedDelay() == 2_000) let requests = runner.requests #expect( requests.filter { @@ -241,11 +238,11 @@ struct RemoteSessionInheritedForwardRecoveryTests { ) #expect( requests.filter { - Self.isControlCommand("cancel", in: $0.arguments) + Self.isControlCommand("exit", in: $0.arguments) }.count == 1 ) #expect(!requests.contains(where: { - Self.isControlCommand("exit", in: $0.arguments) + Self.isControlCommand("cancel", in: $0.arguments) })) _ = await coordinator.stopAndWait(cleanupScope: .transport) @@ -257,8 +254,13 @@ struct RemoteSessionInheritedForwardRecoveryTests { mode: .transientMetadataFailure ) let host = ReverseRelayRecoveryHost() + let clock = ManualBrokerClock() let fixture = try await RemoteSessionReverseRelayStartupTests - .makeCoordinator(host: host, runner: runner) + .makeCoordinator( + host: host, + runner: runner, + clock: clock + ) let coordinator = fixture.coordinator defer { try? FileManager.default.removeItem( @@ -289,33 +291,30 @@ struct RemoteSessionInheritedForwardRecoveryTests { remotePath: "/tmp/cmuxd-remote" ) } - coordinator.queue.sync { - coordinator.startReverseRelayLocked( - remotePath: "/tmp/cmuxd-remote" - ) - } - var statuses = host.daemonStatuses.makeAsyncIterator() #expect(await statuses.next() == readyStatus) + #expect(await clock.nextRequestedDelay() == 2_000) + await clock.resumeNextSleep() + #expect(await statuses.next()?.state == .error) + #expect(await clock.nextRequestedDelay() == 2_000) #expect(await statuses.next()?.state == .error) - #expect(await statuses.next() == readyStatus) let requests = runner.requests #expect( requests.filter { Self.isControlCommand("forward", in: $0.arguments) - }.count == 3 + }.count == 2 ) #expect( requests.filter(Self.isMetadataOwnershipProbe).count == 2 ) #expect( requests.filter { - Self.isControlCommand("cancel", in: $0.arguments) + Self.isControlCommand("exit", in: $0.arguments) }.count == 1 ) #expect(!requests.contains(where: { - Self.isControlCommand("exit", in: $0.arguments) + Self.isControlCommand("cancel", in: $0.arguments) })) _ = await coordinator.stopAndWait(cleanupScope: .transport) @@ -378,18 +377,6 @@ struct RemoteSessionInheritedForwardRecoveryTests { }) } - private static func reverseForward( - in arguments: [String] - ) -> String? { - guard let reverseIndex = arguments.firstIndex(of: "-R") else { - return nil - } - let valueIndex = arguments.index(after: reverseIndex) - return arguments.indices.contains(valueIndex) - ? arguments[valueIndex] - : nil - } - private static func isMetadataOwnershipProbe( _ request: RemoteProcessRequest ) -> Bool { diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift index e6d5b7a95b6f..853ed6789d98 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift @@ -1,3 +1,7 @@ +import CmuxCore +import CmuxFoundation +import CmuxRemoteDaemon +import CmuxRemoteWorkspace import Foundation import Testing @testable import CmuxRemoteSession @@ -63,6 +67,207 @@ struct RemoteSessionInheritedMasterReapTests { _ = await coordinator.stopAndWait(cleanupScope: .transport) } + @MainActor + @Test("A successful reap invalidates every sibling transport") + func successfulReapInvalidatesSiblingTransport() async throws { + let runner = InheritedMasterReapProcessRunner() + let broker = NativeSSHConnectionBroker( + sharingOptions: SSHConnectionSharingOptions(), + clock: RecordingImmediateClock(), + jitterMilliseconds: { 200 }, + cleanupLauncher: { _ in }, + inheritedMasterReapRunner: runner, + controlMasterOwnershipRegistry: + PermissiveNativeSSHControlMasterOwnershipRegistry() + ) + let firstClock = ManualBrokerClock() + let siblingClock = ManualBrokerClock() + let firstFixture = try Self.makeCoordinator( + broker: broker, + runner: runner, + clock: firstClock, + ownerWorkspaceID: UUID() + ) + let siblingFixture = try Self.makeCoordinator( + broker: broker, + runner: runner, + clock: siblingClock, + ownerWorkspaceID: UUID() + ) + let first = firstFixture.coordinator + let sibling = siblingFixture.coordinator + defer { + try? FileManager.default.removeItem( + at: firstFixture.scratchDirectory + ) + try? FileManager.default.removeItem( + at: siblingFixture.scratchDirectory + ) + } + + let events = try #require( + await broker.controlMasterReapEvents( + controlPath: ResolvedControlPathFixture.path + ) + ) + let siblingObserver = Task { + var iterator = events.makeAsyncIterator() + guard let eventID = await iterator.next() else { return } + await withCheckedContinuation { + (continuation: CheckedContinuation) in + sibling.queue.async { + sibling.handleSharedControlMasterReapLocked( + eventID: eventID + ) + continuation.resume() + } + } + } + sibling.queue.sync { + sibling.daemonReady = true + sibling.daemonRemotePath = "/tmp/cmuxd-remote" + sibling.reverseRelayControlMasterForwardSpec = + "127.0.0.1:64045:127.0.0.1:55002" + } + first.queue.sync { + first.daemonReady = true + first.daemonRemotePath = "/tmp/cmuxd-remote" + first.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" + ) + } + + #expect(await firstClock.nextRequestedDelay() == 2_000) + #expect(await siblingClock.nextRequestedDelay() == 2_000) + await siblingObserver.value + #expect(first.queue.sync { + !first.daemonReady && + first.reverseRelayControlMasterForwardSpec == nil + }) + #expect(sibling.queue.sync { + !sibling.daemonReady && + sibling.reverseRelayControlMasterForwardSpec == nil + }) + #expect(runner.requests.filter { + Self.isControlCommand("exit", in: $0.arguments) + }.count == 1) + + _ = await first.stopAndWait(cleanupScope: .transport) + _ = await sibling.stopAndWait(cleanupScope: .transport) + } + + @Test("Stopping detaches from an in-flight inherited-master reap") + func stopDetachesFromInheritedMasterReap() async throws { + let runner = BlockingInheritedMasterReapRunner() + let fixture = try await RemoteSessionReverseRelayStartupTests + .makeCoordinator( + runner: runner, + persistentDaemonSlot: "ssh-persistent-slot" + ) + let coordinator = fixture.coordinator + defer { + try? FileManager.default.removeItem( + at: fixture.scratchDirectory + ) + } + var exitStarts = runner.exitStarts.makeAsyncIterator() + var exitFinishes = runner.exitFinishes.makeAsyncIterator() + + coordinator.queue.sync { + coordinator.daemonReady = true + coordinator.daemonRemotePath = "/tmp/cmuxd-remote" + coordinator.startReverseRelayLocked( + remotePath: "/tmp/cmuxd-remote" + ) + } + #expect(await exitStarts.next() != nil) + + _ = await coordinator.stopAndWait(cleanupScope: .transport) + #expect(coordinator.queue.sync { + coordinator.controlMasterReapState.startupPhase + .allowsRelayLaunch + }) + + runner.finishExit() + #expect(await exitFinishes.next() != nil) + } + + @MainActor + private static func makeCoordinator( + broker: NativeSSHConnectionBroker, + runner: any RemoteSessionProcessRunning, + clock: any RemoteProxyRetryClock, + ownerWorkspaceID: UUID + ) throws -> ( + coordinator: RemoteSessionCoordinator, + scratchDirectory: URL + ) { + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-inherited-master-reap-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory( + at: scratchDirectory, + withIntermediateDirectories: true + ) + let configuration = broker.retainWorkspace( + WorkspaceRemoteConfiguration( + destination: "user@example.test", + port: nil, + identityFile: nil, + sshOptions: [ + "ControlMaster=auto", + "ControlPersist=600", + "ControlPath=\(ResolvedControlPathFixture.path)", + ], + localProxyPort: nil, + relayPort: 64_044, + relayID: "relay-startup-cancellation", + relayToken: String(repeating: "a", count: 64), + localSocketPath: scratchDirectory + .appendingPathComponent("relay.sock").path, + ownerWorkspaceID: ownerWorkspaceID, + terminalStartupCommand: nil, + preserveAfterTerminalExit: true, + persistentDaemonSlot: "ssh-persistent-slot" + ) + ) + return ( + RemoteSessionCoordinator( + host: NoopRemoteSessionHost(), + configuration: configuration, + proxyBroker: SSHOverrideUnusedRemoteProxyBroker(), + connectionBroker: broker, + manifestRepository: RemoteDaemonManifestRepository( + homeDirectory: scratchDirectory + ), + processRunner: runner, + reverseRelayLauncher: RecordingReverseRelayLauncher(), + reachabilityProbe: SSHOverrideNoopReachabilityProbe(), + relayCommandRewriter: + SSHOverridePassthroughRelayCommandRewriter(), + buildInfo: SSHOverrideStubBuildInfo(), + daemonStrings: RemoteDaemonStrings( + missingPersistentPTYCapability: "", + missingRequiredFunctionality: "" + ), + strings: RemoteSessionStrings( + connectedVMNoProxyFormat: "%@", + suspendedDetailFormat: "%@", + reverseRelayUnavailableRetrying: + "test relay unavailable", + reverseRelayPortUnavailableRetrying: + "test relay port unavailable", + controlMasterOwnershipUnavailable: + "test control master unavailable" + ), + clock: clock + ), + scratchDirectory + ) + } + private static func isControlCommand( _ command: String, in arguments: [String] @@ -121,3 +326,53 @@ private final class InheritedMasterReapProcessRunner: }) } } + +private final class BlockingInheritedMasterReapRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let exitStarts: AsyncStream + let exitFinishes: AsyncStream + + private let exitStartContinuation: AsyncStream.Continuation + private let exitFinishContinuation: AsyncStream.Continuation + private let exitGate = DispatchSemaphore(value: 0) + + init() { + (exitStarts, exitStartContinuation) = AsyncStream.makeStream() + (exitFinishes, exitFinishContinuation) = AsyncStream.makeStream() + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: + "remote port forwarding failed for listen port 64044" + ) + } + if Self.isControlCommand("exit", in: request.arguments) { + exitStartContinuation.yield() + exitGate.wait() + exitFinishContinuation.yield() + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + + func finishExit() { + exitGate.signal() + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift index b90c31f0000b..a57becc7a74f 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionReverseRelayStartupTests.swift @@ -83,6 +83,7 @@ struct RemoteSessionReverseRelayStartupTests { clock: RecordingImmediateClock(), jitterMilliseconds: { 200 }, cleanupLauncher: { _ in }, + inheritedMasterReapRunner: effectiveRunner, controlMasterOwnershipRegistry: ownershipRegistry ) let configuration = connectionBroker.retainWorkspace(rawConfiguration) From 3b6d7c268c235120e1a5c12a4213f884f4c43df0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 08:44:28 -0700 Subject: [PATCH 39/39] refactor: isolate inherited master reap types --- .../NativeSSHControlMasterReapAttempt.swift | 81 ++++++++++++++ ...ativeSSHControlMasterReapCoordinator.swift | 101 ++---------------- .../NativeSSHControlMasterReapOutcome.swift | 8 ++ .../BlockingInheritedMasterReapRunner.swift | 53 +++++++++ .../InheritedMasterReapProcessRunner.swift | 52 +++++++++ ...emoteSessionInheritedMasterReapTests.swift | 99 ----------------- 6 files changed, 202 insertions(+), 192 deletions(-) create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapAttempt.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapOutcome.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingInheritedMasterReapRunner.swift create mode 100644 Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedMasterReapProcessRunner.swift diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapAttempt.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapAttempt.swift new file mode 100644 index 000000000000..08d66430417d --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapAttempt.swift @@ -0,0 +1,81 @@ +internal import CmuxRemoteWorkspace +internal import Foundation + +enum NativeSSHControlMasterReapAttempt: Sendable { + case reaped + case deferred(String) + case ignored(String) +} + +@concurrent +func runNativeSSHControlMasterReap( + metadataProbeRequest: RemoteProcessRequest, + exitRequest: RemoteProcessRequest, + processRunner: any RemoteSessionProcessRunning +) async -> NativeSSHControlMasterReapAttempt { + let cancellation = RemoteProcessCancellationOperation() + return await withTaskCancellationHandler { + guard !Task.isCancelled else { + return .deferred("control-master reap cancelled") + } + do { + let probe = try processRunner.run( + metadataProbeRequest, + operation: cancellation + ) + guard probe.status == 0 else { + let detail = nativeSSHControlMasterReapErrorLine( + stderr: probe.stderr, + stdout: probe.stdout + ) ?? "ssh exited \(probe.status)" + return probe.status == 64 + ? .ignored("relay metadata did not match") + : .deferred( + "could not verify relay metadata: \(detail)" + ) + } + guard !Task.isCancelled else { + return .deferred("control-master reap cancelled") + } + let exit = try processRunner.run( + exitRequest, + operation: cancellation + ) + guard exit.status == 0 else { + let detail = nativeSSHControlMasterReapErrorLine( + stderr: exit.stderr, + stdout: exit.stdout + ) ?? "ssh exited \(exit.status)" + return .ignored( + "control-master exit failed: \(detail)" + ) + } + return .reaped + } catch { + return .deferred(error.localizedDescription) + } + } onCancel: { + cancellation.cancel() + } +} + +private func nativeSSHControlMasterReapErrorLine( + stderr: String, + stdout: String +) -> String? { + for text in [stderr, stdout] { + if let line = text + .split(whereSeparator: \.isNewline) + .map(String.init) + .last(where: { + !$0.trimmingCharacters( + in: .whitespacesAndNewlines + ).isEmpty + }) { + return line.trimmingCharacters( + in: .whitespacesAndNewlines + ) + } + } + return nil +} diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift index d93089c41d2c..82a6f2361593 100644 --- a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapCoordinator.swift @@ -3,24 +3,12 @@ internal import CmuxFoundation internal import CmuxRemoteWorkspace internal import Foundation -/// Outcome of one broker-authorized inherited-ControlMaster reap. -enum NativeSSHControlMasterReapOutcome: Sendable, Equatable { - case reaped(eventID: UUID) - case deferred(String) - case ignored(String) -} - /// Broker-owned state for disruptive inherited-ControlMaster reaps. /// /// Authorization, metadata proof, process execution, and sibling invalidation /// form one operation so no caller can prove one socket and exit another. @MainActor final class NativeSSHControlMasterReapCoordinator { - private struct InFlightReap { - let id: UUID - let task: Task - } - private let sharingOptions: SSHConnectionSharingOptions private let processRunner: any RemoteSessionProcessRunning private let eventHub: NativeSSHControlMasterReapEventHub @@ -32,7 +20,12 @@ final class NativeSSHControlMasterReapCoordinator { WorkspaceRemoteConfiguration ] ] = [:] - private var inFlightReaps: [String: InFlightReap] = [:] + private var inFlightReaps: [ + String: ( + id: UUID, + task: Task + ) + ] = [:] nonisolated init( sharingOptions: SSHConnectionSharingOptions, @@ -162,7 +155,7 @@ final class NativeSSHControlMasterReapCoordinator { Never > = Task { defer { authorization.release() } - let attempt = await Self.runReap( + let attempt = await runNativeSSHControlMasterReap( metadataProbeRequest: probeRequest, exitRequest: exitRequest, processRunner: processRunner @@ -179,7 +172,7 @@ final class NativeSSHControlMasterReapCoordinator { return .ignored(detail) } } - inFlightReaps[resolvedControlPath] = InFlightReap( + inFlightReaps[resolvedControlPath] = ( id: reapID, task: task ) @@ -199,82 +192,4 @@ final class NativeSSHControlMasterReapCoordinator { .sshControlMasterLeaseGeneration == generation } - @concurrent - private static func runReap( - metadataProbeRequest: RemoteProcessRequest, - exitRequest: RemoteProcessRequest, - processRunner: any RemoteSessionProcessRunning - ) async -> ReapAttemptOutcome { - let cancellation = RemoteProcessCancellationOperation() - return await withTaskCancellationHandler { - guard !Task.isCancelled else { - return .deferred("control-master reap cancelled") - } - do { - let probe = try processRunner.run( - metadataProbeRequest, - operation: cancellation - ) - guard probe.status == 0 else { - let detail = bestErrorLine( - stderr: probe.stderr, - stdout: probe.stdout - ) ?? "ssh exited \(probe.status)" - return probe.status == 64 - ? .ignored("relay metadata did not match") - : .deferred( - "could not verify relay metadata: \(detail)" - ) - } - guard !Task.isCancelled else { - return .deferred("control-master reap cancelled") - } - let exit = try processRunner.run( - exitRequest, - operation: cancellation - ) - guard exit.status == 0 else { - let detail = bestErrorLine( - stderr: exit.stderr, - stdout: exit.stdout - ) ?? "ssh exited \(exit.status)" - return .ignored( - "control-master exit failed: \(detail)" - ) - } - return .reaped - } catch { - return .deferred(error.localizedDescription) - } - } onCancel: { - cancellation.cancel() - } - } - - private nonisolated static func bestErrorLine( - stderr: String, - stdout: String - ) -> String? { - for text in [stderr, stdout] { - if let line = text - .split(whereSeparator: \.isNewline) - .map(String.init) - .last(where: { - !$0.trimmingCharacters( - in: .whitespacesAndNewlines - ).isEmpty - }) { - return line.trimmingCharacters( - in: .whitespacesAndNewlines - ) - } - } - return nil - } -} - -private enum ReapAttemptOutcome: Sendable { - case reaped - case deferred(String) - case ignored(String) } diff --git a/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapOutcome.swift b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapOutcome.swift new file mode 100644 index 000000000000..c5cb00c15756 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Sources/CmuxRemoteSession/Connection/NativeSSHControlMasterReapOutcome.swift @@ -0,0 +1,8 @@ +internal import Foundation + +/// Outcome of one broker-authorized inherited-ControlMaster reap. +enum NativeSSHControlMasterReapOutcome: Sendable, Equatable { + case reaped(eventID: UUID) + case deferred(String) + case ignored(String) +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingInheritedMasterReapRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingInheritedMasterReapRunner.swift new file mode 100644 index 000000000000..ef52f1632348 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/BlockingInheritedMasterReapRunner.swift @@ -0,0 +1,53 @@ +import CmuxRemoteWorkspace +import Dispatch +@testable import CmuxRemoteSession + +final class BlockingInheritedMasterReapRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let exitStarts: AsyncStream + let exitFinishes: AsyncStream + + private let exitStartContinuation: AsyncStream.Continuation + private let exitFinishContinuation: AsyncStream.Continuation + private let exitGate = DispatchSemaphore(value: 0) + + init() { + (exitStarts, exitStartContinuation) = AsyncStream.makeStream() + (exitFinishes, exitFinishContinuation) = AsyncStream.makeStream() + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: + "remote port forwarding failed for listen port 64044" + ) + } + if Self.isControlCommand("exit", in: request.arguments) { + exitStartContinuation.yield() + exitGate.wait() + exitFinishContinuation.yield() + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + + func finishExit() { + exitGate.signal() + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedMasterReapProcessRunner.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedMasterReapProcessRunner.swift new file mode 100644 index 000000000000..1fddde069df6 --- /dev/null +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/InheritedMasterReapProcessRunner.swift @@ -0,0 +1,52 @@ +import CmuxRemoteWorkspace +import Foundation +@testable import CmuxRemoteSession + +final class InheritedMasterReapProcessRunner: + RemoteSessionProcessRunning, + @unchecked Sendable +{ + let requestStream: AsyncStream + + // lint:allow lock - synchronous test requests append and snapshot only. + private let lock = NSLock() + private var recordedRequests: [RemoteProcessRequest] = [] + private let requestContinuation: + AsyncStream.Continuation + + init() { + (requestStream, requestContinuation) = AsyncStream.makeStream() + } + + var requests: [RemoteProcessRequest] { + lock.withLock { recordedRequests } + } + + func run( + _ request: RemoteProcessRequest, + operation: (any RemoteTransferCancelling)? + ) throws -> RemoteCommandResult { + lock.withLock { + recordedRequests.append(request) + } + requestContinuation.yield(request) + if Self.isControlCommand("forward", in: request.arguments) { + return RemoteCommandResult( + status: 255, + stdout: "", + stderr: + "remote port forwarding failed for listen port 64044" + ) + } + return RemoteCommandResult(status: 0, stdout: "", stderr: "") + } + + private static func isControlCommand( + _ command: String, + in arguments: [String] + ) -> Bool { + arguments.indices.dropLast().contains(where: { + arguments[$0] == "-O" && arguments[$0 + 1] == command + }) + } +} diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift index 853ed6789d98..84e9add0a16d 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteSessionInheritedMasterReapTests.swift @@ -277,102 +277,3 @@ struct RemoteSessionInheritedMasterReapTests { }) } } - -private final class InheritedMasterReapProcessRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - let requestStream: AsyncStream - - // lint:allow lock - synchronous test requests append and snapshot only. - private let lock = NSLock() - private var recordedRequests: [RemoteProcessRequest] = [] - private let requestContinuation: - AsyncStream.Continuation - - init() { - (requestStream, requestContinuation) = AsyncStream.makeStream() - } - - var requests: [RemoteProcessRequest] { - lock.withLock { recordedRequests } - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - lock.withLock { - recordedRequests.append(request) - } - requestContinuation.yield(request) - if Self.isControlCommand("forward", in: request.arguments) { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: - "remote port forwarding failed for listen port 64044" - ) - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - - private static func isControlCommand( - _ command: String, - in arguments: [String] - ) -> Bool { - arguments.indices.dropLast().contains(where: { - arguments[$0] == "-O" && arguments[$0 + 1] == command - }) - } -} - -private final class BlockingInheritedMasterReapRunner: - RemoteSessionProcessRunning, - @unchecked Sendable -{ - let exitStarts: AsyncStream - let exitFinishes: AsyncStream - - private let exitStartContinuation: AsyncStream.Continuation - private let exitFinishContinuation: AsyncStream.Continuation - private let exitGate = DispatchSemaphore(value: 0) - - init() { - (exitStarts, exitStartContinuation) = AsyncStream.makeStream() - (exitFinishes, exitFinishContinuation) = AsyncStream.makeStream() - } - - func run( - _ request: RemoteProcessRequest, - operation: (any RemoteTransferCancelling)? - ) throws -> RemoteCommandResult { - if Self.isControlCommand("forward", in: request.arguments) { - return RemoteCommandResult( - status: 255, - stdout: "", - stderr: - "remote port forwarding failed for listen port 64044" - ) - } - if Self.isControlCommand("exit", in: request.arguments) { - exitStartContinuation.yield() - exitGate.wait() - exitFinishContinuation.yield() - } - return RemoteCommandResult(status: 0, stdout: "", stderr: "") - } - - func finishExit() { - exitGate.signal() - } - - private static func isControlCommand( - _ command: String, - in arguments: [String] - ) -> Bool { - arguments.indices.dropLast().contains(where: { - arguments[$0] == "-O" && arguments[$0 + 1] == command - }) - } -}