Preserve live remote PTYs when relay lease disappears

daemon/remote/README.md

@@ -60,15 +60,15 @@ Remote slot files:
 1. `/tmp/cmuxd-remote-<uid>/cmuxd-<slot-hash>.sock` authenticated Unix socket for stdio proxies.
 2. `~/.cmux/daemon/<version>/<slot>/auth.token` random 32-byte hex token, mode `0600`.
 3. `~/.cmux/daemon/<version>/<slot>/daemon.lock` single-owner lock.
-4. `~/.cmux/daemon/<version>/<slot>/daemon.log` startup and crash diagnostics.
+4. `~/.cmux/daemon/<version>/<slot>/daemon.log` lifecycle and crash diagnostics.
 
 PTY lifecycle:
 1. A local attach creates or reuses a named `pty.*` session in the persistent daemon.
 2. If the local surface closes, the stdio proxy disconnects and its attachment detaches, but the PTY process and bounded scrollback remain in the daemon.
 3. `cmux ssh-session-list` calls `pty.list`; `cmux ssh-session-attach` creates a new local terminal whose startup script calls `ssh-pty-attach --require-existing`.
 4. `cmux ssh-session-cleanup` calls `pty.close` to terminate a persisted PTY session explicitly.
 5. Sessions with no attachments keep their last-known size and are reaped by the daemon idle TTL.
-6. Closing the owning workspace sends an authenticated slot-shutdown request, waits a bounded interval for the daemon lock to be released, and removes the relay's shell-state directory. As defense in depth, a daemon launched with `--persistent-lease-port` observes that exact `~/.cmux/relay/<port>.slot` lease, exits after the observed lease disappears and stdio disconnects, and removes the matching shell-state directory. Older callers that omit the flag retain the prior behavior without unsafe broad lease scanning.
+6. Closing the owning workspace sends an authenticated slot-shutdown request, waits a bounded interval for the daemon lock to be released, and removes the relay's shell-state directory. As defense in depth, a daemon launched with `--persistent-lease-port` observes that exact `~/.cmux/relay/<port>.slot` lease, but retires passively only after the observed lease disappears and both stdio connections and live PTY sessions are empty. A detached live PTY survives lease loss until it exits or is closed explicitly. Older callers that omit the flag retain the prior behavior without unsafe broad lease scanning.
 
 ## Cloud WebSocket PTY transport
 

daemon/remote/cmd/cmuxd-remote/main.go

@@ -1201,35 +1201,43 @@ func servePersistentDaemonWithVerifierConfig(
 	for {
 		now := time.Now()
 		var acceptDeadline time.Time
+		var slotLeaseMissing bool
 		if config.slotLeasePresent != nil {
 			present, err := config.slotLeasePresent()
 			if err == nil {
 				if present {
 					slotLeaseObserved = true
-				} else if slotLeaseObserved && atomic.LoadInt64(&activeConnections) == 0 {
-					if config.slotLeaseRemoved != nil {
-						config.slotLeaseRemoved()
-					}
-					return nil
+				} else if slotLeaseObserved {
+					slotLeaseMissing = true
 				}
 			}
 			acceptDeadline = now.Add(persistentDaemonAcceptPollStep(config))
 		}
+		activity := persistentDaemonActivity{
+			activeConnections: atomic.LoadInt64(&activeConnections),
+		}
+		// Counting sessions takes the hub lock and scans the session map. A live
+		// connection already prevents automatic retirement, so inspect sessions
+		// only when their count can affect the decision.
+		if activity.activeConnections == 0 {
+			activity.activeSessions = hub.activeSessionCount()
+		}
+		var emptyIdleExpired bool
 		if config.emptyIdleTimeout > 0 {
-			isEmpty := atomic.LoadInt64(&activeConnections) == 0 && hub.activeSessionCount() == 0
-			if isEmpty {
+			if activity.isEmpty() {
 				if idleSince.IsZero() {
 					idleSince = now
 				}
 				remaining := config.emptyIdleTimeout - now.Sub(idleSince)
 				if remaining <= 0 {
-					return nil
+					emptyIdleExpired = true
+				} else {
+					idleDeadline := now.Add(minDuration(
+						remaining,
+						persistentDaemonAcceptPollStep(config),
+					))
+					acceptDeadline = earliestNonzeroTime(acceptDeadline, idleDeadline)
 				}
-				idleDeadline := now.Add(minDuration(
-					remaining,
-					persistentDaemonAcceptPollStep(config),
-				))
-				acceptDeadline = earliestNonzeroTime(acceptDeadline, idleDeadline)
 			} else {
 				idleSince = time.Time{}
 				acceptDeadline = earliestNonzeroTime(
@@ -1238,6 +1246,18 @@ func servePersistentDaemonWithVerifierConfig(
 				)
 			}
 		}
+		exitReason := persistentDaemonAutomaticExitReason(
+			activity,
+			slotLeaseMissing,
+			emptyIdleExpired,
+		)
+		if exitReason != "" {
+			logPersistentDaemonExit(stderr, now, exitReason, activity)
+			if exitReason == persistentDaemonExitSlotLeaseRemoved && config.slotLeaseRemoved != nil {
+				config.slotLeaseRemoved()
+			}
+			return nil
+		}
 		if !acceptDeadline.IsZero() {
 			setPersistentDaemonAcceptDeadline(listener, acceptDeadline)
 		}

daemon/remote/cmd/cmuxd-remote/main_test.go

@@ -2013,12 +2013,13 @@ func TestPersistentDaemonServerExitsAfterEmptySlotIdleTimeout(t *testing.T) {
 		t.Fatalf("listen unix: %v", err)
 	}
 
+	var stderr bytes.Buffer
 	done := make(chan error, 1)
 	go func() {
 		done <- servePersistentDaemonWithVerifierConfig(
 			listener,
 			persistentDaemonFixedTokenVerifier("idle-token"),
-			io.Discard,
+			&stderr,
 			persistentDaemonServerConfig{
 				emptyIdleTimeout: 500 * time.Millisecond,
 				acceptPollStep:   25 * time.Millisecond,
@@ -2064,6 +2065,11 @@ func TestPersistentDaemonServerExitsAfterEmptySlotIdleTimeout(t *testing.T) {
 	case <-time.After(2 * time.Second):
 		t.Fatalf("persistent daemon did not stop after empty idle timeout")
 	}
+	requirePersistentDaemonAutomaticExitLog(
+		t,
+		stderr.String(),
+		persistentDaemonExitEmptyIdleTimeout,
+	)
 }
 
 func TestRunStdioSlotRequiresPersistent(t *testing.T) {

daemon/remote/cmd/cmuxd-remote/persistent_lifecycle.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"os"
 	"path/filepath"
@@ -17,6 +18,58 @@ import (
 
 const persistentDaemonShutdownMethod = "daemon.shutdown"
 
+type persistentDaemonActivity struct {
+	activeConnections int64
+	activeSessions    int
+}
+
+func (activity persistentDaemonActivity) isEmpty() bool {
+	return activity.activeConnections == 0 && activity.activeSessions == 0
+}
+
+type persistentDaemonExitReason string
+
+const (
+	persistentDaemonExitSlotLeaseRemoved persistentDaemonExitReason = "slot_lease_removed"
+	persistentDaemonExitEmptyIdleTimeout persistentDaemonExitReason = "empty_idle_timeout"
+)
+
+func persistentDaemonAutomaticExitReason(
+	activity persistentDaemonActivity,
+	slotLeaseMissing bool,
+	emptyIdleExpired bool,
+) persistentDaemonExitReason {
+	if !activity.isEmpty() {
+		return ""
+	}
+	if slotLeaseMissing {
+		return persistentDaemonExitSlotLeaseRemoved
+	}
+	if emptyIdleExpired {
+		return persistentDaemonExitEmptyIdleTimeout
+	}
+	return ""
+}
+
+func logPersistentDaemonExit(
+	stderr io.Writer,
+	now time.Time,
+	reason persistentDaemonExitReason,
+	activity persistentDaemonActivity,
+) {
+	if stderr == nil {
+		return
+	}
+	_, _ = fmt.Fprintf(
+		stderr,
+		"cmuxd-remote persistent daemon exit time=%s reason=%s active_connections=%d active_sessions=%d\n",
+		now.UTC().Format(time.RFC3339Nano),
+		reason,
+		activity.activeConnections,
+		activity.activeSessions,
+	)
+}
+
 const (
 	persistentDaemonStopWaitTimeout = 5 * time.Second
 	persistentDaemonStopRetryStep   = 25 * time.Millisecond

daemon/remote/cmd/cmuxd-remote/persistent_lifecycle_test.go

@@ -342,7 +342,7 @@ func TestWaitForPersistentDaemonStopTimesOutWhenOwnershipDoesNotRelease(t *testi
 	}
 }
 
-func TestPersistentDaemonReapsActivePTYAfterObservedSlotLeaseDisappears(t *testing.T) {
+func TestPersistentDaemonPreservesActivePTYAfterObservedSlotLeaseDisappears(t *testing.T) {
 	socketDir, err := os.MkdirTemp("/tmp", "cmuxd-remote-lease-reap-*")
 	if err != nil {
 		t.Fatalf("create short socket dir: %v", err)
@@ -356,21 +356,30 @@ func TestPersistentDaemonReapsActivePTYAfterObservedSlotLeaseDisappears(t *testi
 
 	var leasePresent atomic.Bool
 	leasePresent.Store(true)
-	leaseChecked := make(chan struct{}, 1)
+	leaseChecked := make(chan bool, 1)
+	leaseRemoved := make(chan struct{}, 1)
+	var stderr bytes.Buffer
 	done := make(chan error, 1)
 	go func() {
 		done <- servePersistentDaemonWithVerifierConfig(
 			listener,
 			persistentDaemonFixedTokenVerifier("lease-token"),
-			io.Discard,
+			&stderr,
 			persistentDaemonServerConfig{
 				acceptPollStep: 10 * time.Millisecond,
 				slotLeasePresent: func() (bool, error) {
+					present := leasePresent.Load()
+					select {
+					case leaseChecked <- present:
+					default:
+					}
+					return present, nil
+				},
+				slotLeaseRemoved: func() {
 					select {
-					case leaseChecked <- struct{}{}:
+					case leaseRemoved <- struct{}{}:
 					default:
 					}
-					return leasePresent.Load(), nil
 				},
 			},
 		)
@@ -389,7 +398,10 @@ func TestPersistentDaemonReapsActivePTYAfterObservedSlotLeaseDisappears(t *testi
 	}()
 
 	select {
-	case <-leaseChecked:
+	case present := <-leaseChecked:
+		if !present {
+			t.Fatalf("persistent daemon observed an absent initial slot lease")
+		}
 	case <-time.After(2 * time.Second):
 		t.Fatalf("persistent daemon did not inspect the slot lease")
 	}
@@ -412,18 +424,118 @@ func TestPersistentDaemonReapsActivePTYAfterObservedSlotLeaseDisappears(t *testi
 	readPersistentTestEvent(t, conn, reader, func(frame map[string]any) bool {
 		return frame["event"] == "pty.ready" && frame["attachment_id"] == "lease-attachment"
 	})
+	for {
+		select {
+		case <-leaseChecked:
+			continue
+		default:
+			goto leaseChecksDrained
+		}
+	}
+
+leaseChecksDrained:
 	_ = conn.Close()
+	for checkedAfterDisconnect := 0; checkedAfterDisconnect < 5; {
+		select {
+		case present := <-leaseChecked:
+			if present {
+				checkedAfterDisconnect++
+			}
+		case err := <-done:
+			serverExited = true
+			t.Fatalf("persistent daemon exited before its slot lease disappeared: %v", err)
+		case <-time.After(2 * time.Second):
+			t.Fatalf("persistent daemon did not continue checking its slot lease after disconnect")
+		}
+	}
 	leasePresent.Store(false)
 
+	for absentChecks := 0; absentChecks < 3; {
+		select {
+		case present := <-leaseChecked:
+			if !present {
+				absentChecks++
+			}
+		case <-leaseRemoved:
+			t.Fatalf("persistent daemon retired while its detached PTY was still active")
+		case err := <-done:
+			serverExited = true
+			t.Fatalf("persistent daemon exited while its detached PTY was still active: %v", err)
+		case <-time.After(2 * time.Second):
+			t.Fatalf("persistent daemon did not continue checking the absent slot lease")
+		}
+	}
+
+	reconnect, reconnectReader, reconnectWriter := openPersistentTestClient(t, socketPath, "lease-token")
+	reattach := persistentTestRPCCall(t, reconnect, reconnectReader, reconnectWriter, rpcRequest{
+		ID:     2,
+		Method: "pty.attach",
+		Params: map[string]any{
+			"session_id":              "lease-session",
+			"attachment_id":           "lease-reattachment",
+			"client_attachment_token": "lease-reattachment-token",
+			"cols":                    100,
+			"rows":                    30,
+			"command":                 "printf 'replacement command must not run\\n'",
+			"require_existing":        true,
+		},
+	})
+	if ok, _ := reattach["ok"].(bool); !ok {
+		t.Fatalf("reattach after slot lease removal failed: %v", reattach)
+	}
+	readPersistentTestEvent(t, reconnect, reconnectReader, func(frame map[string]any) bool {
+		return frame["event"] == "pty.ready" && frame["attachment_id"] == "lease-reattachment"
+	})
+	closePTY := persistentTestRPCCall(t, reconnect, reconnectReader, reconnectWriter, rpcRequest{
+		ID:     3,
+		Method: "pty.close",
+		Params: map[string]any{"session_id": "lease-session"},
+	})
+	if ok, _ := closePTY["ok"].(bool); !ok {
+		t.Fatalf("pty.close after lease removal failed: %v", closePTY)
+	}
+	_ = reconnect.Close()
+
 	select {
 	case err := <-done:
 		serverExited = true
 		if err != nil {
-			t.Fatalf("persistent daemon exited with error: %v", err)
+			t.Fatalf("persistent daemon exited with error after its final PTY closed: %v", err)
 		}
 	case <-time.After(2 * time.Second):
-		t.Fatalf("persistent daemon did not stop after its observed slot lease disappeared")
+		t.Fatalf("persistent daemon did not retire after its absent lease and final PTY close")
+	}
+	requirePersistentDaemonAutomaticExitLog(
+		t,
+		stderr.String(),
+		persistentDaemonExitSlotLeaseRemoved,
+	)
+}
+
+func requirePersistentDaemonAutomaticExitLog(
+	t *testing.T,
+	logOutput string,
+	reason persistentDaemonExitReason,
+) {
+	t.Helper()
+	wantActivity := "reason=" + string(reason) + " active_connections=0 active_sessions=0"
+	for _, line := range strings.Split(logOutput, "\n") {
+		if !strings.Contains(line, wantActivity) {
+			continue
+		}
+		for _, field := range strings.Fields(line) {
+			if !strings.HasPrefix(field, "time=") {
+				continue
+			}
+			timestamp := strings.TrimPrefix(field, "time=")
+			if _, err := time.Parse(time.RFC3339Nano, timestamp); err != nil {
+				t.Fatalf("persistent daemon exit log timestamp %q is not RFC3339Nano: %v", timestamp, err)
+			}
+			return
+		}
+		t.Fatalf("persistent daemon exit log line %q has no time field", line)
 	}
+	t.Fatalf("persistent daemon exit log = %q, want %q", logOutput, wantActivity)
 }
 
 func TestPersistentDaemonSlotLeasePresentMatchesExactRelayPortAndSlot(t *testing.T) {