[v15] Machine ID: Leaf cluster support for SSH multiplexer (#43484)

lib/config/openssh/openssh.go

@@ -271,7 +271,7 @@ Host *.{{ $clusterName }}
     HostKeyAlgorithms {{ if $dot.NewerHostKeyAlgorithmsSupported }}rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,{{ end }}ssh-rsa-cert-v01@openssh.com
     IdentityFile none
     IdentityAgent {{ proxyCommandQuote $dot.AgentSocketPath }}    
-    ProxyCommand {{range $v := $dot.ProxyCommand}}{{ proxyCommandQuote $v }} {{end}}{{ proxyCommandQuote $dot.MuxSocketPath }} '{{ $dot.Data }}'
+    ProxyCommand {{range $v := $dot.ProxyCommand}}{{ proxyCommandQuote $v }} {{end}}{{ proxyCommandQuote $dot.MuxSocketPath }} '%h:%p|{{ $clusterName }}'
     ProxyUseFDPass yes
 {{- end }}
 # End generated Teleport configuration
@@ -286,7 +286,6 @@ type MuxedSSHConfigParameters struct {
 	ProxyCommand    []string
 	MuxSocketPath   string
 	AgentSocketPath string
-	Data            string
 	// Port is the node port to use, defaulting to 3022, if not specified by flag
 	Port int
 }

lib/config/openssh/openssh_test.go

@@ -191,7 +191,6 @@ func TestSSHConfig_GetMuxedSSHConfig(t *testing.T) {
 				MuxSocketPath:   "/opt/machine-id/v1.sock",
 				AgentSocketPath: "/opt/machine-id/agent.sock",
 				ProxyCommand:    []string{"/bin/fdpass-teleport", "foo"},
-				Data:            `%h:%p`,
 			},
 		},
 		{
@@ -204,7 +203,6 @@ func TestSSHConfig_GetMuxedSSHConfig(t *testing.T) {
 				MuxSocketPath:   "/opt/machine-id/v1.sock",
 				AgentSocketPath: "/opt/machine-id/agent.sock",
 				ProxyCommand:    []string{"/bin/fdpass-teleport", "foo"},
-				Data:            `%h:%p`,
 			},
 		},
 		{
@@ -217,7 +215,6 @@ func TestSSHConfig_GetMuxedSSHConfig(t *testing.T) {
 				MuxSocketPath:   "/opt/machine-id/v1.sock",
 				AgentSocketPath: "/opt/machine-id/agent.sock",
 				ProxyCommand:    []string{"/bin/fdpass-teleport", "foo"},
-				Data:            `%h:%p`,
 			},
 		},
 	}

lib/config/openssh/testdata/TestSSHConfig_GetMuxedSSHConfig/legacy_OpenSSH_-_single_cluster.golden

@@ -6,6 +6,6 @@ Host *.example.com
     HostKeyAlgorithms ssh-rsa-cert-v01@openssh.com
     IdentityFile none
     IdentityAgent '/opt/machine-id/agent.sock'    
-    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p'
+    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p|example.com'
     ProxyUseFDPass yes
 # End generated Teleport configuration

lib/config/openssh/testdata/TestSSHConfig_GetMuxedSSHConfig/modern_OpenSSH_-_multiple_clusters.golden

@@ -6,14 +6,14 @@ Host *.example.com
     HostKeyAlgorithms rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com
     IdentityFile none
     IdentityAgent '/opt/machine-id/agent.sock'    
-    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p'
+    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p|example.com'
     ProxyUseFDPass yes
 Host *.example.org
     Port 3022
     UserKnownHostsFile '/opt/machine-id/known_hosts'
     HostKeyAlgorithms rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com
     IdentityFile none
     IdentityAgent '/opt/machine-id/agent.sock'    
-    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p'
+    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p|example.org'
     ProxyUseFDPass yes
 # End generated Teleport configuration

lib/config/openssh/testdata/TestSSHConfig_GetMuxedSSHConfig/modern_OpenSSH_-_single_cluster.golden

@@ -6,6 +6,6 @@ Host *.example.com
     HostKeyAlgorithms rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com
     IdentityFile none
     IdentityAgent '/opt/machine-id/agent.sock'    
-    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p'
+    ProxyCommand '/bin/fdpass-teleport' 'foo' '/opt/machine-id/v1.sock' '%h:%p|example.com'
     ProxyUseFDPass yes
 # End generated Teleport configuration

lib/tbot/service_ssh_multiplexer.go

@@ -146,14 +146,36 @@ func writeIfChanged(ctx context.Context, dest bot.Destination, log *slog.Logger,
 	return dest.Write(ctx, path, data)
 }
 
-func (s *SSHMultiplexerService) writeArtifacts(ctx context.Context, proxyHost string, id *identity.Identity) error {
+func (s *SSHMultiplexerService) getClusterNames(clt *authclient.Client) ([]string, error) {
+	allClusterNames := []string{s.identity.Get().ClusterName}
+	leafClusters, err := clt.GetRemoteClusters()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	for _, lc := range leafClusters {
+		allClusterNames = append(allClusterNames, lc.GetName())
+	}
+
+	return allClusterNames, nil
+}
+
+func (s *SSHMultiplexerService) writeArtifacts(
+	ctx context.Context,
+	proxyHost string,
+	authClient *authclient.Client,
+) error {
 	dest := s.cfg.Destination.(*config.DestinationDirectory)
 
+	clusterNames, err := s.getClusterNames(authClient)
+	if err != nil {
+		return trace.Wrap(err, "fetching cluster names")
+	}
+
 	// Generate known hosts
 	knownHosts, err := ssh.GenerateKnownHosts(
 		ctx,
 		s.botAuthClient,
-		[]string{id.ClusterName},
+		clusterNames,
 		proxyHost,
 	)
 	if err != nil {
@@ -187,10 +209,9 @@ func (s *SSHMultiplexerService) writeArtifacts(ctx context.Context, proxyHost st
 	sshConf := openssh.NewSSHConfig(openssh.GetSystemSSHVersion, nil)
 	err = sshConf.GetMuxedSSHConfig(&sshConfigBuilder, &openssh.MuxedSSHConfigParameters{
 		AppName:         openssh.TbotApp,
-		ClusterNames:    []string{id.ClusterName},
+		ClusterNames:    clusterNames,
 		KnownHostsPath:  filepath.Join(absPath, ssh.KnownHostsName),
 		ProxyCommand:    proxyCommand,
-		Data:            `%h:%p`,
 		MuxSocketPath:   filepath.Join(absPath, sshMuxSocketName),
 		AgentSocketPath: filepath.Join(absPath, agentSocketName),
 	})
@@ -358,10 +379,11 @@ func (s *SSHMultiplexerService) generateIdentity(ctx context.Context) (*identity
 	return id, nil
 }
 
-func (s *SSHMultiplexerService) identityRenewalLoop(ctx context.Context, proxyHost string) error {
+func (s *SSHMultiplexerService) identityRenewalLoop(
+	ctx context.Context, proxyHost string, authClient *authclient.Client,
+) error {
 	reloadCh, unsubscribe := s.reloadBroadcaster.subscribe()
 	defer unsubscribe()
-
 	err := runOnInterval(ctx, runOnIntervalConfig{
 		name: "identity-renewal",
 		f: func(ctx context.Context) error {
@@ -370,7 +392,7 @@ func (s *SSHMultiplexerService) identityRenewalLoop(ctx context.Context, proxyHo
 				return trace.Wrap(err, "generating identity")
 			}
 			s.identity.Set(id)
-			return s.writeArtifacts(ctx, proxyHost, id)
+			return s.writeArtifacts(ctx, proxyHost, authClient)
 		},
 		interval:   s.botCfg.RenewalInterval,
 		retryLimit: renewalRetryLimit,
@@ -514,7 +536,7 @@ func (s *SSHMultiplexerService) Run(ctx context.Context) (err error) {
 	})
 	// Handle identity renewal
 	eg.Go(func() error {
-		return s.identityRenewalLoop(egCtx, proxyHost)
+		return s.identityRenewalLoop(egCtx, proxyHost, authClient)
 	})
 
 	return eg.Wait()
@@ -572,7 +594,13 @@ func (s *SSHMultiplexerService) handleConn(
 	}
 
 	// The first thing downstream will send is the multiplexing request which is
-	// the "[host]:[port]\x00" format.
+	// in the "[host]:[port]|[cluster_name]\x00" format.
+	// The "|[cluster_name]" section is optional and if omitted, the cluster
+	// associated with the bot will be used.
+	//
+	// We choose this format because | is not an acceptable character in
+	// hostnames or ports through OpenSSH.
+	// https://github.com/openssh/openssh-portable/commit/7ef3787c84b6b524501211b11a26c742f829af1a
 	buf := bufio.NewReader(downstream)
 	if !strings.HasSuffix(req, "\x00") {
 		r, err := buf.ReadString('\x00')
@@ -582,20 +610,37 @@ func (s *SSHMultiplexerService) handleConn(
 		req += r
 	}
 	req = req[:len(req)-1] // Drop the NUL.
-	host, port, err := utils.SplitHostPort(req)
+
+	// Split by | to pull out the optionally specified cluster name.
+	// TODO(noah): When we need to add another parameter in future, we should
+	// roll this API to v2 and use a more extensible format.
+	splitReq := strings.Split(req, "|")
+	if len(splitReq) > 2 {
+		return trace.BadParameter(
+			"malformed request, expected at most 2 fields, got %d: %q",
+			len(splitReq), req,
+		)
+	}
+
+	host, port, err := utils.SplitHostPort(splitReq[0])
 	if err != nil {
 		return trace.Wrap(err, "malformed request %q", req)
 	}
 
+	clusterName := s.identity.Get().ClusterName
+	if len(splitReq) > 1 {
+		clusterName = splitReq[1]
+	}
+
 	log := s.log.With(
 		slog.Group("req",
 			"host", host,
 			"port", port,
+			"cluster_name", clusterName,
 		),
 	)
 	log.InfoContext(ctx, "Received multiplexing request")
 
-	clusterName := s.identity.Get().ClusterName
 	expanded, matched := tshConfig.ProxyTemplates.Apply(
 		net.JoinHostPort(host, port),
 	)

lib/tbot/tbot_test.go

@@ -1037,40 +1037,51 @@ func TestBotSSHMultiplexer(t *testing.T) {
 		}
 	}, 10*time.Second, 100*time.Millisecond)
 
-	agentConn, err := net.Dial("unix", filepath.Join(tmpDir, "agent.sock"))
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		agentConn.Close()
-	})
-	agentClient := agent.NewClient(agentConn)
-	callback, err := knownhosts.New(filepath.Join(tmpDir, "known_hosts"))
-	require.NoError(t, err)
-	sshConfig := &ssh.ClientConfig{
-		Auth: []ssh.AuthMethod{
-			ssh.PublicKeysCallback(agentClient.Signers),
-		},
-		User:            currentUser.Username,
-		HostKeyCallback: callback,
+	targets := []string{
+		"server01.root:0\x00",      // Old style target without cluster
+		"server01.root:0|root\x00", // New style target with cluster
+	}
+	for _, target := range targets {
+		target := target
+		t.Run(target, func(t *testing.T) {
+			t.Parallel()
+
+			agentConn, err := net.Dial("unix", filepath.Join(tmpDir, "agent.sock"))
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				agentConn.Close()
+			})
+			agentClient := agent.NewClient(agentConn)
+			callback, err := knownhosts.New(filepath.Join(tmpDir, "known_hosts"))
+			require.NoError(t, err)
+			sshConfig := &ssh.ClientConfig{
+				Auth: []ssh.AuthMethod{
+					ssh.PublicKeysCallback(agentClient.Signers),
+				},
+				User:            currentUser.Username,
+				HostKeyCallback: callback,
+			}
+			conn, err := net.Dial("unix", filepath.Join(tmpDir, "v1.sock"))
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				conn.Close()
+			})
+			_, err = fmt.Fprint(conn, target)
+			require.NoError(t, err)
+			sshConn, sshChan, sshReq, err := ssh.NewClientConn(conn, "server01.root:22", sshConfig)
+			require.NoError(t, err)
+			sshClient := ssh.NewClient(sshConn, sshChan, sshReq)
+			t.Cleanup(func() {
+				sshClient.Close()
+			})
+			sshSess, err := sshClient.NewSession()
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				sshSess.Close()
+			})
+			out, err := sshSess.CombinedOutput("echo hello")
+			require.NoError(t, err)
+			require.Equal(t, "hello\n", string(out))
+		})
 	}
-	conn, err := net.Dial("unix", filepath.Join(tmpDir, "v1.sock"))
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		conn.Close()
-	})
-	_, err = fmt.Fprint(conn, "server01.root:0\x00")
-	require.NoError(t, err)
-	sshConn, sshChan, sshReq, err := ssh.NewClientConn(conn, "server01.root:22", sshConfig)
-	require.NoError(t, err)
-	sshClient := ssh.NewClient(sshConn, sshChan, sshReq)
-	t.Cleanup(func() {
-		sshClient.Close()
-	})
-	sshSess, err := sshClient.NewSession()
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		sshSess.Close()
-	})
-	out, err := sshSess.CombinedOutput("echo hello")
-	require.NoError(t, err)
-	require.Equal(t, "hello\n", string(out))
 }