TLS routing behind LB support for Auth, SSH, Reverse tunnel

api/client/alpn.go

@@ -19,12 +19,28 @@ package client
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"net"
+	"strings"
 	"time"
 
 	"github.com/gravitational/trace"
+
+	"github.com/gravitational/teleport/api/client/webclient"
+	"github.com/gravitational/teleport/api/constants"
 )
 
+// GetClusterCAsFunc is a function to fetch cluster CAs.
+type GetClusterCAsFunc func(ctx context.Context) (*x509.CertPool, error)
+
+// ClusterCAsFromCertPool returns a GetClusterCAsFunc with provided static cert
+// pool.
+func ClusterCAsFromCertPool(cas *x509.CertPool) GetClusterCAsFunc {
+	return func(_ context.Context) (*x509.CertPool, error) {
+		return cas, nil
+	}
+}
+
 // ALPNDialerConfig is the config for ALPNDialer.
 type ALPNDialerConfig struct {
 	// KeepAlivePeriod defines period between keep alives.
@@ -35,12 +51,17 @@ type ALPNDialerConfig struct {
 	TLSConfig *tls.Config
 	// ALPNConnUpgradeRequired specifies if ALPN connection upgrade is required.
 	ALPNConnUpgradeRequired bool
+	// GetClusterCAs is an optional callback function to fetch cluster
+	// CAs when connection upgrade is required. If not provided, it's assumed
+	// the proper CAs are already present in TLSConfig.
+	GetClusterCAs GetClusterCAsFunc
 }
 
 // ALPNDialer is a ContextDialer that dials a connection to the Proxy Service
 // with ALPN and SNI configured in the provided TLSConfig. An ALPN connection
 // upgrade is also performed at the initial connection, if an upgrade is
-// required.
+// required. If the negotiated protocol is a Ping protocol, it will return the
+// de-multiplexed connection without the Ping.
 type ALPNDialer struct {
 	cfg ALPNDialerConfig
 }
@@ -52,25 +73,73 @@ func NewALPNDialer(cfg ALPNDialerConfig) ContextDialer {
 	}
 }
 
-// DialContext implements ContextDialer.
-func (d ALPNDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+func (d *ALPNDialer) shouldUpdateTLSConfig() bool {
+	return d.shouldUpdateServerName() || d.shouldGetClusterCAs()
+}
+
+// shouldUpdateServerName returns true if ServerName is not in the provided TLS
+// config. It will default to the host of the dialing address.
+func (d *ALPNDialer) shouldUpdateServerName() bool {
+	return d.cfg.TLSConfig.ServerName == ""
+}
+
+// shouldGetClusterCAs returns true if RootCAs of the provided TLS config needs
+// to be set to the Teleport cluster CAs.
+//
+// When Teleport Proxy is behind a L7 load balancer, the load balancer
+// usually terminates TLS with public certs, and the Proxy is usually in
+// private subnets with self-signed web certs. During the connection
+// upgrade flow for TLS Routing, instead of serving these self-signed web
+// certs, the TLS Routing handler at the Proxy server will present the
+// Cluster CAs so clients here can still verify the server.
+func (d *ALPNDialer) shouldGetClusterCAs() bool {
+	return d.cfg.ALPNConnUpgradeRequired && d.cfg.TLSConfig.RootCAs == nil && d.cfg.GetClusterCAs != nil
+}
+
+func (d *ALPNDialer) getTLSConfig(ctx context.Context, addr string) (*tls.Config, error) {
 	if d.cfg.TLSConfig == nil {
 		return nil, trace.BadParameter("missing TLS config")
 	}
+	if !d.shouldUpdateTLSConfig() {
+		return d.cfg.TLSConfig, nil
+	}
 
-	dialer := NewDialer(ctx, d.cfg.DialTimeout, d.cfg.DialTimeout, WithTLSConfig(d.cfg.TLSConfig))
-	if d.cfg.ALPNConnUpgradeRequired {
-		dialer = newALPNConnUpgradeDialer(dialer, &tls.Config{
-			InsecureSkipVerify: d.cfg.TLSConfig.InsecureSkipVerify,
-		})
+	var err error
+	tlsConfig := d.cfg.TLSConfig.Clone()
+	if d.shouldGetClusterCAs() {
+		tlsConfig.RootCAs, err = d.cfg.GetClusterCAs(ctx)
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+	}
+	if d.shouldUpdateServerName() {
+		tlsConfig.ServerName, _, err = webclient.ParseHostPort(addr)
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+	}
+	return tlsConfig, nil
+}
+
+// DialContext implements ContextDialer.
+func (d *ALPNDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	tlsConfig, err := d.getTLSConfig(ctx, addr)
+	if err != nil {
+		return nil, trace.Wrap(err)
 	}
 
+	dialer := NewDialer(ctx, d.cfg.DialTimeout, d.cfg.DialTimeout,
+		WithInsecureSkipVerify(d.cfg.TLSConfig.InsecureSkipVerify),
+		WithALPNConnUpgrade(d.cfg.ALPNConnUpgradeRequired),
+		WithALPNConnUpgradePing(shouldALPNConnUpgradeWithPing(tlsConfig)),
+	)
+
 	conn, err := dialer.DialContext(ctx, network, addr)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
 
-	tlsConn := tls.Client(conn, d.cfg.TLSConfig)
+	tlsConn := tls.Client(conn, tlsConfig)
 	if err := tlsConn.HandshakeContext(ctx); err != nil {
 		defer tlsConn.Close()
 		return nil, trace.Wrap(err)
@@ -91,3 +160,25 @@ func DialALPN(ctx context.Context, addr string, cfg ALPNDialerConfig) (*tls.Conn
 	}
 	return tlsConn, nil
 }
+
+// IsALPNPingProtocol checks if the provided protocol is suffixed with Ping.
+func IsALPNPingProtocol(protocol string) bool {
+	return strings.HasSuffix(protocol, constants.ALPNSNIProtocolPingSuffix)
+}
+
+// shouldALPNConnUpgradeWithPing returns true if Ping wrapper is required
+// during connection upgrade.
+func shouldALPNConnUpgradeWithPing(config *tls.Config) bool {
+	for _, proto := range config.NextProtos {
+		switch proto {
+		// Server usually sends SSH keepalives or HTTP2 pings every five
+		// minutes for reverse tunnel and SSH connections. Load balancers
+		// usually have a shorter idle timeout. Thus wrapping the connection
+		// with Ping protocol at the connection upgrade layer to keepalive.
+		case constants.ALPNSNIProtocolReverseTunnel,
+			constants.ALPNSNIProtocolSSH:
+			return true
+		}
+	}
+	return false
+}

api/client/alpn_conn_upgrade.go

@@ -33,6 +33,7 @@ import (
 	"github.com/gravitational/teleport/api/constants"
 	"github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/utils"
+	"github.com/gravitational/teleport/api/utils/pingconn"
 )
 
 // IsALPNConnUpgradeRequired returns true if a tunnel is required through a HTTP
@@ -145,18 +146,20 @@ func isALPNConnUpgradeRequiredByEnv(addr, envValue string) bool {
 type alpnConnUpgradeDialer struct {
 	dialer    ContextDialer
 	tlsConfig *tls.Config
+	withPing  bool
 }
 
 // newALPNConnUpgradeDialer creates a new alpnConnUpgradeDialer.
-func newALPNConnUpgradeDialer(dialer ContextDialer, tlsConfig *tls.Config) ContextDialer {
+func newALPNConnUpgradeDialer(dialer ContextDialer, tlsConfig *tls.Config, withPing bool) ContextDialer {
 	return &alpnConnUpgradeDialer{
 		dialer:    dialer,
 		tlsConfig: tlsConfig,
+		withPing:  withPing,
 	}
 }
 
 // DialContext implements ContextDialer
-func (d alpnConnUpgradeDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+func (d *alpnConnUpgradeDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
 	logrus.Debugf("ALPN connection upgrade for %v.", addr)
 
 	conn, err := d.dialer.DialContext(ctx, network, addr)
@@ -181,47 +184,58 @@ func (d alpnConnUpgradeDialer) DialContext(ctx context.Context, network, addr st
 	}
 
 	tlsConn := tls.Client(conn, cfg)
-
-	err = upgradeConnThroughWebAPI(tlsConn, url.URL{
+	upgradeURL := url.URL{
 		Host:   addr,
 		Scheme: "https",
 		Path:   constants.WebAPIConnUpgrade,
-	})
+	}
+
+	conn, err = upgradeConnThroughWebAPI(tlsConn, upgradeURL, d.upgradeType())
 	if err != nil {
-		defer tlsConn.Close()
-		return nil, trace.Wrap(err)
+		return nil, trace.NewAggregate(tlsConn.Close(), err)
 	}
-	return tlsConn, nil
+	return conn, nil
 }
 
-func upgradeConnThroughWebAPI(conn net.Conn, api url.URL) error {
+func (d *alpnConnUpgradeDialer) upgradeType() string {
+	if d.withPing {
+		return constants.WebAPIConnUpgradeTypeALPNPing
+	}
+	return constants.WebAPIConnUpgradeTypeALPN
+}
+
+func upgradeConnThroughWebAPI(conn net.Conn, api url.URL, upgradeType string) (net.Conn, error) {
 	req, err := http.NewRequest(http.MethodGet, api.String(), nil)
 	if err != nil {
-		return trace.Wrap(err)
+		return nil, trace.Wrap(err)
 	}
 
-	// For now, only "alpn" is supported.
-	req.Header.Add(constants.WebAPIConnUpgradeHeader, constants.WebAPIConnUpgradeTypeALPN)
+	req.Header.Add(constants.WebAPIConnUpgradeHeader, upgradeType)
 
 	// Send the request and check if upgrade is successful.
 	if err = req.Write(conn); err != nil {
-		return trace.Wrap(err)
+		return nil, trace.Wrap(err)
 	}
 	resp, err := http.ReadResponse(bufio.NewReader(conn), req)
 	if err != nil {
-		return trace.Wrap(err)
+		return nil, trace.Wrap(err)
 	}
 	defer resp.Body.Close()
 
 	if http.StatusSwitchingProtocols != resp.StatusCode {
 		if http.StatusNotFound == resp.StatusCode {
-			return trace.NotImplemented(
-				"connection upgrade call to %q failed with status code %v. Please upgrade the server and try again.",
+			return nil, trace.NotImplemented(
+				"connection upgrade call to %q with upgrade type %v failed with status code %v. Please upgrade the server and try again.",
 				constants.WebAPIConnUpgrade,
+				upgradeType,
 				resp.StatusCode,
 			)
 		}
-		return trace.BadParameter("failed to switch Protocols %v", resp.StatusCode)
+		return nil, trace.BadParameter("failed to switch Protocols %v", resp.StatusCode)
+	}
+
+	if upgradeType == constants.WebAPIConnUpgradeTypeALPNPing {
+		return pingconn.New(conn), nil
 	}
-	return nil
+	return conn, nil
 }

api/client/alpn_conn_upgrade_test.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/gravitational/teleport/api/constants"
 	"github.com/gravitational/teleport/api/fixtures"
+	"github.com/gravitational/teleport/api/utils/pingconn"
 )
 
 func TestIsALPNConnUpgradeRequired(t *testing.T) {
@@ -123,42 +124,58 @@ func TestIsALPNConnUpgradeRequiredByEnv(t *testing.T) {
 func TestALPNConnUpgradeDialer(t *testing.T) {
 	t.Parallel()
 
-	t.Run("connection upgraded", func(t *testing.T) {
-		ctx := context.Background()
-
-		server := httptest.NewTLSServer(mockConnUpgradeHandler(t, "alpn", []byte("hello")))
-		t.Cleanup(server.Close)
-		addr, err := url.Parse(server.URL)
-		require.NoError(t, err)
-		pool := x509.NewCertPool()
-		pool.AddCert(server.Certificate())
-
-		tlsConfig := &tls.Config{RootCAs: pool}
-		preDialer := NewDialer(ctx, 0, 5*time.Second)
-		dialer := newALPNConnUpgradeDialer(preDialer, tlsConfig)
-		conn, err := dialer.DialContext(ctx, "tcp", addr.Host)
-		require.NoError(t, err)
-
-		data := make([]byte, 100)
-		n, err := conn.Read(data)
-		require.NoError(t, err)
-		require.Equal(t, string(data[:n]), "hello")
-	})
-
-	t.Run("connection upgrade API not found", func(t *testing.T) {
-		ctx := context.Background()
-
-		server := httptest.NewTLSServer(http.NotFoundHandler())
-		t.Cleanup(server.Close)
-		addr, err := url.Parse(server.URL)
-		require.NoError(t, err)
+	tests := []struct {
+		name          string
+		serverHandler http.Handler
+		withPing      bool
+		wantError     bool
+	}{
+		{
+			name:          "connection upgrade",
+			serverHandler: mockConnUpgradeHandler(t, constants.WebAPIConnUpgradeTypeALPN, []byte("hello")),
+		},
+		{
+			name:          "connection upgrade with ping",
+			serverHandler: mockConnUpgradeHandler(t, constants.WebAPIConnUpgradeTypeALPNPing, []byte("hello")),
+			withPing:      true,
+		},
+		{
+			name:          "connection upgrade API not found",
+			serverHandler: http.NotFoundHandler(),
+			wantError:     true,
+		},
+	}
 
-		tlsConfig := &tls.Config{InsecureSkipVerify: true}
-		preDialer := NewDialer(ctx, 0, 5*time.Second)
-		dialer := newALPNConnUpgradeDialer(preDialer, tlsConfig)
-		_, err = dialer.DialContext(ctx, "tcp", addr.Host)
-		require.Error(t, err)
-	})
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			ctx := context.Background()
+
+			server := httptest.NewTLSServer(test.serverHandler)
+			t.Cleanup(server.Close)
+			addr, err := url.Parse(server.URL)
+			require.NoError(t, err)
+			pool := x509.NewCertPool()
+			pool.AddCert(server.Certificate())
+
+			tlsConfig := &tls.Config{RootCAs: pool}
+			preDialer := newDirectDialer(0, 5*time.Second)
+			dialer := newALPNConnUpgradeDialer(preDialer, tlsConfig, test.withPing)
+			conn, err := dialer.DialContext(ctx, "tcp", addr.Host)
+			if test.wantError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			defer conn.Close()
+
+			data := make([]byte, 100)
+			n, err := conn.Read(data)
+			require.NoError(t, err)
+			require.Equal(t, string(data[:n]), "hello")
+		})
+	}
 }
 
 type mockALPNServer struct {
@@ -218,6 +235,8 @@ func mustStartMockALPNServer(t *testing.T, supportedProtos []string) *mockALPNSe
 // mockConnUpgradeHandler mocks the server side implementation to handle an
 // upgrade request and sends back some data inside the tunnel.
 func mockConnUpgradeHandler(t *testing.T, upgradeType string, write []byte) http.Handler {
+	t.Helper()
+
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, constants.WebAPIConnUpgrade, r.URL.Path)
 		require.Equal(t, upgradeType, r.Header.Get(constants.WebAPIConnUpgradeHeader))
@@ -238,7 +257,18 @@ func mockConnUpgradeHandler(t *testing.T, upgradeType string, write []byte) http
 		require.NoError(t, response.Write(conn))
 
 		// Upgraded.
-		_, err = conn.Write(write)
-		require.NoError(t, err)
+		switch upgradeType {
+		case constants.WebAPIConnUpgradeTypeALPNPing:
+			// Wrap conn with Ping and write some pings.
+			pingConn := pingconn.New(conn)
+			pingConn.WritePing()
+			_, err = pingConn.Write(write)
+			require.NoError(t, err)
+			pingConn.WritePing()
+
+		default:
+			_, err = conn.Write(write)
+			require.NoError(t, err)
+		}
 	})
 }