Fix Machine ID selection of Kubernetes Access address/port

docs/pages/machine-id/access-guides/kubernetes.mdx

@@ -18,9 +18,6 @@ used to access a Kubernetes cluster enrolled with your Teleport cluster.
 
 - If you have not already connected your Kubernetes cluster to Teleport, follow
   the [Kubernetes Access Getting Started Guide](../../kubernetes-access/getting-started.mdx).
-
-(!docs/pages/includes/machine-id/kubernetes-machineidnote.mdx!)
-
 - (!docs/pages/includes/tctl.mdx!)
 - To configure the Kubernetes cluster, your client system will need to have
   `kubectl` installed. See the

docs/pages/machine-id/troubleshooting.mdx

@@ -292,52 +292,3 @@ flag:
 $ tctl bots rm example
 $ tctl bots add example --roles=foo,bar,machine-id-db
 ```
-
-## Kubernetes connections are failing with `Unable to connect to the server: x509: certificate signed by unknown authority`
-
-### Symptoms
-
-A self-hosted Teleport cluster is connecting Machine ID to Kubernetes clusters
-with the following errors. This can happen for non-TLS configured Teleport clusters.
-
-```bash
-E0322 22:53:31.653051    1699 memcache.go:265] couldn't get current server API group list: Get "https://teleport.example.com:443/api?timeout=32s": x509: certificate signed by unknown authority
-```
-
-To confirm the TLS routing mode check the value of the `proxy.tls_routing_enabled`
-key with this command, substituting your proxy address:
-
-```bash
-curl https://teleport.example.com:443/webapi/ping | jq
-```
-
-If the value is `false` then this is a non-TLS routing configuration.
-
-### Explanation
-
-Proxies configured with non-TLS routing use specific ports for various types
-of traffic. That requires that a Kubernetes
-connection use its designated port. Currently Machine ID requires that the Kubernetes
-public address is set to use the correct port. Otherwise it will use the Proxy web port
-which can cause these type of errors.
-
-### Resolution
-
-The Kubernetes public address is via the `kube_public_addr` within the 
-Teleport `proxy_service` configuration by administrators. The proxy will
-require a restart after the configuration is updated.
-
-  ```yaml
-  proxy_service:
-    enabled: true
-    kube_listen_addr: 0.0.0.0:3026
-    kube_public_addr: teleport.example.com:3026
-  ```
-
-Retrieve the configuration listing from the proxy web address to confirm the
-Kubernetes public address is populated in `proxy.kube.public_addr`.
-
-```bash
-curl https://teleport.example.com:443/webapi/ping | jq
-```
-

lib/tbot/config/bot_test.go

@@ -153,7 +153,15 @@ func (p *mockProvider) GenerateHostCert(
 }
 
 func (p *mockProvider) ProxyPing(ctx context.Context) (*webclient.PingResponse, error) {
-	return &webclient.PingResponse{}, nil
+	return &webclient.PingResponse{
+		ClusterName: p.clusterName,
+		Proxy: webclient.ProxySettings{
+			TLSRoutingEnabled: true,
+			SSH: webclient.SSHProxySettings{
+				PublicAddr: p.proxyAddr,
+			},
+		},
+	}, nil
 }
 
 func (p *mockProvider) Config() *BotConfig {

lib/tbot/config/template_kubernetes.go

@@ -27,16 +27,12 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
-	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/client/webclient"
-	"github.com/gravitational/teleport/api/constants"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/client"
-	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/kube/kubeconfig"
 	"github.com/gravitational/teleport/lib/tbot/bot"
 	"github.com/gravitational/teleport/lib/tbot/identity"
-	"github.com/gravitational/teleport/lib/utils"
 )
 
 const defaultKubeconfigPath = "kubeconfig.yaml"
@@ -62,32 +58,12 @@ func (t *templateKubernetes) describe() []FileDescription {
 // kubeconfig.
 type kubernetesStatus struct {
 	clusterAddr           string
-	proxyAddr             string
 	teleportClusterName   string
 	kubernetesClusterName string
 	tlsServerName         string
 	credentials           *client.Key
 }
 
-func getKubeProxyHostPort(authPong *proto.PingResponse, proxyPong *webclient.PingResponse) (string, int, error) {
-	addr := proxyPong.Proxy.Kube.PublicAddr
-	if addr == "" {
-		addr = authPong.ProxyPublicAddr
-	}
-
-	if addr == "" {
-		return "", 0, trace.BadParameter(
-			"Teleport server reported no usable public proxy address")
-	}
-
-	parsed, err := utils.ParseAddr(addr)
-	if err != nil {
-		return "", 0, trace.Wrap(err, "invalid proxy address")
-	}
-
-	return parsed.Host(), parsed.Port(defaults.KubeListenPort), nil
-}
-
 // generateKubeConfig creates a Kubernetes config object with the given cluster
 // config.
 func generateKubeConfig(ks *kubernetesStatus, destPath string, executablePath string) (*clientcmdapi.Config, error) {
@@ -161,31 +137,15 @@ func (t *templateKubernetes) render(
 		return trace.BadParameter("Destination %s must be a directory", destination)
 	}
 
-	// Ping the auth server and proxy to resolve connection addresses.
-	authPong, err := bot.AuthPing(ctx)
-	if err != nil {
-		return trace.Wrap(err)
-	}
-
+	// Ping the proxy to resolve connection addresses.
 	proxyPong, err := bot.ProxyPing(ctx)
 	if err != nil {
 		return trace.Wrap(err)
 	}
-
-	host, port, err := getKubeProxyHostPort(authPong, proxyPong)
+	clusterAddr, tlsServerName, err := selectKubeConnectionMethod(proxyPong)
 	if err != nil {
 		return trace.Wrap(err)
 	}
-	kubeAddr := fmt.Sprintf("https://%s:%d", host, port)
-
-	// Next, determine the TLS routing config (if any)
-	// Note: derived from tool/tsh/kube.go; this impl should defer to it for
-	// future changes.
-	serverName := fmt.Sprintf("%s%s", constants.KubeTeleportProxyALPNPrefix, host)
-	isIPFormat := net.ParseIP(host) != nil
-	if host == "" || isIPFormat {
-		serverName = fmt.Sprintf("%s%s", constants.KubeTeleportProxyALPNPrefix, constants.APIDomain)
-	}
 
 	hostCAs, err := bot.GetCertAuthorities(ctx, types.HostCA)
 	if err != nil {
@@ -198,17 +158,13 @@ func (t *templateKubernetes) render(
 	}
 
 	status := &kubernetesStatus{
-		clusterAddr:           kubeAddr,
-		proxyAddr:             authPong.ProxyPublicAddr,
+		clusterAddr:           clusterAddr,
+		tlsServerName:         tlsServerName,
 		credentials:           key,
-		teleportClusterName:   authPong.ClusterName,
+		teleportClusterName:   proxyPong.ClusterName,
 		kubernetesClusterName: t.clusterName,
 	}
 
-	if proxyPong.Proxy.TLSRoutingEnabled {
-		status.tlsServerName = serverName
-	}
-
 	executablePath, err := t.executablePathGetter()
 	if err != nil {
 		return trace.Wrap(err)
@@ -226,3 +182,45 @@ func (t *templateKubernetes) render(
 
 	return trace.Wrap(destination.Write(ctx, defaultKubeconfigPath, yamlCfg))
 }
+
+// selectKubeConnectionMethod determines the address and SNI that should be
+// put into the kubeconfig file.
+func selectKubeConnectionMethod(proxyPong *webclient.PingResponse) (clusterAddr string, sni string, err error) {
+	// First we check for TLS routing. If this is enabled, we use the Proxy's
+	// PublicAddr, and we must also specify a special SNI.
+	//
+	// Even if KubePublicAddr is specified, we still use the general
+	// PublicAddr when using TLS routing.
+	if proxyPong.Proxy.TLSRoutingEnabled {
+		addr := proxyPong.Proxy.SSH.PublicAddr
+		host, _, err := net.SplitHostPort(proxyPong.Proxy.SSH.PublicAddr)
+		if err != nil {
+			return "", "", trace.Wrap(err, "parsing proxy public_addr")
+		}
+
+		return fmt.Sprintf("https://%s", addr), client.GetKubeTLSServerName(host), nil
+	}
+
+	// Next, we try to use the KubePublicAddr.
+	if proxyPong.Proxy.Kube.PublicAddr != "" {
+		return fmt.Sprintf("https://%s", proxyPong.Proxy.Kube.PublicAddr), "", nil
+	}
+
+	// Finally, we fall back to the main proxy PublicAddr with the port from
+	// KubeListenAddr.
+	if proxyPong.Proxy.Kube.ListenAddr != "" {
+		host, _, err := net.SplitHostPort(proxyPong.Proxy.SSH.PublicAddr)
+		if err != nil {
+			return "", "", trace.Wrap(err, "parsing proxy public_addr")
+		}
+
+		_, port, err := net.SplitHostPort(proxyPong.Proxy.Kube.ListenAddr)
+		if err != nil {
+			return "", "", trace.Wrap(err, "parsing proxy kube_listen_addr")
+		}
+
+		return fmt.Sprintf("https://%s:%s", host, port), "", nil
+	}
+
+	return "", "", trace.BadParameter("unable to determine kubernetes address")
+}

lib/tbot/config/template_kubernetes_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/gravitational/teleport/api/client/webclient"
 	"github.com/gravitational/teleport/lib/tbot/botfs"
 	"github.com/gravitational/teleport/lib/utils/golden"
 )
@@ -88,3 +89,91 @@ func TestTemplateKubernetesRender(t *testing.T) {
 		})
 	}
 }
+
+func Test_selectKubeConnectionMethod(t *testing.T) {
+	tests := []struct {
+		name string
+
+		proxyPing *webclient.PingResponse
+		wantAddr  string
+		wantSNI   string
+	}{
+		{
+			// Copied from my real Teleport Cloud webapi/ping
+			name: "TLS Routing",
+			proxyPing: &webclient.PingResponse{
+				Proxy: webclient.ProxySettings{
+					Kube: webclient.KubeProxySettings{
+						Enabled:    true,
+						ListenAddr: "0.0.0.0:3080",
+					},
+					SSH: webclient.SSHProxySettings{
+						ListenAddr:       "0.0.0.0:3080",
+						TunnelListenAddr: "0.0.0.0:3080",
+						WebListenAddr:    "0.0.0.0:3080",
+						PublicAddr:       "noah.teleport.sh:443",
+					},
+					TLSRoutingEnabled: true,
+				},
+				ClusterName: "noah.teleport.sh",
+			},
+			wantAddr: "https://noah.teleport.sh:443",
+			wantSNI:  "kube-teleport-proxy-alpn.noah.teleport.sh",
+		},
+		{
+			name: "KubePublicAddr specified",
+			proxyPing: &webclient.PingResponse{
+				Proxy: webclient.ProxySettings{
+					Kube: webclient.KubeProxySettings{
+						Enabled:    true,
+						ListenAddr: "0.0.0.0:1337",
+						PublicAddr: "kube.example.com:1337",
+					},
+					SSH: webclient.SSHProxySettings{
+						ListenAddr:       "0.0.0.0:3023",
+						TunnelListenAddr: "0.0.0.0:3024",
+						WebListenAddr:    "0.0.0.0:3080",
+						PublicAddr:       "cluster.example.com:443",
+						SSHPublicAddr:    "cluster.example.com:3023",
+						TunnelPublicAddr: "cluster.example.com:3024",
+					},
+					TLSRoutingEnabled: false,
+				},
+				ClusterName: "cluster.example.com",
+			},
+			wantAddr: "https://kube.example.com:1337",
+		},
+		{
+			// https://github.com/gravitational/teleport/issues/19811
+			name: "Falls back to Kube ListenAddr Port with PublicAddr",
+			proxyPing: &webclient.PingResponse{
+				Proxy: webclient.ProxySettings{
+					Kube: webclient.KubeProxySettings{
+						Enabled:    true,
+						ListenAddr: "0.0.0.0:3026",
+					},
+					SSH: webclient.SSHProxySettings{
+						ListenAddr:       "[::]:3023",
+						TunnelListenAddr: "0.0.0.0:3024",
+						WebListenAddr:    "0.0.0.0:3080",
+						PublicAddr:       "cluster.example.com:5443",
+						SSHPublicAddr:    "cluster.example.com:3023",
+						TunnelPublicAddr: "cluster.example.com:3024",
+					},
+					TLSRoutingEnabled: false,
+				},
+				ClusterName: "cluster.example.com",
+			},
+			wantAddr: "https://cluster.example.com:3026",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			addr, sni, err := selectKubeConnectionMethod(tt.proxyPing)
+			require.NoError(t, err)
+			require.Equal(t, tt.wantAddr, addr)
+			require.Equal(t, tt.wantSNI, sni)
+		})
+	}
+}

lib/tbot/config/testdata/TestTemplateKubernetesRender/absolute_path/kubeconfig.yaml.golden

@@ -3,6 +3,7 @@ clusters:
 - cluster:
     certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLakNDQWhLZ0F3SUJBZ0lRSnRKREpaWkJrZy9hZk04ZDJaSkNUakFOQmdrcWhraUc5dzBCQVFzRkFEQkEKTVJVd0V3WURWUVFLRXd4VVpXeGxjRzl5ZENCUFUxTXhKekFsQmdOVkJBTVRIblJsYkdWd2IzSjBMbXh2WTJGcwphRzl6ZEM1c2IyTmhiR1J2YldGcGJqQWVGdzB4TnpBMU1Ea3hPVFF3TXpaYUZ3MHlOekExTURjeE9UUXdNelphCk1FQXhGVEFUQmdOVkJBb1RERlJsYkdWd2IzSjBJRTlUVXpFbk1DVUdBMVVFQXhNZWRHVnNaWEJ2Y25RdWJHOWoKWVd4b2IzTjBMbXh2WTJGc1pHOXRZV2x1TUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQwpBUUVBdUtGTGFmMmlJSS94RFIrbTJZajZQblVFYStxenF3eHNkTFVqbnVuRlphQVhHK2habTRNbDgwU0NpQmdJCmdUSFFsSnlMSWtUdHVSb0g1YWVNeXoxRVJVQ3RpaTRac1RxRHJqalV5YnhQNHIrNEhWWDZtMzRzNmh3RXI4RmkKZnRzOXBNcDRpUzN0UWd1UmMyOGdQZERvL1Q2VnJKVFZZVWZVVXNORFJ0SXJsQjVPOWlncXFMbnVhWTllcUdpNApQVXgwRzB3UllKcFJ5d29qOEcwSWtwZlFUaVgrQ0FDN2R0NXdzN1pybkdxQ05CTEdpNWJHc2FNbXB0VmJzU0VwCjFUZW5udEY1NFYxaVI0OUlWNUpxRGhtMVMwSG1rbGVvSnpLZGMrNnNQL3hOZXB6OVBKenVGOWQ5TnViVExXZ0IKc0syOFlJdGNtV0hkSFhEL09EeFZhZWhSandJREFRQUJveUF3SGpBT0JnTlZIUThCQWY4RUJBTUNCNEF3REFZRApWUjBUQVFIL0JBSXdBREFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBQVZVNnNOQmRqNzZzYUh3T3hHU2RuRXFRCm8ydE11UjNtc1NNNEY2d0ZLMlVrS2Vwc0Q3Q1lJZi9Qek5TTlVxQTVKSUVVVmVNcUd5aUh1QWJVNEM2NTVuVDEKSXlKWDFELytyNzNzU3A1amJJcFFtMnhvUUdabmo2Zy9LbHR3OE9TT0F3K0RzTUYvUExWcW9XSnAwN3U2ZXcvbQpOeFdzSktjWjVrK3E0ZU14Y2k5bUtSSEhxc3F1V0tYelFsVVJNTkZJK21HYUZ3cktNNGRtemFSMEJFYytpbFN4ClFxVXZRNzRzbXNMSyt6aE5pa21namxHQzVvYjlnOFhraFZBa0pNQWgycmI5b25ETmlSbDY4aUFnY3pQODhtWHUKdk4vbzk4ZHlwenNQeFhtdzZ0a0RxSVJQVUFVYmg0NjVybFk1c0tNbVJnWGkyclVmbC9RVjVuYm96VW8vSFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
     server: https://tele.blackmesa.gov:443
+    tls-server-name: kube-teleport-proxy-alpn.tele.blackmesa.gov
   name: tele.blackmesa.gov-example
 contexts:
 - context:

lib/tbot/config/testdata/TestTemplateKubernetesRender/relative_path/kubeconfig.yaml.golden

@@ -3,6 +3,7 @@ clusters:
 - cluster:
     certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLakNDQWhLZ0F3SUJBZ0lRSnRKREpaWkJrZy9hZk04ZDJaSkNUakFOQmdrcWhraUc5dzBCQVFzRkFEQkEKTVJVd0V3WURWUVFLRXd4VVpXeGxjRzl5ZENCUFUxTXhKekFsQmdOVkJBTVRIblJsYkdWd2IzSjBMbXh2WTJGcwphRzl6ZEM1c2IyTmhiR1J2YldGcGJqQWVGdzB4TnpBMU1Ea3hPVFF3TXpaYUZ3MHlOekExTURjeE9UUXdNelphCk1FQXhGVEFUQmdOVkJBb1RERlJsYkdWd2IzSjBJRTlUVXpFbk1DVUdBMVVFQXhNZWRHVnNaWEJ2Y25RdWJHOWoKWVd4b2IzTjBMbXh2WTJGc1pHOXRZV2x1TUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQwpBUUVBdUtGTGFmMmlJSS94RFIrbTJZajZQblVFYStxenF3eHNkTFVqbnVuRlphQVhHK2habTRNbDgwU0NpQmdJCmdUSFFsSnlMSWtUdHVSb0g1YWVNeXoxRVJVQ3RpaTRac1RxRHJqalV5YnhQNHIrNEhWWDZtMzRzNmh3RXI4RmkKZnRzOXBNcDRpUzN0UWd1UmMyOGdQZERvL1Q2VnJKVFZZVWZVVXNORFJ0SXJsQjVPOWlncXFMbnVhWTllcUdpNApQVXgwRzB3UllKcFJ5d29qOEcwSWtwZlFUaVgrQ0FDN2R0NXdzN1pybkdxQ05CTEdpNWJHc2FNbXB0VmJzU0VwCjFUZW5udEY1NFYxaVI0OUlWNUpxRGhtMVMwSG1rbGVvSnpLZGMrNnNQL3hOZXB6OVBKenVGOWQ5TnViVExXZ0IKc0syOFlJdGNtV0hkSFhEL09EeFZhZWhSandJREFRQUJveUF3SGpBT0JnTlZIUThCQWY4RUJBTUNCNEF3REFZRApWUjBUQVFIL0JBSXdBREFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBQVZVNnNOQmRqNzZzYUh3T3hHU2RuRXFRCm8ydE11UjNtc1NNNEY2d0ZLMlVrS2Vwc0Q3Q1lJZi9Qek5TTlVxQTVKSUVVVmVNcUd5aUh1QWJVNEM2NTVuVDEKSXlKWDFELytyNzNzU3A1amJJcFFtMnhvUUdabmo2Zy9LbHR3OE9TT0F3K0RzTUYvUExWcW9XSnAwN3U2ZXcvbQpOeFdzSktjWjVrK3E0ZU14Y2k5bUtSSEhxc3F1V0tYelFsVVJNTkZJK21HYUZ3cktNNGRtemFSMEJFYytpbFN4ClFxVXZRNzRzbXNMSyt6aE5pa21namxHQzVvYjlnOFhraFZBa0pNQWgycmI5b25ETmlSbDY4aUFnY3pQODhtWHUKdk4vbzk4ZHlwenNQeFhtdzZ0a0RxSVJQVUFVYmg0NjVybFk1c0tNbVJnWGkyclVmbC9RVjVuYm96VW8vSFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
     server: https://tele.blackmesa.gov:443
+    tls-server-name: kube-teleport-proxy-alpn.tele.blackmesa.gov
   name: tele.blackmesa.gov-example
 contexts:
 - context:

lib/tbot/testhelpers/srv.go

@@ -64,6 +64,7 @@ func DefaultConfig(t *testing.T) (*config.FileConfig, []servicecfg.FileDescripto
 			},
 			WebAddr:    testenv.NewTCPListener(t, service.ListenerProxyWeb, &fds),
 			TunAddr:    testenv.NewTCPListener(t, service.ListenerProxyTunnel, &fds),
+			KubeAddr:   testenv.NewTCPListener(t, service.ListenerProxyKube, &fds),
 			PublicAddr: []string{"localhost"}, // ListenerProxyWeb port will be appended
 		},
 		Auth: config.Auth{