Stop registering a Kubernetes cluster named after the Teleport cluster (

#6786)

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 7.0
+
+Teleport 7.0 is a major release with new features, functionality, and bug fixes.
+
+## Breaking Changes
+
+* Proxy services whose configuration includes a `kube_listen_addr` but no `kubernetes` section will no longer publish a Kubernetes cluster named after the Teleport cluster.
+
 ## 6.2
 
 Teleport 6.2 contains new features, improvements, and bug fixes.

integration/kube_integration_test.go

@@ -1139,6 +1139,7 @@ func (s *KubeSuite) teleKubeConfig(hostname string) *service.Config {
 	tconf.Proxy.Kube.Enabled = true
 	tconf.Proxy.Kube.ListenAddr.Addr = net.JoinHostPort(hostname, ports.Pop())
 	tconf.Proxy.Kube.KubeconfigPath = s.kubeConfigPath
+	tconf.Proxy.Kube.LegacyKubeProxy = true
 
 	return tconf
 }

lib/config/configuration.go

@@ -645,6 +645,7 @@ func applyProxyConfig(fc *FileConfig, cfg *service.Config) error {
 	switch {
 	case legacyKube && !newKube:
 		cfg.Proxy.Kube.Enabled = true
+		cfg.Proxy.Kube.LegacyKubeProxy = true
 		if fc.Proxy.Kube.KubeconfigFile != "" {
 			cfg.Proxy.Kube.KubeconfigPath = fc.Proxy.Kube.KubeconfigFile
 		}

lib/config/configuration_test.go

@@ -1131,8 +1131,9 @@ func TestProxyKube(t *testing.T) {
 				Service: Service{EnabledFlag: "yes", ListenAddress: "0.0.0.0:8080"},
 			}},
 			want: service.KubeProxyConfig{
-				Enabled:    true,
-				ListenAddr: *utils.MustParseAddr("0.0.0.0:8080"),
+				Enabled:         true,
+				ListenAddr:      *utils.MustParseAddr("0.0.0.0:8080"),
+				LegacyKubeProxy: true,
 			},
 			checkErr: require.NoError,
 		},
@@ -1144,10 +1145,11 @@ func TestProxyKube(t *testing.T) {
 				PublicAddr:     utils.Strings([]string{"kube.example.com:443"}),
 			}},
 			want: service.KubeProxyConfig{
-				Enabled:        true,
-				ListenAddr:     *utils.MustParseAddr("0.0.0.0:8080"),
-				KubeconfigPath: "/tmp/kubeconfig",
-				PublicAddrs:    []utils.NetAddr{*utils.MustParseAddr("kube.example.com:443")},
+				Enabled:         true,
+				ListenAddr:      *utils.MustParseAddr("0.0.0.0:8080"),
+				KubeconfigPath:  "/tmp/kubeconfig",
+				PublicAddrs:     []utils.NetAddr{*utils.MustParseAddr("kube.example.com:443")},
+				LegacyKubeProxy: true,
 			},
 			checkErr: require.NoError,
 		},

lib/kube/proxy/auth.go

@@ -62,65 +62,81 @@ func TestOnlySkipSelfPermissionCheck(skip bool) {
 // - kubeconfig: a file with a set of k8s endpoints and credentials mapped to
 //   them this is used when kubeconfigPath is set
 //
-// newKubeService changes the loading behavior:
-// - false:
+// serviceType changes the loading behavior:
+// - LegacyProxyService:
 //   - if loading from kubeconfig, only "current-context" is returned; the
 //     returned map key matches tpClusterName
 //   - if no credentials are loaded, no error is returned
 //   - permission self-test failures are only logged
-// - true:
+// - ProxyService:
+//   - no credentials are loaded and no error is returned
+// - KubeService:
 //   - if loading from kubeconfig, all contexts are returned
 //   - if no credentials are loaded, returns an error
 //   - permission self-test failures cause an error to be returned
-func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, kubeClusterName, kubeconfigPath string, newKubeService bool) (map[string]*kubeCreds, error) {
+func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, kubeClusterName, kubeconfigPath string, serviceType KubeServiceType) (map[string]*kubeCreds, error) {
 	log.
 		WithField("kubeconfigPath", kubeconfigPath).
 		WithField("kubeClusterName", kubeClusterName).
-		WithField("newKubeService", newKubeService).
-		Debug("Reading kubernetes creds.")
+		WithField("serviceType", serviceType).
+		Debug("Reading Kubernetes creds.")
+
+	// Proxy service should never have creds, forwards to kube service
+	if serviceType == ProxyService {
+		return map[string]*kubeCreds{}, nil
+	}
 
 	// Load kubeconfig or local pod credentials.
-	cfg, err := kubeutils.GetKubeConfig(kubeconfigPath, newKubeService, kubeClusterName)
+	loadAll := serviceType == KubeService
+	cfg, err := kubeutils.GetKubeConfig(kubeconfigPath, loadAll, kubeClusterName)
 	if err != nil && !trace.IsNotFound(err) {
 		return nil, trace.Wrap(err)
 	}
+
 	if trace.IsNotFound(err) || len(cfg.Contexts) == 0 {
-		if newKubeService {
-			return nil, trace.BadParameter("no kubernetes credentials found; kubernetes_service requires either a valid kubeconfig_path or to run inside of a kubernetes pod")
+		switch serviceType {
+		case KubeService:
+			return nil, trace.BadParameter("no Kubernetes credentials found; Kubernetes_service requires either a valid kubeconfig_path or to run inside of a Kubernetes pod")
+		case LegacyProxyService:
+			log.Debugf("Could not load Kubernetes credentials. This proxy will still handle Kubernetes requests for trusted teleport clusters or Kubernetes nodes in this teleport cluster")
 		}
-		log.Debugf("Could not load kubernetes credentials. This proxy will still handle kubernetes requests for trusted teleport clusters or kubernetes nodes in this teleport cluster")
 		return map[string]*kubeCreds{}, nil
 	}
-	if !newKubeService {
-		// Hack for proxy_service - register a k8s cluster named after the
-		// teleport cluster name to route legacy requests.
+
+	if serviceType == LegacyProxyService {
+		// Hack for legacy proxy service - register a k8s cluster named after
+		// the teleport cluster name to route legacy requests.
 		//
 		// Also, remove all other contexts. Multiple kubeconfig entries are
 		// only supported for kubernetes_service.
-		cfg.Contexts = map[string]*rest.Config{
-			tpClusterName: cfg.Contexts[cfg.CurrentContext],
+		if currentContext, ok := cfg.Contexts[cfg.CurrentContext]; ok {
+			cfg.Contexts = map[string]*rest.Config{
+				tpClusterName: currentContext,
+			}
+		} else {
+			return nil, trace.BadParameter("no Kubernetes current-context found; Kubernetes proxy service requires either a valid kubeconfig_path with a current-context or to run inside of a Kubernetes pod")
 		}
 	}
 
 	res := make(map[string]*kubeCreds, len(cfg.Contexts))
 	// Convert kubeconfig contexts into kubeCreds.
 	for cluster, clientCfg := range cfg.Contexts {
 		log := log.WithField("cluster", cluster)
-		log.Debug("Checking kubernetes impersonation permissions.")
+		log.Debug("Checking Kubernetes impersonation permissions.")
 		client, err := kubernetes.NewForConfig(clientCfg)
 		if err != nil {
-			return nil, trace.Wrap(err, "failed to generate kubernetes client for cluster %q", cluster)
+			return nil, trace.Wrap(err, "failed to generate Kubernetes client for cluster %q", cluster)
 		}
 		// For each loaded cluster, check impersonation permissions. This
 		// failure is only critical for newKubeService.
 		if err := checkImpersonationPermissions(ctx, client.AuthorizationV1().SelfSubjectAccessReviews()); err != nil {
 			// kubernetes_service must have valid RBAC permissions, otherwise
 			// it's pointless.
 			// proxy_service can run without them (e.g. a root proxy).
-			if newKubeService {
+			if serviceType == KubeService {
 				return nil, trace.Wrap(err)
 			}
-			log.WithError(err).Warning("Failed to test the necessary kubernetes permissions. This teleport instance will still handle kubernetes requests towards other kubernetes clusters")
+			log.WithError(err).Warning("Failed to test the necessary Kubernetes permissions. This teleport instance will still handle Kubernetes requests towards other Kubernetes clusters")
 			// We used to recommend users to set a dummy kubeconfig on root
 			// proxies to get kubernetes support working for leaf clusters:
 			// https://community.goteleport.com/t/enabling-teleport-to-act-as-a-kubernetes-proxy-for-trusted-leaf-clusters/418
@@ -131,7 +147,7 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku
 				log.Info("If this is a proxy and you provided a dummy kubeconfig_path, you can remove it from teleport.yaml to get rid of this warning")
 			}
 		} else {
-			log.Debug("Have all necessary kubernetes impersonation permissions.")
+			log.Debug("Have all necessary Kubernetes impersonation permissions.")
 		}
 
 		targetAddr, err := parseKubeHost(clientCfg.Host)
@@ -147,7 +163,7 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku
 			return nil, trace.Wrap(err, "failed to generate transport config from kubeconfig: %v", err)
 		}
 
-		log.Debug("Initialized kubernetes credentials")
+		log.Debug("Initialized Kubernetes credentials")
 		res[cluster] = &kubeCreds{
 			tlsConfig:       tlsConfig,
 			transportConfig: transportConfig,
@@ -164,7 +180,7 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku
 func parseKubeHost(host string) (string, error) {
 	u, err := url.Parse(host)
 	if err != nil {
-		return "", trace.Wrap(err, "failed to parse kubernetes host: %v", err)
+		return "", trace.Wrap(err, "failed to parse Kubernetes host: %v", err)
 	}
 	if _, _, err := net.SplitHostPort(u.Host); err != nil {
 		// add default HTTPS port
@@ -195,10 +211,10 @@ func checkImpersonationPermissions(ctx context.Context, sarClient authztypes.Sel
 			},
 		}, metav1.CreateOptions{})
 		if err != nil {
-			return trace.Wrap(err, "failed to verify impersonation permissions for kubernetes: %v; this may be due to missing the SelfSubjectAccessReview permission on the ClusterRole used by the proxy; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", err)
+			return trace.Wrap(err, "failed to verify impersonation permissions for Kubernetes: %v; this may be due to missing the SelfSubjectAccessReview permission on the ClusterRole used by the proxy; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", err)
 		}
 		if !resp.Status.Allowed {
-			return trace.AccessDenied("proxy can't impersonate kubernetes %s at the cluster level; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", resource)
+			return trace.AccessDenied("proxy can't impersonate Kubernetes %s at the cluster level; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", resource)
 		}
 	}
 	return nil

lib/kube/proxy/auth_test.go

@@ -152,23 +152,30 @@ current-context: foo
 		desc           string
 		kubeconfigPath string
 		kubeCluster    string
-		newService     bool
+		serviceType    KubeServiceType
 		want           map[string]*kubeCreds
 		assertErr      require.ErrorAssertionFunc
 	}{
 		{
-			desc:       "kubernetes_service, no kube creds",
-			newService: true,
-			assertErr:  require.Error,
+			desc:        "kubernetes_service, no kube creds",
+			serviceType: KubeService,
+			assertErr:   require.Error,
 		},
 		{
-			desc:      "proxy_service, no kube creds",
-			assertErr: require.NoError,
-			want:      map[string]*kubeCreds{},
+			desc:        "proxy_service, no kube creds",
+			serviceType: ProxyService,
+			assertErr:   require.NoError,
+			want:        map[string]*kubeCreds{},
+		},
+		{
+			desc:        "legacy proxy_service, no kube creds",
+			serviceType: ProxyService,
+			assertErr:   require.NoError,
+			want:        map[string]*kubeCreds{},
 		},
 		{
 			desc:           "kubernetes_service, with kube creds",
-			newService:     true,
+			serviceType:    KubeService,
 			kubeconfigPath: kubeconfigPath,
 			want: map[string]*kubeCreds{
 				"foo": {
@@ -187,6 +194,14 @@ current-context: foo
 		{
 			desc:           "proxy_service, with kube creds",
 			kubeconfigPath: kubeconfigPath,
+			serviceType:    ProxyService,
+			want:           map[string]*kubeCreds{},
+			assertErr:      require.NoError,
+		},
+		{
+			desc:           "legacy proxy_service, with kube creds",
+			kubeconfigPath: kubeconfigPath,
+			serviceType:    LegacyProxyService,
 			want: map[string]*kubeCreds{
 				teleClusterName: {
 					targetAddr:      "example.com:3026",
@@ -199,7 +214,7 @@ current-context: foo
 	}
 	for _, tt := range tests {
 		t.Run(tt.desc, func(t *testing.T) {
-			got, err := getKubeCreds(ctx, testlog.FailureOnly(t), teleClusterName, "", tt.kubeconfigPath, tt.newService)
+			got, err := getKubeCreds(ctx, testlog.FailureOnly(t), teleClusterName, "", tt.kubeconfigPath, tt.serviceType)
 			tt.assertErr(t, err)
 			if err != nil {
 				return

lib/kube/proxy/forwarder.go

@@ -63,6 +63,23 @@ import (
 	utilexec "k8s.io/client-go/util/exec"
 )
 
+// KubeServiceType specifies a Teleport service type which can forward Kubernetes requests
+type KubeServiceType int
+
+const (
+	// KubeService is a Teleport kubernetes_service. A KubeService always forwards
+	// requests directly to a Kubernetes endpoint.
+	KubeService KubeServiceType = iota
+	// ProxyService is a Teleport proxy_service with kube_listen_addr/
+	// kube_public_addr enabled. A ProxyService always forwards requests to a
+	// Teleport KubeService or LegacyProxyService.
+	ProxyService
+	// LegacyProxyService is a Teleport proxy_service with the kubernetes section
+	// enabled. A LegacyProxyService can forward requests directly to a Kubernetes
+	// endpoint, or to another Teleport LegacyProxyService or KubeService.
+	LegacyProxyService
+)
+
 // ForwarderConfig specifies configuration for proxy forwarder
 type ForwarderConfig struct {
 	// ReverseTunnelSrv is the teleport reverse tunnel server
@@ -94,10 +111,8 @@ type ForwarderConfig struct {
 	Context context.Context
 	// KubeconfigPath is a path to kubernetes configuration
 	KubeconfigPath string
-	// NewKubeService specifies whether to apply the additional kubernetes_service features:
-	// - parsing multiple kubeconfig entries
-	// - enforcing self permission check
-	NewKubeService bool
+	// KubeServiceType specifies which Teleport service type this forwarder is for
+	KubeServiceType KubeServiceType
 	// KubeClusterName is the name of the kubernetes cluster that this
 	// forwarder handles.
 	KubeClusterName string
@@ -157,7 +172,14 @@ func (f *ForwarderConfig) CheckAndSetDefaults() error {
 	if f.Component == "" {
 		f.Component = "kube_forwarder"
 	}
-	if f.KubeClusterName == "" && f.KubeconfigPath == "" {
+	switch f.KubeServiceType {
+	case KubeService:
+	case ProxyService:
+	case LegacyProxyService:
+	default:
+		return trace.BadParameter("unknown value for KubeServiceType")
+	}
+	if f.KubeClusterName == "" && f.KubeconfigPath == "" && f.KubeServiceType == LegacyProxyService {
 		// Running without a kubeconfig and explicit k8s cluster name. Use
 		// teleport cluster name instead, to ask kubeutils.GetKubeConfig to
 		// attempt loading the in-cluster credentials.
@@ -176,7 +198,7 @@ func NewForwarder(cfg ForwarderConfig) (*Forwarder, error) {
 		trace.Component: cfg.Component,
 	})
 
-	creds, err := getKubeCreds(cfg.Context, log, cfg.ClusterName, cfg.KubeClusterName, cfg.KubeconfigPath, cfg.NewKubeService)
+	creds, err := getKubeCreds(cfg.Context, log, cfg.ClusterName, cfg.KubeClusterName, cfg.KubeconfigPath, cfg.KubeServiceType)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}

lib/kube/proxy/server.go

@@ -128,7 +128,8 @@ func NewTLSServer(cfg TLSServerConfig) (*TLSServer, error) {
 	// Only announce when running in an actual kubernetes_service, or when
 	// running in proxy_service with local kube credentials. This means that
 	// proxy_service will pretend to also be kubernetes_service.
-	if cfg.NewKubeService || len(fwd.kubeClusters()) > 0 {
+	if cfg.KubeServiceType == KubeService ||
+		(cfg.KubeServiceType == LegacyProxyService && len(fwd.kubeClusters()) > 0) {
 		log.Debugf("Starting kubernetes_service heartbeats for %q", cfg.Component)
 		server.heartbeat, err = srv.NewHeartbeat(srv.HeartbeatConfig{
 			Mode:            srv.HeartbeatModeKube,
@@ -264,7 +265,7 @@ func (t *TLSServer) GetServerInfo() (services.Resource, error) {
 	// Note: we *don't* want to add suffix for kubernetes_service!
 	// This breaks reverse tunnel routing, which uses server.Name.
 	name := t.ServerID
-	if !t.NewKubeService {
+	if t.KubeServiceType != KubeService {
 		name += "-proxy_service"
 	}
 

lib/service/cfg.go

@@ -441,6 +441,10 @@ type KubeProxyConfig struct {
 
 	// KubeconfigPath is a path to kubeconfig
 	KubeconfigPath string
+
+	// LegacyKubeProxy specifies that this proxy was configured using the
+	// legacy kubernetes section.
+	LegacyKubeProxy bool
 }
 
 // AuthConfig is a configuration of the auth server

lib/service/kubernetes.go

@@ -219,7 +219,7 @@ func (process *TeleportProcess) initKubernetesService(log *logrus.Entry, conn *C
 			Context:           process.ExitContext(),
 			KubeconfigPath:    cfg.Kube.KubeconfigPath,
 			KubeClusterName:   cfg.Kube.KubeClusterName,
-			NewKubeService:    true,
+			KubeServiceType:   kubeproxy.KubeService,
 			Component:         teleport.ComponentKube,
 			StaticLabels:      cfg.Kube.StaticLabels,
 			DynamicLabels:     dynLabels,

lib/service/service.go

@@ -2750,6 +2750,10 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error {
 			return trace.Wrap(err)
 		}
 		component := teleport.Component(teleport.ComponentProxy, teleport.ComponentProxyKube)
+		kubeServiceType := kubeproxy.ProxyService
+		if cfg.Proxy.Kube.LegacyKubeProxy {
+			kubeServiceType = kubeproxy.LegacyProxyService
+		}
 		kubeServer, err = kubeproxy.NewTLSServer(kubeproxy.TLSServerConfig{
 			ForwarderConfig: kubeproxy.ForwarderConfig{
 				Namespace:         defaults.Namespace,
@@ -2765,6 +2769,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error {
 				ClusterOverride:   cfg.Proxy.Kube.ClusterOverride,
 				KubeconfigPath:    cfg.Proxy.Kube.KubeconfigPath,
 				Component:         component,
+				KubeServiceType:   kubeServiceType,
 			},
 			TLS:           tlsConfig,
 			LimiterConfig: cfg.Proxy.Limiter,