From 517169050d2d84ca177aa4c6b4468b0fa4e73c18 Mon Sep 17 00:00:00 2001 From: Nic Klaassen Date: Fri, 7 May 2021 15:13:03 -0700 Subject: [PATCH] Stop registering a Kubernetes cluster named after the Teleport cluster --- CHANGELOG.md | 8 ++++ integration/kube_integration_test.go | 1 + lib/config/configuration.go | 1 + lib/config/configuration_test.go | 14 +++--- lib/kube/proxy/auth.go | 64 +++++++++++++++++----------- lib/kube/proxy/auth_test.go | 33 ++++++++++---- lib/kube/proxy/forwarder.go | 34 ++++++++++++--- lib/kube/proxy/server.go | 5 ++- lib/service/cfg.go | 4 ++ lib/service/kubernetes.go | 2 +- lib/service/service.go | 5 +++ 11 files changed, 123 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8bce505aa804..9ad746ec83a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 7.0 + +Teleport 7.0 is a major release with new features, functionality, and bug fixes. + +## Breaking Changes + +* Proxy services whose configuration includes a `kube_listen_addr` but no `kubernetes` section will no longer publish a Kubernetes cluster named after the Teleport cluster. + ## 6.2 Teleport 6.2 contains new features, improvements, and bug fixes. diff --git a/integration/kube_integration_test.go b/integration/kube_integration_test.go index 5c4e4adfd7270..d32555aab7e79 100644 --- a/integration/kube_integration_test.go +++ b/integration/kube_integration_test.go @@ -1139,6 +1139,7 @@ func (s *KubeSuite) teleKubeConfig(hostname string) *service.Config { tconf.Proxy.Kube.Enabled = true tconf.Proxy.Kube.ListenAddr.Addr = net.JoinHostPort(hostname, ports.Pop()) tconf.Proxy.Kube.KubeconfigPath = s.kubeConfigPath + tconf.Proxy.Kube.LegacyKubeProxy = true return tconf } diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 60297d00e5a8c..3709a8668564a 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -645,6 +645,7 @@ func applyProxyConfig(fc *FileConfig, cfg *service.Config) error { switch { case legacyKube && !newKube: cfg.Proxy.Kube.Enabled = true + cfg.Proxy.Kube.LegacyKubeProxy = true if fc.Proxy.Kube.KubeconfigFile != "" { cfg.Proxy.Kube.KubeconfigPath = fc.Proxy.Kube.KubeconfigFile } diff --git a/lib/config/configuration_test.go b/lib/config/configuration_test.go index 12e672c7a7aac..23b6c4f9102dd 100644 --- a/lib/config/configuration_test.go +++ b/lib/config/configuration_test.go @@ -1131,8 +1131,9 @@ func TestProxyKube(t *testing.T) { Service: Service{EnabledFlag: "yes", ListenAddress: "0.0.0.0:8080"}, }}, want: service.KubeProxyConfig{ - Enabled: true, - ListenAddr: *utils.MustParseAddr("0.0.0.0:8080"), + Enabled: true, + ListenAddr: *utils.MustParseAddr("0.0.0.0:8080"), + LegacyKubeProxy: true, }, checkErr: require.NoError, }, @@ -1144,10 +1145,11 @@ func TestProxyKube(t *testing.T) { PublicAddr: utils.Strings([]string{"kube.example.com:443"}), }}, want: service.KubeProxyConfig{ - Enabled: true, - ListenAddr: *utils.MustParseAddr("0.0.0.0:8080"), - KubeconfigPath: "/tmp/kubeconfig", - PublicAddrs: []utils.NetAddr{*utils.MustParseAddr("kube.example.com:443")}, + Enabled: true, + ListenAddr: *utils.MustParseAddr("0.0.0.0:8080"), + KubeconfigPath: "/tmp/kubeconfig", + PublicAddrs: []utils.NetAddr{*utils.MustParseAddr("kube.example.com:443")}, + LegacyKubeProxy: true, }, checkErr: require.NoError, }, diff --git a/lib/kube/proxy/auth.go b/lib/kube/proxy/auth.go index 3d8f3ad39a091..3c5053028d33c 100644 --- a/lib/kube/proxy/auth.go +++ b/lib/kube/proxy/auth.go @@ -62,43 +62,59 @@ func TestOnlySkipSelfPermissionCheck(skip bool) { // - kubeconfig: a file with a set of k8s endpoints and credentials mapped to // them this is used when kubeconfigPath is set // -// newKubeService changes the loading behavior: -// - false: +// serviceType changes the loading behavior: +// - LegacyProxyService: // - if loading from kubeconfig, only "current-context" is returned; the // returned map key matches tpClusterName // - if no credentials are loaded, no error is returned // - permission self-test failures are only logged -// - true: +// - ProxyService: +// - no credentials are loaded and no error is returned +// - KubeService: // - if loading from kubeconfig, all contexts are returned // - if no credentials are loaded, returns an error // - permission self-test failures cause an error to be returned -func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, kubeClusterName, kubeconfigPath string, newKubeService bool) (map[string]*kubeCreds, error) { +func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, kubeClusterName, kubeconfigPath string, serviceType KubeServiceType) (map[string]*kubeCreds, error) { log. WithField("kubeconfigPath", kubeconfigPath). WithField("kubeClusterName", kubeClusterName). - WithField("newKubeService", newKubeService). - Debug("Reading kubernetes creds.") + WithField("serviceType", serviceType). + Debug("Reading Kubernetes creds.") + + // Proxy service should never have creds, forwards to kube service + if serviceType == ProxyService { + return map[string]*kubeCreds{}, nil + } // Load kubeconfig or local pod credentials. - cfg, err := kubeutils.GetKubeConfig(kubeconfigPath, newKubeService, kubeClusterName) + loadAll := serviceType == KubeService + cfg, err := kubeutils.GetKubeConfig(kubeconfigPath, loadAll, kubeClusterName) if err != nil && !trace.IsNotFound(err) { return nil, trace.Wrap(err) } + if trace.IsNotFound(err) || len(cfg.Contexts) == 0 { - if newKubeService { - return nil, trace.BadParameter("no kubernetes credentials found; kubernetes_service requires either a valid kubeconfig_path or to run inside of a kubernetes pod") + switch serviceType { + case KubeService: + return nil, trace.BadParameter("no Kubernetes credentials found; Kubernetes_service requires either a valid kubeconfig_path or to run inside of a Kubernetes pod") + case LegacyProxyService: + log.Debugf("Could not load Kubernetes credentials. This proxy will still handle Kubernetes requests for trusted teleport clusters or Kubernetes nodes in this teleport cluster") } - log.Debugf("Could not load kubernetes credentials. This proxy will still handle kubernetes requests for trusted teleport clusters or kubernetes nodes in this teleport cluster") return map[string]*kubeCreds{}, nil } - if !newKubeService { - // Hack for proxy_service - register a k8s cluster named after the - // teleport cluster name to route legacy requests. + + if serviceType == LegacyProxyService { + // Hack for legacy proxy service - register a k8s cluster named after + // the teleport cluster name to route legacy requests. // // Also, remove all other contexts. Multiple kubeconfig entries are // only supported for kubernetes_service. - cfg.Contexts = map[string]*rest.Config{ - tpClusterName: cfg.Contexts[cfg.CurrentContext], + if currentContext, ok := cfg.Contexts[cfg.CurrentContext]; ok { + cfg.Contexts = map[string]*rest.Config{ + tpClusterName: currentContext, + } + } else { + return nil, trace.BadParameter("no Kubernetes current-context found; Kubernetes proxy service requires either a valid kubeconfig_path with a current-context or to run inside of a Kubernetes pod") } } @@ -106,10 +122,10 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku // Convert kubeconfig contexts into kubeCreds. for cluster, clientCfg := range cfg.Contexts { log := log.WithField("cluster", cluster) - log.Debug("Checking kubernetes impersonation permissions.") + log.Debug("Checking Kubernetes impersonation permissions.") client, err := kubernetes.NewForConfig(clientCfg) if err != nil { - return nil, trace.Wrap(err, "failed to generate kubernetes client for cluster %q", cluster) + return nil, trace.Wrap(err, "failed to generate Kubernetes client for cluster %q", cluster) } // For each loaded cluster, check impersonation permissions. This // failure is only critical for newKubeService. @@ -117,10 +133,10 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku // kubernetes_service must have valid RBAC permissions, otherwise // it's pointless. // proxy_service can run without them (e.g. a root proxy). - if newKubeService { + if serviceType == KubeService { return nil, trace.Wrap(err) } - log.WithError(err).Warning("Failed to test the necessary kubernetes permissions. This teleport instance will still handle kubernetes requests towards other kubernetes clusters") + log.WithError(err).Warning("Failed to test the necessary Kubernetes permissions. This teleport instance will still handle Kubernetes requests towards other Kubernetes clusters") // We used to recommend users to set a dummy kubeconfig on root // proxies to get kubernetes support working for leaf clusters: // https://community.goteleport.com/t/enabling-teleport-to-act-as-a-kubernetes-proxy-for-trusted-leaf-clusters/418 @@ -131,7 +147,7 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku log.Info("If this is a proxy and you provided a dummy kubeconfig_path, you can remove it from teleport.yaml to get rid of this warning") } } else { - log.Debug("Have all necessary kubernetes impersonation permissions.") + log.Debug("Have all necessary Kubernetes impersonation permissions.") } targetAddr, err := parseKubeHost(clientCfg.Host) @@ -147,7 +163,7 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku return nil, trace.Wrap(err, "failed to generate transport config from kubeconfig: %v", err) } - log.Debug("Initialized kubernetes credentials") + log.Debug("Initialized Kubernetes credentials") res[cluster] = &kubeCreds{ tlsConfig: tlsConfig, transportConfig: transportConfig, @@ -164,7 +180,7 @@ func getKubeCreds(ctx context.Context, log logrus.FieldLogger, tpClusterName, ku func parseKubeHost(host string) (string, error) { u, err := url.Parse(host) if err != nil { - return "", trace.Wrap(err, "failed to parse kubernetes host: %v", err) + return "", trace.Wrap(err, "failed to parse Kubernetes host: %v", err) } if _, _, err := net.SplitHostPort(u.Host); err != nil { // add default HTTPS port @@ -195,10 +211,10 @@ func checkImpersonationPermissions(ctx context.Context, sarClient authztypes.Sel }, }, metav1.CreateOptions{}) if err != nil { - return trace.Wrap(err, "failed to verify impersonation permissions for kubernetes: %v; this may be due to missing the SelfSubjectAccessReview permission on the ClusterRole used by the proxy; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", err) + return trace.Wrap(err, "failed to verify impersonation permissions for Kubernetes: %v; this may be due to missing the SelfSubjectAccessReview permission on the ClusterRole used by the proxy; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", err) } if !resp.Status.Allowed { - return trace.AccessDenied("proxy can't impersonate kubernetes %s at the cluster level; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", resource) + return trace.AccessDenied("proxy can't impersonate Kubernetes %s at the cluster level; please make sure that proxy has all the necessary permissions: https://goteleport.com/teleport/docs/kubernetes-ssh/#impersonation", resource) } } return nil diff --git a/lib/kube/proxy/auth_test.go b/lib/kube/proxy/auth_test.go index 0a10e432cfe7c..3847ae4ed9594 100644 --- a/lib/kube/proxy/auth_test.go +++ b/lib/kube/proxy/auth_test.go @@ -152,23 +152,30 @@ current-context: foo desc string kubeconfigPath string kubeCluster string - newService bool + serviceType KubeServiceType want map[string]*kubeCreds assertErr require.ErrorAssertionFunc }{ { - desc: "kubernetes_service, no kube creds", - newService: true, - assertErr: require.Error, + desc: "kubernetes_service, no kube creds", + serviceType: KubeService, + assertErr: require.Error, }, { - desc: "proxy_service, no kube creds", - assertErr: require.NoError, - want: map[string]*kubeCreds{}, + desc: "proxy_service, no kube creds", + serviceType: ProxyService, + assertErr: require.NoError, + want: map[string]*kubeCreds{}, + }, + { + desc: "legacy proxy_service, no kube creds", + serviceType: ProxyService, + assertErr: require.NoError, + want: map[string]*kubeCreds{}, }, { desc: "kubernetes_service, with kube creds", - newService: true, + serviceType: KubeService, kubeconfigPath: kubeconfigPath, want: map[string]*kubeCreds{ "foo": { @@ -187,6 +194,14 @@ current-context: foo { desc: "proxy_service, with kube creds", kubeconfigPath: kubeconfigPath, + serviceType: ProxyService, + want: map[string]*kubeCreds{}, + assertErr: require.NoError, + }, + { + desc: "legacy proxy_service, with kube creds", + kubeconfigPath: kubeconfigPath, + serviceType: LegacyProxyService, want: map[string]*kubeCreds{ teleClusterName: { targetAddr: "example.com:3026", @@ -199,7 +214,7 @@ current-context: foo } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { - got, err := getKubeCreds(ctx, testlog.FailureOnly(t), teleClusterName, "", tt.kubeconfigPath, tt.newService) + got, err := getKubeCreds(ctx, testlog.FailureOnly(t), teleClusterName, "", tt.kubeconfigPath, tt.serviceType) tt.assertErr(t, err) if err != nil { return diff --git a/lib/kube/proxy/forwarder.go b/lib/kube/proxy/forwarder.go index d2dc2dde2eca6..ab4bdf665f4dc 100644 --- a/lib/kube/proxy/forwarder.go +++ b/lib/kube/proxy/forwarder.go @@ -63,6 +63,23 @@ import ( utilexec "k8s.io/client-go/util/exec" ) +// KubeServiceType specifies a Teleport service type which can forward Kubernetes requests +type KubeServiceType int + +const ( + // KubeService is a Teleport kubernetes_service. A KubeService always forwards + // requests directly to a Kubernetes endpoint. + KubeService KubeServiceType = iota + // ProxyService is a Teleport proxy_service with kube_listen_addr/ + // kube_public_addr enabled. A ProxyService always forwards requests to a + // Teleport KubeService or LegacyProxyService. + ProxyService + // LegacyProxyService is a Teleport proxy_service with the kubernetes section + // enabled. A LegacyProxyService can forward requests directly to a Kubernetes + // endpoint, or to another Teleport LegacyProxyService or KubeService. + LegacyProxyService +) + // ForwarderConfig specifies configuration for proxy forwarder type ForwarderConfig struct { // ReverseTunnelSrv is the teleport reverse tunnel server @@ -94,10 +111,8 @@ type ForwarderConfig struct { Context context.Context // KubeconfigPath is a path to kubernetes configuration KubeconfigPath string - // NewKubeService specifies whether to apply the additional kubernetes_service features: - // - parsing multiple kubeconfig entries - // - enforcing self permission check - NewKubeService bool + // KubeServiceType specifies which Teleport service type this forwarder is for + KubeServiceType KubeServiceType // KubeClusterName is the name of the kubernetes cluster that this // forwarder handles. KubeClusterName string @@ -157,7 +172,14 @@ func (f *ForwarderConfig) CheckAndSetDefaults() error { if f.Component == "" { f.Component = "kube_forwarder" } - if f.KubeClusterName == "" && f.KubeconfigPath == "" { + switch f.KubeServiceType { + case KubeService: + case ProxyService: + case LegacyProxyService: + default: + return trace.BadParameter("unknown value for KubeServiceType") + } + if f.KubeClusterName == "" && f.KubeconfigPath == "" && f.KubeServiceType == LegacyProxyService { // Running without a kubeconfig and explicit k8s cluster name. Use // teleport cluster name instead, to ask kubeutils.GetKubeConfig to // attempt loading the in-cluster credentials. @@ -176,7 +198,7 @@ func NewForwarder(cfg ForwarderConfig) (*Forwarder, error) { trace.Component: cfg.Component, }) - creds, err := getKubeCreds(cfg.Context, log, cfg.ClusterName, cfg.KubeClusterName, cfg.KubeconfigPath, cfg.NewKubeService) + creds, err := getKubeCreds(cfg.Context, log, cfg.ClusterName, cfg.KubeClusterName, cfg.KubeconfigPath, cfg.KubeServiceType) if err != nil { return nil, trace.Wrap(err) } diff --git a/lib/kube/proxy/server.go b/lib/kube/proxy/server.go index 3097ef49a65a2..b75032e1b61b1 100644 --- a/lib/kube/proxy/server.go +++ b/lib/kube/proxy/server.go @@ -128,7 +128,8 @@ func NewTLSServer(cfg TLSServerConfig) (*TLSServer, error) { // Only announce when running in an actual kubernetes_service, or when // running in proxy_service with local kube credentials. This means that // proxy_service will pretend to also be kubernetes_service. - if cfg.NewKubeService || len(fwd.kubeClusters()) > 0 { + if cfg.KubeServiceType == KubeService || + (cfg.KubeServiceType == LegacyProxyService && len(fwd.kubeClusters()) > 0) { log.Debugf("Starting kubernetes_service heartbeats for %q", cfg.Component) server.heartbeat, err = srv.NewHeartbeat(srv.HeartbeatConfig{ Mode: srv.HeartbeatModeKube, @@ -264,7 +265,7 @@ func (t *TLSServer) GetServerInfo() (services.Resource, error) { // Note: we *don't* want to add suffix for kubernetes_service! // This breaks reverse tunnel routing, which uses server.Name. name := t.ServerID - if !t.NewKubeService { + if t.KubeServiceType != KubeService { name += "-proxy_service" } diff --git a/lib/service/cfg.go b/lib/service/cfg.go index fbd9cdbf80f4d..14fff9fc7142d 100644 --- a/lib/service/cfg.go +++ b/lib/service/cfg.go @@ -441,6 +441,10 @@ type KubeProxyConfig struct { // KubeconfigPath is a path to kubeconfig KubeconfigPath string + + // LegacyKubeProxy specifies that this proxy was configured using the + // legacy kubernetes section. + LegacyKubeProxy bool } // AuthConfig is a configuration of the auth server diff --git a/lib/service/kubernetes.go b/lib/service/kubernetes.go index e704fbe7fdc5f..cb0ba4c98d4e2 100644 --- a/lib/service/kubernetes.go +++ b/lib/service/kubernetes.go @@ -219,7 +219,7 @@ func (process *TeleportProcess) initKubernetesService(log *logrus.Entry, conn *C Context: process.ExitContext(), KubeconfigPath: cfg.Kube.KubeconfigPath, KubeClusterName: cfg.Kube.KubeClusterName, - NewKubeService: true, + KubeServiceType: kubeproxy.KubeService, Component: teleport.ComponentKube, StaticLabels: cfg.Kube.StaticLabels, DynamicLabels: dynLabels, diff --git a/lib/service/service.go b/lib/service/service.go index 5bcb760451e6c..2cce545b48cbb 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -2750,6 +2750,10 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { return trace.Wrap(err) } component := teleport.Component(teleport.ComponentProxy, teleport.ComponentProxyKube) + kubeServiceType := kubeproxy.ProxyService + if cfg.Proxy.Kube.LegacyKubeProxy { + kubeServiceType = kubeproxy.LegacyProxyService + } kubeServer, err = kubeproxy.NewTLSServer(kubeproxy.TLSServerConfig{ ForwarderConfig: kubeproxy.ForwarderConfig{ Namespace: defaults.Namespace, @@ -2765,6 +2769,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { ClusterOverride: cfg.Proxy.Kube.ClusterOverride, KubeconfigPath: cfg.Proxy.Kube.KubeconfigPath, Component: component, + KubeServiceType: kubeServiceType, }, TLS: tlsConfig, LimiterConfig: cfg.Proxy.Limiter,