Read kubernetes config from kubeconfig

Fixes #1986

When deployed outside of the kubernetes cluster,
teleport now reads all configuration from kubernetes
config file, supplied via parameter.

Auth server then passes information about
target api server back to the proxy.

integration/kube_integration_test.go

@@ -29,7 +29,6 @@ import (
 	"net/url"
 	"os"
 	"os/user"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -66,19 +65,18 @@ var _ = check.Suite(&KubeSuite{})
 
 type KubeSuite struct {
 	*kubernetes.Clientset
-	kubeConfig *rest.Config
-	ports      utils.PortList
-	me         *user.User
+
+	ports utils.PortList
+	me    *user.User
 	// priv/pub pair to avoid re-generating it
 	priv []byte
 	pub  []byte
 
-	// kubeCACert is a certificate of a kubernetes certificate authority
-	kubeCACert []byte
-	// kubeCACertPath is a temp file to hold CA certificate authority contents
-	kubeCACertPath string
-	// kubeAPIHost is a host:port with kubernetes API server
-	kubeAPIHost string
+	// kubeconfigPath is a path to valid kubeconfig
+	kubeConfigPath string
+
+	// kubeConfig is a kubernetes config struct
+	kubeConfig *rest.Config
 }
 
 func (s *KubeSuite) SetUpSuite(c *check.C) {
@@ -108,16 +106,12 @@ func (s *KubeSuite) SetUpSuite(c *check.C) {
 		c.Skip("Skipping Kubernetes test suite.")
 	}
 
-	kubeConfigPath := os.Getenv("KUBECONFIG")
-	s.Clientset, s.kubeConfig, err = kubeutils.GetKubeClient(kubeConfigPath)
-	c.Assert(err, check.IsNil)
-
-	u, err := url.Parse(s.kubeConfig.Host)
-	c.Assert(err, check.IsNil)
-	s.kubeAPIHost = u.Host
-	if _, _, err := net.SplitHostPort(u.Host); err != nil {
-		s.kubeAPIHost = fmt.Sprintf("%v:443", u.Host)
+	s.kubeConfigPath = os.Getenv(teleport.EnvKubeConfig)
+	if s.kubeConfigPath == "" {
+		c.Fatal("This test requires path to valid kubeconfig")
 	}
+	s.Clientset, s.kubeConfig, err = kubeutils.GetKubeClient(s.kubeConfigPath)
+	c.Assert(err, check.IsNil)
 
 	ns := newNamespace(testNamespace)
 	_, err = s.Core().Namespaces().Create(ns)
@@ -126,32 +120,6 @@ func (s *KubeSuite) SetUpSuite(c *check.C) {
 			c.Fatalf("Failed to create namespace: %v.", err)
 		}
 	}
-
-	// fetch certificate authority cert, by grabbing it
-	// from the DNS app pod that is always running
-	pods, err := s.Core().Pods(kubeSystemNamespace).List(metav1.ListOptions{
-		LabelSelector: kubeDNSLabels.AsSelector().String(),
-	})
-	c.Assert(err, check.IsNil)
-	if len(pods.Items) == 0 {
-		c.Fatalf("Failed to find kube-dns pods.")
-	}
-	log.Infof("Found %v pods", len(pods.Items))
-	pod := pods.Items[0]
-
-	out := &bytes.Buffer{}
-	err = kubeExec(s.kubeConfig, kubeExecArgs{
-		podName:      pod.Name,
-		podNamespace: pod.Namespace,
-		container:    kubeDNSContainer,
-		command:      []string{"/bin/cat", teleport.KubeCAPath},
-		stdout:       out,
-	})
-	c.Assert(err, check.IsNil)
-	s.kubeCACert = out.Bytes()
-	s.kubeCACertPath = filepath.Join(c.MkDir(), "ca.pem")
-	err = ioutil.WriteFile(s.kubeCACertPath, s.kubeCACert, 0444)
-	c.Assert(err, check.IsNil)
 }
 
 const (
@@ -216,14 +184,13 @@ func (s *KubeSuite) TestKubeExec(c *check.C) {
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    kubeDNSContainer,
-		command:      []string{"/bin/cat", teleport.KubeCAPath},
+		command:      []string{"/bin/cat", "/var/run/secrets/kubernetes.io/serviceaccount/namespace"},
 		stdout:       out,
 	})
 	c.Assert(err, check.IsNil)
-	// this is the same certificate authority file fetched
-	// during suite setup process, so the output should be identical)
+
 	data := out.Bytes()
-	c.Assert(string(data), check.Equals, string(s.kubeCACert))
+	c.Assert(string(data), check.Equals, pod.Namespace)
 
 	// interactive command, allocate pty
 	term := NewTerminal(250)
@@ -467,14 +434,13 @@ func (s *KubeSuite) TestKubeTrustedClusters(c *check.C) {
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    kubeDNSContainer,
-		command:      []string{"/bin/cat", teleport.KubeCAPath},
+		command:      []string{"/bin/cat", "/var/run/secrets/kubernetes.io/serviceaccount/namespace"},
 		stdout:       out,
 	})
 	c.Assert(err, check.IsNil)
-	// this is the same certificate authority file fetched
-	// during suite setup process, so the output should be identical)
+
 	data := out.Bytes()
-	c.Assert(string(data), check.Equals, string(s.kubeCACert))
+	c.Assert(string(data), check.Equals, string(pod.Namespace))
 
 	// interactive command, allocate pty
 	term := NewTerminal(250)
@@ -566,8 +532,7 @@ func (s *KubeSuite) teleKubeConfig(hostname string) *service.Config {
 	// set kubernetes specific parameters
 	tconf.Proxy.Kube.Enabled = true
 	tconf.Proxy.Kube.ListenAddr.Addr = net.JoinHostPort(hostname, s.ports.Pop())
-	tconf.Proxy.Kube.APIAddr.Addr = s.kubeAPIHost
-	tconf.Auth.KubeCACertPath = s.kubeCACertPath
+	tconf.Auth.KubeconfigPath = s.kubeConfigPath
 
 	return tconf
 }

lib/auth/auth.go

@@ -82,9 +82,6 @@ func NewAuthServer(cfg *InitConfig, opts ...AuthServerOption) (*AuthServer, erro
 	if cfg.AuditLog == nil {
 		cfg.AuditLog = events.NewDiscardAuditLog()
 	}
-	if cfg.KubeCACertPath == "" {
-		cfg.KubeCACertPath = teleport.KubeCAPath
-	}
 
 	closeCtx, cancelFunc := context.WithCancel(context.TODO())
 	as := AuthServer{
@@ -104,7 +101,7 @@ func NewAuthServer(cfg *InitConfig, opts ...AuthServerOption) (*AuthServer, erro
 		githubClients:        make(map[string]*githubClient),
 		cancelFunc:           cancelFunc,
 		closeCtx:             closeCtx,
-		kubeCACertPath:       cfg.KubeCACertPath,
+		kubeconfigPath:       cfg.KubeconfigPath,
 	}
 	for _, o := range opts {
 		o(&as)
@@ -157,8 +154,8 @@ type AuthServer struct {
 	// cipherSuites is a list of ciphersuites that the auth server supports.
 	cipherSuites []uint16
 
-	// kubeCACertPath is a path to PEM encoded kubernetes CA certificate
-	kubeCACertPath string
+	// kubeconfigPath is a path to PEM encoded kubernetes CA certificate
+	kubeconfigPath string
 }
 
 // runPeriodicOperations runs some periodic bookkeeping operations

lib/auth/init.go

@@ -127,8 +127,8 @@ type InitConfig struct {
 	// CipherSuites is a list of ciphersuites that the auth server supports.
 	CipherSuites []uint16
 
-	// KubeCACertPath is an optional path to kubernetes CA certificate authority
-	KubeCACertPath string
+	// KubeconfigPath is an optional path to kubernetes config file
+	KubeconfigPath string
 }
 
 // Init instantiates and configures an instance of AuthServer

lib/auth/kube.go

@@ -17,16 +17,22 @@ limitations under the License.
 package auth
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net"
+	"net/url"
+	"os"
 
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/kube/authority"
+	kubeutils "github.com/gravitational/teleport/lib/kube/utils"
 	"github.com/gravitational/teleport/lib/modules"
 	"github.com/gravitational/teleport/lib/services"
 	"github.com/gravitational/teleport/lib/tlsca"
 
 	"github.com/gravitational/trace"
+	"k8s.io/client-go/kubernetes"
 )
 
 // KubeCSR is a kubernetes CSR request
@@ -53,6 +59,21 @@ type KubeCSRResponse struct {
 	Cert []byte `json:"cert"`
 	// CertAuthorities is a list of PEM block with trusted cert authorities
 	CertAuthorities [][]byte `json:"cert_authorities"`
+	// TargetAddr is an optional target address
+	// of the kubernetes API server that can be set
+	// in the kubeconfig
+	TargetAddr string `json:"target_addr"`
+}
+
+type kubeCreds struct {
+	// clt is a working kubernetes client
+	clt *kubernetes.Clientset
+	// caPEM is a PEM encoded certificate authority
+	// of the kubernetes API server
+	caPEM []byte
+	// targetAddr is a target address of the
+	// kubernetes cluster read from config
+	targetAddr string
 }
 
 // ProcessKubeCSR processes CSR request against Kubernetes CA, returns
@@ -66,19 +87,25 @@ func (s *AuthServer) ProcessKubeCSR(req KubeCSR) (*KubeCSRResponse, error) {
 		return nil, trace.Wrap(err)
 	}
 
-	// generate cluster with local kubernetes cluster
 	if req.ClusterName == s.clusterName.GetClusterName() {
-		caPEM, err := ioutil.ReadFile(s.kubeCACertPath)
+		log.Debugf("Generating certificate for local Kubernetes cluster.")
+
+		kubeCreds, err := s.getKubeClient()
 		if err != nil {
 			return nil, trace.Wrap(err)
 		}
-		log.Debugf("Generating certificate with local Kubernetes cluster.")
-		cert, err := authority.ProcessCSR(req.CSR, caPEM)
+
+		cert, err := authority.ProcessCSR(kubeCreds.clt, req.CSR)
 		if err != nil {
 			return nil, trace.Wrap(err)
 		}
-		return &KubeCSRResponse{Cert: cert.Cert, CertAuthorities: [][]byte{cert.CA}}, nil
+		return &KubeCSRResponse{
+			Cert:            cert,
+			CertAuthorities: [][]byte{kubeCreds.caPEM},
+			TargetAddr:      kubeCreds.targetAddr,
+		}, nil
 	}
+
 	// Certificate for remote cluster is a user certificate
 	// with special provisions.
 	log.Debugf("Generating certificate for remote Kubernetes cluster.")
@@ -144,3 +171,78 @@ func (s *AuthServer) ProcessKubeCSR(req KubeCSR) (*KubeCSRResponse, error) {
 	}
 	return re, nil
 }
+
+func (s *AuthServer) getKubeClient() (*kubeCreds, error) {
+	// no kubeconfig is set, assume auth server is running in the cluster
+	if s.kubeconfigPath == "" {
+		caPEM, err := ioutil.ReadFile(teleport.KubeCAPath)
+		if err != nil {
+			return nil, trace.BadParameter(`auth server assumed that it is 
+running in a kubernetes cluster, but %v mounted in pods could not be read: %v, 
+set kubeconfig_path if auth server is running outside of the cluster`, teleport.KubeCAPath, err)
+		}
+
+		clt, cfg, err := kubeutils.GetKubeClient(os.Getenv(teleport.EnvKubeConfig))
+		if err != nil {
+			return nil, trace.BadParameter(`auth server assumed that it is 
+running in a kubernetes cluster, but could not init in-cluster kubernetes client`, err)
+		}
+
+		targetAddr, err := parseKubeHost(cfg.Host)
+		if err != nil {
+			return nil, trace.Wrap(err, "failed to parse kubernetes host")
+		}
+
+		return &kubeCreds{
+			clt:        clt,
+			caPEM:      caPEM,
+			targetAddr: targetAddr,
+		}, nil
+	}
+
+	log.Debugf("Reading configuration from kubeconfig file %v.", s.kubeconfigPath)
+
+	clt, cfg, err := kubeutils.GetKubeClient(s.kubeconfigPath)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	targetAddr, err := parseKubeHost(cfg.Host)
+	if err != nil {
+		return nil, trace.Wrap(err, "failed to parse kubernetes host")
+	}
+
+	var caPEM []byte
+	if len(cfg.CAData) == 0 {
+		if cfg.CAFile == "" {
+			return nil, trace.BadParameter("can't find trusted certificates in %v", s.kubeconfigPath)
+		}
+		caPEM, err = ioutil.ReadFile(cfg.CAFile)
+		if err != nil {
+			return nil, trace.BadParameter("failed to read trusted certificates from %v: %v", cfg.CAFile, err)
+		}
+	} else {
+		caPEM = cfg.CAData
+	}
+
+	return &kubeCreds{
+		clt:        clt,
+		caPEM:      caPEM,
+		targetAddr: targetAddr,
+	}, nil
+}
+
+// parseKubeHost parses and formats kubernetes hostname
+// to host:port format, if no port it set,
+// it assumes default HTTPS port
+func parseKubeHost(host string) (string, error) {
+	u, err := url.Parse(host)
+	if err != nil {
+		return "", trace.Wrap(err, "failed to parse kubernetes host")
+	}
+	if _, _, err := net.SplitHostPort(u.Host); err != nil {
+		// add default HTTPS port
+		return fmt.Sprintf("%v:443", u.Host), nil
+	}
+	return u.Host, nil
+}

lib/config/configuration.go

@@ -310,8 +310,8 @@ func applyAuthConfig(fc *FileConfig, cfg *service.Config) error {
 	var err error
 
 	// passhtrough custom certificate authority file
-	if fc.Auth.KubeCACertFile != "" {
-		cfg.Auth.KubeCACertPath = fc.Auth.KubeCACertFile
+	if fc.Auth.KubeconfigFile != "" {
+		cfg.Auth.KubeconfigPath = fc.Auth.KubeconfigFile
 	}
 	cfg.Auth.EnableProxyProtocol, err = utils.ParseOnOff("proxy_protocol", fc.Auth.ProxyProtocol, true)
 	if err != nil {
@@ -514,13 +514,6 @@ func applyProxyConfig(fc *FileConfig, cfg *service.Config) error {
 		}
 		cfg.Proxy.Kube.ListenAddr = *addr
 	}
-	if fc.Proxy.Kube.APIAddr != "" {
-		addr, err := utils.ParseHostPortAddr(fc.Proxy.Kube.APIAddr, 443)
-		if err != nil {
-			return trace.Wrap(err)
-		}
-		cfg.Proxy.Kube.APIAddr = *addr
-	}
 	if len(fc.Proxy.Kube.PublicAddr) != 0 {
 		addrs, err := fc.Proxy.Kube.PublicAddr.Addrs(defaults.KubeProxyListenPort)
 		if err != nil {

lib/config/fileconf.go

@@ -67,7 +67,7 @@ var (
 		"proxy_service":           true,
 		"auth_service":            true,
 		"kubernetes":              true,
-		"kubernetes_ca_cert_file": true,
+		"kubeconfig_file":         true,
 		"auth_token":              true,
 		"auth_servers":            true,
 		"domain_name":             true,
@@ -91,7 +91,6 @@ var (
 		"tunnel_listen_addr":      true,
 		"ssh_listen_addr":         true,
 		"listen_addr":             true,
-		"api_addr":                false,
 		"ca_cert_file":            false,
 		"https_key_file":          true,
 		"https_cert_file":         true,
@@ -541,10 +540,10 @@ type Auth struct {
 	// if true, connections with expired client certificates will get disconnected
 	DisconnectExpiredCert services.Bool `yaml:"disconnect_expired_cert"`
 
-	// KubeCACertFile is a path to kubernetes certificate authority certificate file,
-	// used in cases when auth server is deployed outside of the kubernetes
-	// cluster.
-	KubeCACertFile string `yaml:"kubernetes_ca_cert_file,omitempty"`
+	// KubeconfigFile is an optional path to kubeconfig file,
+	// if specified, teleport will use API server address and
+	// trusted certificate authority information from it
+	KubeconfigFile string `yaml:"kubeconfig_file,omitempty"`
 }
 
 // TrustedCluster struct holds configuration values under "trusted_clusters" key
@@ -746,8 +745,6 @@ type Proxy struct {
 type Kube struct {
 	// Service is a generic service configuration section
 	Service `yaml:",inline"`
-	// KubeAPIAddr is an address of a target kubernetes API server
-	APIAddr string `yaml:"api_addr,omitempty"`
 	// PublicAddr is a publicly advertised address of the kubernetes proxy
 	PublicAddr utils.Strings `yaml:"public_addr,omitempty"`
 }