Add support for a profile specific kubeconfig file. (#7840)

api/utils/keypaths/keypaths.go

@@ -18,6 +18,7 @@ limitations under the License.
 package keypaths
 
 import (
+	"fmt"
 	"path/filepath"
 	"strings"
 )
@@ -45,39 +46,44 @@ const (
 	dbDirSuffix = "-db"
 	// kubeDirSuffix is the suffix of a sub-directory where kube TLS certs are stored.
 	kubeDirSuffix = "-kube"
+	// kubeConfigSuffix is the suffix of a kubeconfig file stored under the keys directory.
+	kubeConfigSuffix = "-kubeconfig"
 )
 
 // Here's the file layout of all these keypaths.
-// ~/.tsh/							 --> default base directory
-// ├── known_hosts                   --> trusted certificate authorities (their keys) in a format similar to known_hosts
-// └── keys							 --> session keys directory
-//    ├── one.example.com            --> Proxy hostname
-//    │   ├── certs.pem              --> TLS CA certs for the Teleport CA
-//    │   ├── foo                    --> RSA Private Key for user "foo"
-//    │   ├── foo.pub                --> Public Key
-//    │   ├── foo-x509.pem           --> TLS client certificate for Auth Server
-//    │   ├── foo-ssh                --> SSH certs for user "foo"
-//    │   │   ├── root-cert.pub      --> SSH cert for Teleport cluster "root"
-//    │   │   └── leaf-cert.pub      --> SSH cert for Teleport cluster "leaf"
-//    │   ├── foo-app                --> Database access certs for user "foo"
-//    │   │   ├── root               --> Database access certs for cluster "root"
-//    │   │   │   ├── appA-x509.pem  --> TLS cert for app service "appA"
-//    │   │   │   └── appB-x509.pem  --> TLS cert for app service "appB"
-//    │   │   └── leaf               --> Database access certs for cluster "leaf"
-//    │   │       └── appC-x509.pem  --> TLS cert for app service "appC"
-//    │   ├── foo-db                 --> App access certs for user "foo"
-//    │   │   ├── root               --> App access certs for cluster "root"
-//    │   │   │   ├── dbA-x509.pem   --> TLS cert for database service "dbA"
-//    │   │   │   └── dbB-x509.pem   --> TLS cert for database service "dbB"
-//    │   │   └── leaf               --> App access certs for cluster "leaf"
-//    │   │       └── dbC-x509.pem   --> TLS cert for database service "dbC"
-//    │   └── foo-kube               --> Kubernetes certs for user "foo"
-//    │       ├── root               --> Kubernetes certs for Teleport cluster "root"
-//    │       │   ├── kubeA-x509.pem --> TLS cert for Kubernetes cluster "kubeA"
-//    │       │   └── kubeB-x509.pem --> TLS cert for Kubernetes cluster "kubeB"
-//    │       └── leaf               --> Kubernetes certs for Teleport cluster "leaf"
-//    │           └── kubeC-x509.pem --> TLS cert for Kubernetes cluster "kubeC"
-//    └── two.example.com			 --> Additional proxy host entries follow the same format
+// ~/.tsh/							   --> default base directory
+// ├── known_hosts                     --> trusted certificate authorities (their keys) in a format similar to known_hosts
+// └── keys							   --> session keys directory
+//    ├── one.example.com              --> Proxy hostname
+//    │   ├── certs.pem                --> TLS CA certs for the Teleport CA
+//    │   ├── foo                      --> RSA Private Key for user "foo"
+//    │   ├── foo.pub                  --> Public Key
+//    │   ├── foo-x509.pem             --> TLS client certificate for Auth Server
+//    │   ├── foo-ssh                  --> SSH certs for user "foo"
+//    │   │   ├── root-cert.pub        --> SSH cert for Teleport cluster "root"
+//    │   │   └── leaf-cert.pub        --> SSH cert for Teleport cluster "leaf"
+//    │   ├── foo-app                  --> Database access certs for user "foo"
+//    │   │   ├── root                 --> Database access certs for cluster "root"
+//    │   │   │   ├── appA-x509.pem    --> TLS cert for app service "appA"
+//    │   │   │   └── appB-x509.pem    --> TLS cert for app service "appB"
+//    │   │   └── leaf                 --> Database access certs for cluster "leaf"
+//    │   │       └── appC-x509.pem    --> TLS cert for app service "appC"
+//    │   ├── foo-db                   --> App access certs for user "foo"
+//    │   │   ├── root                 --> App access certs for cluster "root"
+//    │   │   │   ├── dbA-x509.pem     --> TLS cert for database service "dbA"
+//    │   │   │   └── dbB-x509.pem     --> TLS cert for database service "dbB"
+//    │   │   └── leaf                 --> App access certs for cluster "leaf"
+//    │   │       └── dbC-x509.pem     --> TLS cert for database service "dbC"
+//    │   └── foo-kube                 --> Kubernetes certs for user "foo"
+//    │       ├── root                 --> Kubernetes certs for Teleport cluster "root"
+//    │       │   ├── kubeA-kubeconfig --> standalone kubeconfig for Kubernetes cluster "kubeA"
+//    │       │   ├── kubeA-x509.pem   --> TLS cert for Kubernetes cluster "kubeA"
+//    │       │   ├── kubeB-kubeconfig --> standalone kubeconfig for Kubernetes cluster "kubeB"
+//    │       │   └── kubeB-x509.pem   --> TLS cert for Kubernetes cluster "kubeB"
+//    │       └── leaf                 --> Kubernetes certs for Teleport cluster "leaf"
+//    │           ├── kubeC-kubeconfig --> standalone kubeconfig for Kubernetes cluster "kubeC"
+//    │           └── kubeC-x509.pem   --> TLS cert for Kubernetes cluster "kubeC"
+//    └── two.example.com			   --> Additional proxy host entries follow the same format
 //		  ...
 
 // KeyDir returns the path to the keys directory.
@@ -178,15 +184,15 @@ func AppCertPath(baseDir, proxy, username, cluster, appname string) string {
 	return filepath.Join(AppCertDir(baseDir, proxy, username, cluster), appname+fileExtTLSCert)
 }
 
-// DatabaseDir returns the path to the user's kube directory
+// DatabaseDir returns the path to the user's database directory
 // for the given proxy.
 //
 // <baseDir>/keys/<proxy>/<username>-db
 func DatabaseDir(baseDir, proxy, username string) string {
 	return filepath.Join(ProxyKeyDir(baseDir, proxy), username+dbDirSuffix)
 }
 
-// DatabaseCertDir returns the path to the user's kube cert directory
+// DatabaseCertDir returns the path to the user's database cert directory
 // for the given proxy and cluster.
 //
 // <baseDir>/keys/<proxy>/<username>-db/<cluster>
@@ -195,7 +201,7 @@ func DatabaseCertDir(baseDir, proxy, username, cluster string) string {
 }
 
 // DatabaseCertPath returns the path to the user's TLS certificate
-// for the given proxy, cluster, and kube cluster.
+// for the given proxy, cluster, and database.
 //
 // <baseDir>/keys/<proxy>/<username>-db/<cluster>/<dbname>-x509.pem
 func DatabaseCertPath(baseDir, proxy, username, cluster, dbname string) string {
@@ -226,16 +232,36 @@ func KubeCertPath(baseDir, proxy, username, cluster, kubename string) string {
 	return filepath.Join(KubeCertDir(baseDir, proxy, username, cluster), kubename+fileExtTLSCert)
 }
 
+// KubeConfigPath returns the path to the user's standalone kubeconfig
+// for the given proxy, cluster, and kube cluster.
+//
+// <baseDir>/keys/<proxy>/<username>-kube/<cluster>/<kubename>-kubeconfig
+func KubeConfigPath(baseDir, proxy, username, cluster, kubename string) string {
+	return filepath.Join(KubeCertDir(baseDir, proxy, username, cluster), kubename+kubeConfigSuffix)
+}
+
+// IsProfileKubeConfigPath makes a best effort attempt to check if the given
+// path is a profile specific kubeconfig path generated by this package.
+func IsProfileKubeConfigPath(path string) (bool, error) {
+	if path == "" {
+		return false, nil
+	}
+	// Split path on sessionKeyDir since we can't do filepath.Match with baseDir
+	splitPath := strings.Split(path, "/"+sessionKeyDir+"/")
+	match := fmt.Sprintf("*/*%v/*/*%v", kubeDirSuffix, kubeConfigSuffix)
+	return filepath.Match(match, splitPath[len(splitPath)-1])
+}
+
 // IdentitySSHCertPath returns the path to the identity file's SSH certificate.
 //
 // <identity-file-dir>/<path>-cert.pub
 func IdentitySSHCertPath(path string) string {
 	return path + fileExtSSHCert
 }
 
-// TrimPathSuffix trims the suffix/extension off of the given cert path.
+// TrimCertPathSuffix returns the given path with any cert suffix/extension trimmed off.
 func TrimCertPathSuffix(path string) string {
-	path = strings.TrimSuffix(path, fileExtTLSCert)
-	path = strings.TrimSuffix(path, fileExtSSHCert)
-	return path
+	trimmedPath := strings.TrimSuffix(path, fileExtTLSCert)
+	trimmedPath = strings.TrimSuffix(trimmedPath, fileExtSSHCert)
+	return trimmedPath
 }

api/utils/keypaths/keypaths_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2021 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package keypaths defines several keypaths used by multiple Teleport services.
+package keypaths_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/gravitational/teleport/api/utils/keypaths"
+)
+
+func TestIsProfileKubeConfigPath(t *testing.T) {
+	path := ""
+	isKubeConfig, err := keypaths.IsProfileKubeConfigPath(path)
+	require.NoError(t, err)
+	require.False(t, isKubeConfig)
+
+	path = keypaths.KubeCertPath("~/tsh", "proxy", "user", "cluster", "kube")
+	isKubeConfig, err = keypaths.IsProfileKubeConfigPath(path)
+	require.NoError(t, err)
+	require.False(t, isKubeConfig)
+
+	path = keypaths.KubeConfigPath("~/tsh", "proxy", "user", "cluster", "kube")
+	isKubeConfig, err = keypaths.IsProfileKubeConfigPath(path)
+	require.NoError(t, err)
+	require.True(t, isKubeConfig)
+
+	path = keypaths.KubeConfigPath("keys/keys/keys", "proxy", "user", "cluster", "kube")
+	isKubeConfig, err = keypaths.IsProfileKubeConfigPath(path)
+	require.NoError(t, err)
+	require.True(t, isKubeConfig)
+}

lib/client/api.go

@@ -403,35 +403,42 @@ func (p *ProfileStatus) IsExpired(clock clockwork.Clock) bool {
 
 // CACertPath returns path to the CA certificate for this profile.
 //
-// It's stored in ~/.tsh/keys/<proxy>/certs.pem by default.
+// It's stored in <profile-dir>/keys/<proxy>/certs.pem by default.
 func (p *ProfileStatus) CACertPath() string {
 	return keypaths.TLSCAsPath(p.Dir, p.Name)
 }
 
 // KeyPath returns path to the private key for this profile.
 //
-// It's kept in ~/.tsh/keys/<proxy>/<user>.
+// It's kept in <profile-dir>/keys/<proxy>/<user>.
 func (p *ProfileStatus) KeyPath() string {
 	return keypaths.UserKeyPath(p.Dir, p.Name, p.Username)
 }
 
 // DatabaseCertPath returns path to the specified database access certificate
 // for this profile.
 //
-// It's kept in ~/.tsh/keys/<proxy>/<user>-db/<cluster>/<name>-x509.pem
+// It's kept in <profile-dir>/keys/<proxy>/<user>-db/<cluster>/<name>-x509.pem
 func (p *ProfileStatus) DatabaseCertPath(name string) string {
 	return keypaths.DatabaseCertPath(p.Dir, p.Name, p.Username, p.Cluster, name)
 }
 
 // AppCertPath returns path to the specified app access certificate
 // for this profile.
 //
-// It's kept in ~/.tsh/keys/<proxy>/<user>-app/<cluster>/<name>-x509.pem
+// It's kept in <profile-dir>/keys/<proxy>/<user>-app/<cluster>/<name>-x509.pem
 func (p *ProfileStatus) AppCertPath(name string) string {
 	return keypaths.AppCertPath(p.Dir, p.Name, p.Username, p.Cluster, name)
 
 }
 
+// KubeConfigPath returns path to the specified kubeconfig for this profile.
+//
+// It's kept in <profile-dir>/keys/<proxy>/<user>-kube/<cluster>/<name>-kubeconfig
+func (p *ProfileStatus) KubeConfigPath(name string) string {
+	return keypaths.KubeConfigPath(p.Dir, p.Name, p.Username, p.Cluster, name)
+}
+
 // DatabaseServices returns a list of database service names for this profile.
 func (p *ProfileStatus) DatabaseServices() (result []string) {
 	for _, db := range p.Databases {

lib/client/keystore.go

@@ -309,12 +309,14 @@ func (fs *FSLocalKeyStore) updateKeyWithCerts(o CertOption, key *Key) error {
 			return trace.ConvertSystemError(err)
 		}
 		for _, certFile := range certFiles {
-			data, err := ioutil.ReadFile(filepath.Join(certPath, certFile.Name()))
-			if err != nil {
-				return trace.ConvertSystemError(err)
-			}
 			name := keypaths.TrimCertPathSuffix(certFile.Name())
-			certDataMap[name] = data
+			if isCert := name != certFile.Name(); isCert {
+				data, err := ioutil.ReadFile(filepath.Join(certPath, certFile.Name()))
+				if err != nil {
+					return trace.ConvertSystemError(err)
+				}
+				certDataMap[name] = data
+			}
 		}
 		return o.updateKeyWithMap(key, certDataMap)
 	}

lib/kube/kubeconfig/kubeconfig.go

@@ -216,7 +216,7 @@ func Save(path string, config clientcmdapi.Config) error {
 // missing.
 func finalPath(customPath string) (string, error) {
 	if customPath == "" {
-		customPath = pathFromEnv()
+		customPath = PathFromEnv()
 	}
 	finalPath, err := utils.EnsureLocalPath(customPath, teleport.KubeConfigDir, teleport.KubeConfigFile)
 	if err != nil {
@@ -225,8 +225,8 @@ func finalPath(customPath string) (string, error) {
 	return finalPath, nil
 }
 
-// pathFromEnv extracts location of kubeconfig from the environment.
-func pathFromEnv() string {
+// PathFromEnv extracts location of kubeconfig from the environment.
+func PathFromEnv() string {
 	kubeconfig := os.Getenv(teleport.EnvKubeConfig)
 
 	// The KUBECONFIG environment variable is a list. On Windows it's

tool/tsh/kube.go

@@ -19,12 +19,15 @@ package main
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/gravitational/kingpin"
 	"github.com/gravitational/trace"
 
+	"github.com/gravitational/teleport/api/profile"
 	apiutils "github.com/gravitational/teleport/api/utils"
+	"github.com/gravitational/teleport/api/utils/keypaths"
 	"github.com/gravitational/teleport/lib/asciitable"
 	"github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/kube/kubeconfig"
@@ -208,6 +211,9 @@ func newKubeLoginCommand(parent *kingpin.CmdClause) *kubeLoginCommand {
 }
 
 func (c *kubeLoginCommand) run(cf *CLIConf) error {
+	// Set CLIConf.KubernetesCluster so that the kube cluster's context is automatically selected.
+	cf.KubernetesCluster = c.kubeCluster
+
 	tc, err := makeClient(cf, true)
 	if err != nil {
 		return trace.Wrap(err)
@@ -232,14 +238,20 @@ func (c *kubeLoginCommand) run(cf *CLIConf) error {
 		//
 		// Re-generate kubeconfig contexts and try selecting this kube cluster
 		// again.
-		if err := updateKubeConfig(cf, tc); err != nil {
-			return trace.Wrap(err)
-		}
-		if err := kubeconfig.SelectContext(currentTeleportCluster, c.kubeCluster); err != nil {
+		if err := updateKubeConfig(cf, tc, ""); err != nil {
 			return trace.Wrap(err)
 		}
 	}
 
+	// Generate a profile specific kubeconfig which can be used
+	// by setting the kubeconfig environment variable (with `tsh env`)
+	profileKubeconfigPath := keypaths.KubeConfigPath(
+		profile.FullProfilePath(cf.HomePath), tc.WebProxyHost(), tc.Username, currentTeleportCluster, c.kubeCluster,
+	)
+	if err := updateKubeConfig(cf, tc, profileKubeconfigPath); err != nil {
+		return trace.Wrap(err)
+	}
+
 	fmt.Printf("Logged into kubernetes cluster %q\n", c.kubeCluster)
 	return nil
 }
@@ -339,8 +351,9 @@ func buildKubeConfigUpdate(cf *CLIConf, kubeStatus *kubernetesStatus) (*kubeconf
 }
 
 // updateKubeConfig adds Teleport configuration to the users's kubeconfig based on the CLI
-// parameters and the kubernetes services in the current Teleport cluster.
-func updateKubeConfig(cf *CLIConf, tc *client.TeleportClient) error {
+// parameters and the kubernetes services in the current Teleport cluster. If no path for
+// the kubeconfig is given, it will use environment values or known defaults to get a path.
+func updateKubeConfig(cf *CLIConf, tc *client.TeleportClient, path string) error {
 	// Fetch proxy's advertised ports to check for k8s support.
 	if _, err := tc.Ping(cf.Context); err != nil {
 		return trace.Wrap(err)
@@ -360,7 +373,24 @@ func updateKubeConfig(cf *CLIConf, tc *client.TeleportClient) error {
 		return trace.Wrap(err)
 	}
 
-	return trace.Wrap(kubeconfig.Update("", *values))
+	if path == "" {
+		path = kubeconfig.PathFromEnv()
+	}
+
+	// If this is a profile specific kubeconfig, we only need
+	// to put the selected kube cluster into the kubeconfig.
+	isKubeConfig, err := keypaths.IsProfileKubeConfigPath(path)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	if isKubeConfig {
+		if !strings.Contains(path, cf.KubernetesCluster) {
+			return trace.BadParameter("profile specific kubeconfig is in use, run 'eval $(tsh env --unset)' to switch contexts to another kube cluster")
+		}
+		values.Exec.KubeClusters = []string{cf.KubernetesCluster}
+	}
+
+	return trace.Wrap(kubeconfig.Update(path, *values))
 }
 
 // Required magic boilerplate to use the k8s encoder.