forwarder.go

/*
Copyright 2018-2019 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package proxy

import (
	"context"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/auth"
	"github.com/gravitational/teleport/lib/defaults"
	"github.com/gravitational/teleport/lib/events"
	"github.com/gravitational/teleport/lib/httplib"
	"github.com/gravitational/teleport/lib/reversetunnel"
	"github.com/gravitational/teleport/lib/services"
	"github.com/gravitational/teleport/lib/session"
	"github.com/gravitational/teleport/lib/srv"
	"github.com/gravitational/teleport/lib/sshca"
	"github.com/gravitational/teleport/lib/utils"

	"github.com/gravitational/oxy/forward"
	kubeutils "github.com/gravitational/teleport/lib/kube/utils"
	"github.com/gravitational/trace"
	"github.com/gravitational/ttlmap"
	"github.com/jonboulle/clockwork"
	"github.com/julienschmidt/httprouter"
	log "github.com/sirupsen/logrus"
	"golang.org/x/crypto/ssh"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/remotecommand"
	"k8s.io/client-go/transport/spdy"
	utilexec "k8s.io/client-go/util/exec"
)

// ForwarderConfig specifies configuration for proxy forwarder
type ForwarderConfig struct {
	// Tunnel is the teleport reverse tunnel server
	Tunnel reversetunnel.Server
	// ClusterName is a local cluster name
	ClusterName string
	// Keygen points to a key generator implementation
	Keygen sshca.Authority
	// Auth authenticates user
	Auth auth.Authorizer
	// Client is a proxy client
	Client auth.ClientI
	// TargetAddr is a target address
	TargetAddr string
	// DataDir is a data dir to store logs
	DataDir string
	// Namespace is a namespace of the proxy server (not a K8s namespace)
	Namespace string
	// AccessPoint is a caching access point to auth server
	// for caching common requests to the backend
	AccessPoint auth.AccessPoint
	// AuditLog is audit log to send events to
	AuditLog events.IAuditLog
	// ServerID is a unique ID of a proxy server
	ServerID string
	// ClusterOverride if set, routes all requests
	// to the cluster name, used in tests
	ClusterOverride string
	// Context passes the optional external context
	// passing global close to all forwarder operations
	Context context.Context
	// KubeconfigPath is a path to kubernetes configuration
	KubeconfigPath string
	// Clock is a server clock, could be overriden in tests
	Clock clockwork.Clock
}

// CheckAndSetDefaults checks and sets default values
func (f *ForwarderConfig) CheckAndSetDefaults() error {
	if f.Client == nil {
		return trace.BadParameter("missing parameter Client")
	}
	if f.AccessPoint == nil {
		return trace.BadParameter("missing parameter AccessPoint")
	}
	if f.Auth == nil {
		return trace.BadParameter("missing parameter Auth")
	}
	if f.Tunnel == nil {
		return trace.BadParameter("missing parameter Tunnel")
	}
	if f.ClusterName == "" {
		return trace.BadParameter("missing parameter LocalCluster")
	}
	if f.Keygen == nil {
		return trace.BadParameter("missing parameter Keygen")
	}
	if f.DataDir == "" {
		return trace.BadParameter("missing parameter DataDir")
	}
	if f.ServerID == "" {
		return trace.BadParameter("missing parameter ServerID")
	}
	if f.TargetAddr == "" {
		f.TargetAddr = teleport.KubeServiceAddr
	}
	if f.Namespace == "" {
		f.Namespace = defaults.Namespace
	}
	if f.Context == nil {
		f.Context = context.TODO()
	}
	if f.Clock == nil {
		f.Clock = clockwork.NewRealClock()
	}
	return nil
}

// NewForwarder returns new instance of Kubernetes request
// forwarding proxy.
func NewForwarder(cfg ForwarderConfig) (*Forwarder, error) {
	if err := cfg.CheckAndSetDefaults(); err != nil {
		return nil, trace.Wrap(err)
	}

	creds, err := getKubeCreds(cfg.KubeconfigPath)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	clusterSessions, err := ttlmap.New(defaults.ClientCacheSize)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	closeCtx, close := context.WithCancel(cfg.Context)
	fwd := &Forwarder{
		creds: *creds,
		Entry: log.WithFields(log.Fields{
			trace.Component: teleport.Component(teleport.ComponentKube),
		}),
		Router:          *httprouter.New(),
		ForwarderConfig: cfg,
		clusterSessions: clusterSessions,
		activeRequests:  make(map[string]context.Context),
		ctx:             closeCtx,
		close:           close,
	}

	fwd.POST("/api/:ver/namespaces/:podNamespace/pods/:podName/exec", fwd.withAuth(fwd.exec))
	fwd.GET("/api/:ver/namespaces/:podNamespace/pods/:podName/exec", fwd.withAuth(fwd.exec))

	fwd.POST("/api/:ver/namespaces/:podNamespace/pods/:podName/attach", fwd.withAuth(fwd.exec))
	fwd.GET("/api/:ver/namespaces/:podNamespace/pods/:podName/attach", fwd.withAuth(fwd.exec))

	fwd.POST("/api/:ver/namespaces/:podNamespace/pods/:podName/portforward", fwd.withAuth(fwd.portForward))
	fwd.GET("/api/:ver/namespaces/:podNamespace/pods/:podName/portforward", fwd.withAuth(fwd.portForward))

	fwd.NotFound = fwd.withAuthStd(fwd.catchAll)

	if cfg.ClusterOverride != "" {
		fwd.Debugf("Cluster override is set, forwarder will send all requests to remote cluster %v.", cfg.ClusterOverride)
	}
	return fwd, nil
}

// Forwarder intercepts kubernetes requests, acting as Kubernetes API proxy.
// it blindly forwards most of the requests on HTTPS protocol layer,
// however some requests like exec sessions it intercepts and records.
type Forwarder struct {
	sync.Mutex
	*log.Entry
	httprouter.Router
	ForwarderConfig
	// clusterSessions is an expiring cache associated with authenticated
	// user connected to a remote cluster, session is invalidated
	// if user changes kubernetes groups via RBAC or cache has expired
	// TODO(klizhentas): flush certs on teleport CA rotation?
	clusterSessions *ttlmap.TTLMap
	// activeRequests is a map used to serialize active CSR requests to the auth server
	activeRequests map[string]context.Context
	// close is a close function
	close context.CancelFunc
	// ctx is a global context signalling exit
	ctx context.Context
	// creds contain kubernetes credentials shared with a proxy process,
	// could be a service account token or client X509 credentials
	creds kubeCreds
}

// Close signals close to all outstanding or background operations
// to complete
func (f *Forwarder) Close() error {
	f.close()
	return nil
}

// authContext is a context of authenticated user,
// contains information about user, target cluster and authenticated groups
type authContext struct {
	auth.AuthContext
	kubeGroups    []string
	cluster       cluster
	clusterConfig services.ClusterConfig
	// clientIdleTimeout sets information on client idle timeout
	clientIdleTimeout time.Duration
	// disconnectExpiredCert time.Time
	disconnectExpiredCert time.Time
	// sessionTTL specifies the duration of the user's session
	sessionTTL time.Duration
}

func (c authContext) String() string {
	return fmt.Sprintf("user: %v, groups: %v, cluster: %v", c.User.GetName(), c.kubeGroups, c.cluster.GetName())
}

func (c *authContext) key() string {
	// it is important that the context key contains user, kubernetes groups and certificate expiry,
	// so that new logins with different parameters will not reuse this context
	return fmt.Sprintf("%v:%v:%v:%v", c.cluster.GetName(), c.User.GetName(), c.kubeGroups, c.disconnectExpiredCert.UTC().Unix())
}

// cluster represents cluster information, name of the cluster
// target address and custom dialer
type cluster struct {
	remoteAddr utils.NetAddr
	reversetunnel.RemoteSite
	targetAddr string
	isRemote   bool
}

func (c *cluster) Dial(_, _ string) (net.Conn, error) {
	return c.RemoteSite.DialTCP(
		&c.remoteAddr,
		&utils.NetAddr{AddrNetwork: "tcp", Addr: c.targetAddr})
}

func (c *cluster) DialWithContext(ctx context.Context, _, _ string) (net.Conn, error) {
	return c.RemoteSite.DialTCP(
		&c.remoteAddr,
		&utils.NetAddr{AddrNetwork: "tcp", Addr: c.targetAddr})
}

// handlerWithAuthFunc is http handler with passed auth context
type handlerWithAuthFunc func(ctx *authContext, w http.ResponseWriter, r *http.Request, p httprouter.Params) (interface{}, error)

// handlerWithAuthFuncStd is http handler with passed auth context
type handlerWithAuthFuncStd func(ctx *authContext, w http.ResponseWriter, r *http.Request) (interface{}, error)

// authenticate function authenticates request
func (f *Forwarder) authenticate(req *http.Request) (*authContext, error) {
	const accessDeniedMsg = "[00] access denied"

	var isRemoteUser bool
	userTypeI := req.Context().Value(auth.ContextUser)
	switch userTypeI.(type) {
	case auth.LocalUser:

	case auth.RemoteUser:
		isRemoteUser = true
	default:
		f.Warningf("Denying proxy access to unsupported user type: %T.", userTypeI)
		return nil, trace.AccessDenied(accessDeniedMsg)
	}

	userContext, err := f.Auth.Authorize(req.Context())
	if err != nil {
		switch {
		// propagate connection problem error so we can differentiate
		// between connection failed and access denied
		case trace.IsConnectionProblem(err):
			return nil, trace.ConnectionProblem(err, "[07] failed to connect to the database")
		case trace.IsAccessDenied(err):
			// don't print stack trace, just log the warning
			f.Warn(err)
			return nil, trace.AccessDenied(accessDeniedMsg)
		default:
			f.Warn(trace.DebugReport(err))
			return nil, trace.AccessDenied(accessDeniedMsg)
		}
	}
	peers := req.TLS.PeerCertificates
	if len(peers) > 1 {
		// when turning intermediaries on, don't forget to verify
		// https://github.com/kubernetes/kubernetes/pull/34524/files#diff-2b283dde198c92424df5355f39544aa4R59
		return nil, trace.AccessDenied("access denied: intermediaries are not supported")
	}
	if len(peers) == 0 {
		return nil, trace.AccessDenied("access denied: only mutual TLS authentication is supported")
	}
	clientCert := peers[0]
	authContext, err := f.setupContext(*userContext, req, isRemoteUser, clientCert.NotAfter)
	if err != nil {
		f.Warn(err.Error())
		return nil, trace.AccessDenied(accessDeniedMsg)
	}
	return authContext, nil
}

func (f *Forwarder) withAuthStd(handler handlerWithAuthFuncStd) http.HandlerFunc {
	return httplib.MakeStdHandler(func(w http.ResponseWriter, req *http.Request) (interface{}, error) {
		authContext, err := f.authenticate(req)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		return handler(authContext, w, req)
	})
}

func (f *Forwarder) withAuth(handler handlerWithAuthFunc) httprouter.Handle {
	return httplib.MakeHandler(func(w http.ResponseWriter, req *http.Request, p httprouter.Params) (interface{}, error) {
		authContext, err := f.authenticate(req)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		return handler(authContext, w, req, p)
	})
}

func (f *Forwarder) setupContext(ctx auth.AuthContext, req *http.Request, isRemoteUser bool, certExpires time.Time) (*authContext, error) {
	roles := ctx.Checker

	clusterConfig, err := f.AccessPoint.GetClusterConfig()
	if err != nil {
		return nil, trace.Wrap(err)
	}

	// adjust session ttl to the smaller of two values: the session
	// ttl requested in tsh or the session ttl for the role.
	sessionTTL := roles.AdjustSessionTTL(time.Hour)

	// check signing TTL and return a list of allowed logins
	kubeGroups, err := roles.CheckKubeGroups(sessionTTL)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	var isRemoteCluster bool
	targetCluster, err := f.Tunnel.GetSite(f.ClusterName)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	for _, remoteCluster := range f.Tunnel.GetSites() {
		encodedName := kubeutils.EncodeClusterName(remoteCluster.GetName())
		if strings.HasPrefix(req.Host, remoteCluster.GetName()+".") || strings.HasPrefix(req.Host, encodedName+".") {
			f.Debugf("Going to proxy to cluster: %v based on matching host prefix %v.", remoteCluster.GetName(), req.Host)
			targetCluster = remoteCluster
			isRemoteCluster = remoteCluster.GetName() != f.ClusterName
			break
		}
		if f.ClusterOverride != "" && f.ClusterOverride == remoteCluster.GetName() {
			f.Debugf("Going to proxy to cluster: %v based on override %v.", remoteCluster.GetName(), f.ClusterOverride)
			targetCluster = remoteCluster
			isRemoteCluster = remoteCluster.GetName() != f.ClusterName
			f.Debugf("Override isRemoteCluster: %v %v %v", isRemoteCluster, remoteCluster.GetName(), f.ClusterName)
			break
		}
	}
	if targetCluster.GetName() != f.ClusterName && isRemoteUser {
		return nil, trace.AccessDenied("access denied: remote user can not access remote cluster")
	}

	authCtx := &authContext{
		clientIdleTimeout: roles.AdjustClientIdleTimeout(clusterConfig.GetClientIdleTimeout()),
		sessionTTL:        sessionTTL,
		AuthContext:       ctx,
		kubeGroups:        kubeGroups,
		clusterConfig:     clusterConfig,
		cluster: cluster{
			remoteAddr: utils.NetAddr{AddrNetwork: "tcp", Addr: req.RemoteAddr},
			RemoteSite: targetCluster,
			targetAddr: f.TargetAddr,
			isRemote:   isRemoteCluster,
		},
	}

	disconnectExpiredCert := roles.AdjustDisconnectExpiredCert(clusterConfig.GetDisconnectExpiredCert())
	if !certExpires.IsZero() && disconnectExpiredCert {
		authCtx.disconnectExpiredCert = certExpires
	}

	return authCtx, nil
}

// exec forwards all exec requests to the target server, captures
// all output from the session
func (f *Forwarder) exec(ctx *authContext, w http.ResponseWriter, req *http.Request, p httprouter.Params) (interface{}, error) {
	f.Debugf("Exec %v.", req.URL.String())
	q := req.URL.Query()
	request := remoteCommandRequest{
		podNamespace:       p.ByName("podNamespace"),
		podName:            p.ByName("podName"),
		containerName:      q.Get("container"),
		cmd:                q["command"],
		stdin:              utils.AsBool(q.Get("stdin")),
		stdout:             utils.AsBool(q.Get("stdout")),
		stderr:             utils.AsBool(q.Get("stderr")),
		tty:                utils.AsBool(q.Get("tty")),
		httpRequest:        req,
		httpResponseWriter: w,
		context:            req.Context(),
	}

	var recorder events.SessionRecorder
	sessionID := session.NewID()
	var err error
	if request.tty {
		// create session recorder
		// get the audit log from the server and create a session recorder. this will
		// be a discard audit log if the proxy is in recording mode and a teleport
		// node so we don't create double recordings.
		recorder, err = events.NewForwardRecorder(events.ForwardRecorderConfig{
			DataDir:        filepath.Join(f.DataDir, teleport.LogsDir),
			SessionID:      sessionID,
			Namespace:      f.Namespace,
			RecordSessions: ctx.clusterConfig.GetSessionRecording() != services.RecordOff,
			Component:      teleport.Component(teleport.ComponentSession, teleport.ComponentKube),
			ForwardTo:      f.AuditLog,
		})
		if err != nil {
			return nil, trace.Wrap(err)
		}
		defer recorder.Close()
		request.onResize = func(resize remotecommand.TerminalSize) {
			params := session.TerminalParams{
				W: int(resize.Width),
				H: int(resize.Height),
			}
			// Build the resize event.
			resizeEvent := events.EventFields{
				events.EventProtocol:  events.EventProtocolKube,
				events.EventType:      events.ResizeEvent,
				events.EventNamespace: f.Namespace,
				events.SessionEventID: sessionID,
				events.EventLogin:     ctx.User.GetName(),
				events.EventUser:      ctx.User.GetName(),
				events.TerminalSize:   params.Serialize(),
			}

			// Report the updated window size to the event log (this is so the sessions
			// can be replayed correctly).
			recorder.GetAuditLog().EmitAuditEvent(events.ResizeEvent, resizeEvent)
		}
	}

	sess, err := f.getOrCreateClusterSession(*ctx)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	if request.tty {
		// Emit "new session created" event. There are no initial terminal
		// parameters per k8s protocol, so set up with any default
		termParams := session.TerminalParams{
			W: 100,
			H: 100,
		}
		recorder.GetAuditLog().EmitAuditEvent(events.SessionStartEvent, events.EventFields{
			events.EventProtocol:   events.EventProtocolKube,
			events.EventNamespace:  f.Namespace,
			events.SessionEventID:  string(sessionID),
			events.SessionServerID: f.ServerID,
			events.EventLogin:      ctx.User.GetName(),
			events.EventUser:       ctx.User.GetName(),
			events.LocalAddr:       sess.cluster.targetAddr,
			events.RemoteAddr:      req.RemoteAddr,
			events.TerminalSize:    termParams.Serialize(),
		})
	}

	if err := f.setupForwardingHeaders(ctx, sess, req); err != nil {
		return nil, trace.Wrap(err)
	}

	proxy, err := createRemoteCommandProxy(request)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	defer proxy.Close()

	f.Debugf("Created streams, getting executor.")

	executor, err := f.getExecutor(*ctx, sess, req)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	streamOptions := proxy.options()

	if request.tty {
		// capture stderr and stdout writes to session recorder
		streamOptions.Stdout = utils.NewBroadcastWriter(streamOptions.Stdout, recorder)
		streamOptions.Stderr = utils.NewBroadcastWriter(streamOptions.Stderr, recorder)
	}

	err = executor.Stream(streamOptions)
	if err := proxy.sendStatus(err); err != nil {
		f.Warningf("Failed to send status: %v. Exec command was aborted by client.", err)
		return nil, trace.Wrap(err)
	}

	if request.tty {
		// send an event indicating that this session has ended
		recorder.GetAuditLog().EmitAuditEvent(events.SessionEndEvent, events.EventFields{
			events.EventProtocol:  events.EventProtocolKube,
			events.SessionEventID: sessionID,
			events.EventUser:      ctx.User.GetName(),
			events.EventNamespace: f.Namespace,
		})
	} else {
		f.Debugf("No tty, sending exec event.")
		// send an exec event
		fields := events.EventFields{
			events.EventProtocol:    events.EventProtocolKube,
			events.ExecEventCommand: strings.Join(request.cmd, " "),
			events.EventLogin:       ctx.User.GetName(),
			events.EventUser:        ctx.User.GetName(),
			events.LocalAddr:        sess.cluster.targetAddr,
			events.RemoteAddr:       req.RemoteAddr,
			events.EventNamespace:   f.Namespace,
		}
		if err != nil {
			fields[events.ExecEventError] = err.Error()
			if exitErr, ok := err.(utilexec.ExitError); ok && exitErr.Exited() {
				fields[events.ExecEventCode] = fmt.Sprintf("%d", exitErr.ExitStatus())
			}
		}
		f.AuditLog.EmitAuditEvent(events.ExecEvent, fields)
	}

	f.Debugf("Exited successfully.")
	return nil, nil
}

// portForward starts port forwarding to the remote cluster
func (f *Forwarder) portForward(ctx *authContext, w http.ResponseWriter, req *http.Request, p httprouter.Params) (interface{}, error) {
	f.Debugf("Port forward: %v. req headers: %v", req.URL.String(), req.Header)
	sess, err := f.getOrCreateClusterSession(*ctx)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	if err := f.setupForwardingHeaders(ctx, sess, req); err != nil {
		f.Debugf("DENIED Port forward: %v.", req.URL.String())
		return nil, trace.Wrap(err)
	}

	dialer, err := f.getDialer(*ctx, sess, req)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	onPortForward := func(addr string, success bool) {
		f.AuditLog.EmitAuditEvent(events.PortForwardEvent, events.EventFields{
			events.EventProtocol:      events.EventProtocolKube,
			events.PortForwardAddr:    addr,
			events.PortForwardSuccess: success,
			events.EventLogin:         ctx.User.GetName(),
			events.EventUser:          ctx.User.GetName(),
			events.LocalAddr:          sess.cluster.targetAddr,
			events.RemoteAddr:         req.RemoteAddr,
		})
	}

	q := req.URL.Query()
	request := portForwardRequest{
		podNamespace:       p.ByName("podNamespace"),
		podName:            p.ByName("podName"),
		ports:              q["ports"],
		context:            req.Context(),
		httpRequest:        req,
		httpResponseWriter: w,
		onPortForward:      onPortForward,
		targetDialer:       dialer,
	}
	f.Debugf("Starting %v.", request)
	err = runPortForwarding(request)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	f.Debugf("Done %v.", request)
	return nil, nil
}

func (f *Forwarder) setupForwardingHeaders(ctx *authContext, sess *clusterSession, req *http.Request) error {
	for header := range req.Header {
		if strings.HasPrefix(header, "Impersonate-") {
			return trace.AccessDenied("impersonation request has been denied")
		}
	}

	// Setup scheme, override target URL to the destination address
	req.URL.Scheme = "https"
	req.URL.Host = sess.cluster.targetAddr
	req.RequestURI = req.URL.Path + "?" + req.URL.RawQuery

	// add origin headers so the service consuming the request on the other site
	// is aware of where it came from
	req.Header.Add("X-Forwarded-Proto", "https")
	req.Header.Add("X-Forwarded-Host", req.Host)
	req.Header.Add("X-Forwarded-Path", req.URL.Path)

	if !ctx.cluster.isRemote {
		req.Header.Add("Impersonate-User", ctx.User.GetName())
		f.Debugf("Impersonate User: %v", ctx.User)
		for _, group := range ctx.kubeGroups {
			req.Header.Add("Impersonate-Group", group)
			f.Debugf("Impersonate Group: %v", group)
		}
		if f.creds.cfg.BearerToken != "" {
			f.Debugf("Using Bearer Token Auth")
			req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", f.creds.cfg.BearerToken))
		}
	}
	return nil
}

// catchAll forwards all HTTP requests to the target k8s API server
func (f *Forwarder) catchAll(ctx *authContext, w http.ResponseWriter, req *http.Request) (interface{}, error) {
	sess, err := f.getOrCreateClusterSession(*ctx)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	if err := f.setupForwardingHeaders(ctx, sess, req); err != nil {
		return nil, trace.Wrap(err)
	}
	sess.forwarder.ServeHTTP(w, req)
	return nil, nil
}

func (f *Forwarder) getExecutor(ctx authContext, sess *clusterSession, req *http.Request) (remotecommand.Executor, error) {
	upgradeRoundTripper := NewSpdyRoundTripperWithDialer(roundTripperConfig{
		ctx:             req.Context(),
		authCtx:         ctx,
		dial:            sess.DialWithContext,
		tlsConfig:       sess.tlsConfig,
		followRedirects: true,
		bearerToken:     f.creds.cfg.BearerToken,
	})
	return remotecommand.NewSPDYExecutorForTransports(upgradeRoundTripper, upgradeRoundTripper, req.Method, req.URL)
}

func (f *Forwarder) getDialer(ctx authContext, sess *clusterSession, req *http.Request) (httpstream.Dialer, error) {
	upgradeRoundTripper := NewSpdyRoundTripperWithDialer(roundTripperConfig{
		ctx:             req.Context(),
		authCtx:         ctx,
		dial:            sess.DialWithContext,
		tlsConfig:       sess.tlsConfig,
		followRedirects: true,
		bearerToken:     f.creds.cfg.BearerToken,
	})
	client := &http.Client{
		Transport: upgradeRoundTripper,
	}

	return spdy.NewDialer(upgradeRoundTripper, client, req.Method, req.URL), nil
}

// clusterSession contains authenticated user session to the target cluster:
// x509 short lived credentials, forwarding proxies and other data
type clusterSession struct {
	authContext
	parent    *Forwarder
	tlsConfig *tls.Config
	forwarder *forward.Forwarder
}

func (s *clusterSession) monitorConn(conn net.Conn, err error) (net.Conn, error) {
	if err != nil {
		return nil, trace.Wrap(err)
	}
	if s.disconnectExpiredCert.IsZero() && s.clientIdleTimeout == 0 {
		return conn, nil
	}
	ctx, cancel := context.WithCancel(s.parent.ctx)
	tc := &trackingConn{
		Conn:   conn,
		clock:  s.parent.Clock,
		ctx:    ctx,
		cancel: cancel,
	}

	mon, err := srv.NewMonitor(srv.MonitorConfig{
		DisconnectExpiredCert: s.disconnectExpiredCert,
		ClientIdleTimeout:     s.clientIdleTimeout,
		Clock:                 s.parent.Clock,
		Tracker:               tc,
		Conn:                  tc,
		Context:               ctx,
		TeleportUser:          s.User.GetName(),
		ServerID:              s.parent.ServerID,
		Audit:                 s.parent.AuditLog,
		Entry:                 s.parent.Entry,
	})
	if err != nil {
		tc.Close()
		return nil, trace.Wrap(err)
	}
	go mon.Start()
	return tc, nil
}

func (s *clusterSession) Dial(network, addr string) (net.Conn, error) {
	return s.monitorConn(s.cluster.Dial(network, addr))
}

func (s *clusterSession) DialWithContext(ctx context.Context, network, addr string) (net.Conn, error) {
	return s.monitorConn(s.cluster.DialWithContext(ctx, network, addr))
}

type trackingConn struct {
	sync.RWMutex
	net.Conn
	clock      clockwork.Clock
	lastActive time.Time
	ctx        context.Context
	cancel     context.CancelFunc
}

// Read reads data from the connection.
// Read can be made to time out and return an Error with Timeout() == true
// after a fixed time limit; see SetDeadline and SetReadDeadline.
func (t *trackingConn) Read(b []byte) (int, error) {
	n, err := t.Conn.Read(b)
	t.UpdateClientActivity()
	return n, err
}

func (t *trackingConn) Close() error {
	t.cancel()
	return t.Conn.Close()
}

// GetClientLastActive returns time when client was last active
func (t *trackingConn) GetClientLastActive() time.Time {
	t.RLock()
	defer t.RUnlock()
	return t.lastActive
}

// UpdateClientActivity sets last recorded client activity
func (t *trackingConn) UpdateClientActivity() {
	t.Lock()
	defer t.Unlock()
	t.lastActive = t.clock.Now().UTC()
}

func (f *Forwarder) getOrCreateClusterSession(ctx authContext) (*clusterSession, error) {
	client := f.getClusterSession(ctx)
	if client != nil {
		return client, nil
	}
	return f.serializedNewClusterSession(ctx)
}

func (f *Forwarder) getClusterSession(ctx authContext) *clusterSession {
	f.Lock()
	defer f.Unlock()
	creds, ok := f.clusterSessions.Get(ctx.key())
	if ok {
		return creds.(*clusterSession)
	}
	return nil
}

func (f *Forwarder) serializedNewClusterSession(authContext authContext) (*clusterSession, error) {
	ctx, cancel := f.getOrCreateRequestContext(authContext.key())
	if cancel != nil {
		f.Debugf("Requesting new creds for %v.", authContext)
		defer cancel()
		return f.newClusterSession(authContext)
	}
	// cancel == nil means that another request is in progress, so simply wait until
	// it finishes or fails
	f.Debugf("Another request is in progress for %v, waiting until it gets completed.", authContext)
	select {
	case <-ctx.Done():
		sess := f.getClusterSession(authContext)
		if sess == nil {
			return nil, trace.BadParameter("failed to request certificate, try again")
		}
		return sess, nil
	case <-f.ctx.Done():
		return nil, trace.BadParameter("forwarder is closing, aborting the request")
	}
}

func (f *Forwarder) newClusterSession(ctx authContext) (*clusterSession, error) {
	pool := x509.NewCertPool()
	tlsConfig := &tls.Config{
		RootCAs: pool,
	}
	// for remote clusters, retrieve a client certificate
	if ctx.cluster.isRemote {
		response, err := f.requestCertificate(ctx)
		if err != nil {
			f.Warningf("Failed to get certificate for %v: %v.", ctx, err)
			return nil, trace.AccessDenied("access denied: failed to authenticate with auth server")
		}
		cert, err := tls.X509KeyPair(response.cert, response.key)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		for _, certAuthority := range response.certAuthorities {
			ok := pool.AppendCertsFromPEM(certAuthority)
			if !ok {
				return nil, trace.BadParameter("failed to append certs from PEM")
			}
		}
		tlsConfig.Certificates = []tls.Certificate{cert}
		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
	} else {
		ok := pool.AppendCertsFromPEM(f.creds.caPEM)
		if !ok {
			return nil, trace.BadParameter("failed to append certs from PEM")
		}
		if f.creds.cert != nil {
			f.Debugf("Local Cluster: using TLS client certificates.")
			tlsConfig.Certificates = []tls.Certificate{*f.creds.cert}
			tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
		}
	}
	tlsConfig.BuildNameToCertificate()

	// remote clusters use special hardcoded URL,
	// and use a special dialer
	if ctx.cluster.isRemote {
		ctx.cluster.targetAddr = reversetunnel.RemoteKubeProxy
	} else {
		// auth server supplied target API address to dial,
		// use it for dialing
		if f.creds.targetAddr != "" {
			ctx.cluster.targetAddr = f.creds.targetAddr
		} else { // otherwise, use supplied defaults
			ctx.cluster.targetAddr = f.TargetAddr
		}
	}

	sess := &clusterSession{
		parent:      f,
		authContext: ctx,
		tlsConfig:   tlsConfig,
	}

	fwd, err := forward.New(
		forward.FlushInterval(100*time.Millisecond),
		forward.RoundTripper(f.newTransport(sess.Dial, tlsConfig)),
		forward.WebsocketDial(sess.Dial),
		forward.Logger(log.StandardLogger()),
	)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	sess.forwarder = fwd

	f.Lock()
	defer f.Unlock()

	sessI, ok := f.clusterSessions.Get(ctx.key())
	if ok {
		return sessI.(*clusterSession), nil
	}

	f.clusterSessions.Set(ctx.key(), sess, ctx.sessionTTL)
	f.Debugf("Created new session for %v.", ctx)
	return sess, nil
}

// DialFunc is a network dialer function that returns a network connection
type DialFunc func(string, string) (net.Conn, error)

func (f *Forwarder) newTransport(dial DialFunc, tlsConfig *tls.Config) *http.Transport {
	return &http.Transport{
		Dial:            dial,
		TLSClientConfig: tlsConfig,
		// Increase the size of the connection pool. This substantially improves the
		// performance of Teleport under load as it reduces the number of TLS
		// handshakes performed.
		MaxIdleConns:        defaults.HTTPMaxIdleConns,
		MaxIdleConnsPerHost: defaults.HTTPMaxIdleConnsPerHost,
		// IdleConnTimeout defines the maximum amount of time before idle connections
		// are closed. Leaving this unset will lead to connections open forever and
		// will cause memory leaks in a long running process.
		IdleConnTimeout: defaults.HTTPIdleTimeout,
	}
}

type bundle struct {
	cert            []byte
	key             []byte
	certAuthorities [][]byte
	targetAddr      string
}

// getOrCreateRequestContext creates a new certificate request for a given context,
// if there is no active CSR request in progress, or returns an existing one.
// if the new context has been created, cancel function is returned as a
// second argument. Caller should call this function to signal that CSR has been
// completed or failed.
func (f *Forwarder) getOrCreateRequestContext(key string) (context.Context, context.CancelFunc) {
	f.Lock()
	defer f.Unlock()
	ctx, ok := f.activeRequests[key]
	if ok {
		return ctx, nil
	}
	ctx, cancel := context.WithCancel(context.TODO())
	f.activeRequests[key] = ctx
	return ctx, func() {
		cancel()
		f.Lock()
		defer f.Unlock()
		delete(f.activeRequests, key)
	}
}

func (f *Forwarder) requestCertificate(ctx authContext) (*bundle, error) {
	f.Debugf("Requesting K8s cert for %v.", ctx)
	keyPEM, _, err := f.Keygen.GenerateKeyPair("")
	if err != nil {
		return nil, trace.Wrap(err)
	}

	privateKey, err := ssh.ParseRawPrivateKey(keyPEM)
	if err != nil {
		return nil, trace.Wrap(err, "failed to parse private key")
	}

	csr := &x509.CertificateRequest{
		Subject: pkix.Name{
			CommonName:   ctx.User.GetName(),
			Organization: ctx.kubeGroups,
		},
	}
	csrBytes, err := x509.CreateCertificateRequest(rand.Reader, csr, privateKey)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	csrPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})

	response, err := f.Client.ProcessKubeCSR(auth.KubeCSR{
		Username:    ctx.User.GetName(),
		ClusterName: ctx.cluster.GetName(),
		CSR:         csrPEM,
	})
	if err != nil {
		return nil, trace.Wrap(err)
	}
	f.Debugf("Received valid K8s cert for %v.", ctx)
	return &bundle{
		cert:            response.Cert,
		certAuthorities: response.CertAuthorities,
		targetAddr:      response.TargetAddr,
		key:             keyPEM,
	}, nil
}

type kubeCreds struct {
	// clt is a working kubernetes client
	cfg *rest.Config
	// caPEM is a PEM encoded certificate authority
	// of the kubernetes API server
	caPEM []byte
	// targetAddr is a target address of the
	// kubernetes cluster read from config
	targetAddr string
	// cert is a client certificate
	cert *tls.Certificate
}

func getKubeCreds(kubeconfigPath string) (*kubeCreds, error) {
	// no kubeconfig is set, assume auth server is running in the cluster
	if kubeconfigPath == "" {
		caPEM, err := ioutil.ReadFile(teleport.KubeCAPath)
		if err != nil {
			return nil, trace.BadParameter(`auth server assumed that it is 
running in a kubernetes cluster, but %v mounted in pods could not be read: %v, 
set kubeconfig_path if auth server is running outside of the cluster`, teleport.KubeCAPath, err)
		}

		cfg, err := kubeutils.GetKubeConfig(os.Getenv(teleport.EnvKubeConfig))
		if err != nil {
			return nil, trace.BadParameter(`auth server assumed that it is 
running in a kubernetes cluster, but could not init in-cluster kubernetes client: %v`, err)
		}

		targetAddr, err := parseKubeHost(cfg.Host)
		if err != nil {
			return nil, trace.Wrap(err, "failed to parse kubernetes host %q", cfg.Host)
		}

		return &kubeCreds{
			cfg:        cfg,
			caPEM:      caPEM,
			targetAddr: targetAddr,
		}, nil
	}

	log.Debugf("Reading configuration from kubeconfig file %v.", kubeconfigPath)

	cfg, err := kubeutils.GetKubeConfig(kubeconfigPath)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	targetAddr, err := parseKubeHost(cfg.Host)
	if err != nil {
		return nil, trace.Wrap(err, "failed to parse kubernetes host")
	}

	var caPEM []byte
	if len(cfg.CAData) == 0 {
		if cfg.CAFile == "" {
			return nil, trace.BadParameter("can't find trusted certificates in %v", kubeconfigPath)
		}
		caPEM, err = ioutil.ReadFile(cfg.CAFile)
		if err != nil {
			return nil, trace.BadParameter("failed to read trusted certificates from %v: %v", cfg.CAFile, err)
		}
	} else {
		caPEM = cfg.CAData
	}

	creds := &kubeCreds{
		cfg:        cfg,
		caPEM:      caPEM,
		targetAddr: targetAddr,
	}

	if len(cfg.CertData) != 0 && len(cfg.KeyData) != 0 {
		cert, err := tls.X509KeyPair(cfg.CertData, cfg.KeyData)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		creds.cert = &cert
	}
	return creds, nil
}

// parseKubeHost parses and formats kubernetes hostname
// to host:port format, if no port it set,
// it assumes default HTTPS port
func parseKubeHost(host string) (string, error) {
	u, err := url.Parse(host)
	if err != nil {
		return "", trace.Wrap(err, "failed to parse kubernetes host")
	}
	if _, _, err := net.SplitHostPort(u.Host); err != nil {
		// add default HTTPS port
		return fmt.Sprintf("%v:443", u.Host), nil
	}
	return u.Host, nil
}