Adding support for server to authenticate agent

Makefile

@@ -34,6 +34,11 @@ DOCKER_CLI_EXPERIMENTAL ?= enabled
 ## --------------------------------------
 ## Testing
 ## --------------------------------------
+mock_gen:
+	mkdir -p proto/agent/mocks
+	mockgen sigs.k8s.io/apiserver-network-proxy/proto/agent AgentService_ConnectServer > proto/agent/mocks/agent_mock.go
+	cat hack/go-license-header.txt proto/agent/mocks/agent_mock.go > proto/agent/mocks/agent_mock.licensed.go
+	mv proto/agent/mocks/agent_mock.licensed.go proto/agent/mocks/agent_mock.go
 
 .PHONY: test
 test:
@@ -68,7 +73,7 @@ bin/proxy-server: bin cmd/proxy/main.go proto/agent/agent.pb.go proto/proxy.pb.g
 ## --------------------------------------
 
 .PHONY: gen
-gen: proto/agent/agent.pb.go proto/proxy.pb.go
+gen: proto/agent/agent.pb.go proto/proxy.pb.go mock_gen
 
 proto/agent/agent.pb.go: proto/agent/agent.proto
 	protoc -I proto proto/agent/agent.proto --go_out=plugins=grpc:proto
@@ -244,4 +249,4 @@ release-alias-tag: # Adds the tag to the last build tag. BASE_REF comes from the
 
 .PHONY: clean
 clean:
-	rm -rf proto/agent/agent.pb.go proto/proxy.pb.go easy-rsa.tar.gz easy-rsa-master cfssl cfssljson certs bin
+	rm -rf proto/agent/agent.pb.go proto/proxy.pb.go easy-rsa.tar.gz easy-rsa-master cfssl cfssljson certs bin proto/agent/mocks

cmd/agent/main.go

@@ -68,16 +68,20 @@ type GrpcProxyAgentOptions struct {
 	syncInterval      time.Duration
 	probeInterval     time.Duration
 	reconnectInterval time.Duration
+
+	// file contains service account authorization token for enabling proxy-server token based authorization
+	serviceAccountTokenPath string
 }
 
 func (o *GrpcProxyAgentOptions) ClientSetConfig(dialOption grpc.DialOption) *agentclient.ClientSetConfig {
 	return &agentclient.ClientSetConfig{
-		Address:           fmt.Sprintf("%s:%d", o.proxyServerHost, o.proxyServerPort),
-		AgentID:           o.agentID,
-		SyncInterval:      o.syncInterval,
-		ProbeInterval:     o.probeInterval,
-		ReconnectInterval: o.reconnectInterval,
-		DialOption:        dialOption,
+		Address:                 fmt.Sprintf("%s:%d", o.proxyServerHost, o.proxyServerPort),
+		AgentID:                 o.agentID,
+		SyncInterval:            o.syncInterval,
+		ProbeInterval:           o.probeInterval,
+		ReconnectInterval:       o.reconnectInterval,
+		DialOption:              dialOption,
+		ServiceAccountTokenPath: o.serviceAccountTokenPath,
 	}
 }
 
@@ -92,6 +96,7 @@ func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet {
 	flags.DurationVar(&o.syncInterval, "sync-interval", o.syncInterval, "The interval by which the agent periodically checks that it has connections to all instances of the proxy server.")
 	flags.DurationVar(&o.probeInterval, "probe-interval", o.probeInterval, "The interval by which the agent periodically checks if its connections to the proxy server are ready.")
 	flags.DurationVar(&o.reconnectInterval, "reconnect-interval", o.reconnectInterval, "The interval by which the agent tries to reconnect.")
+	flags.StringVar(&o.serviceAccountTokenPath, "service-account-token-path", o.serviceAccountTokenPath, "If non-empty proxy agent uses this token to prove its identity to the proxy server.")
 	return flags
 }
 
@@ -105,6 +110,7 @@ func (o *GrpcProxyAgentOptions) Print() {
 	klog.Warningf("SyncInterval set to %v.\n", o.syncInterval)
 	klog.Warningf("ProbeInterval set to %v.\n", o.probeInterval)
 	klog.Warningf("ReconnectInterval set to %v.\n", o.reconnectInterval)
+	klog.Warningf("ServiceAccountTokenPath set to \"%s\".\n", o.serviceAccountTokenPath)
 }
 
 func (o *GrpcProxyAgentOptions) Validate() error {
@@ -132,20 +138,26 @@ func (o *GrpcProxyAgentOptions) Validate() error {
 	if o.proxyServerPort <= 0 {
 		return fmt.Errorf("proxy server port %d must be greater than 0", o.proxyServerPort)
 	}
+	if o.serviceAccountTokenPath != "" {
+		if _, err := os.Stat(o.serviceAccountTokenPath); os.IsNotExist(err) {
+			return fmt.Errorf("error checking service account token path %s, got %v", o.serviceAccountTokenPath, err)
+		}
+	}
 	return nil
 }
 
 func newGrpcProxyAgentOptions() *GrpcProxyAgentOptions {
 	o := GrpcProxyAgentOptions{
-		agentCert:         "",
-		agentKey:          "",
-		caCert:            "",
-		proxyServerHost:   "127.0.0.1",
-		proxyServerPort:   8091,
-		agentID:           uuid.New().String(),
-		syncInterval:      5 * time.Second,
-		probeInterval:     5 * time.Second,
-		reconnectInterval: 5 * time.Second,
+		agentCert:               "",
+		agentKey:                "",
+		caCert:                  "",
+		proxyServerHost:         "127.0.0.1",
+		proxyServerPort:         8091,
+		agentID:                 uuid.New().String(),
+		syncInterval:            5 * time.Second,
+		probeInterval:           5 * time.Second,
+		reconnectInterval:       5 * time.Second,
+		serviceAccountTokenPath: "",
 	}
 	return &o
 }

cmd/proxy/main.go

@@ -29,14 +29,15 @@ import (
 	"os/signal"
 	"syscall"
 
-	"k8s.io/klog"
-
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"k8s.io/klog"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/agent/agentserver"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/util"
 	"sigs.k8s.io/apiserver-network-proxy/proto/agent"
@@ -86,6 +87,14 @@ type ProxyRunOptions struct {
 	serverID string
 	// Number of proxy server instances, should be 1 unless it is a HA proxy server.
 	serverCount uint
+	// Agent pod's namespace for token-based agent authentication
+	agentNamespace string
+	// Agent pod's service account for token-based agent authentication
+	agentServiceAccount string
+	// Token's audience for token-based agent authentication
+	authenticationAudience string
+	// Path to kubeconfig (used by kubernetes client)
+	kubeconfigPath string
 }
 
 func (o *ProxyRunOptions) Flags() *pflag.FlagSet {
@@ -103,6 +112,10 @@ func (o *ProxyRunOptions) Flags() *pflag.FlagSet {
 	flags.UintVar(&o.adminPort, "admin-port", o.adminPort, "Port we listen for admin connections on.")
 	flags.StringVar(&o.serverID, "server-id", o.serverID, "The unique ID of this server.")
 	flags.UintVar(&o.serverCount, "server-count", o.serverCount, "The number of proxy server instances, should be 1 unless it is an HA server.")
+	flags.StringVar(&o.agentNamespace, "agent-namespace", o.agentNamespace, "Expected agent's namespace during agent authentication (used with agent-service-account, authentication-audience, kubeconfig).")
+	flags.StringVar(&o.agentServiceAccount, "agent-service-account", o.agentServiceAccount, "Expected agent's service account during agent authentication (used with agent-namespace, authentication-audience, kubeconfig).")
+	flags.StringVar(&o.kubeconfigPath, "kubeconfig", o.kubeconfigPath, "absolute path to the kubeconfig file (used with agent-namespace, agent-service-account, authentication-audience).")
+	flags.StringVar(&o.authenticationAudience, "authentication-audience", o.authenticationAudience, "Expected agent's token authentication audience (used with agent-namespace, agent-service-account, kubeconfig).")
 	return flags
 }
 
@@ -120,6 +133,10 @@ func (o *ProxyRunOptions) Print() {
 	klog.Warningf("Admin port set to %d.\n", o.adminPort)
 	klog.Warningf("ServerID set to %s.\n", o.serverID)
 	klog.Warningf("ServerCount set to %d.\n", o.serverCount)
+	klog.Warningf("AgentNamespace set to %q.\n", o.agentNamespace)
+	klog.Warningf("AgentServiceAccount set to %q.\n", o.agentServiceAccount)
+	klog.Warningf("AuthenticationAudience set to %q.\n", o.authenticationAudience)
+	klog.Warningf("KubeconfigPath set to %q.\n", o.kubeconfigPath)
 }
 
 func (o *ProxyRunOptions) Validate() error {
@@ -202,24 +219,48 @@ func (o *ProxyRunOptions) Validate() error {
 	if o.adminPort < 1024 {
 		return fmt.Errorf("please do not try to use reserved port %d for the admin port", o.adminPort)
 	}
+
+	// validate agent authentication params
+	// all 4 parametes must be empty or must have value (except kubeconfigPath that might be empty)
+	if o.agentNamespace != "" || o.agentServiceAccount != "" || o.authenticationAudience != "" || o.kubeconfigPath != "" {
+		if o.agentNamespace == "" {
+			return fmt.Errorf("agentNamespace cannot be empty when agent authentication is enabled")
+		}
+		if o.agentServiceAccount == "" {
+			return fmt.Errorf("agentServiceAccount cannot be empty when agent authentication is enabled")
+		}
+		if o.authenticationAudience == "" {
+			return fmt.Errorf("authenticationAudience cannot be empty when agent authentication is enabled")
+		}
+		if o.kubeconfigPath != "" {
+			if _, err := os.Stat(o.kubeconfigPath); os.IsNotExist(err) {
+				return fmt.Errorf("error checking kubeconfigPath %q, got %v", o.kubeconfigPath, err)
+			}
+		}
+	}
+
 	return nil
 }
 
 func newProxyRunOptions() *ProxyRunOptions {
 	o := ProxyRunOptions{
-		serverCert:    "",
-		serverKey:     "",
-		serverCaCert:  "",
-		clusterCert:   "",
-		clusterKey:    "",
-		clusterCaCert: "",
-		mode:          "grpc",
-		udsName:       "",
-		serverPort:    8090,
-		agentPort:     8091,
-		adminPort:     8092,
-		serverID:      uuid.New().String(),
-		serverCount:   1,
+		serverCert:             "",
+		serverKey:              "",
+		serverCaCert:           "",
+		clusterCert:            "",
+		clusterKey:             "",
+		clusterCaCert:          "",
+		mode:                   "grpc",
+		udsName:                "",
+		serverPort:             8090,
+		agentPort:              8091,
+		adminPort:              8092,
+		serverID:               uuid.New().String(),
+		serverCount:            1,
+		agentNamespace:         "",
+		agentServiceAccount:    "",
+		kubeconfigPath:         "",
+		authenticationAudience: "",
 	}
 	return &o
 }
@@ -247,7 +288,29 @@ func (p *Proxy) run(o *ProxyRunOptions) error {
 		return fmt.Errorf("failed to validate server options with %v", err)
 	}
 	ctx, cancel := context.WithCancel(context.Background())
-	server := agentserver.NewProxyServer(o.serverID, int(o.serverCount))
+	defer cancel()
+
+	var k8sClient *kubernetes.Clientset
+	if o.agentNamespace != "" {
+		config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfigPath)
+		if err != nil {
+			return fmt.Errorf("failed to load kubernetes client config: %v", err)
+		}
+
+		k8sClient, err = kubernetes.NewForConfig(config)
+		if err != nil {
+			return fmt.Errorf("failed to create kubernetes clientset: %v", err)
+		}
+	}
+
+	authOpt := &agentserver.AgentTokenAuthenticationOptions{
+		Enabled:                o.agentNamespace != "",
+		AgentNamespace:         o.agentNamespace,
+		AgentServiceAccount:    o.agentServiceAccount,
+		KubernetesClient:       k8sClient,
+		AuthenticationAudience: o.authenticationAudience,
+	}
+	server := agentserver.NewProxyServer(o.serverID, int(o.serverCount), authOpt)
 
 	klog.Info("Starting master server for client connections.")
 	masterStop, err := p.runMasterServer(ctx, o, server)
@@ -274,7 +337,6 @@ func (p *Proxy) run(o *ProxyRunOptions) error {
 	if masterStop != nil {
 		masterStop()
 	}
-	cancel()
 
 	return nil
 }

examples/kubernetes/README.md

@@ -20,16 +20,42 @@ KUBIA_IP=$(kubectl get svc kubia -o=jsonpath='{.spec.clusterIP}')
 PROXY_IMAGE=$(docker images | grep "proxy-server-" -m1 | awk '{print $1}')
 AGENT_IMAGE=$(docker images | grep "proxy-agent-" -m1 | awk '{print $1}')
 TEST_CLIENT_IMAGE=$(docker images | grep "proxy-test-client-" -m1 | awk '{print $1}')
+SERVER_TOKEN=$(./examples/kubernetes/token_generation.sh 32)
 CLUSTER_CERT=<yourdirectory/server.crt>
 CLUSTER_KEY=</yourdirectory/server.key>
 ```
 
-#### GKE specific configuration
+#### GCE sample configuration
 ```bash
 CLUSTER_CERT=/etc/srv/kubernetes/pki/apiserver.crt
 CLUSTER_KEY=/etc/srv/kubernetes/pki/apiserver.key
 ```
 
+# Register SERVER_TOKEN in [static-token-file](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file)
+Append the output of the following line to the [static-token-file](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file) and restart **kube-apiserver** on the master
+```bash
+echo "${SERVER_TOKEN},system:konnectivity-server,uid:system:konnectivity-server"
+```
+
+#### GCE sample configuration
+1. [static-token-file](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file) location is: **/etc/srv/kubernetes/known_tokens.csv**
+
+1. Restart kube-apiserver
+```bash
+K8S_API_PID=$(sudo crictl ps | grep kube-apiserver | awk '{ print $1; }') 
+sudo crictl stop ${K8S_API_PID}
+```
+
+# Save following config at /etc/srv/kubernetes/konnectivity-server/kubeconfig on master VM
+```bash
+SERVER_TOKEN=${SERVER_TOKEN} envsubst < examples/kubernetes/kubeconfig
+```
+
+# Create a clusterrolebinding allowing proxy-server authenticate proxy-client
+```bash
+kubectl create clusterrolebinding --user system:konnectivity-server --clusterrole system:auth-delegator system:konnectivity-server
+```
+
 # Start **proxy-server** as a [static pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) with following configuration
 ```bash
 TAG=${TAG} PROXY_IMAGE=${PROXY_IMAGE} CLUSTER_CERT=${CLUSTER_CERT} CLUSTER_KEY=${CLUSTER_KEY} envsubst <  examples/kubernetes/konnectivity-server.yaml

examples/kubernetes/konnectivity-agent.yaml

@@ -1,8 +1,14 @@
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: konnectivity-agent
+  namespace: kube-system
+--- 
+apiVersion: v1
 kind: Pod
 metadata:
   name: konnectivity-agent
-  namespace: default
+  namespace: kube-system
   annotations:
     scheduler.alpha.kubernetes.io/critical-pod: ''
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
@@ -20,6 +26,7 @@ spec:
       "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
       "--proxy-server-host=${CLUSTER_IP}",
       "--proxy-server-port=8091",
+      "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
       ]
     livenessProbe:
       httpGet:
@@ -33,3 +40,14 @@ spec:
       limits:
         cpu: 50m
         memory: 30Mi
+    volumeMounts:
+      - mountPath: /var/run/secrets/tokens
+        name: konnectivity-agent-token
+  serviceAccountName: konnectivity-agent
+  volumes:
+  - name: konnectivity-agent-token
+    projected:
+      sources:
+      - serviceAccountToken:
+          path: konnectivity-agent-token
+          audience: system:konnectivity-server

examples/kubernetes/konnectivity-server.yaml

@@ -19,13 +19,17 @@ spec:
       "--log-file=/var/log/konnectivity-server.log",
       "--logtostderr=false",
       "--log-file-max-size=0",
-      "--uds-name=/etc/srv/kubernetes/konnectivity/konnectivity-server.socket",
+      "--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket",
       "--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt",
       "--cluster-key=/etc/srv/kubernetes/pki/apiserver.key",
       "--server-port=0",
       "--agent-port=8091",
       "--admin-port=8092",
-      "--mode=http-connect"
+      "--mode=http-connect",
+      "--agent-namespace=kube-system",
+      "--agent-service-account=konnectivity-agent",
+      "--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig",
+      "--authentication-audience=system:konnectivity-server",
       ]
     livenessProbe:
       httpGet:
@@ -53,7 +57,7 @@ spec:
       mountPath: /etc/srv/kubernetes/pki
       readOnly: true
     - name: konnectivity-home
-      mountPath: /etc/srv/kubernetes/konnectivity
+      mountPath: /etc/srv/kubernetes/konnectivity-server
   volumes:
   - name: varlogkonnectivityserver
     hostPath:
@@ -64,6 +68,5 @@ spec:
       path: /etc/srv/kubernetes/pki
   - name: konnectivity-home
     hostPath:
-      path: /etc/srv/kubernetes/konnectivity
+      path: /etc/srv/kubernetes/konnectivity-server
       type: DirectoryOrCreate
-