Adds Envoy Deployment Support (#200)

Signed-off-by: danehans <[email protected]>

Signed-off-by: danehans <[email protected]>

internal/infrastructure/kubernetes/deployment.go

@@ -0,0 +1,218 @@
+package kubernetes
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/pointer"
+
+	"github.com/envoyproxy/gateway/internal/ir"
+)
+
+const (
+	// envoyContainerName is the name of the Envoy container.
+	envoyContainerName = "envoy"
+	// envoyNsEnvVar is the name of the contour namespace environment variable.
+	envoyNsEnvVar = "ENVOY_GATEWAY_NAMESPACE"
+	// envoyPodEnvVar is the name of the Envoy pod name environment variable.
+	envoyPodEnvVar = "ENVOY_POD_NAME"
+	// envoyCfgVolName is the name of the Envoy configuration volume.
+	envoyCfgVolName = "envoy-config"
+	// envoyCfgVolMntDir is the directory name of the Envoy configuration volume.
+	envoyCfgVolMntDir = "config"
+	// envoyCfgFileName is the name of the Envoy configuration file.
+	envoyCfgFileName = "envoy.json"
+	// envoyHTTPPort is the container port number of Envoy's HTTP endpoint.
+	envoyHTTPPort = int32(8080)
+	// envoyHTTPSPort is the container port number of Envoy's HTTPS endpoint.
+	envoyHTTPSPort = int32(8443)
+)
+
+// createDeploymentIfNeeded creates a Deployment based on the provided infra, if
+// it doesn't exist in the kube api server.
+func (i *Infra) createDeploymentIfNeeded(ctx context.Context, infra *ir.Infra) error {
+	current, err := i.getDeployment(ctx, infra)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			deploy, err := i.createDeployment(ctx, infra)
+			if err != nil {
+				return err
+			}
+			if err := i.addResource(deploy); err != nil {
+				return err
+			}
+			return nil
+		}
+		return err
+	}
+
+	if err := i.addResource(current); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getDeployment gets the Deployment for the provided infra from the kube api.
+func (i *Infra) getDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Deployment, error) {
+	ns := i.Namespace
+	name := infra.Proxy.Name
+	key := types.NamespacedName{
+		Namespace: ns,
+		Name:      infra.GetProxyInfra().ObjectName(),
+	}
+	deploy := new(appsv1.Deployment)
+	if err := i.Client.Get(ctx, key, deploy); err != nil {
+		return nil, fmt.Errorf("failed to get deployment %s/%s: %w", ns, name, err)
+	}
+
+	return deploy, nil
+}
+
+// expectedDeployment returns the expected Deployment based on the provided infra.
+func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
+	containers := expectedContainers(infra)
+
+	deployment := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: i.Namespace,
+			Name:      infra.GetProxyInfra().ObjectName(),
+			Labels:    envoyLabels(),
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: pointer.Int32(1),
+			Selector: EnvoyPodSelector(),
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: EnvoyPodSelector().MatchLabels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: containers,
+					Volumes: []corev1.Volume{
+						{
+							Name: envoyCfgVolName,
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+					ServiceAccountName:            infra.Proxy.ObjectName(),
+					AutomountServiceAccountToken:  pointer.BoolPtr(false),
+					TerminationGracePeriodSeconds: pointer.Int64Ptr(int64(300)),
+					DNSPolicy:                     corev1.DNSClusterFirst,
+					RestartPolicy:                 corev1.RestartPolicyAlways,
+					SchedulerName:                 "default-scheduler",
+				},
+			},
+		},
+	}
+
+	return deployment
+}
+
+func expectedContainers(infra *ir.Infra) []corev1.Container {
+	ports := []corev1.ContainerPort{
+		{
+			Name:          "http",
+			ContainerPort: envoyHTTPPort,
+			Protocol:      corev1.ProtocolTCP,
+		},
+		{
+			Name:          "https",
+			ContainerPort: envoyHTTPSPort,
+			Protocol:      corev1.ProtocolTCP,
+		},
+	}
+
+	containers := []corev1.Container{
+		{
+			Name:            envoyContainerName,
+			Image:           infra.Proxy.Image,
+			ImagePullPolicy: corev1.PullIfNotPresent,
+			Command: []string{
+				"envoy",
+			},
+			Args: []string{
+				"--config-path",
+				filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName),
+				fmt.Sprintf("--service-cluster $(%s)", envoyNsEnvVar),
+				fmt.Sprintf("--service-node $(%s)", envoyPodEnvVar),
+				"--log-level info",
+			},
+			Env: []corev1.EnvVar{
+				{
+					Name: envoyNsEnvVar,
+					ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{
+							APIVersion: "v1",
+							FieldPath:  "metadata.namespace",
+						},
+					},
+				},
+				{
+					Name: envoyPodEnvVar,
+					ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{
+							APIVersion: "v1",
+							FieldPath:  "metadata.name",
+						},
+					},
+				},
+			},
+			Ports: ports,
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      envoyCfgVolName,
+					MountPath: filepath.Join("/", envoyCfgVolMntDir),
+					ReadOnly:  true,
+				},
+			},
+			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
+			TerminationMessagePath:   "/dev/termination-log",
+		},
+	}
+
+	return containers
+}
+
+// createDeployment creates a Deployment in the kube api server based on the provided
+// infra, if it doesn't exist.
+func (i *Infra) createDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Deployment, error) {
+	expected := i.expectedDeployment(infra)
+	err := i.Client.Create(ctx, expected)
+	if err != nil {
+		if kerrors.IsAlreadyExists(err) {
+			return expected, nil
+		}
+		return nil, fmt.Errorf("failed to create deployment %s/%s: %w",
+			expected.Namespace, expected.Name, err)
+	}
+
+	return expected, nil
+}
+
+// EnvoyPodSelector returns a label selector using "control-plane: envoy-gateway" as the
+// key/value pair.
+//
+// TODO: Update k/v pair to use gatewayclass controller name to distinguish between
+//       multiple Envoy Gateways.
+func EnvoyPodSelector() *metav1.LabelSelector {
+	return &metav1.LabelSelector{
+		MatchLabels: envoyLabels(),
+	}
+}
+
+// envoyLabels returns the labels used for Envoy.
+func envoyLabels() map[string]string {
+	return map[string]string{"control-plane": "envoy-gateway"}
+}

internal/infrastructure/kubernetes/deployment_test.go

@@ -0,0 +1,155 @@
+package kubernetes
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"github.com/envoyproxy/gateway/internal/envoygateway"
+	"github.com/envoyproxy/gateway/internal/ir"
+)
+
+func checkEnvVar(t *testing.T, deploy *appsv1.Deployment, container, name string) {
+	t.Helper()
+
+	for i, c := range deploy.Spec.Template.Spec.Containers {
+		if c.Name == container {
+			for _, envVar := range deploy.Spec.Template.Spec.Containers[i].Env {
+				if envVar.Name == name {
+					return
+				}
+			}
+		}
+	}
+
+	t.Errorf("deployment is missing environment variable %q", name)
+}
+
+func checkContainer(t *testing.T, deploy *appsv1.Deployment, name string, expect bool) *corev1.Container {
+	t.Helper()
+
+	if deploy.Spec.Template.Spec.Containers == nil {
+		t.Error("deployment has no containers")
+	}
+
+	for _, container := range deploy.Spec.Template.Spec.Containers {
+		if container.Name == name {
+			if expect {
+				return &container
+			}
+			t.Errorf("deployment has unexpected %q container", name)
+		}
+	}
+
+	if expect {
+		t.Errorf("deployment has no %q container", name)
+	}
+	return nil
+}
+
+func checkLabels(t *testing.T, deploy *appsv1.Deployment, expected map[string]string) {
+	t.Helper()
+
+	if apiequality.Semantic.DeepEqual(deploy.Labels, expected) {
+		return
+	}
+
+	t.Errorf("deployment has unexpected %q labels", deploy.Labels)
+}
+
+func checkContainerHasPort(t *testing.T, deploy *appsv1.Deployment, port int32) {
+	t.Helper()
+
+	for _, c := range deploy.Spec.Template.Spec.Containers {
+		for _, p := range c.Ports {
+			if p.ContainerPort == port {
+				return
+			}
+		}
+	}
+	t.Errorf("container is missing containerPort %q", port)
+}
+
+func checkContainerImage(t *testing.T, container *corev1.Container, image string) {
+	t.Helper()
+
+	if container.Image == image {
+		return
+	}
+	t.Errorf("container is missing image %q", image)
+}
+
+func TestExpectedDeployment(t *testing.T) {
+	cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build()
+	kube := NewInfra(cli)
+	infra := ir.NewInfra()
+	deploy := kube.expectedDeployment(infra)
+
+	// Check container details, i.e. env vars, labels, etc. for the deployment are as expected.
+	container := checkContainer(t, deploy, envoyContainerName, true)
+	checkContainerImage(t, container, ir.DefaultProxyImage)
+	checkEnvVar(t, deploy, envoyContainerName, envoyNsEnvVar)
+	checkEnvVar(t, deploy, envoyContainerName, envoyPodEnvVar)
+	checkLabels(t, deploy, deploy.Labels)
+
+	// Check container ports for the deployment are as expected.
+	ports := []int32{envoyHTTPPort, envoyHTTPSPort}
+	for _, port := range ports {
+		checkContainerHasPort(t, deploy, port)
+	}
+}
+
+func TestCreateDeploymentIfNeeded(t *testing.T) {
+	kube := NewInfra(nil)
+	infra := ir.NewInfra()
+	deploy := kube.expectedDeployment(infra)
+	deploy.ResourceVersion = "1"
+
+	testCases := []struct {
+		name    string
+		in      *ir.Infra
+		current *appsv1.Deployment
+		out     *Resources
+		expect  bool
+	}{
+		{
+			name: "create deployment",
+			in:   infra,
+			out: &Resources{
+				Deployment: deploy,
+			},
+			expect: true,
+		},
+		{
+			name:    "deployment exists",
+			in:      infra,
+			current: deploy,
+			out: &Resources{
+				Deployment: deploy,
+			},
+			expect: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.current != nil {
+				kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build()
+			} else {
+				kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build()
+			}
+			err := kube.createDeploymentIfNeeded(context.Background(), tc.in)
+			if !tc.expect {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.out.Deployment, kube.Resources.Deployment)
+			}
+		})
+	}
+}