From 7db79f02905852e7e0370b1b361c3e1b385b7dc4 Mon Sep 17 00:00:00 2001 From: danehans Date: Thu, 11 Aug 2022 10:15:49 -0700 Subject: [PATCH] Adds Envoy Deployment Support Signed-off-by: danehans --- .../infrastructure/kubernetes/deployment.go | 218 ++++++++++++++++++ .../kubernetes/deployment_test.go | 155 +++++++++++++ internal/infrastructure/kubernetes/infra.go | 28 ++- internal/ir/infra.go | 2 +- 4 files changed, 394 insertions(+), 9 deletions(-) create mode 100644 internal/infrastructure/kubernetes/deployment.go create mode 100644 internal/infrastructure/kubernetes/deployment_test.go diff --git a/internal/infrastructure/kubernetes/deployment.go b/internal/infrastructure/kubernetes/deployment.go new file mode 100644 index 00000000000..72c193fdb6c --- /dev/null +++ b/internal/infrastructure/kubernetes/deployment.go @@ -0,0 +1,218 @@ +package kubernetes + +import ( + "context" + "fmt" + "path/filepath" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + + "github.com/envoyproxy/gateway/internal/ir" +) + +const ( + // envoyContainerName is the name of the Envoy container. + envoyContainerName = "envoy" + // envoyNsEnvVar is the name of the contour namespace environment variable. + envoyNsEnvVar = "ENVOY_GATEWAY_NAMESPACE" + // envoyPodEnvVar is the name of the Envoy pod name environment variable. + envoyPodEnvVar = "ENVOY_POD_NAME" + // envoyCfgVolName is the name of the Envoy configuration volume. + envoyCfgVolName = "envoy-config" + // envoyCfgVolMntDir is the directory name of the Envoy configuration volume. + envoyCfgVolMntDir = "config" + // envoyCfgFileName is the name of the Envoy configuration file. + envoyCfgFileName = "envoy.json" + // envoyHTTPPort is the container port number of Envoy's HTTP endpoint. + envoyHTTPPort = int32(8080) + // envoyHTTPSPort is the container port number of Envoy's HTTPS endpoint. + envoyHTTPSPort = int32(8443) +) + +// createDeploymentIfNeeded creates a Deployment based on the provided infra, if +// it doesn't exist in the kube api server. +func (i *Infra) createDeploymentIfNeeded(ctx context.Context, infra *ir.Infra) error { + current, err := i.getDeployment(ctx, infra) + if err != nil { + if kerrors.IsNotFound(err) { + deploy, err := i.createDeployment(ctx, infra) + if err != nil { + return err + } + if err := i.addResource(deploy); err != nil { + return err + } + return nil + } + return err + } + + if err := i.addResource(current); err != nil { + return err + } + + return nil +} + +// getDeployment gets the Deployment for the provided infra from the kube api. +func (i *Infra) getDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Deployment, error) { + ns := i.Namespace + name := infra.Proxy.Name + key := types.NamespacedName{ + Namespace: ns, + Name: infra.GetProxyInfra().ObjectName(), + } + deploy := new(appsv1.Deployment) + if err := i.Client.Get(ctx, key, deploy); err != nil { + return nil, fmt.Errorf("failed to get deployment %s/%s: %w", ns, name, err) + } + + return deploy, nil +} + +// expectedDeployment returns the expected Deployment based on the provided infra. +func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment { + containers := expectedContainers(infra) + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: infra.GetProxyInfra().ObjectName(), + Labels: envoyLabels(), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Selector: EnvoyPodSelector(), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: EnvoyPodSelector().MatchLabels, + }, + Spec: corev1.PodSpec{ + Containers: containers, + Volumes: []corev1.Volume{ + { + Name: envoyCfgVolName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + ServiceAccountName: infra.Proxy.ObjectName(), + AutomountServiceAccountToken: pointer.BoolPtr(false), + TerminationGracePeriodSeconds: pointer.Int64Ptr(int64(300)), + DNSPolicy: corev1.DNSClusterFirst, + RestartPolicy: corev1.RestartPolicyAlways, + SchedulerName: "default-scheduler", + }, + }, + }, + } + + return deployment +} + +func expectedContainers(infra *ir.Infra) []corev1.Container { + ports := []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: envoyHTTPPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "https", + ContainerPort: envoyHTTPSPort, + Protocol: corev1.ProtocolTCP, + }, + } + + containers := []corev1.Container{ + { + Name: envoyContainerName, + Image: infra.Proxy.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "envoy", + }, + Args: []string{ + "--config-path", + filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName), + fmt.Sprintf("--service-cluster $(%s)", envoyNsEnvVar), + fmt.Sprintf("--service-node $(%s)", envoyPodEnvVar), + "--log-level info", + }, + Env: []corev1.EnvVar{ + { + Name: envoyNsEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: envoyPodEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, + Ports: ports, + VolumeMounts: []corev1.VolumeMount{ + { + Name: envoyCfgVolName, + MountPath: filepath.Join("/", envoyCfgVolMntDir), + ReadOnly: true, + }, + }, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + TerminationMessagePath: "/dev/termination-log", + }, + } + + return containers +} + +// createDeployment creates a Deployment in the kube api server based on the provided +// infra, if it doesn't exist. +func (i *Infra) createDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Deployment, error) { + expected := i.expectedDeployment(infra) + err := i.Client.Create(ctx, expected) + if err != nil { + if kerrors.IsAlreadyExists(err) { + return expected, nil + } + return nil, fmt.Errorf("failed to create deployment %s/%s: %w", + expected.Namespace, expected.Name, err) + } + + return expected, nil +} + +// EnvoyPodSelector returns a label selector using "control-plane: envoy-gateway" as the +// key/value pair. +// +// TODO: Update k/v pair to use gatewayclass controller name to distinguish between +// multiple Envoy Gateways. +func EnvoyPodSelector() *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: envoyLabels(), + } +} + +// envoyLabels returns the labels used for Envoy. +func envoyLabels() map[string]string { + return map[string]string{"control-plane": "envoy-gateway"} +} diff --git a/internal/infrastructure/kubernetes/deployment_test.go b/internal/infrastructure/kubernetes/deployment_test.go new file mode 100644 index 00000000000..c47f5bfbaff --- /dev/null +++ b/internal/infrastructure/kubernetes/deployment_test.go @@ -0,0 +1,155 @@ +package kubernetes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/ir" +) + +func checkEnvVar(t *testing.T, deploy *appsv1.Deployment, container, name string) { + t.Helper() + + for i, c := range deploy.Spec.Template.Spec.Containers { + if c.Name == container { + for _, envVar := range deploy.Spec.Template.Spec.Containers[i].Env { + if envVar.Name == name { + return + } + } + } + } + + t.Errorf("deployment is missing environment variable %q", name) +} + +func checkContainer(t *testing.T, deploy *appsv1.Deployment, name string, expect bool) *corev1.Container { + t.Helper() + + if deploy.Spec.Template.Spec.Containers == nil { + t.Error("deployment has no containers") + } + + for _, container := range deploy.Spec.Template.Spec.Containers { + if container.Name == name { + if expect { + return &container + } + t.Errorf("deployment has unexpected %q container", name) + } + } + + if expect { + t.Errorf("deployment has no %q container", name) + } + return nil +} + +func checkLabels(t *testing.T, deploy *appsv1.Deployment, expected map[string]string) { + t.Helper() + + if apiequality.Semantic.DeepEqual(deploy.Labels, expected) { + return + } + + t.Errorf("deployment has unexpected %q labels", deploy.Labels) +} + +func checkContainerHasPort(t *testing.T, deploy *appsv1.Deployment, port int32) { + t.Helper() + + for _, c := range deploy.Spec.Template.Spec.Containers { + for _, p := range c.Ports { + if p.ContainerPort == port { + return + } + } + } + t.Errorf("container is missing containerPort %q", port) +} + +func checkContainerImage(t *testing.T, container *corev1.Container, image string) { + t.Helper() + + if container.Image == image { + return + } + t.Errorf("container is missing image %q", image) +} + +func TestExpectedDeployment(t *testing.T) { + cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() + kube := NewInfra(cli) + infra := ir.NewInfra() + deploy := kube.expectedDeployment(infra) + + // Check container details, i.e. env vars, labels, etc. for the deployment are as expected. + container := checkContainer(t, deploy, envoyContainerName, true) + checkContainerImage(t, container, ir.DefaultProxyImage) + checkEnvVar(t, deploy, envoyContainerName, envoyNsEnvVar) + checkEnvVar(t, deploy, envoyContainerName, envoyPodEnvVar) + checkLabels(t, deploy, deploy.Labels) + + // Check container ports for the deployment are as expected. + ports := []int32{envoyHTTPPort, envoyHTTPSPort} + for _, port := range ports { + checkContainerHasPort(t, deploy, port) + } +} + +func TestCreateDeploymentIfNeeded(t *testing.T) { + kube := NewInfra(nil) + infra := ir.NewInfra() + deploy := kube.expectedDeployment(infra) + deploy.ResourceVersion = "1" + + testCases := []struct { + name string + in *ir.Infra + current *appsv1.Deployment + out *Resources + expect bool + }{ + { + name: "create deployment", + in: infra, + out: &Resources{ + Deployment: deploy, + }, + expect: true, + }, + { + name: "deployment exists", + in: infra, + current: deploy, + out: &Resources{ + Deployment: deploy, + }, + expect: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.current != nil { + kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + } else { + kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + } + err := kube.createDeploymentIfNeeded(context.Background(), tc.in) + if !tc.expect { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.out.Deployment, kube.Resources.Deployment) + } + }) + } +} diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index aa81159f0f7..34e6ba1bc6d 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,16 +26,23 @@ type Infra struct { // Resources are managed Kubernetes resources. type Resources struct { ServiceAccount *corev1.ServiceAccount + Deployment *appsv1.Deployment } // NewInfra returns a new Infra. func NewInfra(cli client.Client) *Infra { return &Infra{ - mu: sync.Mutex{}, - Client: cli, - Resources: &Resources{ - ServiceAccount: new(corev1.ServiceAccount), - }, + mu: sync.Mutex{}, + Client: cli, + Resources: newResources(), + } +} + +// newResources returns a new Resources. +func newResources() *Resources { + return &Resources{ + ServiceAccount: new(corev1.ServiceAccount), + Deployment: new(appsv1.Deployment), } } @@ -50,6 +58,8 @@ func (i *Infra) addResource(obj client.Object) error { switch o := obj.(type) { case *corev1.ServiceAccount: i.Resources.ServiceAccount = o + case *appsv1.Deployment: + i.Resources.Deployment = o default: return fmt.Errorf("unexpected object kind %s", obj.GetObjectKind()) } @@ -68,14 +78,16 @@ func (i *Infra) CreateInfra(ctx context.Context, infra *ir.Infra) error { } if i.Resources == nil { - i.Resources = &Resources{ - ServiceAccount: new(corev1.ServiceAccount), - } + i.Resources = newResources() } if err := i.createServiceAccountIfNeeded(ctx, infra); err != nil { return err } + if err := i.createDeploymentIfNeeded(ctx, infra); err != nil { + return err + } + return nil } diff --git a/internal/ir/infra.go b/internal/ir/infra.go index 94f8e2fa928..869fded9b49 100644 --- a/internal/ir/infra.go +++ b/internal/ir/infra.go @@ -158,7 +158,7 @@ func ValidateProxyInfra(pInfra *ProxyInfra) error { return utilerrors.NewAggregate(errs) } -// ObjectName returns the name of proxy infrastructure objects. +// ObjectName returns the name of the proxy infrastructure object. func (p *ProxyInfra) ObjectName() string { if len(p.Name) == 0 { return fmt.Sprintf("envoy-%s", DefaultProxyName)