Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adds Envoy Bootstrap Config #206

Merged
merged 1 commit into from
Aug 23, 2022
Merged

Adds Envoy Bootstrap Config #206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 112 additions & 17 deletions internal/infrastructure/kubernetes/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package kubernetes

import (
"bytes"
"context"
"fmt"
"path/filepath"
"text/template"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -27,13 +29,102 @@ const (
// envoyCfgVolMntDir is the directory name of the Envoy configuration volume.
envoyCfgVolMntDir = "config"
// envoyCfgFileName is the name of the Envoy configuration file.
envoyCfgFileName = "envoy.json"
envoyCfgFileName = "bootstrap.yaml"
// envoyHTTPPort is the container port number of Envoy's HTTP endpoint.
envoyHTTPPort = int32(8080)
// envoyHTTPSPort is the container port number of Envoy's HTTPS endpoint.
envoyHTTPSPort = int32(8443)
)

var bootstrapTmpl = template.Must(template.New(envoyCfgFileName).Parse(`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd consider putting this in a separate bootstrap.yaml file and saying

//go:embed bootstrap.yaml
const string bootstrapTmplStr

var bootstrapTmpl = template.Must(template.New(envoyCfgFileName).Parse(bootstrapTmplStr))

so that the file is easier to edit, with editors recognizing it as yaml.

admin:
access_log_path: /dev/null
address:
socket_address:
address: 127.0.0.1
port_value: 19000
dynamic_resources:
cds_config:
resource_api_version: V3
api_config_source:
api_type: GRPC
transport_api_version: V3
grpc_services:
- envoy_grpc:
cluster_name: xds_cluster
set_node_on_first_message_only: true
lds_config:
resource_api_version: V3
api_config_source:
api_type: GRPC
transport_api_version: V3
grpc_services:
- envoy_grpc:
cluster_name: xds_cluster
set_node_on_first_message_only: true
node:
cluster: envoy-gateway-system
id: envoy-default
static_resources:
clusters:
- connect_timeout: 1s
load_assignment:
cluster_name: xds_cluster
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: {{ .XdsServerAddress }}
port_value: 18000
http2_protocol_options: {}
name: xds_cluster
type: STRICT_DNS
layered_runtime:
layers:
- name: runtime-0
rtds_layer:
rtds_config:
resource_api_version: V3
api_config_source:
transport_api_version: V3
api_type: GRPC
grpc_services:
envoy_grpc:
cluster_name: xds_cluster
name: runtime-0
`))

var (
// envoyGatewayService is the name of the Envoy Gateway service.
envoyGatewayService = "envoy-gateway"
)
danehans marked this conversation as resolved.
Show resolved Hide resolved

// envoyBootstrap defines the envoy Bootstrap configuration.
type bootstrapConfig struct {
// parameters defines configurable bootstrap configuration parameters.
parameters bootstrapParameters
// rendered is the rendered bootstrap configuration.
rendered string
}

// envoyBootstrap defines the envoy Bootstrap configuration.
type bootstrapParameters struct {
// XdsServerAddress is the address of the XDS Server that Envoy is managed by.
XdsServerAddress string
}

// render the stringified bootstrap config in yaml format.
func (b *bootstrapConfig) render() error {
buf := new(bytes.Buffer)
if err := bootstrapTmpl.Execute(buf, b.parameters); err != nil {
return fmt.Errorf("failed to render bootstrap config: %v", err)
}
b.rendered = buf.String()
Comment on lines +119 to +123
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we never need access it as a []byte or mutate it (other than append), then it'd be more efficient to use a strings.Builder instead of a bytes.Buffer.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#225


return nil
}

// createDeploymentIfNeeded creates a Deployment based on the provided infra, if
// it doesn't exist in the kube api server.
func (i *Infra) createDeploymentIfNeeded(ctx context.Context, infra *ir.Infra) error {
Expand Down Expand Up @@ -76,8 +167,11 @@ func (i *Infra) getDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Dep
}

// expectedDeployment returns the expected Deployment based on the provided infra.
func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
containers := expectedContainers(infra)
func (i *Infra) expectedDeployment(infra *ir.Infra) (*appsv1.Deployment, error) {
containers, err := expectedContainers(infra)
if err != nil {
return nil, err
}

deployment := &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Expand All @@ -97,15 +191,7 @@ func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
Labels: EnvoyPodSelector().MatchLabels,
},
Spec: corev1.PodSpec{
Containers: containers,
Volumes: []corev1.Volume{
{
Name: envoyCfgVolName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
},
Containers: containers,
ServiceAccountName: infra.Proxy.ObjectName(),
AutomountServiceAccountToken: pointer.BoolPtr(false),
TerminationGracePeriodSeconds: pointer.Int64Ptr(int64(300)),
Expand All @@ -117,10 +203,10 @@ func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
},
}

return deployment
return deployment, nil
}

func expectedContainers(infra *ir.Infra) []corev1.Container {
func expectedContainers(infra *ir.Infra) ([]corev1.Container, error) {
ports := []corev1.ContainerPort{
{
Name: "http",
Expand All @@ -134,6 +220,11 @@ func expectedContainers(infra *ir.Infra) []corev1.Container {
},
}

cfg := bootstrapConfig{parameters: bootstrapParameters{XdsServerAddress: envoyGatewayService}}
if err := cfg.render(); err != nil {
return nil, err
}

containers := []corev1.Container{
{
Name: envoyContainerName,
Expand All @@ -147,6 +238,7 @@ func expectedContainers(infra *ir.Infra) []corev1.Container {
filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName),
fmt.Sprintf("--service-cluster $(%s)", envoyNsEnvVar),
fmt.Sprintf("--service-node $(%s)", envoyPodEnvVar),
fmt.Sprintf("--config-yaml %s", cfg.rendered),
"--log-level info",
},
Env: []corev1.EnvVar{
Expand Down Expand Up @@ -182,15 +274,18 @@ func expectedContainers(infra *ir.Infra) []corev1.Container {
},
}

return containers
return containers, nil
}

// createDeployment creates a Deployment in the kube api server based on the provided
// infra, if it doesn't exist.
func (i *Infra) createDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Deployment, error) {
expected := i.expectedDeployment(infra)
err := i.Client.Create(ctx, expected)
expected, err := i.expectedDeployment(infra)
if err != nil {
return nil, err
}

if err := i.Client.Create(ctx, expected); err != nil {
if kerrors.IsAlreadyExists(err) {
return expected, nil
}
Expand Down
24 changes: 22 additions & 2 deletions internal/infrastructure/kubernetes/deployment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package kubernetes

import (
"context"
"fmt"
"testing"

"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -52,6 +53,17 @@ func checkContainer(t *testing.T, deploy *appsv1.Deployment, name string, expect
return nil
}

func checkContainerHasArg(t *testing.T, container *corev1.Container, arg string) {
t.Helper()

for _, a := range container.Args {
if a == arg {
return
}
}
t.Errorf("container is missing argument %q", arg)
}

func checkLabels(t *testing.T, deploy *appsv1.Deployment, expected map[string]string) {
t.Helper()

Expand Down Expand Up @@ -88,7 +100,8 @@ func TestExpectedDeployment(t *testing.T) {
cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build()
kube := NewInfra(cli)
infra := ir.NewInfra()
deploy := kube.expectedDeployment(infra)
deploy, err := kube.expectedDeployment(infra)
require.NoError(t, err)

// Check container details, i.e. env vars, labels, etc. for the deployment are as expected.
container := checkContainer(t, deploy, envoyContainerName, true)
Expand All @@ -97,6 +110,12 @@ func TestExpectedDeployment(t *testing.T) {
checkEnvVar(t, deploy, envoyContainerName, envoyPodEnvVar)
checkLabels(t, deploy, deploy.Labels)

// Create a bootstrap config, render it into an arg, and ensure it's as expected.
cfg := &bootstrapConfig{parameters: bootstrapParameters{XdsServerAddress: envoyGatewayService}}
err = cfg.render()
require.NoError(t, err)
checkContainerHasArg(t, container, fmt.Sprintf("--config-yaml %s", cfg.rendered))

// Check container ports for the deployment are as expected.
ports := []int32{envoyHTTPPort, envoyHTTPSPort}
for _, port := range ports {
Expand All @@ -107,7 +126,8 @@ func TestExpectedDeployment(t *testing.T) {
func TestCreateDeploymentIfNeeded(t *testing.T) {
kube := NewInfra(nil)
infra := ir.NewInfra()
deploy := kube.expectedDeployment(infra)
deploy, err := kube.expectedDeployment(infra)
require.NoError(t, err)
deploy.ResourceVersion = "1"

testCases := []struct {
Expand Down