From 8aa0e97805d41e03654aa1c20feba38e248fac76 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Fri, 1 Jul 2022 00:51:08 -0400 Subject: [PATCH 01/10] Initial CNI installer & plugin (#1304) * Get structure in place and CNI installer & plugin building --- control-plane/go.mod | 2 ++ .../install-cni/testdata/10-kindnet.conflist.alreadyinserted | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/control-plane/go.mod b/control-plane/go.mod index 80f09eabdf..51d2f69b28 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -52,6 +52,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/containernetworking/cni v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661 // indirect github.com/digitalocean/godo v1.10.0 // indirect @@ -69,6 +70,7 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gophercloud/gophercloud v0.1.0 // indirect + github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220622202014-ebf6603ca2ab // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect diff --git a/control-plane/subcommand/install-cni/testdata/10-kindnet.conflist.alreadyinserted b/control-plane/subcommand/install-cni/testdata/10-kindnet.conflist.alreadyinserted index 0520180fde..3c108d766b 100644 --- a/control-plane/subcommand/install-cni/testdata/10-kindnet.conflist.alreadyinserted +++ b/control-plane/subcommand/install-cni/testdata/10-kindnet.conflist.alreadyinserted @@ -26,7 +26,6 @@ { "cni_bin_dir": "/opt/cni/bin", "cni_net_dir": "/etc/cni/net.d", - "dns_service": "", "kubeconfig": "ZZZ-consul-cni-kubeconfig", "log_level": "info", "multus": false, From 26aad37678dd21b94ca529c8bd1ebea05a2fc307 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Tue, 16 Aug 2022 11:17:15 -0400 Subject: [PATCH 02/10] CNI File watcher and pre applying rules setup (#1345) * Add file watcher to CNI installer to watch for config file changes and repair breakages. * Wait for CNI config file to show up on the host file system before attempting to install consul-cni configuration. * Add some code to get ready for the next PR that applying iptables rules * Unit tests for installer and plugin scenarios --- control-plane/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/control-plane/go.mod b/control-plane/go.mod index 51d2f69b28..80f09eabdf 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -52,7 +52,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/containernetworking/cni v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661 // indirect github.com/digitalocean/godo v1.10.0 // indirect @@ -70,7 +69,6 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gophercloud/gophercloud v0.1.0 // indirect - github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220622202014-ebf6603ca2ab // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect From 595c49c9d5632fb44b28cbd380ed072160560e90 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Fri, 1 Jul 2022 00:51:08 -0400 Subject: [PATCH 03/10] Initial CNI installer & plugin (#1304) * Get structure in place and CNI installer & plugin building --- control-plane/Dockerfile | 1 + control-plane/go.mod | 2 ++ 2 files changed, 3 insertions(+) diff --git a/control-plane/Dockerfile b/control-plane/Dockerfile index c70bcedbb5..459b5f974d 100644 --- a/control-plane/Dockerfile +++ b/control-plane/Dockerfile @@ -69,6 +69,7 @@ CMD /bin/${BIN_NAME} FROM alpine:3.16 AS release-default ARG BIN_NAME=consul-k8s-control-plane +ARG CNI_BIN_NAME=consul-cni ARG PRODUCT_VERSION LABEL name=${BIN_NAME} \ diff --git a/control-plane/go.mod b/control-plane/go.mod index 80f09eabdf..51d2f69b28 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -52,6 +52,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/containernetworking/cni v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661 // indirect github.com/digitalocean/godo v1.10.0 // indirect @@ -69,6 +70,7 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gophercloud/gophercloud v0.1.0 // indirect + github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220622202014-ebf6603ca2ab // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect From 0f332955862aff71e43d89ffeba7299c5fa706c0 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Tue, 16 Aug 2022 11:17:15 -0400 Subject: [PATCH 04/10] CNI File watcher and pre applying rules setup (#1345) * Add file watcher to CNI installer to watch for config file changes and repair breakages. * Wait for CNI config file to show up on the host file system before attempting to install consul-cni configuration. * Add some code to get ready for the next PR that applying iptables rules * Unit tests for installer and plugin scenarios --- control-plane/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/control-plane/go.mod b/control-plane/go.mod index 51d2f69b28..80f09eabdf 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -52,7 +52,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/containernetworking/cni v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661 // indirect github.com/digitalocean/godo v1.10.0 // indirect @@ -70,7 +69,6 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gophercloud/gophercloud v0.1.0 // indirect - github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220622202014-ebf6603ca2ab // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect From 888e6d8fceb4c85643cc991aeca397373ad96c98 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Mon, 27 Jun 2022 10:19:17 -0400 Subject: [PATCH 05/10] CNI Helm Charts Add helm charts for CNI installer --- .../install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden | 1 + 1 file changed, 1 insertion(+) diff --git a/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden b/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden index dcb29b3a2e..6a588677ce 100644 --- a/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden +++ b/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden @@ -15,4 +15,5 @@ preferences: {} users: - name: consul-cni user: + as-user-extra: null token: eyJhbGciOiJSUzI1NiIsImtp From 71c2e5eae3aff3855a6ff8808cccbb6340489d5a Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Fri, 1 Jul 2022 00:51:08 -0400 Subject: [PATCH 06/10] Initial CNI installer & plugin (#1304) * Get structure in place and CNI installer & plugin building --- .../install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden | 1 - 1 file changed, 1 deletion(-) diff --git a/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden b/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden index 6a588677ce..dcb29b3a2e 100644 --- a/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden +++ b/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden @@ -15,5 +15,4 @@ preferences: {} users: - name: consul-cni user: - as-user-extra: null token: eyJhbGciOiJSUzI1NiIsImtp From 20e5e025fd31f99b13b6213dfeb53cde28eed017 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Thu, 7 Jul 2022 12:40:20 -0400 Subject: [PATCH 07/10] CNI File watcher and pre applying rules setup increase limits for CNI plugin so that it runs on GKE add annotations for transparent proxy status (enabled, waiting) Initial setup (CNI_ARGS) for getting information to the CNI plugin file watcher for config file changes and for when the config file does not exists added wait for annotation to be used before applying ipconfig traffic redirection Co-Authored-By: Thomas Eckert --- control-plane/cni/go.sum | 2 -- control-plane/cni/main.go | 17 +++++++++++++++++ control-plane/connect-inject/annotations.go | 7 +++++++ control-plane/connect-inject/mesh_webhook.go | 12 ++++++++++++ control-plane/go.mod | 4 ++-- control-plane/go.sum | 5 ++++- 6 files changed, 42 insertions(+), 5 deletions(-) diff --git a/control-plane/cni/go.sum b/control-plane/cni/go.sum index 03309565a1..7f748dc34c 100644 --- a/control-plane/cni/go.sum +++ b/control-plane/cni/go.sum @@ -219,8 +219,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/control-plane/cni/main.go b/control-plane/cni/main.go index 7ae31d9e8f..97e006991d 100644 --- a/control-plane/cni/main.go +++ b/control-plane/cni/main.go @@ -6,7 +6,9 @@ import ( "fmt" "net" "path/filepath" + "time" + "github.com/cenkalti/backoff" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" @@ -155,6 +157,7 @@ func (c *Command) cmdAdd(args *skel.CmdArgs) error { result := prevResult logger.Debug("consul-cni previous result", "result", result) + // Connect to kubernetes. ctx := context.Background() if c.client == nil { @@ -228,6 +231,20 @@ func skipTrafficRedirection(pod corev1.Pod) bool { return false } +// waitForAnnotation waits for an annotation to be available. Returns immediately if the annotation exists. +func waitForAnnotation(pod corev1.Pod, annotation string, retries uint64) bool { + var err error + err = backoff.Retry(func() error { + var ok bool + _, ok = pod.Annotations[annotation] + if !ok { + return fmt.Errorf("annotation %s does not exist yet", annotation) + } + return err + }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), retries)) + return err == nil +} + // parseAnnotation parses the cni-proxy-config annotation into an iptables.Config object. func parseAnnotation(pod corev1.Pod, annotation string) (iptables.Config, error) { anno, ok := pod.Annotations[annotation] diff --git a/control-plane/connect-inject/annotations.go b/control-plane/connect-inject/annotations.go index d1b9c544a7..38aad78a92 100644 --- a/control-plane/connect-inject/annotations.go +++ b/control-plane/connect-inject/annotations.go @@ -5,6 +5,10 @@ const ( // a pod after an injection is done. keyInjectStatus = "consul.hashicorp.com/connect-inject-status" + // keyTransparentProxyStatus is the key of the annotation that is added to + // a pod when transparent proxy is done. + keyTransparentProxyStatus = "consul.hashicorp.com/transparent-proxy-status" + // keyManagedBy is the key of the label that is added to pods managed // by the Endpoints controller. This is to support upgrading from consul-k8s // without Endpoints controller to consul-k8s with Endpoints controller @@ -167,6 +171,9 @@ const ( // injected is used as the annotation value for keyInjectStatus and annotationInjected. injected = "injected" + // enabled is used as the annotation value for keyTransparentProxyStatus. + enabled = "enabled" + // endpointsController is the value for keyManagedBy. managedByValue = "consul-k8s-endpoints-controller" ) diff --git a/control-plane/connect-inject/mesh_webhook.go b/control-plane/connect-inject/mesh_webhook.go index dc0e407ed7..6298c9220f 100644 --- a/control-plane/connect-inject/mesh_webhook.go +++ b/control-plane/connect-inject/mesh_webhook.go @@ -365,6 +365,18 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi // and does not need to be checked for being a nil value. pod.Annotations[keyInjectStatus] = injected + tproxyEnabled, err := transparentProxyEnabled(*ns, pod, w.EnableTransparentProxy) + if err != nil { + w.Log.Error(err, "error determining if transparent proxy is enabled", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error determining if transparent proxy is enabled: %s", err)) + } + + // Add an annotation to the pod sets transparent-proxy-status to enabled or disabled. Used by the CNI plugin + // to determine if it should traffic redirect or not + if tproxyEnabled { + pod.Annotations[keyTransparentProxyStatus] = enabled + } + // Add annotations for metrics. if err = w.prometheusAnnotations(&pod); err != nil { w.Log.Error(err, "error configuring prometheus annotations", "request name", req.Name) diff --git a/control-plane/go.mod b/control-plane/go.mod index 80f09eabdf..7919804509 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -13,8 +13,8 @@ require ( github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220803144027-ad547faeffa5 github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f github.com/hashicorp/go-hclog v0.16.1 - github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/serf v0.9.7 + github.com/hashicorp/go-multierror v1.1.0 + github.com/hashicorp/serf v0.9.6 github.com/kr/text v0.2.0 github.com/miekg/dns v1.1.41 github.com/mitchellh/cli v1.1.0 diff --git a/control-plane/go.sum b/control-plane/go.sum index 7ad24f0fc4..bdcd9b8a68 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -351,7 +351,6 @@ github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/S github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f h1:7WFMVeuJQp6BkzuTv9O52pzwtEFVUJubKYN+zez8eTI= github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f/go.mod h1:D4eo8/CN92vm9/9UDG+ldX1/fMFa4kpl8qzyTolus8o= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.1 h1:IVQwpTGNRRIHafnTs2dQLIk4ENtneRIEEJWOVDqz99o= github.com/hashicorp/go-hclog v0.16.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -361,6 +360,7 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= @@ -392,6 +392,8 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 h1:O/pT5C1Q3mVXMyuqg7yuAWUg/jMZR1/0QTzTRdNR6Uw= @@ -853,6 +855,7 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 9606072aec2a5258431ad9c509b2957404b86601 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Tue, 9 Aug 2022 14:11:01 -0400 Subject: [PATCH 08/10] main_test: remove config specfic tests and check if rules is not empty when a valid annotation is used Add redirect traffic config and apply it in the plugin --- .../build-support/functions/20-build.sh | 4 +- control-plane/cni/go.sum | 2 + control-plane/cni/main.go | 73 +-- control-plane/cni/main_test.go | 124 +++-- control-plane/connect-inject/annotations.go | 4 + .../connect-inject/container_init.go | 20 +- .../connect-inject/container_init_test.go | 4 +- .../connect-inject/endpoints_controller.go | 5 +- control-plane/connect-inject/mesh_webhook.go | 10 + .../connect-inject/mesh_webhook_test.go | 8 +- .../connect-inject/redirect_traffic.go | 115 +++++ .../connect-inject/redirect_traffic_test.go | 447 ++++++++++++++++++ control-plane/go.mod | 5 +- control-plane/go.sum | 5 +- .../subcommand/install-cni/command_test.go | 2 +- 15 files changed, 737 insertions(+), 91 deletions(-) create mode 100644 control-plane/connect-inject/redirect_traffic.go create mode 100644 control-plane/connect-inject/redirect_traffic_test.go diff --git a/control-plane/build-support/functions/20-build.sh b/control-plane/build-support/functions/20-build.sh index 39d816631b..f2f9f05c45 100644 --- a/control-plane/build-support/functions/20-build.sh +++ b/control-plane/build-support/functions/20-build.sh @@ -255,14 +255,14 @@ function build_consul_local { return 1 fi else - status "Building sequentially with go install" + status "Building sequentially with go build" for os in ${build_os} do for arch in ${build_arch} do outdir="pkg.bin.new/${extra_dir}${os}_${arch}" osarch="${os}/${arch}" - if test "${osarch}" == "darwin/arm" -o "${osarch}" == "darwin/arm64" -o "${osarch}" == "freebsd/arm64" -o "${osarch}" == "windows/arm" -o "${osarch}" == "windows/arm64" + if test "${osarch}" == "darwin/arm" -o "${osarch}" == "freebsd/arm64" -o "${osarch}" == "windows/arm" -o "${osarch}" == "windows/arm64" then continue fi diff --git a/control-plane/cni/go.sum b/control-plane/cni/go.sum index 7f748dc34c..03309565a1 100644 --- a/control-plane/cni/go.sum +++ b/control-plane/cni/go.sum @@ -219,6 +219,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/control-plane/cni/main.go b/control-plane/cni/main.go index 97e006991d..c0685efb52 100644 --- a/control-plane/cni/main.go +++ b/control-plane/cni/main.go @@ -6,9 +6,7 @@ import ( "fmt" "net" "path/filepath" - "time" - "github.com/cenkalti/backoff" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" @@ -44,13 +42,16 @@ const ( // indicate the status of the CNI plugin. complete = "complete" - // annotationTrafficRedirection stores iptables.Config information so that the CNI plugin can use it to apply + // annotationRedirectTraffic stores iptables.Config information so that the CNI plugin can use it to apply // iptables rules. - annotationTrafficRedirection = "consul.hashicorp.com/traffic-redirection-config" + annotationRedirectTraffic = "consul.hashicorp.com/redirect-traffic-config" ) type Command struct { + // client is a kubernetes client client kubernetes.Interface + // iptablesProvider is the Provider that will apply iptables rules. Used for testing. + iptablesProvider iptables.Provider } type CNIArgs struct { @@ -157,7 +158,6 @@ func (c *Command) cmdAdd(args *skel.CmdArgs) error { result := prevResult logger.Debug("consul-cni previous result", "result", result) - // Connect to kubernetes. ctx := context.Background() if c.client == nil { @@ -180,34 +180,57 @@ func (c *Command) cmdAdd(args *skel.CmdArgs) error { // Skip traffic redirection if the correct annotations are not on the pod. if skipTrafficRedirection(*pod) { - logger.Debug("skipping traffic redirect on un-injected pod: %s", pod.Name) + logger.Debug("skipping traffic redirection because the pod is either not injected or transparent proxy is disabled: %s", pod.Name) return types.PrintResult(result, cfg.CNIVersion) } - err = c.updateTransparentProxyStatusAnnotation(pod, podNamespace, waiting) + // We do not throw an error here because kubernetes will often throw a benign error where the pod has been + // updated in between the get and update of the annotation. Eventually kubernetes will update the annotation + ok := c.updateTransparentProxyStatusAnnotation(pod, podNamespace, waiting) + if !ok { + logger.Info("unable to update %s pod annotation to waiting", keyTransparentProxyStatus) + } + + // Parse the cni-proxy-config annotation into an iptables.Config object. + iptablesCfg, err := parseAnnotation(*pod, annotationRedirectTraffic) if err != nil { - return fmt.Errorf("error adding waiting annotation: %s", err) + return err } - // TODO: Insert redirect here + // Set NetNS passed through the CNI. + iptablesCfg.NetNS = args.Netns + + // Set the provider to a fake provider in testing, otherwise use the default iptables.Provider + if c.iptablesProvider != nil { + iptablesCfg.IptablesProvider = c.iptablesProvider + } - err = c.updateTransparentProxyStatusAnnotation(pod, podNamespace, complete) + // Apply the iptables rules. + err = iptables.Setup(iptablesCfg) if err != nil { - return fmt.Errorf("error adding complete annotation: %s", err) + return fmt.Errorf("could not apply iptables setup: %v", err) } + // We do not throw an error here because kubernetes will often throw a benign error where the pod has been + // updated in between the get and update of the annotation. Eventually kubernetes will update the annotation + ok = c.updateTransparentProxyStatusAnnotation(pod, podNamespace, complete) + if !ok { + logger.Info("unable to update %s pod annotation to complete", keyTransparentProxyStatus) + } + + logger.Debug("traffic redirect rules applied to pod: %s", pod.Name) // Pass through the result for the next plugin even though we are the final plugin in the chain. return types.PrintResult(result, cfg.CNIVersion) } // cmdDel is called for DELETE requests. -func cmdDel(args *skel.CmdArgs) error { +func cmdDel(_ *skel.CmdArgs) error { // Nothing to do but this function will still be called as part of the CNI specification. return nil } // cmdCheck is called for CHECK requests. -func cmdCheck(args *skel.CmdArgs) error { +func cmdCheck(_ *skel.CmdArgs) error { // Nothing to do but this function will still be called as part of the CNI specification. return nil } @@ -231,20 +254,6 @@ func skipTrafficRedirection(pod corev1.Pod) bool { return false } -// waitForAnnotation waits for an annotation to be available. Returns immediately if the annotation exists. -func waitForAnnotation(pod corev1.Pod, annotation string, retries uint64) bool { - var err error - err = backoff.Retry(func() error { - var ok bool - _, ok = pod.Annotations[annotation] - if !ok { - return fmt.Errorf("annotation %s does not exist yet", annotation) - } - return err - }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), retries)) - return err == nil -} - // parseAnnotation parses the cni-proxy-config annotation into an iptables.Config object. func parseAnnotation(pod corev1.Pod, annotation string) (iptables.Config, error) { anno, ok := pod.Annotations[annotation] @@ -260,13 +269,9 @@ func parseAnnotation(pod corev1.Pod, annotation string) (iptables.Config, error) } // updateTransparentProxyStatusAnnotation updates the transparent-proxy-status annotation. We use it as a simple inicator of -// CNI status on the pod. -func (c *Command) updateTransparentProxyStatusAnnotation(pod *corev1.Pod, namespace, status string) error { +// CNI status on the pod. Failing is not fatal. +func (c *Command) updateTransparentProxyStatusAnnotation(pod *corev1.Pod, namespace, status string) bool { pod.Annotations[keyTransparentProxyStatus] = status _, err := c.client.CoreV1().Pods(namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("error adding annotation to pod: %s", err) - } - - return nil + return err == nil } diff --git a/control-plane/cni/main_test.go b/control-plane/cni/main_test.go index 7ef171cbe8..740e15c646 100644 --- a/control-plane/cni/main_test.go +++ b/control-plane/cni/main_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "testing" "github.com/containernetworking/cni/pkg/skel" @@ -19,76 +20,137 @@ const ( defaultNamespace = "default" ) -func TestRun_cmdAdd(t *testing.T) { - t.Parallel() +type fakeIptablesProvider struct { + rules []string +} - cmd := &Command{ - client: fake.NewSimpleClientset(), - } +func (f *fakeIptablesProvider) AddRule(name string, args ...string) { + var rule []string + rule = append(rule, name) + rule = append(rule, args...) + + f.rules = append(f.rules, strings.Join(rule, " ")) +} + +func (f *fakeIptablesProvider) ApplyRules() error { + return nil +} + +func (f *fakeIptablesProvider) Rules() []string { + return f.rules +} + +func Test_cmdAdd(t *testing.T) { + t.Parallel() cases := []struct { name string + cmd *Command podName string stdInData string - configuredPod func(*corev1.Pod) *corev1.Pod + configuredPod func(*corev1.Pod, *Command) *corev1.Pod + expectedRules bool expectedErr error }{ { name: "K8S_POD_NAME missing from CNI args, should throw error", + cmd: &Command{}, podName: "", stdInData: goodStdinData, - configuredPod: func(pod *corev1.Pod) *corev1.Pod { + configuredPod: func(pod *corev1.Pod, cmd *Command) *corev1.Pod { return pod }, - expectedErr: fmt.Errorf("not running in a pod, namespace and pod should have values"), + expectedErr: fmt.Errorf("not running in a pod, namespace and pod should have values"), + expectedRules: false, // Rules won't be applied because the command will throw an error first }, { - name: "Missing prevResult in stdin data, should throw error", + name: "Missing prevResult in stdin data, should throw error", + cmd: &Command{ + client: fake.NewSimpleClientset(), + }, podName: "missing-prev-result", stdInData: missingPrevResultStdinData, - configuredPod: func(pod *corev1.Pod) *corev1.Pod { - _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + configuredPod: func(pod *corev1.Pod, cmd *Command) *corev1.Pod { + _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) require.NoError(t, err) return pod }, - expectedErr: fmt.Errorf("must be called as final chained plugin"), + expectedErr: fmt.Errorf("must be called as final chained plugin"), + expectedRules: false, // Rules won't be applied because the command will throw an error first }, { - name: "Missing IPs in prevResult in stdin data, should throw error", + name: "Missing IPs in prevResult in stdin data, should throw error", + cmd: &Command{ + client: fake.NewSimpleClientset(), + }, podName: "corrupt-prev-result", stdInData: missingIPsStdinData, - configuredPod: func(pod *corev1.Pod) *corev1.Pod { - _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + configuredPod: func(pod *corev1.Pod, cmd *Command) *corev1.Pod { + _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) require.NoError(t, err) return pod }, - expectedErr: fmt.Errorf("got no container IPs"), + expectedErr: fmt.Errorf("got no container IPs"), + expectedRules: false, // Rules won't be applied because the command will throw an error first }, - { - name: "Pod with traffic redirection annotation, should apply redirect", - podName: "pod-with-annotation", + name: "Pod with incorrect traffic redirection annotation, should throw error", + cmd: &Command{ + client: fake.NewSimpleClientset(), + }, + podName: "pod-with-incorrect-annotation", stdInData: goodStdinData, - configuredPod: func(pod *corev1.Pod) *corev1.Pod { - _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + configuredPod: func(pod *corev1.Pod, cmd *Command) *corev1.Pod { + pod.Annotations[keyInjectStatus] = "true" + pod.Annotations[keyTransparentProxyStatus] = "enabled" + pod.Annotations[annotationRedirectTraffic] = "{foo}" + _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) require.NoError(t, err) - pod.Annotations[annotationTrafficRedirection] = "{foo}" - _, err = cmd.client.CoreV1().Pods(defaultNamespace).Update(context.Background(), pod, metav1.UpdateOptions{}) + return pod + }, + expectedErr: fmt.Errorf("could not unmarshal %s annotation for %s pod", annotationRedirectTraffic, "pod-with-incorrect-annotation"), + expectedRules: false, // Rules won't be applied because the command will throw an error first + }, + { + name: "Pod with correct annotations, should create redirect traffic rules", + cmd: &Command{ + client: fake.NewSimpleClientset(), + iptablesProvider: &fakeIptablesProvider{}, + }, + podName: "pod-no-proxy-outbound-port", + stdInData: goodStdinData, + configuredPod: func(pod *corev1.Pod, cmd *Command) *corev1.Pod { + pod.Annotations[keyInjectStatus] = "true" + pod.Annotations[keyTransparentProxyStatus] = "enabled" + cfg := iptables.Config{ + ProxyUserID: "123", + ProxyInboundPort: 20000, + } + iptablesConfigJson, err := json.Marshal(&cfg) + require.NoError(t, err) + pod.Annotations[annotationRedirectTraffic] = string(iptablesConfigJson) + _, err = cmd.client.CoreV1().Pods(defaultNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) require.NoError(t, err) return pod }, - expectedErr: nil, + expectedErr: nil, + expectedRules: true, // Rules will be applied }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - _ = c.configuredPod(minimalPod(c.podName)) - actual := cmd.cmdAdd(minimalSkelArgs(c.podName, defaultNamespace, c.stdInData)) - require.Equal(t, c.expectedErr, actual) + _ = c.configuredPod(minimalPod(c.podName), c.cmd) + err := c.cmd.cmdAdd(minimalSkelArgs(c.podName, defaultNamespace, c.stdInData)) + require.Equal(t, c.expectedErr, err) + + // Check to see that rules have been generated + if c.expectedErr == nil && c.expectedRules { + require.NotEmpty(t, c.cmd.iptablesProvider.Rules()) + } }) } } @@ -152,7 +214,7 @@ func TestParseAnnotation(t *testing.T) { }{ { name: "Pod with iptables.Config annotation", - annotation: annotationTrafficRedirection, + annotation: annotationRedirectTraffic, configurePod: func(pod *corev1.Pod) *corev1.Pod { // Use iptables.Config so that if the Config struct ever changes that the test is still valid cfg := iptables.Config{ProxyUserID: "1234"} @@ -160,7 +222,7 @@ func TestParseAnnotation(t *testing.T) { if err != nil { t.Fatalf("could not marshal iptables config: %v", err) } - pod.Annotations[annotationTrafficRedirection] = string(j) + pod.Annotations[annotationRedirectTraffic] = string(j) return pod }, expected: iptables.Config{ @@ -170,12 +232,12 @@ func TestParseAnnotation(t *testing.T) { }, { name: "Pod without iptables.Config annotation", - annotation: annotationTrafficRedirection, + annotation: annotationRedirectTraffic, configurePod: func(pod *corev1.Pod) *corev1.Pod { return pod }, expected: iptables.Config{}, - err: fmt.Errorf("could not find %s annotation for %s pod", annotationTrafficRedirection, defaultPodName), + err: fmt.Errorf("could not find %s annotation for %s pod", annotationRedirectTraffic, defaultPodName), }, } for _, c := range cases { diff --git a/control-plane/connect-inject/annotations.go b/control-plane/connect-inject/annotations.go index 38aad78a92..fa35959160 100644 --- a/control-plane/connect-inject/annotations.go +++ b/control-plane/connect-inject/annotations.go @@ -152,6 +152,10 @@ const ( // to point to the Envoy proxy when running in Transparent Proxy mode. annotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes" + // annotationRedirectTraffic stores iptables.Config information so that the CNI plugin can use it to apply + // iptables rules. + annotationRedirectTraffic = "consul.hashicorp.com/redirect-traffic-config" + // annotationOriginalPod is the value of the pod before being overwritten by the consul // webhook/meshWebhook. annotationOriginalPod = "consul.hashicorp.com/original-pod" diff --git a/control-plane/connect-inject/container_init.go b/control-plane/connect-inject/container_init.go index 955d4a0b50..3c94797d58 100644 --- a/control-plane/connect-inject/container_init.go +++ b/control-plane/connect-inject/container_init.go @@ -14,13 +14,13 @@ import ( ) const ( - InjectInitCopyContainerName = "copy-consul-bin" - InjectInitContainerName = "consul-connect-inject-init" - rootUserAndGroupID = 0 - envoyUserAndGroupID = 5995 - copyContainerUserAndGroupID = 5996 - netAdminCapability = "NET_ADMIN" - dnsServiceHostEnvSuffix = "DNS_SERVICE_HOST" + InjectInitCopyContainerName = "copy-consul-bin" + InjectInitContainerName = "consul-connect-inject-init" + rootUserAndGroupID = 0 + envoyUserAndGroupID = 5995 + initContainersUserAndGroupID = 5996 + netAdminCapability = "NET_ADMIN" + dnsServiceHostEnvSuffix = "DNS_SERVICE_HOST" ) type initContainerCommandData struct { @@ -62,7 +62,7 @@ type initContainerCommandData struct { EnableTransparentProxy bool // EnableCNI configures this init container to skip the redirect-traffic command as traffic - // redirection is handled by the CNI plugin on pod creation + // redirection is handled by the CNI plugin on pod creation. EnableCNI bool // TProxyExcludeInboundPorts is a list of inbound ports to exclude from traffic redirection via @@ -122,8 +122,8 @@ func (w *MeshWebhook) initCopyContainer() corev1.Container { if !w.EnableOpenShift { container.SecurityContext = &corev1.SecurityContext{ // Set RunAsUser because the default user for the consul container is root and we want to run non-root. - RunAsUser: pointer.Int64(copyContainerUserAndGroupID), - RunAsGroup: pointer.Int64(copyContainerUserAndGroupID), + RunAsUser: pointer.Int64(initContainersUserAndGroupID), + RunAsGroup: pointer.Int64(initContainersUserAndGroupID), RunAsNonRoot: pointer.Bool(true), ReadOnlyRootFilesystem: pointer.Bool(true), } diff --git a/control-plane/connect-inject/container_init_test.go b/control-plane/connect-inject/container_init_test.go index 6f80ff37a8..c24c2017e5 100644 --- a/control-plane/connect-inject/container_init_test.go +++ b/control-plane/connect-inject/container_init_test.go @@ -1215,8 +1215,8 @@ func TestHandlerInitCopyContainer(t *testing.T) { require.Nil(t, container.SecurityContext) } else { expectedSecurityContext := &corev1.SecurityContext{ - RunAsUser: pointer.Int64(copyContainerUserAndGroupID), - RunAsGroup: pointer.Int64(copyContainerUserAndGroupID), + RunAsUser: pointer.Int64(initContainersUserAndGroupID), + RunAsGroup: pointer.Int64(initContainersUserAndGroupID), RunAsNonRoot: pointer.Bool(true), ReadOnlyRootFilesystem: pointer.Bool(true), } diff --git a/control-plane/connect-inject/endpoints_controller.go b/control-plane/connect-inject/endpoints_controller.go index 47e30ee0b9..04c8e70a26 100644 --- a/control-plane/connect-inject/endpoints_controller.go +++ b/control-plane/connect-inject/endpoints_controller.go @@ -57,6 +57,9 @@ const ( // exposedPathsStartupPortsRangeStart is the start of the port range that we will use as // the ListenerPort for the Expose configuration of the proxy registration for a startup probe. exposedPathsStartupPortsRangeStart = 20500 + + // proxyDefaultInboundPort is the default inbound port for the proxy. + proxyDefaultInboundPort = 20000 ) type EndpointsController struct { @@ -493,7 +496,7 @@ func (r *EndpointsController) createServiceRegistrations(pod corev1.Pod, service } proxyConfig.Upstreams = upstreams - proxyPort := 20000 + proxyPort := proxyDefaultInboundPort if idx := getMultiPortIdx(pod, serviceEndpoints); idx >= 0 { proxyPort += idx } diff --git a/control-plane/connect-inject/mesh_webhook.go b/control-plane/connect-inject/mesh_webhook.go index 6298c9220f..eef1187835 100644 --- a/control-plane/connect-inject/mesh_webhook.go +++ b/control-plane/connect-inject/mesh_webhook.go @@ -404,6 +404,16 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error overwriting readiness or liveness probes: %s", err)) } + // When CNI and tproxy are enabled, we add an annotation to the pod that contains the iptables config so that the CNI + // plugin can apply redirect traffic rules on the pod. + if w.EnableCNI && tproxyEnabled { + if err := w.addRedirectTrafficConfigAnnotation(&pod, *ns); err != nil { + // todo: update this error message + w.Log.Error(err, "error configuring annotation for CNI traffic redirection", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring annotation for CNI traffic redirection: %s", err)) + } + } + // Marshall the pod into JSON after it has the desired envs, annotations, labels, // sidecars and initContainers appended to it. updatedPodJson, err := json.Marshal(pod) diff --git a/control-plane/connect-inject/mesh_webhook_test.go b/control-plane/connect-inject/mesh_webhook_test.go index 8b37462506..0f21b6dca7 100644 --- a/control-plane/connect-inject/mesh_webhook_test.go +++ b/control-plane/connect-inject/mesh_webhook_test.go @@ -718,6 +718,11 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(keyInjectStatus), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(keyTransparentProxyStatus), + }, + { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(annotationOriginalPod), @@ -1868,7 +1873,6 @@ func TestOverwriteProbes(t *testing.T) { require.Equal(t, c.expStartupPort[i], container.StartupProbe.HTTPGet.Port.IntValue()) } } - }) } } @@ -1904,9 +1908,7 @@ func TestHandler_checkUnsupportedMultiPortCases(t *testing.T) { require.Error(t, err) require.Equal(t, tt.expErr, err.Error()) }) - } - } // encodeRaw is a helper to encode some data into a RawExtension. diff --git a/control-plane/connect-inject/redirect_traffic.go b/control-plane/connect-inject/redirect_traffic.go new file mode 100644 index 0000000000..895b5befbe --- /dev/null +++ b/control-plane/connect-inject/redirect_traffic.go @@ -0,0 +1,115 @@ +package connectinject + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + + "github.com/hashicorp/consul/sdk/iptables" + corev1 "k8s.io/api/core/v1" +) + +// addRedirectTrafficConfigAnnotation creates an iptables.Config based on proxy configuration. +// iptables.Config: +// ConsulDNSIP: an environment variable named RESOURCE_PREFIX_DNS_SERVICE_HOST where RESOURCE_PREFIX is the consul.fullname in helm. +// ProxyUserID: a constant set in Annotations +// ProxyInboundPort: the service port or bind port +// ProxyOutboundPort: default transparent proxy outbound port or transparent proxy outbound listener port +// ExcludeInboundPorts: prometheus, envoy stats, expose paths, checks and excluded pod annotations +// ExcludeOutboundPorts: pod annotations +// ExcludeOutboundCIDRs: pod annotations +// ExcludeUIDs: pod annotations +func (w *MeshWebhook) addRedirectTrafficConfigAnnotation(pod *corev1.Pod, ns corev1.Namespace) error { + cfg := iptables.Config{ + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + } + + // Set the proxy's inbound port. + cfg.ProxyInboundPort = proxyDefaultInboundPort + + // Set the proxy's outbound port. + cfg.ProxyOutboundPort = iptables.DefaultTProxyOutboundPort + + // If metrics are enabled, get the prometheusScrapePort and exclude it from the inbound ports + enableMetrics, err := w.MetricsConfig.enableMetrics(*pod) + if err != nil { + return err + } + if enableMetrics { + prometheusScrapePort, err := w.MetricsConfig.prometheusScrapePort(*pod) + if err != nil { + return err + } + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, prometheusScrapePort) + } + + // Exclude any overwritten liveness/readiness/startup ports from redirection. + overwriteProbes, err := shouldOverwriteProbes(*pod, w.TProxyOverwriteProbes) + if err != nil { + return err + } + + if overwriteProbes { + for i, container := range pod.Spec.Containers { + // skip the "envoy-sidecar" container from having its probes overridden + if container.Name == envoySidecarContainer { + continue + } + if container.LivenessProbe != nil && container.LivenessProbe.HTTPGet != nil { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposedPathsLivenessPortsRangeStart+i)) + } + if container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposedPathsReadinessPortsRangeStart+i)) + } + if container.StartupProbe != nil && container.StartupProbe.HTTPGet != nil { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposedPathsStartupPortsRangeStart+i)) + } + } + } + + // Inbound ports + excludeInboundPorts := splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeInboundPorts, *pod) + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, excludeInboundPorts...) + + // Outbound ports + excludeOutboundPorts := splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundPorts, *pod) + cfg.ExcludeOutboundPorts = append(cfg.ExcludeOutboundPorts, excludeOutboundPorts...) + + // Outbound CIDRs + excludeOutboundCIDRs := splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundCIDRs, *pod) + cfg.ExcludeOutboundCIDRs = append(cfg.ExcludeOutboundCIDRs, excludeOutboundCIDRs...) + + // UIDs + excludeUIDs := splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeUIDs, *pod) + cfg.ExcludeUIDs = append(cfg.ExcludeUIDs, excludeUIDs...) + + // Add init container user ID to exclude from traffic redirection. + cfg.ExcludeUIDs = append(cfg.ExcludeUIDs, strconv.Itoa(initContainersUserAndGroupID)) + + dnsEnabled, err := consulDNSEnabled(ns, *pod, w.EnableConsulDNS) + if err != nil { + return err + } + + var consulDNSClusterIP string + if dnsEnabled { + // If Consul DNS is enabled, we find the environment variable that has the value + // of the ClusterIP of the Consul DNS Service. constructDNSServiceHostName returns + // the name of the env variable whose value is the ClusterIP of the Consul DNS Service. + consulDNSClusterIP = os.Getenv(w.constructDNSServiceHostName()) + if consulDNSClusterIP == "" { + return fmt.Errorf("environment variable %s not found", w.constructDNSServiceHostName()) + } + cfg.ConsulDNSIP = consulDNSClusterIP + } + + iptablesConfigJson, err := json.Marshal(&cfg) + if err != nil { + return fmt.Errorf("could not marshal iptables config: %w", err) + } + + pod.Annotations[annotationRedirectTraffic] = string(iptablesConfigJson) + + return nil +} diff --git a/control-plane/connect-inject/redirect_traffic_test.go b/control-plane/connect-inject/redirect_traffic_test.go new file mode 100644 index 0000000000..838f1c0c25 --- /dev/null +++ b/control-plane/connect-inject/redirect_traffic_test.go @@ -0,0 +1,447 @@ +package connectinject + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "testing" + + mapset "github.com/deckarep/golang-set" + logrtest "github.com/go-logr/logr/testing" + "github.com/hashicorp/consul/sdk/iptables" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + defaultPodName = "fakePod" + defaultNamespace = "default" + resourcePrefix = "CONSUL" + dnsEnvVariable = "CONSUL_DNS_SERVICE_HOST" + dnsIP = "127.0.0.1" +) + +func TestAddRedirectTrafficConfig(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{ + Group: "", + Version: "v1", + }, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + cases := []struct { + name string + webhook MeshWebhook + pod *corev1.Pod + namespace corev1.Namespace + dnsEnabled bool + expCfg iptables.Config + expErr error + }{ + { + name: "basic bare minimum pod", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + }, + }, + { + name: "metrics enabled", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationEnableMetrics: "true", + annotationPrometheusScrapePort: "13373", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"13373"}, + }, + }, + { + name: "metrics enabled with incorrect annotation", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationEnableMetrics: "invalid", + annotationPrometheusScrapePort: "13373", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"13373"}, + }, + expErr: fmt.Errorf("%s annotation value of %s was invalid: %s", annotationEnableMetrics, "invalid", "strconv.ParseBool: parsing \"invalid\": invalid syntax"), + }, + { + name: "overwrite probes, transparent proxy annotation set", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationTransparentProxyOverwriteProbes: "true", + keyTransparentProxy: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(exposedPathsLivenessPortsRangeStart), + }, + }, + }, + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{strconv.Itoa(exposedPathsLivenessPortsRangeStart)}, + }, + }, + { + name: "exclude inbound ports", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationTProxyExcludeInboundPorts: "1111,11111", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"1111", "11111"}, + }, + }, + { + name: "exclude outbound ports", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationTProxyExcludeOutboundPorts: "2222,22222", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeOutboundPorts: []string{"2222", "22222"}, + }, + }, + { + name: "exclude outbound CIDRs", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationTProxyExcludeOutboundCIDRs: "3.3.3.3,3.3.3.3/24", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{strconv.Itoa(initContainersUserAndGroupID)}, + ExcludeOutboundCIDRs: []string{"3.3.3.3", "3.3.3.3/24"}, + }, + }, + { + name: "exclude UIDs", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationTProxyExcludeUIDs: "4444,44444", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"4444", "44444", strconv.Itoa(initContainersUserAndGroupID)}, + }, + }, + { + name: "exclude inbound ports, outbound ports, outbound CIDRs, and UIDs", + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + annotationTProxyExcludeInboundPorts: "1111,11111", + annotationTProxyExcludeOutboundPorts: "2222,22222", + annotationTProxyExcludeOutboundCIDRs: "3.3.3.3,3.3.3.3/24", + annotationTProxyExcludeUIDs: "4444,44444", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeInboundPorts: []string{"1111", "11111"}, + ExcludeOutboundPorts: []string{"2222", "22222"}, + ExcludeOutboundCIDRs: []string{"3.3.3.3", "3.3.3.3/24"}, + ExcludeUIDs: []string{"4444", "44444", strconv.Itoa(initContainersUserAndGroupID)}, + }, + }, + { + name: "dns enabled", + dnsEnabled: true, + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + ResourcePrefix: resourcePrefix, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + keyConsulDNS: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: dnsIP, + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{strconv.Itoa(initContainersUserAndGroupID)}, + }, + }, + { + name: "dns annotation set but environment variable missing", + dnsEnabled: false, + webhook: MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + ResourcePrefix: resourcePrefix, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + keyConsulDNS: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: dnsIP, + ProxyUserID: strconv.Itoa(envoyUserAndGroupID), + ProxyInboundPort: proxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{strconv.Itoa(initContainersUserAndGroupID)}, + }, + expErr: fmt.Errorf("environment variable %s not found", dnsEnvVariable), + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if c.dnsEnabled { + os.Setenv(dnsEnvVariable, dnsIP) + } else { + os.Setenv(dnsEnvVariable, "") + } + err := c.webhook.addRedirectTrafficConfigAnnotation(c.pod, c.namespace) + require.Equal(t, c.expErr, err) + + // Only compare annotation and iptables config on successful runs + if c.expErr == nil { + anno, ok := c.pod.Annotations[annotationRedirectTraffic] + require.Equal(t, ok, true) + + actualConfig := iptables.Config{} + json.Unmarshal([]byte(anno), &actualConfig) + require.Equal(t, c.expCfg, actualConfig) + } + }) + } +} diff --git a/control-plane/go.mod b/control-plane/go.mod index 7919804509..d9046e243f 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -10,11 +10,10 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/hashicorp/consul/api v1.10.1-0.20220822180451-60c82757ea35 github.com/hashicorp/consul/sdk v0.11.0 - github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220803144027-ad547faeffa5 github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f github.com/hashicorp/go-hclog v0.16.1 - github.com/hashicorp/go-multierror v1.1.0 - github.com/hashicorp/serf v0.9.6 + github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/serf v0.9.7 github.com/kr/text v0.2.0 github.com/miekg/dns v1.1.41 github.com/mitchellh/cli v1.1.0 diff --git a/control-plane/go.sum b/control-plane/go.sum index bdcd9b8a68..7ad24f0fc4 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -351,6 +351,7 @@ github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/S github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f h1:7WFMVeuJQp6BkzuTv9O52pzwtEFVUJubKYN+zez8eTI= github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f/go.mod h1:D4eo8/CN92vm9/9UDG+ldX1/fMFa4kpl8qzyTolus8o= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.1 h1:IVQwpTGNRRIHafnTs2dQLIk4ENtneRIEEJWOVDqz99o= github.com/hashicorp/go-hclog v0.16.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -360,7 +361,6 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= @@ -392,8 +392,6 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 h1:O/pT5C1Q3mVXMyuqg7yuAWUg/jMZR1/0QTzTRdNR6Uw= @@ -855,7 +853,6 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/control-plane/subcommand/install-cni/command_test.go b/control-plane/subcommand/install-cni/command_test.go index adb8f3b2c9..a7e97a4aa9 100644 --- a/control-plane/subcommand/install-cni/command_test.go +++ b/control-plane/subcommand/install-cni/command_test.go @@ -77,7 +77,7 @@ func TestRun_DirectoryWatcher(t *testing.T) { require.Equal(r, string(expected), string(actual)) }) - t.Log("File event 2: config file changed and consul-cni is not last in the plugin list.Should detect and fix.") + t.Log("File event 2: config file changed and consul-cni is not last in the plugin list. Should detect and fix.") err = replaceFile(notLastConfigFile, filepath.Join(tempDir, configFile)) require.NoError(t, err) time.Sleep(50 * time.Millisecond) From da211afb6482880473e0e71a80c0408c71a3454a Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Tue, 23 Aug 2022 23:48:06 -0400 Subject: [PATCH 09/10] bring in the rest of things from Irynas branch that I missed --- control-plane/build-support/functions/20-build.sh | 2 +- control-plane/connect-inject/container_init.go | 2 ++ control-plane/connect-inject/container_init_test.go | 3 +++ control-plane/go.mod | 3 ++- control-plane/go.sum | 7 ++++--- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/control-plane/build-support/functions/20-build.sh b/control-plane/build-support/functions/20-build.sh index f2f9f05c45..ddde7b6acf 100644 --- a/control-plane/build-support/functions/20-build.sh +++ b/control-plane/build-support/functions/20-build.sh @@ -287,7 +287,7 @@ function build_consul_local { else OS_BIN_EXTENSION="" fi - CGO_ENABLED=0 GOOS=${os} GOARCH=${arch} go install -ldflags "${GOLDFLAGS}" -tags "${GOTAGS}" && cp "${MAIN_GOPATH}/bin/${GOBIN_EXTRA}"/control-plane${OS_BIN_EXTENSION} "${outdir}/${bin_name}" + CGO_ENABLED=0 GOOS=${os} GOARCH=${arch} go build -ldflags "${GOLDFLAGS}" -tags "${GOTAGS}" -o "${outdir}/${bin_name}" if test $? -ne 0 then err "ERROR: Failed to build Consul for ${osarch}" diff --git a/control-plane/connect-inject/container_init.go b/control-plane/connect-inject/container_init.go index 3c94797d58..e0ef413011 100644 --- a/control-plane/connect-inject/container_init.go +++ b/control-plane/connect-inject/container_init.go @@ -315,6 +315,8 @@ func (w *MeshWebhook) containerInit(namespace corev1.Namespace, pod corev1.Pod, } } else { container.SecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64(initContainersUserAndGroupID), + RunAsGroup: pointer.Int64(initContainersUserAndGroupID), RunAsNonRoot: pointer.Bool(true), Privileged: pointer.Bool(false), Capabilities: &corev1.Capabilities{ diff --git a/control-plane/connect-inject/container_init_test.go b/control-plane/connect-inject/container_init_test.go index c24c2017e5..22f14c1f73 100644 --- a/control-plane/connect-inject/container_init_test.go +++ b/control-plane/connect-inject/container_init_test.go @@ -416,6 +416,9 @@ func TestHandlerContainerInit_transparentProxy(t *testing.T) { Add: []corev1.Capability{netAdminCapability}, } } else { + + expectedSecurityContext.RunAsUser = pointer.Int64(initContainersUserAndGroupID) + expectedSecurityContext.RunAsGroup = pointer.Int64(initContainersUserAndGroupID) expectedSecurityContext.RunAsNonRoot = pointer.Bool(true) expectedSecurityContext.Privileged = pointer.Bool(false) expectedSecurityContext.Capabilities = &corev1.Capabilities{ diff --git a/control-plane/go.mod b/control-plane/go.mod index d9046e243f..1be842ad95 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-logr/logr v0.4.0 github.com/google/go-cmp v0.5.7 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220824032443-9606072aec2a github.com/hashicorp/consul/api v1.10.1-0.20220822180451-60c82757ea35 github.com/hashicorp/consul/sdk v0.11.0 github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f @@ -19,7 +20,7 @@ require ( github.com/mitchellh/cli v1.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.4.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 go.uber.org/zap v1.19.0 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 diff --git a/control-plane/go.sum b/control-plane/go.sum index 7ad24f0fc4..b42a9dca6b 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -333,8 +333,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220803144027-ad547faeffa5 h1:JSJVjhdSWIunNiRYMDZLxY77lKP191TL9uoHlZ3dZd8= -github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220803144027-ad547faeffa5/go.mod h1:0pYEBqw/7L6UjMJvAxXe/0S4llxee+0DekXRwPhng2k= +github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220824032443-9606072aec2a h1:cBwGTYV84M0W8PwHX6+PywY8NiP3dzA7JaZzs9+LdiA= +github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220824032443-9606072aec2a/go.mod h1:aw35GB76URgbtxaSSMxbOetbG7YEHHPkIX3/SkTBaWc= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.10.1-0.20220822180451-60c82757ea35 h1:csNww5qBHaFqsX1eMEKVvmJ4dhqcXWj0sCkbccsSsHc= github.com/hashicorp/consul/api v1.10.1-0.20220822180451-60c82757ea35/go.mod h1:bcaw5CSZ7NE9qfOfKCI1xb7ZKjzu/MyvQkCLTfqLqxQ= @@ -627,8 +627,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tencentcloud/tencentcloud-sdk-go v3.0.83+incompatible h1:8uRvJleFpqLsO77WaAh2UrasMOzd8MxXrNj20e7El+Q= github.com/tencentcloud/tencentcloud-sdk-go v3.0.83+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= From 76092765cb5a6c47a00a86697cab48805b17d3b9 Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Wed, 24 Aug 2022 00:33:07 -0400 Subject: [PATCH 10/10] missing nodes from connect inject cluster role --- charts/consul/templates/connect-inject-clusterrole.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/consul/templates/connect-inject-clusterrole.yaml b/charts/consul/templates/connect-inject-clusterrole.yaml index 565f146d02..afb485c227 100644 --- a/charts/consul/templates/connect-inject-clusterrole.yaml +++ b/charts/consul/templates/connect-inject-clusterrole.yaml @@ -19,7 +19,7 @@ rules: - get {{- end }} - apiGroups: [ "" ] - resources: [ "endpoints", "services", "namespaces" ] + resources: [ "endpoints", "services", "namespaces", "nodes" ] verbs: - "get" - "list"