CNI Accptance test on Kind

.circleci/config.yml

@@ -48,7 +48,6 @@ commands:
             wget https://get.helm.sh/helm-v3.7.0-linux-amd64.tar.gz
             tar -zxvf helm-v3.7.0-linux-amd64.tar.gz
             sudo mv linux-amd64/helm /usr/local/bin/helm
-
   create-kind-clusters:
     parameters:
       version:
@@ -59,6 +58,18 @@ commands:
           command: |
             kind create cluster --name dc1 --image kindest/node:<< parameters.version >>
             kind create cluster --name dc2 --image kindest/node:<< parameters.version >>
+  create-kind-cni-clusters:
+    parameters:
+      version:
+        type: string
+    steps:
+      - run:
+          name: Create CNI kind clusters
+          command: |
+            kind create cluster --config=acceptance/framework/environment/cni-kind/kind.config --name dc1 --image kindest/node:<< parameters.version >>
+            make kind-cni-calico
+            kind create cluster --config=acceptance/framework/environment/cni-kind/kind.config --name dc2 --image kindest/node:<< parameters.version >>
+            make kind-cni-calico
   run-acceptance-tests:
     parameters:
       failfast:
@@ -878,6 +889,42 @@ jobs:
           fail_only: true
           failure_message: "Acceptance tests against Kind with Kubernetes v1.23 failed. Check the logs at: ${CIRCLE_BUILD_URL}"
 
+  acceptance-kind-cni-1-23:
+    parallelism: 6 
+    environment:
+      - TEST_RESULTS: /tmp/test-results
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: xlarge
+    steps:
+      - checkout
+      - install-prereqs
+      - create-kind-cni-clusters:
+          version: "v1.23.0"
+      - restore_cache:
+          keys:
+            - consul-helm-modcache-v2-{{ checksum "acceptance/go.mod" }}
+      - run:
+          name: go mod download
+          working_directory: *acceptance-mod-path
+          command: go mod download
+      - save_cache:
+          key: consul-helm-modcache-v2-{{ checksum "acceptance/go.mod" }}
+          paths:
+            - ~/.go_workspace/pkg/mod
+      - run: mkdir -p $TEST_RESULTS
+      - run-acceptance-tests:
+          additional-flags: -use-kind -kubecontext="kind-dc1" -secondary-kubecontext="kind-dc2" -enable-transparent-proxy -enable-cni
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+      - slack/status:
+          # temporarily sending to #cni-acceptance-tests channel
+          channel: C03V3K0040G
+          fail_only: true
+          failure_message: "Acceptance tests for CNI against Kind with Kubernetes v1.23 failed. Check the logs at: ${CIRCLE_BUILD_URL}"
+
   acceptance-kind-1-23-consul-nightly-1-11:
     environment:
       - TEST_RESULTS: /tmp/test-results
@@ -1007,18 +1054,22 @@ workflows:
           context: consul-ci
           requires:
             - dev-upload-docker
+      - acceptance-kind-cni-1-23:
+          context: consul-ci
+          requires:
+            - dev-upload-docker
       - acceptance-tproxy:
           context: consul-ci
           requires:
             - dev-upload-docker
   nightly-acceptance-tests:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - main
+    # triggers:
+    #   - schedule:
+    #       cron: "0 0 * * *"
+    #       filters:
+    #         branches:
+    #           only:
+    #             - main
     jobs:
       - build-distro:
           OS: "linux"
@@ -1027,29 +1078,33 @@ workflows:
       - dev-upload-docker:
           requires:
             - build-distros-linux
-      - cleanup-gcp-resources
-      - cleanup-azure-resources
-      - cleanup-eks-resources
+      # - cleanup-gcp-resources
+      # - cleanup-azure-resources
+      # - cleanup-eks-resources
       # Disable until we can use UBI images.
       #      - acceptance-openshift:
       #          requires:
       #          - cleanup-azure-resources
-      - acceptance-gke-1-20:
-          requires:
-            - cleanup-gcp-resources
-            - dev-upload-docker
-      - acceptance-eks-1-19:
-          requires:
-            - cleanup-eks-resources
-            - dev-upload-docker
-      - acceptance-aks-1-21:
-          requires:
-            - cleanup-azure-resources
-            - dev-upload-docker
-      - acceptance-kind-1-23:
+      # - acceptance-gke-1-20:
+      #     requires:
+      #       - cleanup-gcp-resources
+      #       - dev-upload-docker
+      # - acceptance-eks-1-19:
+      #     requires:
+      #       - cleanup-eks-resources
+      #       - dev-upload-docker
+      # - acceptance-aks-1-21:
+      #     requires:
+      #       - cleanup-azure-resources
+      #       - dev-upload-docker
+      # - acceptance-kind-1-23:
+      #     requires:
+      #       - dev-upload-docker
+      - acceptance-kind-cni-1-23:
           requires:
             - dev-upload-docker
 
+
 #  nightly-acceptance-tests-consul:
 #    triggers:
 #      - schedule:

Makefile

@@ -66,7 +66,14 @@ cni-plugin-lint:
 ctrl-generate: get-controller-gen ## Run CRD code generation.
 	cd control-plane; $(CONTROLLER_GEN) object:headerFile="build-support/controller/boilerplate.go.txt" paths="./..."
 
-
+# Helper target for doing local cni acceptance testing
+kind-cni:
+	kind delete cluster --name dc1
+	kind delete cluster --name dc2
+	kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc1 --image kindest/node:v1.23.6
+	make kind-cni-calico
+	kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc2 --image kindest/node:v1.23.6
+	make kind-cni-calico
 
 
 # ===========> CLI Targets
@@ -75,13 +82,20 @@ cli-lint: ## Run linter in the control-plane directory.
 	cd cli; golangci-lint run -c ../.golangci.yml
 
 
-
-
 # ===========> Acceptance Tests Targets
 
 acceptance-lint: ## Run linter in the control-plane directory.
 	cd acceptance; golangci-lint run -c ../.golangci.yml
 
+# For CNI acceptance tests, the calico CNI pluging needs to be installed on Kind. Our consul-cni plugin will not work 
+# without another plugin installed first
+kind-cni-calico:
+	kubectl create namespace calico-system ||true
+	kubectl create -f https://docs.projectcalico.org/archive/v3.24/manifests/tigera-operator.yaml
+	# Sleeps are needed as installs can happen too quickly for Kind to handle it
+	@sleep 30
+	kubectl create -f https://docs.projectcalico.org/archive/v3.24/manifests/custom-resources.yaml
+	@sleep 20 
 
 # ===========> Shared Targets
 

acceptance/framework/config/config.go

@@ -38,6 +38,8 @@ type TestConfig struct {
 
 	EnablePodSecurityPolicies bool
 
+	EnableCNI bool
+
 	EnableTransparentProxy bool
 
 	DisablePeering bool
@@ -83,6 +85,10 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) {
 		setIfNotEmpty(helmValues, "global.enablePodSecurityPolicies", "true")
 	}
 
+	if t.EnableCNI {
+		setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true")
+	}
+
 	setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy))
 
 	setIfNotEmpty(helmValues, "global.image", t.ConsulImage)

acceptance/framework/config/config_test.go

@@ -107,6 +107,16 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) {
 				"connectInject.transparentProxy.defaultEnabled": "true",
 			},
 		},
+		{
+			"sets connectInject.cni.enabled helm value to true when -enable-cni is set",
+			TestConfig{
+				EnableCNI: true,
+			},
+			map[string]string{
+				"connectInject.cni.enabled":                     "true",
+				"connectInject.transparentProxy.defaultEnabled": "false",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -146,7 +156,6 @@ func TestConfig_HelmValuesFromConfig_EntImage(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.consulImage, func(t *testing.T) {
-
 			// Write values.yaml to a temp dir which will then get parsed.
 			valuesYAML := fmt.Sprintf(`global:
   image: %s

acceptance/framework/environment/cni-kind/kind.config

@@ -0,0 +1,10 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # Calicos default subnet. Needed for Calico to run on kind
+  podSubnet: 192.168.0.0/16
+  serviceSubnet: 10.110.0.0/16
+  # The default kind.net CNI plugin will not be installed
+  disableDefaultCNI: true 
+nodes:
+- role: control-plane

acceptance/framework/flags/flags.go

@@ -27,6 +27,8 @@ type TestFlags struct {
 
 	flagEnablePodSecurityPolicies bool
 
+	flagEnableCNI bool
+
 	flagEnableTransparentProxy bool
 
 	flagConsulImage    string
@@ -85,6 +87,10 @@ func (t *TestFlags) init() {
 	flag.BoolVar(&t.flagEnablePodSecurityPolicies, "enable-pod-security-policies", false,
 		"If true, the test suite will run tests with pod security policies enabled.")
 
+	flag.BoolVar(&t.flagEnableCNI, "enable-cni", false,
+		"If true, the test suite will run tests with consul-cni plugin enabled. "+
+			"In general, this will only run against tests that are mesh related (connect, mesh-gateway, peering, etc")
+
 	flag.BoolVar(&t.flagEnableTransparentProxy, "enable-transparent-proxy", false,
 		"If true, the test suite will run tests with transparent proxy enabled. "+
 			"This applies only to tests that enable connectInject.")
@@ -142,6 +148,8 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig {
 
 		EnablePodSecurityPolicies: t.flagEnablePodSecurityPolicies,
 
+		EnableCNI: t.flagEnableCNI,
+
 		EnableTransparentProxy: t.flagEnableTransparentProxy,
 
 		DisablePeering: t.flagDisablePeering,

acceptance/tests/basic/basic_test.go

@@ -19,6 +19,10 @@ import (
 // servers and clients, works by creating a kv entry
 // and subsequently reading it from Consul.
 func TestBasicInstallation(t *testing.T) {
+	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and installing CNI is not a basic installation")
+	}
 	cases := []struct {
 		secure      bool
 		autoEncrypt bool

acceptance/tests/consul-dns/consul_dns_test.go

@@ -17,6 +17,10 @@ import (
 const podName = "dns-pod"
 
 func TestConsulDNS(t *testing.T) {
+	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set")
+	}
 	for _, secure := range []bool{false, true} {
 		name := fmt.Sprintf("secure: %t", secure)
 		t.Run(name, func(t *testing.T) {

acceptance/tests/controller/controller_namespaces_test.go

@@ -33,6 +33,9 @@ const (
 // and non-auto-encrypt secure installations, so testing just one is enough.
 func TestControllerNamespaces(t *testing.T) {
 	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and controller is already tested with regular tproxy")
+	}
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}

acceptance/tests/controller/controller_test.go

@@ -28,7 +28,9 @@ const (
 
 func TestController(t *testing.T) {
 	cfg := suite.Config()
-
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and controller is already tested with regular tproxy")
+	}
 	cases := []struct {
 		secure      bool
 		autoEncrypt bool

acceptance/tests/partitions/partitions_sync_test.go

@@ -24,6 +24,9 @@ func TestPartitions_Sync(t *testing.T) {
 	env := suite.Environment()
 	cfg := suite.Config()
 
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set")
+	}
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}

acceptance/tests/snapshot-agent/snapshot_agent_k8s_secret_test.go

@@ -28,6 +28,9 @@ import (
 // a command line arg or an environment variable.
 func TestSnapshotAgent_K8sSecret(t *testing.T) {
 	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and snapshot agent is already tested with regular tproxy")
+	}
 	ctx := suite.Environment().DefaultContext(t)
 	kubectlOptions := ctx.KubectlOptions(t)
 	ns := kubectlOptions.Namespace

acceptance/tests/snapshot-agent/snapshot_agent_vault_test.go

@@ -27,6 +27,9 @@ import (
 // a command line arg or an environment variable.
 func TestSnapshotAgent_Vault(t *testing.T) {
 	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and snapshot agent is already tested with regular tproxy")
+	}
 	ctx := suite.Environment().DefaultContext(t)
 	kubectlOptions := ctx.KubectlOptions(t)
 	ns := kubectlOptions.Namespace

acceptance/tests/sync/sync_catalog_namespaces_test.go

@@ -25,6 +25,9 @@ const staticServerService = "static-server"
 // and non-auto-encrypt secure installations, so testing just one is enough.
 func TestSyncCatalogNamespaces(t *testing.T) {
 	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and sync catalog is already tested with regular tproxy")
+	}
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}

acceptance/tests/sync/sync_catalog_test.go

@@ -18,6 +18,10 @@ import (
 // The test will create a test service and a pod and will
 // wait for the service to be synced *to* consul.
 func TestSyncCatalog(t *testing.T) {
+	cfg := suite.Config()
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and sync catalog is already tested with regular tproxy")
+	}
 	cases := []struct {
 		name       string
 		helmValues map[string]string

charts/consul/test/unit/cni-daemonset.bats

@@ -135,7 +135,7 @@ rollingUpdate:
       --set 'connectInject.enabled=true' \
       . | tee /dev/stderr |
       yq -rc '.spec.template.spec.containers[0].resources' | tee /dev/stderr)
-  [ "${actual}" = '{"limits":{"cpu":"75m","memory":"75Mi"},"requests":{"cpu":"50m","memory":"50Mi"}}' ]
+  [ "${actual}" = '{"limits":{"cpu":"100m","memory":"100Mi"},"requests":{"cpu":"75m","memory":"75Mi"}}' ]
 }
 
 @test "cni/DaemonSet: resources can be overridden" {

charts/consul/test/unit/connect-inject-clusterrole.bats

@@ -32,7 +32,7 @@ load _helpers
 #--------------------------------------------------------------------
 # rules
 
-@test "connectInject/ClusterRole: sets get, list, and watch access to endpoints, services, and namespaces in all api groups" {
+@test "connectInject/ClusterRole: sets get, list, and watch access to endpoints, services, namespaces and nodes in all api groups" {
   cd `chart_dir`
   local object=$(helm template \
       -s templates/connect-inject-clusterrole.yaml  \
@@ -51,6 +51,9 @@ load _helpers
   local actual=$(echo $object | yq -r '.resources[| index("namespaces")' | tee /dev/stderr)
   [ "${actual}" != null ]
 
+  local actual=$(echo $object | yq -r '.resources[| index("nodes")' | tee /dev/stderr)
+  [ "${actual}" != null ]
+
   local actual=$(echo $object | yq -r '.apiGroups[0]' | tee /dev/stderr)
   [ "${actual}" = "" ]