CNI Accptance test on Kind

.circleci/config.yml

@@ -48,7 +48,6 @@ commands:
             wget https://get.helm.sh/helm-v3.7.0-linux-amd64.tar.gz
             tar -zxvf helm-v3.7.0-linux-amd64.tar.gz
             sudo mv linux-amd64/helm /usr/local/bin/helm
-
   create-kind-clusters:
     parameters:
       version:
@@ -59,6 +58,18 @@ commands:
           command: |
             kind create cluster --name dc1 --image kindest/node:<< parameters.version >>
             kind create cluster --name dc2 --image kindest/node:<< parameters.version >>
+  create-kind-cni-clusters:
+    parameters:
+      version:
+        type: string
+    steps:
+      - run:
+          name: Create CNI kind clusters
+          command: |
+            kind create cluster --config=acceptance/framework/environment/cni-kind/kind.config --name dc1 --image kindest/node:<< parameters.version >>
+            make kind-cni-calico
+            kind create cluster --config=acceptance/framework/environment/cni-kind/kind.config --name dc2 --image kindest/node:<< parameters.version >>
+            make kind-cni-calico
   run-acceptance-tests:
     parameters:
       failfast:
@@ -547,6 +558,38 @@ jobs:
       - store_artifacts:
           path: /tmp/test-results
 
+  acceptance-tproxy-cni:
+    environment:
+      - TEST_RESULTS: /tmp/test-results
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: xlarge
+    parallelism: 6
+    steps:
+      - checkout
+      - install-prereqs
+      - create-kind-clusters:
+          version: "v1.22.4"
+      - restore_cache:
+          keys:
+            - consul-helm-modcache-v2-{{ checksum "acceptance/go.mod" }}
+      - run:
+          name: go mod download
+          working_directory: *acceptance-mod-path
+          command: go mod download
+      - save_cache:
+          key: consul-helm-modcache-v2-{{ checksum "acceptance/go.mod" }}
+          paths:
+            - ~/.go_workspace/pkg/mod
+      - run: mkdir -p $TEST_RESULTS
+      - run-acceptance-tests:
+          failfast: true
+          additional-flags: -use-kind -kubecontext="kind-dc1" -secondary-kubecontext="kind-dc2" -enable-transparent-proxy -enable-cni
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+
   ##########################
   # CLEANUP CLOUD RESOURCES
   ##########################
@@ -878,6 +921,42 @@ jobs:
           fail_only: true
           failure_message: "Acceptance tests against Kind with Kubernetes v1.23 failed. Check the logs at: ${CIRCLE_BUILD_URL}"
 
+  acceptance-kind-cni-1-23:
+    parallelism: 6 
+    environment:
+      - TEST_RESULTS: /tmp/test-results
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: xlarge
+    steps:
+      - checkout
+      - install-prereqs
+      - create-kind-cni-clusters:
+          version: "v1.23.0"
+      - restore_cache:
+          keys:
+            - consul-helm-modcache-v2-{{ checksum "acceptance/go.mod" }}
+      - run:
+          name: go mod download
+          working_directory: *acceptance-mod-path
+          command: go mod download
+      - save_cache:
+          key: consul-helm-modcache-v2-{{ checksum "acceptance/go.mod" }}
+          paths:
+            - ~/.go_workspace/pkg/mod
+      - run: mkdir -p $TEST_RESULTS
+      - run-acceptance-tests:
+          additional-flags: -use-kind -kubecontext="kind-dc1" -secondary-kubecontext="kind-dc2" -enable-transparent-proxy -enable-cni
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+      - slack/status:
+          # temporarily sending to #cni-acceptance-tests channel
+          channel: C03V3K0040G
+          fail_only: true
+          failure_message: "Acceptance tests for CNI against Kind with Kubernetes v1.23 failed. Check the logs at: ${CIRCLE_BUILD_URL}"
+
   acceptance-kind-1-23-consul-nightly-1-11:
     environment:
       - TEST_RESULTS: /tmp/test-results
@@ -1007,6 +1086,10 @@ workflows:
           context: consul-ci
           requires:
             - dev-upload-docker
+      - acceptance-tproxy-cni:
+          context: consul-ci
+          requires:
+            - dev-upload-docker
       - acceptance-tproxy:
           context: consul-ci
           requires:
@@ -1049,6 +1132,10 @@ workflows:
       - acceptance-kind-1-23:
           requires:
             - dev-upload-docker
+      - acceptance-kind-cni-1-23:
+          requires:
+            - dev-upload-docker
+
 
 #  nightly-acceptance-tests-consul:
 #    triggers:

Makefile

@@ -66,7 +66,14 @@ cni-plugin-lint:
 ctrl-generate: get-controller-gen ## Run CRD code generation.
 	cd control-plane; $(CONTROLLER_GEN) object:headerFile="build-support/controller/boilerplate.go.txt" paths="./..."
 
-
+# Helper target for doing local cni acceptance testing
+kind-cni:
+	kind delete cluster --name dc1
+	kind delete cluster --name dc2
+	kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc1 --image kindest/node:v1.23.6
+	make kind-cni-calico
+	kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc2 --image kindest/node:v1.23.6
+	make kind-cni-calico
 
 
 # ===========> CLI Targets
@@ -75,13 +82,20 @@ cli-lint: ## Run linter in the control-plane directory.
 	cd cli; golangci-lint run -c ../.golangci.yml
 
 
-
-
 # ===========> Acceptance Tests Targets
 
 acceptance-lint: ## Run linter in the control-plane directory.
 	cd acceptance; golangci-lint run -c ../.golangci.yml
 
+# For CNI acceptance tests, the calico CNI pluging needs to be installed on Kind. Our consul-cni plugin will not work 
+# without another plugin installed first
+kind-cni-calico:
+	kubectl create namespace calico-system ||true
+	kubectl create -f $(CURDIR)/acceptance/framework/environment/cni-kind/tigera-operator.yaml
+	# Sleeps are needed as installs can happen too quickly for Kind to handle it
+	@sleep 30
+	kubectl create -f $(CURDIR)/acceptance/framework/environment/cni-kind/custom-resources.yaml
+	@sleep 20 
 
 # ===========> Shared Targets
 

acceptance/framework/config/config.go

@@ -38,6 +38,8 @@ type TestConfig struct {
 
 	EnablePodSecurityPolicies bool
 
+	EnableCNI bool
+
 	EnableTransparentProxy bool
 
 	DisablePeering bool
@@ -83,6 +85,10 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) {
 		setIfNotEmpty(helmValues, "global.enablePodSecurityPolicies", "true")
 	}
 
+	if t.EnableCNI {
+		setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true")
+	}
+
 	setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy))
 
 	setIfNotEmpty(helmValues, "global.image", t.ConsulImage)

acceptance/framework/config/config_test.go

@@ -107,6 +107,16 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) {
 				"connectInject.transparentProxy.defaultEnabled": "true",
 			},
 		},
+		{
+			"sets connectInject.cni.enabled helm value to true when -enable-cni is set",
+			TestConfig{
+				EnableCNI: true,
+			},
+			map[string]string{
+				"connectInject.cni.enabled":                     "true",
+				"connectInject.transparentProxy.defaultEnabled": "false",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -146,7 +156,6 @@ func TestConfig_HelmValuesFromConfig_EntImage(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.consulImage, func(t *testing.T) {
-
 			// Write values.yaml to a temp dir which will then get parsed.
 			valuesYAML := fmt.Sprintf(`global:
   image: %s

acceptance/framework/environment/cni-kind/custom-resources.yaml

@@ -0,0 +1,27 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+    - blockSize: 26
+      cidr: 192.168.0.0/16
+      encapsulation: VXLANCrossSubnet
+      natOutgoing: Enabled
+      nodeSelector: all()
+
+---
+
+# This section configures the Calico API server.
+# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer 
+metadata: 
+  name: default 
+spec: {}
+

acceptance/framework/environment/cni-kind/kind.config

@@ -0,0 +1,10 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # Calicos default subnet. Needed for Calico to run on kind
+  podSubnet: 192.168.0.0/16
+  serviceSubnet: 10.110.0.0/16
+  # The default kind.net CNI plugin will not be installed
+  disableDefaultCNI: true 
+nodes:
+- role: control-plane