diff --git a/.circleci/config.yml b/.circleci/config.yml index 792776a2a3..4affddc3a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -716,6 +716,75 @@ jobs: fail_only: true failure_message: "GKE acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}" + acceptance-gke-cni-1-20: + parallelism: 6 + environment: + - TEST_RESULTS: /tmp/test-results + docker: + # This image is built from test/docker/Test.dockerfile + - image: docker.mirror.hashicorp.services/hashicorpdev/consul-helm-test:0.11.0 + + steps: + - run: + name: Exit if forked PR + command: | + if [ -n "$CIRCLE_PR_NUMBER" ]; then + echo "Skipping acceptance tests for forked PRs; marking step successful." + circleci step halt + fi + + - checkout + + - run: + name: terraform init & apply + working_directory: *gke-terraform-path + command: | + terraform init + echo "${GOOGLE_CREDENTIALS}" | gcloud auth activate-service-account --key-file=- + + # On GKE, we're setting the build number instead of build URL because label values + # cannot contain '/'. + terraform apply \ + -var project=${CLOUDSDK_CORE_PROJECT} \ + -var init_cli=true \ + -var cluster_count=2 \ + -var labels="{\"build_number\": \"$CIRCLE_BUILD_NUM\"}" \ + -auto-approve + + primary_kubeconfig=$(terraform output -json | jq -r .kubeconfigs.value[0]) + secondary_kubeconfig=$(terraform output -json | jq -r .kubeconfigs.value[1]) + + echo "export primary_kubeconfig=$primary_kubeconfig" >> $BASH_ENV + echo "export secondary_kubeconfig=$secondary_kubeconfig" >> $BASH_ENV + + # Restore go module cache if there is one + - restore_cache: + keys: + - consul-helm-acceptance-modcache-v2-{{ checksum "acceptance/go.mod" }} + + - run: mkdir -p $TEST_RESULTS + + - run-acceptance-tests: + additional-flags: -use-gke -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-pod-security-policies -enable-transparent-proxy -enable-cni + + - store_test_results: + path: /tmp/test-results + - store_artifacts: + path: /tmp/test-results + + - run: + name: terraform destroy + working_directory: *gke-terraform-path + command: | + terraform destroy -var project=${CLOUDSDK_CORE_PROJECT} -auto-approve + when: always + + - slack/status: + # temporarily sending to #cni-acceptance-tests channel + channel: C03V3K0040G + fail_only: true + failure_message: "GKE CNI acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}" + acceptance-aks-1-21: parallelism: 6 environment: @@ -922,7 +991,7 @@ jobs: failure_message: "Acceptance tests against Kind with Kubernetes v1.23 failed. Check the logs at: ${CIRCLE_BUILD_URL}" acceptance-kind-cni-1-23: - parallelism: 6 + parallelism: 6 environment: - TEST_RESULTS: /tmp/test-results machine: @@ -1072,10 +1141,10 @@ workflows: version: 2 test-and-build: jobs: - # Build this one control-plane binary so that acceptance and acceptance-tproxy will run - # The rest of these CircleCI jobs have been migrated to Github Actions. We need to wait until - # the summer of 2022 for larger puplic Github Action VMs be available before the acceptance tests can - # be moved + # Build this one control-plane binary so that acceptance and acceptance-tproxy will run + # The rest of these CircleCI jobs have been migrated to Github Actions. We need to wait until + # the summer of 2022 for larger puplic Github Action VMs be available before the acceptance tests can + # be moved - build-distro: OS: "linux" ARCH: "amd64 arm64" @@ -1124,6 +1193,10 @@ workflows: requires: - cleanup-gcp-resources - dev-upload-docker + - acceptance-gke-cni-1-20: + requires: + - cleanup-gcp-resources + - dev-upload-docker - acceptance-eks-1-19: requires: - cleanup-eks-resources @@ -1138,7 +1211,9 @@ workflows: - acceptance-kind-cni-1-23: requires: - dev-upload-docker - + - acceptance-kind-cni-1-23: + requires: + - dev-upload-docker nightly-acceptance-tests-consul: triggers: diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index 4c4d9036aa..8177347a7a 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -54,6 +54,7 @@ type TestConfig struct { DebugDirectory string UseKind bool + UseGKE bool helmChartPath string } @@ -88,6 +89,10 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { if t.EnableCNI { setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true") + // GKE is currently the only cloud provider that uses a different CNI bin dir. + if t.UseGKE { + setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin") + } } setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy)) diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index d65372ebfa..81f8131efb 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -42,6 +42,7 @@ type TestFlags struct { flagDebugDirectory string flagUseKind bool + flagUseGKE bool flagDisablePeering bool @@ -106,6 +107,9 @@ func (t *TestFlags) init() { flag.BoolVar(&t.flagUseKind, "use-kind", false, "If true, the tests will assume they are running against a local kind cluster(s).") + flag.BoolVar(&t.flagUseGKE, "use-gke", false, + "If true, the tests will assume they are running against a GKE cluster(s).") + flag.BoolVar(&t.flagDisablePeering, "disable-peering", false, "If true, the peering tests will not run.") @@ -165,5 +169,6 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { NoCleanupOnFailure: t.flagNoCleanupOnFailure, DebugDirectory: tempDir, UseKind: t.flagUseKind, + UseGKE: t.flagUseGKE, } } diff --git a/charts/consul/templates/cni-clusterrole.yaml b/charts/consul/templates/cni-clusterrole.yaml index 744a4af864..84551205c0 100644 --- a/charts/consul/templates/cni-clusterrole.yaml +++ b/charts/consul/templates/cni-clusterrole.yaml @@ -20,4 +20,11 @@ rules: - watch - patch - update +- apiGroups: ["policy"] + resources: + - podsecuritypolicies + resourceNames: + - {{ template "consul.fullname" . }}-cni + verbs: + - use {{- end }} diff --git a/charts/consul/templates/cni-podsecuritypolicy.yaml b/charts/consul/templates/cni-podsecuritypolicy.yaml index ddc75913ae..15b96bc230 100644 --- a/charts/consul/templates/cni-podsecuritypolicy.yaml +++ b/charts/consul/templates/cni-podsecuritypolicy.yaml @@ -12,8 +12,8 @@ metadata: component: cni spec: privileged: true - # Required to prevent escalations to root. - allowPrivilegeEscalation: false + # GKE requires that allowPrivilegeEscalation:true if privileged: true. + allowPrivilegeEscalation: true volumes: - hostPath - secret