peering: expose servers over Kubernetes service

.circleci/config.yml

@@ -706,7 +706,7 @@ jobs:
       - run: mkdir -p $TEST_RESULTS
 
       - run-acceptance-tests:
-          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -disable-peering -enable-transparent-proxy
+          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-transparent-proxy
 
       - store_test_results:
           path: /tmp/test-results

acceptance/framework/consul/helm_cluster.go

@@ -510,7 +510,6 @@ func configureSCCs(t *testing.T, client kubernetes.Interface, cfg *config.TestCo
 func defaultValues() map[string]string {
 	values := map[string]string{
 		"server.replicas":              "1",
-		"server.bootstrapExpect":       "1",
 		"connectInject.envoyExtraArgs": "--log-level debug",
 		"connectInject.logLevel":       "debug",
 		// Disable DNS since enabling it changes the policy for the anonymous token,

acceptance/framework/consul/helm_cluster_test.go

@@ -24,7 +24,6 @@ func TestNewHelmCluster(t *testing.T) {
 			helmValues: map[string]string{},
 			want: map[string]string{
 				"global.image":                                  "test-config-image",
-				"server.bootstrapExpect":                        "1",
 				"server.replicas":                               "1",
 				"connectInject.envoyExtraArgs":                  "--log-level debug",
 				"connectInject.logLevel":                        "debug",

acceptance/framework/k8s/deploy.go

@@ -96,7 +96,7 @@ func CheckStaticServerConnectionMultipleFailureMessages(t *testing.T, options *k
 		expectedOutput = expectedSuccessOutput
 	}
 
-	retrier := &retry.Timer{Timeout: 80 * time.Second, Wait: 2 * time.Second}
+	retrier := &retry.Timer{Timeout: 160 * time.Second, Wait: 2 * time.Second}
 
 	args := []string{"exec", "deploy/" + sourceApp, "-c", sourceApp, "--", "curl", "-vvvsSf"}
 	args = append(args, curlArgs...)

acceptance/tests/partitions/partitions_connect_test.go

@@ -123,6 +123,7 @@ func TestPartitions_Connect(t *testing.T) {
 				serverHelmValues["global.adminPartitions.service.nodePort.https"] = "30000"
 				serverHelmValues["meshGateway.service.type"] = "NodePort"
 				serverHelmValues["meshGateway.service.nodePort"] = "30100"
+				serverHelmValues["server.exposeService.type"] = "NodePort"
 			}
 
 			releaseName := helpers.RandomName()

acceptance/tests/peering/peering_connect_namespaces_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"testing"
+	"time"
 
 	terratestk8s "github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
@@ -95,7 +96,7 @@ func TestPeering_ConnectNamespaces(t *testing.T) {
 				"global.peering.enabled":        "true",
 				"global.enableConsulNamespaces": "true",
 
-				"global.image": "thisisnotashwin/consul@sha256:b1d3f59406adf5fb9a3bee4ded058e619d3a186e83b2e2dc14d6da3f28a7073d",
+				"global.image": "ndhanushkodi/consul-dev:ent-backoff-fix",
 
 				"global.tls.enabled":           "true",
 				"global.tls.httpsOnly":         strconv.FormatBool(c.ACLsAndAutoEncryptEnabled),
@@ -122,13 +123,19 @@ func TestPeering_ConnectNamespaces(t *testing.T) {
 				"global.datacenter": staticServerPeer,
 			}
 
+			if !cfg.UseKind {
+				staticServerPeerHelmValues["server.replicas"] = "3"
+			}
+
 			// On Kind, there are no load balancers but since all clusters
 			// share the same node network (docker bridge), we can use
 			// a NodePort service so that we can access node(s) in a different Kind cluster.
 			if cfg.UseKind {
 				staticServerPeerHelmValues["server.exposeGossipAndRPCPorts"] = "true"
 				staticServerPeerHelmValues["meshGateway.service.type"] = "NodePort"
 				staticServerPeerHelmValues["meshGateway.service.nodePort"] = "30100"
+				staticServerPeerHelmValues["server.exposeService.type"] = "NodePort"
+				staticServerPeerHelmValues["server.exposeService.nodePort.grpc"] = "30200"
 			}
 
 			releaseName := helpers.RandomName()
@@ -143,10 +150,16 @@ func TestPeering_ConnectNamespaces(t *testing.T) {
 				"global.datacenter": staticClientPeer,
 			}
 
+			if !cfg.UseKind {
+				staticClientPeerHelmValues["server.replicas"] = "3"
+			}
+
 			if cfg.UseKind {
 				staticClientPeerHelmValues["server.exposeGossipAndRPCPorts"] = "true"
 				staticClientPeerHelmValues["meshGateway.service.type"] = "NodePort"
 				staticClientPeerHelmValues["meshGateway.service.nodePort"] = "30100"
+				staticClientPeerHelmValues["server.exposeService.type"] = "NodePort"
+				staticClientPeerHelmValues["server.exposeService.nodePort.grpc"] = "30200"
 			}
 
 			helpers.MergeMaps(staticClientPeerHelmValues, commonHelmValues)
@@ -162,7 +175,8 @@ func TestPeering_ConnectNamespaces(t *testing.T) {
 			})
 
 			// Ensure the secret is created.
-			retry.Run(t, func(r *retry.R) {
+			timer := &retry.Timer{Timeout: 1 * time.Minute, Wait: 1 * time.Second}
+			retry.RunWith(timer, t, func(r *retry.R) {
 				acceptorSecretResourceVersion, err := k8s.RunKubectlAndGetOutputE(t, staticClientPeerClusterContext.KubectlOptions(t), "get", "peeringacceptor", "server", "-o", "jsonpath={.status.secret.resourceVersion}")
 				require.NoError(r, err)
 				require.NotEmpty(r, acceptorSecretResourceVersion)

acceptance/tests/peering/peering_connect_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"testing"
+	"time"
 
 	terratestk8s "github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
@@ -54,7 +55,7 @@ func TestPeering_Connect(t *testing.T) {
 			commonHelmValues := map[string]string{
 				"global.peering.enabled": "true",
 
-				"global.image": "thisisnotashwin/consul@sha256:b1d3f59406adf5fb9a3bee4ded058e619d3a186e83b2e2dc14d6da3f28a7073d",
+				"global.image": "ndhanushkodi/consul-dev:ent-backoff-fix",
 
 				"global.tls.enabled":           "true",
 				"global.tls.httpsOnly":         strconv.FormatBool(c.ACLsAndAutoEncryptEnabled),
@@ -77,13 +78,19 @@ func TestPeering_Connect(t *testing.T) {
 				"global.datacenter": staticServerPeer,
 			}
 
+			if !cfg.UseKind {
+				staticServerPeerHelmValues["server.replicas"] = "3"
+			}
+
 			// On Kind, there are no load balancers but since all clusters
 			// share the same node network (docker bridge), we can use
 			// a NodePort service so that we can access node(s) in a different Kind cluster.
 			if cfg.UseKind {
 				staticServerPeerHelmValues["server.exposeGossipAndRPCPorts"] = "true"
 				staticServerPeerHelmValues["meshGateway.service.type"] = "NodePort"
 				staticServerPeerHelmValues["meshGateway.service.nodePort"] = "30100"
+				staticServerPeerHelmValues["server.exposeService.type"] = "NodePort"
+				staticServerPeerHelmValues["server.exposeService.nodePort.grpc"] = "30200"
 			}
 
 			releaseName := helpers.RandomName()
@@ -98,10 +105,16 @@ func TestPeering_Connect(t *testing.T) {
 				"global.datacenter": staticClientPeer,
 			}
 
+			if !cfg.UseKind {
+				staticServerPeerHelmValues["server.replicas"] = "3"
+			}
+
 			if cfg.UseKind {
 				staticClientPeerHelmValues["server.exposeGossipAndRPCPorts"] = "true"
 				staticClientPeerHelmValues["meshGateway.service.type"] = "NodePort"
 				staticClientPeerHelmValues["meshGateway.service.nodePort"] = "30100"
+				staticClientPeerHelmValues["server.exposeService.type"] = "NodePort"
+				staticClientPeerHelmValues["server.exposeService.nodePort.grpc"] = "30200"
 			}
 
 			helpers.MergeMaps(staticClientPeerHelmValues, commonHelmValues)
@@ -117,7 +130,8 @@ func TestPeering_Connect(t *testing.T) {
 			})
 
 			// Ensure the secret is created.
-			retry.Run(t, func(r *retry.R) {
+			timer := &retry.Timer{Timeout: 1 * time.Minute, Wait: 1 * time.Second}
+			retry.RunWith(timer, t, func(r *retry.R) {
 				acceptorSecretResourceVersion, err := k8s.RunKubectlAndGetOutputE(t, staticClientPeerClusterContext.KubectlOptions(t), "get", "peeringacceptor", "server", "-o", "jsonpath={.status.secret.resourceVersion}")
 				require.NoError(r, err)
 				require.NotEmpty(r, acceptorSecretResourceVersion)

acceptance/tests/vault/vault_partitions_test.go

@@ -338,6 +338,7 @@ func TestVault_Partitions(t *testing.T) {
 		serverHelmValues["global.adminPartitions.service.nodePort.https"] = "30000"
 		serverHelmValues["meshGateway.service.type"] = "NodePort"
 		serverHelmValues["meshGateway.service.nodePort"] = "30100"
+		serverHelmValues["server.exposeService.type"] = "NodePort"
 	}
 
 	helpers.MergeMaps(serverHelmValues, commonHelmValues)

charts/consul/templates/connect-inject-clusterrole.yaml

@@ -19,7 +19,7 @@ rules:
   - get
 {{- end }}
 - apiGroups: [ "" ]
-  resources: [ "pods", "endpoints", "services", "namespaces" ]
+  resources: [ "pods", "endpoints", "services", "namespaces", "nodes" ]
   verbs:
   - "get"
   - "list"

charts/consul/templates/connect-inject-deployment.yaml

@@ -11,6 +11,8 @@
 {{- if .Values.global.lifecycleSidecarContainer }}{{ fail "global.lifecycleSidecarContainer has been renamed to global.consulSidecarContainer. Please set values using global.consulSidecarContainer." }}{{ end }}
 {{ template "consul.validateVaultWebhookCertConfiguration" . }}
 {{- template "consul.reservedNamesFailer" (list .Values.connectInject.consulNamespaces.consulDestinationNamespace "connectInject.consulNamespaces.consulDestinationNamespace") }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- $serverExposeServiceEnabled := (or (and (ne (.Values.server.exposeService.enabled | toString) "-") .Values.server.exposeService.enabled) (and (eq (.Values.server.exposeService.enabled | toString) "-") (or .Values.global.peering.enabled .Values.global.adminPartitions.enabled))) -}}
 # The deployment for running the Connect sidecar injector
 apiVersion: apps/v1
 kind: Deployment
@@ -129,6 +131,7 @@ spec:
                 -consul-k8s-image="{{ default .Values.global.imageK8S .Values.connectInject.image }}" \
                 -release-name="{{ .Release.Name }}" \
                 -release-namespace="{{ .Release.Namespace }}" \
+                -resource-prefix={{ template "consul.fullname" . }} \
                 -listen=:8080 \
                 {{- if .Values.connectInject.transparentProxy.defaultEnabled }}
                 -default-enable-transparent-proxy=true \
@@ -137,6 +140,11 @@ spec:
                 {{- end }}
                 {{- if .Values.global.peering.enabled }}
                 -enable-peering=true \
+                {{- if (eq .Values.global.peering.tokenGeneration.serverAddresses.source "") }}
+                {{- if (and $serverEnabled $serverExposeServiceEnabled) }}
+                -read-server-expose-service=true \
+                {{- end }}
+                {{- end }}
                 {{- end }}
                 {{- if .Values.global.openshift.enabled }}
                 -enable-openshift \
@@ -146,7 +154,6 @@ spec:
                 {{- else }}
                 -transparent-proxy-default-overwrite-probes=false \
                 {{- end }}
-                -resource-prefix={{ template "consul.fullname" . }} \
                 {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }}
                 -enable-consul-dns=true \
                 {{- end }}

charts/consul/templates/expose-servers-service.yaml

@@ -0,0 +1,63 @@
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- $serverExposeServiceEnabled := (or (and (ne (.Values.server.exposeService.enabled | toString) "-") .Values.server.exposeService.enabled) (and (eq (.Values.server.exposeService.enabled | toString) "-") (or .Values.global.peering.enabled .Values.global.adminPartitions.enabled))) -}}
+{{- if (and $serverEnabled $serverExposeServiceEnabled) }}
+
+# Service with an external IP to reach Consul servers.
+# Used for exposing gRPC port for peering and ports for client partitions to discover servers.
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "consul.fullname" . }}-expose-servers
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: server
+  annotations:
+    {{- if .Values.server.exposeService.annotations }}
+    {{ tpl .Values.server.exposeService.annotations . | nindent 4 | trim }}
+    {{- end }}
+spec:
+  type: "{{ .Values.server.exposeService.type }}"
+  ports:
+    {{- if (or (not .Values.global.tls.enabled) (not .Values.global.tls.httpsOnly)) }}
+    - name: http
+      port: 8500
+      targetPort: 8500
+      {{ if (and (eq .Values.server.exposeService.type "NodePort") .Values.server.exposeService.nodePort.http) }}
+      nodePort: {{ .Values.server.exposeService.nodePort.http }}
+      {{- end }}
+    {{- end }}
+    {{- if .Values.global.tls.enabled }}
+    - name: https
+      port: 8501
+      targetPort: 8501
+      {{ if (and (eq .Values.server.exposeService.type "NodePort") .Values.server.exposeService.nodePort.https) }}
+      nodePort: {{ .Values.server.exposeService.nodePort.https }}
+      {{- end }}
+    {{- end }}
+    - name: serflan
+      port: 8301
+      targetPort: 8301
+      {{ if (and (eq .Values.server.exposeService.type "NodePort") .Values.server.exposeService.nodePort.serf) }}
+      nodePort: {{ .Values.server.exposeService.nodePort.serf }}
+      {{- end }}
+    - name: rpc
+      port: 8300
+      targetPort: 8300
+      {{ if (and (eq .Values.server.exposeService.type "NodePort") .Values.server.exposeService.nodePort.rpc) }}
+      nodePort: {{ .Values.server.exposeService.nodePort.rpc }}
+      {{- end }}
+    - name: grpc
+      port: 8502
+      targetPort: 8503
+      {{ if (and (eq .Values.server.exposeService.type "NodePort") .Values.server.exposeService.nodePort.grpc) }}
+      nodePort: {{ .Values.server.exposeService.nodePort.grpc }}
+      {{- end }}
+  selector:
+    app: {{ template "consul.name" . }}
+    release: "{{ .Release.Name }}"
+    component: server
+{{- end }}

charts/consul/test/unit/connect-inject-deployment.bats

@@ -1831,6 +1831,70 @@ EOF
   [[ "$output" =~ "setting global.peering.enabled to true requires connectInject.enabled to be true" ]]
 }
 
+@test "connectInject/Deployment: -read-server-expose-service=true is set when global.peering.enabled is true and global.peering.tokenGeneration.serverAddresses.source is empty" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'connectInject.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command | any(contains("-read-server-expose-service=true"))' | tee /dev/stderr)
+
+  [ "${actual}" = "true" ]
+}
+
+@test "connectInject/Deployment: -read-server-expose-service=true is set when servers are enabled and peering is enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'global.enabled=false' \
+      --set 'server.enabled=true' \
+      --set 'client.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command | any(contains("-read-server-expose-service=true"))' | tee /dev/stderr)
+
+  [ "${actual}" = "true" ]
+}
+
+@test "connectInject/Deployment: -read-server-expose-service is not set when servers are disabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'server.enabled=false' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command | any(contains("-read-server-expose-service=true"))' | tee /dev/stderr)
+
+  [ "${actual}" = "false" ]
+}
+
+@test "connectInject/Deployment: -read-server-expose-service is not set when peering is disabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'connectInject.enabled=true' \
+      --set 'global.peering.enabled=false' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command | any(contains("-read-server-expose-service=true"))' | tee /dev/stderr)
+
+  [ "${actual}" = "false" ]
+}
+
+@test "connectInject/Deployment: -read-server-expose-service is not set when global.peering.tokenGeneration.serverAddresses.source is not equal to empty string" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'connectInject.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      --set 'global.peering.tokenGeneration.serverAddresses.source="notempty"' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command | any(contains("-read-server-expose-service=true"))' | tee /dev/stderr)
+
+  [ "${actual}" = "false" ]
+}
 
 #--------------------------------------------------------------------
 # openshift