diff --git a/.changelog/3693.txt b/.changelog/3693.txt new file mode 100644 index 0000000000..b26e6da0a4 --- /dev/null +++ b/.changelog/3693.txt @@ -0,0 +1,3 @@ +```release-note:improvement +catalog: Topology zone and region information is now read from the Kubernetes endpoints and associated node and added to registered consul services under Metadata. +``` \ No newline at end of file diff --git a/.changelog/3718.txt b/.changelog/3718.txt new file mode 100644 index 0000000000..9e7cd4f59a --- /dev/null +++ b/.changelog/3718.txt @@ -0,0 +1,4 @@ +```release-note:breaking-change +api-gateway: The api-gateway stanza located under .Values.api-gateway was deprecated in +1.16.0 of Consul and is being removed as of 1.19.0 in favor of connectInject.apiGateway. +``` \ No newline at end of file diff --git a/.changelog/3741.txt b/.changelog/3741.txt new file mode 100644 index 0000000000..496f73597b --- /dev/null +++ b/.changelog/3741.txt @@ -0,0 +1,14 @@ +```release-note:security +Upgrade to use Go `1.21.8`. This resolves CVEs +[CVE-2024-24783](https://nvd.nist.gov/vuln/detail/CVE-2024-24783) (`crypto/x509`). +[CVE-2023-45290](https://nvd.nist.gov/vuln/detail/CVE-2023-45290) (`net/http`). +[CVE-2023-45289](https://nvd.nist.gov/vuln/detail/CVE-2023-45289) (`net/http`, `net/http/cookiejar`). +[CVE-2024-24785](https://nvd.nist.gov/vuln/detail/CVE-2024-24785) (`html/template`). +[CVE-2024-24784](https://nvd.nist.gov/vuln/detail/CVE-2024-24784) (`net/mail`). +``` + +```release-note:security +Update the Consul Build Go base image to `alpine3.19`. This resolves CVEs +[CVE-2023-52425](https://nvd.nist.gov/vuln/detail/CVE-2023-52425) +[CVE-2023-52426⁠](https://nvd.nist.gov/vuln/detail/CVE-2023-52426) +``` \ No newline at end of file diff --git a/.changelog/3758.txt b/.changelog/3758.txt new file mode 100644 index 0000000000..5c4c528f04 --- /dev/null +++ b/.changelog/3758.txt @@ -0,0 +1,4 @@ +```release-note:bug +control-plane: fix an issue where ACL tokens would prematurely be deleted and services would be deregistered if there +was a K8s API error fetching the pod. +``` diff --git a/.changelog/3779.txt b/.changelog/3779.txt new file mode 100644 index 0000000000..946fcca208 --- /dev/null +++ b/.changelog/3779.txt @@ -0,0 +1,3 @@ +```release-note:bug +api-gateway: Fix order of initialization for creating ACL role/policy to avoid error logs in consul. +``` diff --git a/.changelog/3811.txt b/.changelog/3811.txt new file mode 100644 index 0000000000..e333a9df84 --- /dev/null +++ b/.changelog/3811.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +api-gateway: Expose prometheus scrape metrics on api-gateway pods. +``` diff --git a/.go-version b/.go-version index 8819d012ce..428abfd24f 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.21.7 +1.21.8 diff --git a/.release/security-scan.hcl b/.release/security-scan.hcl index 257a2954a2..5ae5e513d2 100644 --- a/.release/security-scan.hcl +++ b/.release/security-scan.hcl @@ -39,6 +39,8 @@ binary { # NET-8174 (2024-02-26): Missing YAML Content Leads To Panic (requires malicious plugin) "GHSA-r53h-jv2g-vpx6", "CVE-2024-26147", # alias + "GHSA-jw44-4f3j-q396", # Tracked in NET-8174 + "CVE-2019-25210" # alias ] } } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b06c27d8a..562f9c89ed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ 1. [Webhook](#webhook) 1. [Update command.go](#update-commandgo) 1. [Generating YAML](#generating-yaml) - 1. [Updating consul-helm](#updating-consul-helm) + 1. [Updating consul-helm](#updating-helm-chart) 1. [Testing a new CRD](#testing-a-new-crd) 1. [Update Consul K8s acceptance tests](#update-consul-k8s-acceptance-tests) 1. [Adding a new ACL Token](#adding-a-new-acl-token) diff --git a/acceptance/framework/consul/helm_cluster.go b/acceptance/framework/consul/helm_cluster.go index 55239eec03..52cf7683c1 100644 --- a/acceptance/framework/consul/helm_cluster.go +++ b/acceptance/framework/consul/helm_cluster.go @@ -130,6 +130,9 @@ func NewHelmCluster( func (h *HelmCluster) Create(t *testing.T) { t.Helper() + // check and remove any CRDs with finalizers + helpers.GetCRDRemoveFinalizers(t, h.helmOptions.KubectlOptions) + // Make sure we delete the cluster if we receive an interrupt signal and // register cleanup so that we delete the cluster when test finishes. helpers.Cleanup(t, h.noCleanupOnFailure, h.noCleanup, func() { diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go index 4b8e98034a..0871532426 100644 --- a/acceptance/framework/helpers/helpers.go +++ b/acceptance/framework/helpers/helpers.go @@ -16,6 +16,7 @@ import ( "time" "github.com/gruntwork-io/terratest/modules/helm" + "github.com/gruntwork-io/terratest/modules/k8s" terratestLogger "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" @@ -196,42 +197,55 @@ type Command struct { Logger *terratestLogger.Logger } -func RunCommand(t testutil.TestingTB, command Command) (string, error) { +type cmdResult struct { + output string + err error +} + +func RunCommand(t testutil.TestingTB, options *k8s.KubectlOptions, command Command) (string, error) { t.Helper() - cmd, err := exec.Command(command.Command, command.Args...).CombinedOutput() - // Check and remove finalizers in crds in the namespace + resultCh := make(chan *cmdResult, 1) + + go func() { + output, err := exec.Command(command.Command, command.Args...).CombinedOutput() + resultCh <- &cmdResult{output: string(output), err: err} + }() + + // might not be needed for _, arg := range command.Args { if strings.Contains(arg, "delete") { - errCh := make(chan error) go func() { - errCh <- getCRDRemoveFinalizers(t) + GetCRDRemoveFinalizers(t, options) }() - if err := <-errCh; err != nil { - return "", err - } } } - return string(cmd), err + select { + case res := <-resultCh: + if res.err != nil { + logger.Logf(t, "Output: %v.", res.output) + } + return res.output, res.err + // Sometimes this func runs for too long handle timeout if needed. + case <-time.After(320 * time.Second): + GetCRDRemoveFinalizers(t, options) + logger.Logf(t, "RunCommand timed out") + return "", nil + } } // getCRDRemoveFinalizers gets CRDs with finalizers and removes them. -func getCRDRemoveFinalizers(t testutil.TestingTB) error { +func GetCRDRemoveFinalizers(t testutil.TestingTB, options *k8s.KubectlOptions) { t.Helper() - // Get CRD names with finalizers - crdNames, err := getCRDsWithFinalizers() + crdNames, err := getCRDsWithFinalizers(options) if err != nil { - return err + logger.Logf(t, "Unable to get CRDs with finalizers, %v.", err) } - // Remove finalizers for each CRD with finalizers if len(crdNames) > 0 { - if err := removeFinalizers(crdNames); err != nil { - return err - } + removeFinalizers(t, options, crdNames) } - return nil } // CRD struct to parse CRD JSON output. @@ -244,15 +258,19 @@ type CRD struct { } `json:"items"` } -// getCRDsWithFinalizers gets CRDs with finalizers. -func getCRDsWithFinalizers() ([]string, error) { - cmd := exec.Command("kubectl", "get", "crd", "-o=json") +func getCRDsWithFinalizers(options *k8s.KubectlOptions) ([]string, error) { + cmdArgs := createCmdArgs(options) + args := []string{"get", "crd", "-o=json"} - output, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("error executing command: %v", err) + cmdArgs = append(cmdArgs, args...) + command := Command{ + Command: "kubectl", + Args: cmdArgs, + Env: options.Env, } + output, err := exec.Command(command.Command, command.Args...).CombinedOutput() + var crds CRD if err := json.Unmarshal(output, &crds); err != nil { return nil, fmt.Errorf("error parsing JSON: %v", err) @@ -265,19 +283,40 @@ func getCRDsWithFinalizers() ([]string, error) { } } - return crdNames, nil + return crdNames, err } // removeFinalizers removes finalizers from CRDs. -func removeFinalizers(crdNames []string) error { +func removeFinalizers(t testutil.TestingTB, options *k8s.KubectlOptions, crdNames []string) { + cmdArgs := createCmdArgs(options) for _, crd := range crdNames { - cmd := exec.Command("kubectl", "patch", "crd", crd, "--type=json", "-p=[{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}]") + args := []string{"patch", "crd", crd, "--type=json", "-p=[{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}]"} - err := cmd.Run() + cmdArgs = append(cmdArgs, args...) + command := Command{ + Command: "kubectl", + Args: cmdArgs, + Env: options.Env, + } + + _, err := exec.Command(command.Command, command.Args...).CombinedOutput() if err != nil { - return fmt.Errorf("error removing finalizers from CRD %s: %v", crd, err) + logger.Logf(t, "Unable to remove finalizers, proceeding anyway: %v.", err) } fmt.Printf("Finalizers removed from CRD %s\n", crd) } - return nil +} + +func createCmdArgs(options *k8s.KubectlOptions) []string { + var cmdArgs []string + if options.ContextName != "" { + cmdArgs = append(cmdArgs, "--context", options.ContextName) + } + if options.ConfigPath != "" { + cmdArgs = append(cmdArgs, "--kubeconfig", options.ConfigPath) + } + if options.Namespace != "" { + cmdArgs = append(cmdArgs, "--namespace", options.Namespace) + } + return cmdArgs } diff --git a/acceptance/framework/k8s/deploy.go b/acceptance/framework/k8s/deploy.go index 3bb5465298..e1d9f01a80 100644 --- a/acceptance/framework/k8s/deploy.go +++ b/acceptance/framework/k8s/deploy.go @@ -129,10 +129,7 @@ func CheckStaticServerConnectionMultipleFailureMessages(t *testing.T, options *k expectedOutput = expectedSuccessOutput } - retrier := &retry.Counter{ - Count: 10, - Wait: 2 * time.Second, - } + retrier := &retry.Counter{Count: 30, Wait: 2 * time.Second} args := []string{"exec", resourceType + sourceApp, "-c", sourceApp, "--", "curl", "-vvvsSf"} args = append(args, curlArgs...) diff --git a/acceptance/framework/k8s/kubectl.go b/acceptance/framework/k8s/kubectl.go index c94caaa86e..4fe54b9d5c 100644 --- a/acceptance/framework/k8s/kubectl.go +++ b/acceptance/framework/k8s/kubectl.go @@ -68,7 +68,7 @@ func RunKubectlAndGetOutputWithLoggerE(t testutil.TestingTB, options *k8s.Kubect var output string var err error retry.RunWith(counter, t, func(r *retry.R) { - output, err = helpers.RunCommand(r, command) + output, err = helpers.RunCommand(r, options, command) if err != nil { // Want to retry on errors connecting to actual Kube API because // these are intermittent. diff --git a/acceptance/tests/api-gateway/api_gateway_test.go b/acceptance/tests/api-gateway/api_gateway_test.go index bbbaa569d4..13cf517382 100644 --- a/acceptance/tests/api-gateway/api_gateway_test.go +++ b/acceptance/tests/api-gateway/api_gateway_test.go @@ -107,11 +107,14 @@ func TestAPIGateway_Basic(t *testing.T) { logger.Log(t, "creating static-client pod") k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.RunKubectl(t, ctx.KubectlOptions(t), "wait", "--for=condition=available", "--timeout=5m", fmt.Sprintf("deploy/%s", "static-server")) + logger.Log(t, "patching route to target http server") k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"name":"static-server","port":80}]}]}}`, "--type=merge") logger.Log(t, "creating target tcp server") k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server-tcp") + k8s.RunKubectl(t, ctx.KubectlOptions(t), "wait", "--for=condition=available", "--timeout=5m", fmt.Sprintf("deploy/%s", "static-server-tcp")) logger.Log(t, "creating tcp-route") k8s.RunKubectl(t, ctx.KubectlOptions(t), "apply", "-f", "../fixtures/cases/api-gateways/tcproute/route.yaml") diff --git a/acceptance/tests/connect/connect_proxy_lifecycle_test.go b/acceptance/tests/connect/connect_proxy_lifecycle_test.go index 7a4b3b383f..95502c9869 100644 --- a/acceptance/tests/connect/connect_proxy_lifecycle_test.go +++ b/acceptance/tests/connect/connect_proxy_lifecycle_test.go @@ -11,14 +11,15 @@ import ( "testing" "time" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/hashicorp/consul-k8s/acceptance/framework/connhelper" "github.com/hashicorp/consul-k8s/acceptance/framework/consul" "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" - "github.com/hashicorp/consul/sdk/testutil/retry" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type LifecycleShutdownConfig struct { @@ -36,24 +37,22 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { cfg := suite.Config() cfg.SkipWhenOpenshiftAndCNI(t) - t.Skipf("TODO(flaky-1.17): NET-XXXX") - for _, testCfg := range []LifecycleShutdownConfig{ {secure: false, helmValues: map[string]string{ helmDrainListenersKey: "true", - helmGracePeriodSecondsKey: "15", + helmGracePeriodSecondsKey: "5", }}, {secure: true, helmValues: map[string]string{ helmDrainListenersKey: "true", - helmGracePeriodSecondsKey: "15", + helmGracePeriodSecondsKey: "5", }}, {secure: false, helmValues: map[string]string{ helmDrainListenersKey: "false", - helmGracePeriodSecondsKey: "15", + helmGracePeriodSecondsKey: "5", }}, {secure: true, helmValues: map[string]string{ helmDrainListenersKey: "false", - helmGracePeriodSecondsKey: "15", + helmGracePeriodSecondsKey: "5", }}, {secure: false, helmValues: map[string]string{ helmDrainListenersKey: "false", @@ -80,8 +79,8 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { gracePeriodSeconds, err = strconv.ParseInt(val, 10, 64) require.NoError(t, err) } else { - // Half of the helm default to speed tests up - gracePeriodSeconds = 15 + // 5s should be a good amount of time to confirm the pod doesn't terminate + gracePeriodSeconds = 5 } name := fmt.Sprintf("secure: %t, drainListeners: %t, gracePeriodSeconds: %d", testCfg.secure, drainListenersEnabled, gracePeriodSeconds) @@ -140,7 +139,8 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { require.Len(t, pods.Items, 1) clientPodName := pods.Items[0].Name - var terminationGracePeriod int64 = 60 + // We should terminate the pods shortly after envoy gracefully shuts down in our 5s test cases. + var terminationGracePeriod int64 = 6 logger.Logf(t, "killing the %q pod with %dseconds termination grace period", clientPodName, terminationGracePeriod) err = ctx.KubernetesClient(t).CoreV1().Pods(ns).Delete(context.Background(), clientPodName, metav1.DeleteOptions{GracePeriodSeconds: &terminationGracePeriod}) require.NoError(t, err) @@ -155,23 +155,29 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { } if gracePeriodSeconds > 0 { - // Ensure outbound requests are still successful during grace - // period. - retry.RunWith(&retry.Timer{Timeout: time.Duration(gracePeriodSeconds) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { - output, err := k8s.RunKubectlAndGetOutputE(r, ctx.KubectlOptions(r), args...) - require.NoError(r, err) - require.Condition(r, func() bool { - exists := false - if strings.Contains(output, "curl: (7) Failed to connect") { - exists = true + // Ensure outbound requests are still successful during grace period. + gracePeriodTimer := time.NewTimer(time.Duration(gracePeriodSeconds) * time.Second) + gracePeriodLoop: + for { + select { + case <-gracePeriodTimer.C: + break gracePeriodLoop + default: + output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) + require.NoError(t, err) + require.True(t, !strings.Contains(output, "curl: (7) Failed to connect")) + + // If listener draining is disabled, ensure inbound + // requests are accepted during grace period. + if !drainListenersEnabled { + connHelper.TestConnectionSuccess(t, connhelper.ConnHelperOpts{}) } - return !exists - }) - }) + // TODO: check that the connection is unsuccessful when drainListenersEnabled is true + // dans note: I found it isn't sufficient to use the existing TestConnectionFailureWithoutIntention - // If listener draining is enabled, ensure inbound - // requests are rejected during grace period. - // connHelper.TestConnectionSuccess(t) + time.Sleep(2 * time.Second) + } + } } else { // Ensure outbound requests fail because proxy has terminated retry.RunWith(&retry.Timer{Timeout: time.Duration(terminationGracePeriod) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { @@ -188,7 +194,10 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { } logger.Log(t, "ensuring pod is deregistered after termination") - retry.Run(t, func(r *retry.R) { + // We wait an arbitrarily long time here. With the deployment rollout creating additional endpoints reconciles, + // This can cause the re-queued reconcile used to come back and clean up the service registration to be re-re-queued at + // 2-3X the intended grace period. + retry.RunWith(&retry.Timer{Timeout: time.Duration(30) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { for _, name := range []string{ "static-client", "static-client-sidecar-proxy", diff --git a/acceptance/tests/sync/sync_catalog_test.go b/acceptance/tests/sync/sync_catalog_test.go index 4bd5f91039..7398ed4e06 100644 --- a/acceptance/tests/sync/sync_catalog_test.go +++ b/acceptance/tests/sync/sync_catalog_test.go @@ -9,13 +9,14 @@ import ( "testing" "time" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/require" + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/sdk/testutil/retry" - "github.com/stretchr/testify/require" ) // Test that sync catalog works in both the default installation and @@ -105,7 +106,7 @@ func TestSyncCatalogWithIngress(t *testing.T) { ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ "syncCatalog.enabled": "true", - "syncCatalog.ingres.enabled": "true", + "syncCatalog.ingress.enabled": "true", "global.tls.enabled": strconv.FormatBool(c.secure), "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), } diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl index ca87485a78..f830e18c26 100644 --- a/charts/consul/templates/_helpers.tpl +++ b/charts/consul/templates/_helpers.tpl @@ -505,7 +505,6 @@ Fails if global.experiments.resourceAPIs is set along with any of these unsuppor - meshGateway.enabled - ingressGateways.enabled - terminatingGateways.enabled -- apiGateway.enabled Usage: {{ template "consul.validateResourceAPIs" . }} @@ -538,9 +537,6 @@ Usage: {{ template "consul.validateResourceAPIs" . }} {{- if (and (mustHas "resource-apis" .Values.global.experiments) .Values.terminatingGateways.enabled ) }} {{fail "When the value global.experiments.resourceAPIs is set, terminatingGateways.enabled is currently unsupported."}} {{- end }} -{{- if (and (mustHas "resource-apis" .Values.global.experiments) .Values.apiGateway.enabled ) }} -{{fail "When the value global.experiments.resourceAPIs is set, apiGateway.enabled is currently unsupported."}} -{{- end }} {{- end }} {{/* diff --git a/charts/consul/templates/api-gateway-controller-clusterrole.yaml b/charts/consul/templates/api-gateway-controller-clusterrole.yaml deleted file mode 100644 index eac2bd1f69..0000000000 --- a/charts/consul/templates/api-gateway-controller-clusterrole.yaml +++ /dev/null @@ -1,265 +0,0 @@ -{{- if .Values.apiGateway.enabled }} -# The ClusterRole to enable the API Gateway controller to access required api endpoints. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ template "consul.fullname" . }}-api-gateway-controller - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller -rules: -- apiGroups: - - api-gateway.consul.hashicorp.com - resources: - - gatewayclassconfigs - verbs: - - get - - list - - update - - watch -- apiGroups: - - api-gateway.consul.hashicorp.com - resources: - - gatewayclassconfigs/finalizers - verbs: - - update -- apiGroups: - - api-gateway.consul.hashicorp.com - resources: - - meshservices - verbs: - - get - - list - - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - get - - list - - update - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - update -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - configmaps/status - verbs: - - get - - patch - - update -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - referencegrants - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - referencepolicies - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses/finalizers - verbs: - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses/status - verbs: - - get - - patch - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - gateways - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gateways/finalizers - verbs: - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - gateways/status - verbs: - - get - - patch - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - httproutes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - httproutes/finalizers - verbs: - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - httproutes/status - verbs: - - get - - patch - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - tcproutes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - tcproutes/finalizers - verbs: - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - tcproutes/status - verbs: - - get - - patch - - update -{{- if .Values.global.enablePodSecurityPolicies }} -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - verbs: - - create - - get - - list - - watch -{{- end }} -{{- end }} diff --git a/charts/consul/templates/api-gateway-controller-clusterrolebinding.yaml b/charts/consul/templates/api-gateway-controller-clusterrolebinding.yaml deleted file mode 100644 index d083a08129..0000000000 --- a/charts/consul/templates/api-gateway-controller-clusterrolebinding.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.apiGateway.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ template "consul.fullname" . }}-api-gateway-controller - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ template "consul.fullname" . }}-api-gateway-controller -subjects: -- kind: ServiceAccount - name: {{ template "consul.fullname" . }}-api-gateway-controller - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/charts/consul/templates/api-gateway-controller-deployment.yaml b/charts/consul/templates/api-gateway-controller-deployment.yaml deleted file mode 100644 index 453be66054..0000000000 --- a/charts/consul/templates/api-gateway-controller-deployment.yaml +++ /dev/null @@ -1,306 +0,0 @@ -{{- if .Values.apiGateway.enabled }} -{{- if not .Values.client.grpc }}{{ fail "client.grpc must be true for api gateway" }}{{ end }} -{{- if not .Values.apiGateway.image}}{{ fail "apiGateway.image must be set to enable api gateway" }}{{ end }} -{{- if and .Values.global.adminPartitions.enabled (not .Values.global.enableConsulNamespaces) }}{{ fail "global.enableConsulNamespaces must be true if global.adminPartitions.enabled=true" }}{{ end }} -{{ template "consul.validateRequiredCloudSecretsExist" . }} -{{ template "consul.validateCloudSecretKeys" . }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "consul.fullname" . }}-api-gateway-controller - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller - {{- if .Values.global.extraLabels }} - {{- toYaml .Values.global.extraLabels | nindent 4 }} - {{- end }} -spec: - replicas: {{ .Values.apiGateway.controller.replicas }} - selector: - matchLabels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - release: {{ .Release.Name }} - component: api-gateway-controller - template: - metadata: - annotations: - consul.hashicorp.com/connect-inject: "false" - consul.hashicorp.com/mesh-inject: "false" - {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }} - "vault.hashicorp.com/agent-init-first": "true" - "vault.hashicorp.com/agent-inject": "true" - "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }} - "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }} - "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }} - {{- if .Values.global.secretsBackend.vault.agentAnnotations }} - {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }} - {{ end }} - {{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}} - "vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}" - {{- end }} - {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }} - "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}" - "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}" - {{- end }} - {{- end }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - release: {{ .Release.Name }} - component: api-gateway-controller - {{- if .Values.global.extraLabels }} - {{- toYaml .Values.global.extraLabels | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ template "consul.fullname" . }}-api-gateway-controller - containers: - - name: api-gateway-controller - image: {{ .Values.apiGateway.image }} - ports: - - containerPort: 9090 - name: sds - protocol: TCP - env: - {{- if or (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) .Values.client.enabled }} - {{- if .Values.global.tls.enabled }} - - name: CONSUL_CACERT - {{- /* When Vault is being used as a secrets backend, auto-encrypt must be enabled. Since clients use a separate - root CA from servers when auto-encrypt is enabled, and our controller communicates with the agent when clients are - enabled, we only use the Vault server CA if clients are disabled and our controller will be communicating w/ the server. */}} - {{- if and (not .Values.client.enabled) .Values.global.secretsBackend.vault.enabled }} - value: /vault/secrets/serverca.crt - {{- else }} - value: /consul/tls/ca/tls.crt - {{- end }} - {{- end }} - {{- end }} - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - {{- if .Values.global.acls.manageSystemACLs }} - - name: CONSUL_HTTP_TOKEN_FILE - value: "/consul/login/acl-token" - # CONSUL_LOGIN_DATACENTER is passed to the gateway that gets created. The controller does not use this to log in - - name: CONSUL_LOGIN_DATACENTER - value: {{ .Values.global.datacenter }} - {{- end }} - - name: CONSUL_HTTP_ADDR - {{- if .Values.client.enabled }} - {{/* - We use client agent nodes if we have them to support backwards compatibility for Consul API Gateway - v0.4 and older, which requires connectivity between the registered Consul agent node and a - deployment for health checking (originating from the Consul node). Always leveraging the agents in - the case that they're explicitly opted into allows us to support users with agent node + - "externalServers" configuration upgrading a Helm chart without upgrading API gateways. - */}} - {{- if .Values.global.tls.enabled }} - value: $(HOST_IP):8501 - {{- else }} - value: $(HOST_IP):8500 - {{- end }} - {{- else if .Values.externalServers.enabled }} - {{/* - "externalServers" specified and running in "agentless" mode, this will only work with - Consul API Gateway v0.5 or newer - */}} - value: {{ first .Values.externalServers.hosts }}:{{ .Values.externalServers.httpsPort }} - {{- else }} - {{/* - We have local network connectivity between deployments and the internal cluster, this - should be supported in all versions of Consul API Gateway - */}} - {{- if .Values.global.tls.enabled }} - value: {{ template "consul.fullname" . }}-server:8501 - {{- else }} - value: {{ template "consul.fullname" . }}-server:8500 - {{- end }} - {{- end }} - - name: CONSUL_HTTP_SSL - value: "{{ .Values.global.tls.enabled }}" - {{- if and (not .Values.client.enabled) .Values.externalServers.enabled .Values.externalServers.tlsServerName }} - - name: CONSUL_TLS_SERVER_NAME - value: {{ .Values.externalServers.tlsServerName }} - {{- end }} - {{- if .Values.global.adminPartitions.enabled }} - - name: CONSUL_PARTITION - value: {{ .Values.global.adminPartitions.name }} - {{- if .Values.global.acls.manageSystemACLs }} - - name: CONSUL_LOGIN_PARTITION - value: {{ .Values.global.adminPartitions.name }} - {{- end }} - {{- end }} - {{- if not .Values.client.enabled }} - - name: CONSUL_DYNAMIC_SERVER_DISCOVERY - value: "true" - {{- end }} - command: - - "/bin/sh" - - "-ec" - - | - exec consul-api-gateway server \ - -sds-server-host {{ template "consul.fullname" . }}-api-gateway-controller.{{ .Release.Namespace }}.svc \ - -k8s-namespace {{ .Release.Namespace }} \ - {{- if .Values.global.enableConsulNamespaces }} - {{- if .Values.connectInject.consulNamespaces.consulDestinationNamespace }} - -consul-destination-namespace={{ .Values.connectInject.consulNamespaces.consulDestinationNamespace }} \ - {{- end }} - {{- if .Values.connectInject.consulNamespaces.mirroringK8S }} - -mirroring-k8s=true \ - {{- if .Values.connectInject.consulNamespaces.mirroringK8SPrefix }} - -mirroring-k8s-prefix={{ .Values.connectInject.consulNamespaces.mirroringK8SPrefix }} \ - {{- end }} - {{- end }} - {{- end }} - {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }} - -primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \ - {{- end }} - -log-level {{ default .Values.global.logLevel .Values.apiGateway.logLevel }} \ - -log-json={{ .Values.global.logJSON }} - volumeMounts: - {{- if .Values.global.acls.manageSystemACLs }} - - name: consul-bin - mountPath: /consul-bin - {{- end }} - {{- if or (not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled)) .Values.client.enabled }} - {{- if .Values.global.tls.enabled }} - {{- if and .Values.client.enabled .Values.global.tls.enableAutoEncrypt }} - - name: consul-auto-encrypt-ca-cert - {{- else }} - - name: consul-ca-cert - {{- end }} - mountPath: /consul/tls/ca - readOnly: true - {{- end }} - {{- end }} - - mountPath: /consul/login - name: consul-data - readOnly: true - {{- if .Values.apiGateway.resources }} - resources: - {{- toYaml .Values.apiGateway.resources | nindent 12 }} - {{- end }} - {{- if .Values.global.acls.manageSystemACLs }} - lifecycle: - preStop: - exec: - command: ["/consul-bin/consul", "logout" ] - {{- end }} - volumes: - {{- if .Values.global.acls.manageSystemACLs }} - - name: consul-bin - emptyDir: { } - {{- end }} - {{- if .Values.global.tls.enabled }} - {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }} - - name: consul-ca-cert - secret: - {{- if .Values.global.tls.caCert.secretName }} - secretName: {{ .Values.global.tls.caCert.secretName }} - {{- else }} - secretName: {{ template "consul.fullname" . }}-ca-cert - {{- end }} - items: - - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }} - path: tls.crt - {{- end }} - {{- if .Values.global.tls.enableAutoEncrypt }} - - name: consul-auto-encrypt-ca-cert - emptyDir: - medium: "Memory" - {{- end }} - {{- end }} - - name: consul-data - emptyDir: - medium: "Memory" - {{- if or .Values.global.acls.manageSystemACLs (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }} - initContainers: - {{- if .Values.global.acls.manageSystemACLs }} - - name: copy-consul-bin - image: {{ .Values.global.image | quote }} - command: - - cp - - /bin/consul - - /consul-bin/consul - volumeMounts: - - name: consul-bin - mountPath: /consul-bin - {{- if .Values.apiGateway.initCopyConsulContainer }} - {{- if .Values.apiGateway.initCopyConsulContainer.resources }} - resources: {{ toYaml .Values.apiGateway.initCopyConsulContainer.resources | nindent 12 }} - {{- end }} - {{- end }} - {{- end }} - {{- if (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }} - {{- include "consul.getAutoEncryptClientCA" . | nindent 6 }} - {{- end }} - {{- if .Values.global.acls.manageSystemACLs }} - - name: api-gateway-controller-acl-init - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: CONSUL_LOGIN_META - value: "component=api-gateway-controller,pod=$(NAMESPACE)/$(POD_NAME)" - - name: CONSUL_LOGIN_DATACENTER - {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }} - value: {{ .Values.global.federation.primaryDatacenter }} - {{- else }} - value: {{ .Values.global.datacenter }} - {{- end}} - {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }} - image: {{ .Values.global.imageK8S }} - volumeMounts: - - mountPath: /consul/login - name: consul-data - readOnly: false - {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }} - {{- if .Values.global.tls.enabled }} - - name: consul-ca-cert - mountPath: /consul/tls/ca - readOnly: true - {{- end }} - {{- end }} - command: - - "/bin/sh" - - "-ec" - - | - exec consul-k8s-control-plane acl-init \ - {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }} - -auth-method-name={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \ - {{- else }} - -auth-method-name={{ template "consul.fullname" . }}-k8s-component-auth-method \ - {{- end }} - -log-level={{ default .Values.global.logLevel .Values.apiGateway.logLevel }} \ - -log-json={{ .Values.global.logJSON }} - resources: - requests: - memory: "25Mi" - cpu: "50m" - limits: - memory: "25Mi" - cpu: "50m" - {{- end }} - {{- end }} - {{- if .Values.apiGateway.controller.priorityClassName }} - priorityClassName: {{ .Values.apiGateway.controller.priorityClassName | quote }} - {{- end }} - {{- if .Values.apiGateway.controller.nodeSelector }} - nodeSelector: - {{ tpl .Values.apiGateway.controller.nodeSelector . | indent 8 | trim }} - {{- end }} - {{- if .Values.apiGateway.controller.tolerations }} - tolerations: - {{ tpl .Values.apiGateway.controller.tolerations . | indent 8 | trim }} - {{- end }} -{{- end }} diff --git a/charts/consul/templates/api-gateway-controller-podsecuritypolicy.yaml b/charts/consul/templates/api-gateway-controller-podsecuritypolicy.yaml deleted file mode 100644 index 390d084303..0000000000 --- a/charts/consul/templates/api-gateway-controller-podsecuritypolicy.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if and .Values.apiGateway.enabled .Values.global.enablePodSecurityPolicies }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ template "consul.fullname" . }}-api-gateway-controller - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller -spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - requiredDropCapabilities: - - ALL - # Allow core volume types. - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - readOnlyRootFilesystem: true -{{- end }} diff --git a/charts/consul/templates/api-gateway-controller-service.yaml b/charts/consul/templates/api-gateway-controller-service.yaml deleted file mode 100644 index aa79ff9fc3..0000000000 --- a/charts/consul/templates/api-gateway-controller-service.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if .Values.apiGateway.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "consul.fullname" . }}-api-gateway-controller - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller - annotations: - {{- if .Values.apiGateway.controller.service.annotations }} - {{ tpl .Values.apiGateway.controller.service.annotations . | nindent 4 | trim }} - {{- end }} -spec: - ports: - - name: sds - port: 9090 - protocol: TCP - targetPort: 9090 - selector: - app: {{ template "consul.name" . }} - release: "{{ .Release.Name }}" - component: api-gateway-controller -{{- end }} diff --git a/charts/consul/templates/api-gateway-controller-serviceaccount.yaml b/charts/consul/templates/api-gateway-controller-serviceaccount.yaml deleted file mode 100644 index 98292a8dbe..0000000000 --- a/charts/consul/templates/api-gateway-controller-serviceaccount.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.apiGateway.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ template "consul.fullname" . }}-api-gateway-controller - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller - {{- if .Values.apiGateway.serviceAccount.annotations }} - annotations: - {{ tpl .Values.apiGateway.serviceAccount.annotations . | nindent 4 | trim }} - {{- end }} -{{- with .Values.global.imagePullSecrets }} -imagePullSecrets: -{{- range . }} - - name: {{ .name }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/consul/templates/api-gateway-gatewayclass.yaml b/charts/consul/templates/api-gateway-gatewayclass.yaml deleted file mode 100644 index d9ba85e633..0000000000 --- a/charts/consul/templates/api-gateway-gatewayclass.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if (and .Values.apiGateway.enabled .Values.apiGateway.managedGatewayClass.enabled) }} -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: GatewayClass -metadata: - name: consul-api-gateway - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller -spec: - controllerName: hashicorp.com/consul-api-gateway-controller - parametersRef: - group: api-gateway.consul.hashicorp.com - kind: GatewayClassConfig - name: consul-api-gateway -{{- end }} diff --git a/charts/consul/templates/api-gateway-gatewayclassconfig.yaml b/charts/consul/templates/api-gateway-gatewayclassconfig.yaml deleted file mode 100644 index ba0e6c63db..0000000000 --- a/charts/consul/templates/api-gateway-gatewayclassconfig.yaml +++ /dev/null @@ -1,84 +0,0 @@ -{{- if (and .Values.apiGateway.enabled .Values.apiGateway.managedGatewayClass.enabled) }} -apiVersion: api-gateway.consul.hashicorp.com/v1alpha1 -kind: GatewayClassConfig -metadata: - name: consul-api-gateway - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway -spec: - consul: - {{- if .Values.client.enabled }} - {{/* - We use client agent nodes if we have them to support backwards compatibility in <=0.4 releases which - require connectivity between the registered Consul agent node and a deployment for health checking - (originating from the Consul node). Always leveraging the agents in the case that they're explicitly - opted into allows us to support users with agent node + "externalServers" configuration upgrading a - helm chart without upgrading api gateways. Otherwise, using "externalServers" when provided - without local agents will break gateways <=0.4. - */}} - address: $(HOST_IP) - {{- else if .Values.externalServers.enabled }} - {{/* - "externalServers" specified and running in "agentless" mode, this will only work 0.5+ - */}} - address: {{ first .Values.externalServers.hosts }} - {{- else }} - {{/* - We have local network connectivity between deployments and the internal cluster, this - should be supported in all versions of api-gateway - */}} - address: {{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc - {{- end }} - authentication: - {{- if .Values.global.acls.manageSystemACLs }} - managed: true - method: {{ template "consul.fullname" . }}-k8s-auth-method - {{- if .Values.global.enablePodSecurityPolicies }} - podSecurityPolicy: {{ template "consul.fullname" . }}-api-gateway - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - scheme: https - {{- else }} - scheme: http - {{- end }} - ports: - {{- if .Values.externalServers.enabled }} - grpc: {{ .Values.externalServers.grpcPort }} - http: {{ .Values.externalServers.httpsPort }} - {{- else }} - grpc: 8502 - {{- if .Values.global.tls.enabled }} - http: 8501 - {{- else }} - http: 8500 - {{- end }} - {{- end }} - {{- with .Values.apiGateway.managedGatewayClass.deployment }} - deployment: - {{- toYaml . | nindent 4 }} - {{- end }} - image: - consulAPIGateway: {{ .Values.apiGateway.image }} - envoy: {{ .Values.apiGateway.imageEnvoy }} - {{- if .Values.apiGateway.managedGatewayClass.nodeSelector }} - nodeSelector: - {{ tpl .Values.apiGateway.managedGatewayClass.nodeSelector . | indent 4 | trim }} - {{- end }} - {{- if .Values.apiGateway.managedGatewayClass.tolerations }} - tolerations: - {{ tpl .Values.apiGateway.managedGatewayClass.tolerations . | indent 4 | trim }} - {{- end }} - {{- if .Values.apiGateway.managedGatewayClass.copyAnnotations.service }} - copyAnnotations: - service: - {{ tpl .Values.apiGateway.managedGatewayClass.copyAnnotations.service.annotations . | nindent 6 | trim }} - {{- end }} - serviceType: {{ .Values.apiGateway.managedGatewayClass.serviceType }} - useHostPorts: {{ .Values.apiGateway.managedGatewayClass.useHostPorts }} - logLevel: {{ default .Values.global.logLevel .Values.apiGateway.managedGatewayClass.logLevel }} -{{- end }} diff --git a/charts/consul/templates/api-gateway-podsecuritypolicy.yaml b/charts/consul/templates/api-gateway-podsecuritypolicy.yaml deleted file mode 100644 index 48f826f995..0000000000 --- a/charts/consul/templates/api-gateway-podsecuritypolicy.yaml +++ /dev/null @@ -1,45 +0,0 @@ -{{- if and .Values.apiGateway.enabled .Values.global.enablePodSecurityPolicies }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ template "consul.fullname" . }}-api-gateway - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: api-gateway-controller -spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - requiredDropCapabilities: - - ALL - # Allow core volume types. - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - allowedCapabilities: - - NET_BIND_SERVICE - hostNetwork: false - hostIPC: false - hostPID: false - hostPorts: - - max: 65535 - min: 1025 - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - readOnlyRootFilesystem: true -{{- end }} diff --git a/charts/consul/templates/crd-gatewayclassconfigs-v1.yaml b/charts/consul/templates/crd-gatewayclassconfigs-v1.yaml index 130db72a22..41023c19dc 100644 --- a/charts/consul/templates/crd-gatewayclassconfigs-v1.yaml +++ b/charts/consul/templates/crd-gatewayclassconfigs-v1.yaml @@ -131,6 +131,23 @@ spec: for gateway containers format: int32 type: integer + metrics: + description: Metrics defines how to configure the metrics for a gateway. + properties: + enabled: + description: Enable metrics for this class of gateways. If unspecified, + will inherit behavior from the global Helm configuration. + type: boolean + path: + description: The path used for metrics. + type: string + port: + description: The port used for metrics. + format: int32 + maximum: 65535 + minimum: 1024 + type: integer + type: object nodeSelector: additionalProperties: type: string diff --git a/charts/consul/templates/crd-terminatinggateways.yaml b/charts/consul/templates/crd-terminatinggateways.yaml index 565aa63381..cd53122e9d 100644 --- a/charts/consul/templates/crd-terminatinggateways.yaml +++ b/charts/consul/templates/crd-terminatinggateways.yaml @@ -73,6 +73,10 @@ spec: to use for TLS connections from the gateway to the linked service. type: string + disableAutoHostRewrite: + description: DisableAutoHostRewrite disables terminating gateways + auto host rewrite feature when set to true. + type: boolean keyFile: description: KeyFile is the optional path to a private key to use for TLS connections from the gateway to the linked service. diff --git a/charts/consul/templates/gateway-resources-job.yaml b/charts/consul/templates/gateway-resources-job.yaml index 5934372ed3..e43efc8a9a 100644 --- a/charts/consul/templates/gateway-resources-job.yaml +++ b/charts/consul/templates/gateway-resources-job.yaml @@ -1,3 +1,4 @@ +{{- if .Values.apiGateway}}{{fail "[DEPRECATED and REMOVED] the apiGateway stanza is no longer supported as of Consul 1.19.0. Use connectInject.apiGateway instead."}}{{- end -}} {{- if .Values.connectInject.enabled }} apiVersion: batch/v1 kind: Job @@ -51,29 +52,6 @@ spec: - -heritage={{ .Release.Service }} - -release-name={{ .Release.Name }} - -component=api-gateway - {{- if .Values.apiGateway.enabled }} # Override values from the old stanza. To be removed after ~1.18 (t-eckert 2023-05-19) NET-6263 - {{- if .Values.apiGateway.managedGatewayClass.deployment }} - {{- if .Values.apiGateway.managedGatewayClass.deployment.defaultInstances }} - - -deployment-default-instances={{ .Values.apiGateway.managedGatewayClass.deployment.defaultInstances }} - {{- end}} - {{- if .Values.apiGateway.managedGatewayClass.deployment.maxInstances }} - - -deployment-max-instances={{ .Values.apiGateway.managedGatewayClass.deployment.maxInstances }} - {{- end}} - {{- if .Values.apiGateway.managedGatewayClass.deployment.minInstances }} - - -deployment-min-instances={{ .Values.apiGateway.managedGatewayClass.deployment.minInstances }} - {{- end}} - {{- end}} - {{- if .Values.apiGateway.managedGatewayClass.nodeSelector }} - - -node-selector={{ .Values.apiGateway.managedGatewayClass.nodeSelector }} - {{- end }} - {{- if .Values.apiGateway.managedGatewayClass.tolerations }} - - -tolerations={{ .Values.apiGateway.managedGatewayClass.tolerations }} - {{- end }} - {{- if .Values.apiGateway.managedGatewayClass.copyAnnotations.service }} - - -service-annotations={{ .Values.apiGateway.managedGatewayClass.copyAnnotations.service.annotations }} - {{- end }} - - -service-type={{ .Values.apiGateway.managedGatewayClass.serviceType }} - {{- else }} # the new stanza {{- if .Values.connectInject.apiGateway.managedGatewayClass.deployment }} {{- if .Values.connectInject.apiGateway.managedGatewayClass.deployment.defaultInstances }} - -deployment-default-instances={{ .Values.connectInject.apiGateway.managedGatewayClass.deployment.defaultInstances }} @@ -101,7 +79,15 @@ spec: - -openshift-scc-name={{ .Values.connectInject.apiGateway.managedGatewayClass.openshiftSCCName }} {{- end }} - -map-privileged-container-ports={{ .Values.connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts }} - {{- end}} + {{- if (ne (.Values.connectInject.apiGateway.managedGatewayClass.metrics.enabled | toString) "-") }} + - -enable-metrics={{ .Values.connectInject.apiGateway.managedGatewayClass.metrics.enabled | toString }} + {{- end }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass.metrics.path }} + - -metrics-path={{ .Values.connectInject.apiGateway.managedGatewayClass.metrics.path }} + {{- end }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass.metrics.port }} + - -metrics-port={{ .Values.connectInject.apiGateway.managedGatewayClass.metrics.port }} + {{- end }} resources: requests: memory: "50Mi" diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml index 2e798a54d5..ca10cb3e34 100644 --- a/charts/consul/templates/server-acl-init-job.yaml +++ b/charts/consul/templates/server-acl-init-job.yaml @@ -304,10 +304,6 @@ spec: -partition-token-file=/vault/secrets/partition-token \ {{- end }} - {{- if .Values.apiGateway.enabled }} - -api-gateway-controller=true \ - {{- end }} - {{- if .Values.global.enableConsulNamespaces }} -enable-namespaces=true \ {{- /* syncCatalog must be enabled to set sync flags */}} diff --git a/charts/consul/templates/sync-catalog-clusterrole.yaml b/charts/consul/templates/sync-catalog-clusterrole.yaml index 585b5ad171..89ea9f3c5c 100644 --- a/charts/consul/templates/sync-catalog-clusterrole.yaml +++ b/charts/consul/templates/sync-catalog-clusterrole.yaml @@ -14,7 +14,19 @@ rules: - apiGroups: [ "" ] resources: - services - - endpoints + verbs: + - get + - list + - watch +{{- if .Values.syncCatalog.toK8S }} + - update + - patch + - delete + - create +{{- end }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices verbs: - get - list @@ -45,4 +57,4 @@ rules: - get - list - watch -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/consul/test/unit/api-gateway-controller-clusterrole.bats b/charts/consul/test/unit/api-gateway-controller-clusterrole.bats deleted file mode 100644 index f26fdfeebd..0000000000 --- a/charts/consul/test/unit/api-gateway-controller-clusterrole.bats +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/ClusterRole: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-clusterrole.yaml \ - . -} - -@test "apiGateway/ClusterRole: enabled with apiGateway.enabled=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-clusterrole.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/ClusterRole: can use podsecuritypolicies with apiGateway.enabled=true and global.enablePodSecurityPolicies=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-clusterrole.yaml \ - --set 'global.enablePodSecurityPolicies=true' \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.rules[] | select((.resources[0] == "podsecuritypolicies") and (.verbs[0] == "use")) | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/ClusterRole: can create roles and rolebindings with apiGateway.enabled=true and global.enablePodSecurityPolicies=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-clusterrole.yaml \ - --set 'global.enablePodSecurityPolicies=true' \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.rules[] | select((.resources[0] == "roles") and (.resources[1] == "rolebindings") and (.verbs | contains(["create","get","list","watch"]))) | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} diff --git a/charts/consul/test/unit/api-gateway-controller-clusterrolebinding.bats b/charts/consul/test/unit/api-gateway-controller-clusterrolebinding.bats deleted file mode 100644 index 3dfd94c36f..0000000000 --- a/charts/consul/test/unit/api-gateway-controller-clusterrolebinding.bats +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/ClusterRoleBinding: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-clusterrolebinding.yaml \ - . -} - -@test "apiGateway/ClusterRoleBinding: enabled with global.enabled false" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-clusterrolebinding.yaml \ - --set 'global.enabled=false' \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq -s 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} diff --git a/charts/consul/test/unit/api-gateway-controller-deployment.bats b/charts/consul/test/unit/api-gateway-controller-deployment.bats deleted file mode 100755 index 696d5f7cbb..0000000000 --- a/charts/consul/test/unit/api-gateway-controller-deployment.bats +++ /dev/null @@ -1,1754 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/Deployment: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - . -} - -@test "apiGateway/Deployment: fails if no image is set" { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - . - [ "$status" -eq 1 ] - [[ "$output" =~ "apiGateway.image must be set to enable api gateway" ]] -} - -@test "apiGateway/Deployment: disable with apiGateway.enabled" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=false' \ - . -} - -@test "apiGateway/Deployment: disable with global.enabled" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'global.enabled=false' \ - . -} - -@test "apiGateway/Deployment: enable namespaces" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.enableConsulNamespaces=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | join(" ") | contains("-consul-destination-namespace=default")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: enable namespace mirroring" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'connectInject.consulNamespaces.mirroringK8S=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | join(" ") | contains("-mirroring-k8s=true")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: enable namespace mirroring prefixes" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'connectInject.consulNamespaces.mirroringK8S=true' \ - --set 'connectInject.consulNamespaces.mirroringK8SPrefix=foo' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | join(" ") | contains("-mirroring-k8s-prefix=foo")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: container image overrides" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].image' | tee /dev/stderr) - [ "${actual}" = "\"bar\"" ] -} - -@test "apiGateway/Deployment: SDS host set correctly" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | join(" ") | contains("-sds-server-host release-name-consul-api-gateway-controller.default.svc")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -#-------------------------------------------------------------------- -# nodeSelector - -@test "apiGateway/Deployment: nodeSelector is not set by default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.spec.template.spec.nodeSelector' | tee /dev/stderr) - [ "${actual}" = "null" ] -} - -@test "apiGateway/Deployment: specified nodeSelector" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.controller.nodeSelector=testing' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.nodeSelector' | tee /dev/stderr) - [ "${actual}" = "testing" ] -} - -#-------------------------------------------------------------------- -# global.tls.enabled - -@test "apiGateway/Deployment: Adds tls-ca-cert volume when global.tls.enabled is true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.volumes[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" != "" ] -} - -@test "apiGateway/Deployment: Adds tls-ca-cert volumeMounts when global.tls.enabled is true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" != "" ] -} - -@test "apiGateway/Deployment: can overwrite CA secret with the provided one" { - cd `chart_dir` - local ca_cert_volume=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.caCert.secretName=foo-ca-cert' \ - --set 'global.tls.caCert.secretKey=key' \ - --set 'global.tls.caKey.secretName=foo-ca-key' \ - --set 'global.tls.caKey.secretKey=key' \ - . | tee /dev/stderr | - yq '.spec.template.spec.volumes[] | select(.name=="consul-ca-cert")' | tee /dev/stderr) - - # check that the provided ca cert secret is attached as a volume - local actual - actual=$(echo $ca_cert_volume | jq -r '.secret.secretName' | tee /dev/stderr) - [ "${actual}" = "foo-ca-cert" ] - - # check that the volume uses the provided secret key - actual=$(echo $ca_cert_volume | jq -r '.secret.items[0].key' | tee /dev/stderr) - [ "${actual}" = "key" ] -} - -#-------------------------------------------------------------------- -# global.tls.enableAutoEncrypt - -@test "apiGateway/Deployment: consul-auto-encrypt-ca-cert volume is added when TLS with auto-encrypt is enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.volumes[] | select(.name == "consul-auto-encrypt-ca-cert") | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: consul-auto-encrypt-ca-cert volumeMount is added when TLS with auto-encrypt is enabled with clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-auto-encrypt-ca-cert") | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: consul-ca-cert volumeMount is added when TLS with auto-encrypt is enabled without clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert") | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: get-auto-encrypt-client-ca init container is created when TLS with auto-encrypt is enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "get-auto-encrypt-client-ca") | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: adds both init containers when TLS with auto-encrypt and ACLs + namespaces are enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers | length == 3' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: consul-ca-cert volume is not added if externalServers.enabled=true and externalServers.useSystemRoots=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=foo.com' \ - --set 'externalServers.useSystemRoots=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.volumes[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -#-------------------------------------------------------------------- -# global.acls.manageSystemACLs - -@test "apiGateway/Deployment: consul-logout preStop hook is added when ACLs are enabled" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]] | any(contains("logout"))' | tee /dev/stderr) - [ "${object}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_TOKEN_FILE is not set when acls are disabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[1].name] | any(contains("CONSUL_HTTP_TOKEN_FILE"))' | tee /dev/stderr) - [ "${actual}" = "false" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_TOKEN_FILE is set when acls are enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[1].name] | any(contains("CONSUL_HTTP_TOKEN_FILE"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_LOGIN_DATACENTER is set when acls are enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[2].name] | any(contains("CONSUL_LOGIN_DATACENTER"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: init container is created when global.acls.manageSystemACLs=true" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1]' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.name' | tee /dev/stderr) - [ "${actual}" = "api-gateway-controller-acl-init" ] - - local actual=$(echo $object | - yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[0].name] | any(contains("NAMESPACE"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[1].name] | any(contains("POD_NAME"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[2].name] | any(contains("CONSUL_LOGIN_META"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[2].value] | any(contains("component=api-gateway-controller,pod=$(NAMESPACE)/$(POD_NAME)"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[3].name] | any(contains("CONSUL_LOGIN_DATACENTER"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq -r '[.env[8].value] | any(contains("5s"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command and environment with tls enabled" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.consulAPITimeout=5s' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "api-gateway-controller-acl-init")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "NAMESPACE") | [.valueFrom.fieldRef.fieldPath] | any(contains("metadata.namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "POD_NAME") | [.valueFrom.fieldRef.fieldPath] | any(contains("metadata.name"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_LOGIN_META") | [.value] | any(contains("component=api-gateway-controller,pod=$(NAMESPACE)/$(POD_NAME)"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_ADDRESSES") | [.value] | any(contains("release-name-consul-server.default.svc"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_GRPC_PORT") | [.value] | any(contains("8502"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_HTTP_PORT") | [.value] | any(contains("8501"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_DATACENTER") | [.value] | any(contains("dc1"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_API_TIMEOUT") | [.value] | any(contains("5s"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_USE_TLS") | [.value] | any(contains("true"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_CACERT_FILE") | [.value] | any(contains("/consul/tls/ca/tls.crt"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.volumeMounts[] | select(.name == "consul-ca-cert") | [.mountPath] | any(contains("/consul/tls/ca"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.volumeMounts[] | select(.name == "consul-data") | [.mountPath] | any(contains("/consul/login"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command with Partitions enabled" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'global.adminPartitions.enabled=true' \ - --set 'global.adminPartitions.name=default' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "api-gateway-controller-acl-init")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq -r '.command | any(contains("-auth-method-name=release-name-consul-k8s-component-auth-method"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "NAMESPACE") | [.valueFrom.fieldRef.fieldPath] | any(contains("metadata.namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "POD_NAME") | [.valueFrom.fieldRef.fieldPath] | any(contains("metadata.name"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_LOGIN_META") | [.value] | any(contains("component=api-gateway-controller,pod=$(NAMESPACE)/$(POD_NAME)"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_ADDRESSES") | [.value] | any(contains("release-name-consul-server.default.svc"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_GRPC_PORT") | [.value] | any(contains("8502"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_HTTP_PORT") | [.value] | any(contains("8501"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_DATACENTER") | [.value] | any(contains("dc1"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_API_TIMEOUT") | [.value] | any(contains("5s"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_PARTITION") | [.value] | any(contains("default"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_LOGIN_PARTITION") | [.value] | any(contains("default"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_USE_TLS") | [.value] | any(contains("true"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_CACERT_FILE") | [.value] | any(contains("/consul/tls/ca/tls.crt"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.volumeMounts[] | select(.name == "consul-ca-cert") | [.mountPath] | any(contains("/consul/tls/ca"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.volumeMounts[] | select(.name == "consul-data") | [.mountPath] | any(contains("/consul/login"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: consul login datacenter is set to primary when when federation enabled in non-primary datacenter" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'meshGateway.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.datacenter=dc1' \ - --set 'global.federation.enabled=true' \ - --set 'global.federation.primaryDatacenter=dc2' \ - --set 'global.tls.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1]' | tee /dev/stderr) - - local actual=$(echo $object | - yq '[.env[3].name] | any(contains("CONSUL_LOGIN_DATACENTER"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[3].value] | any(contains("dc2"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: primary-datacenter flag provided when federation enabled in non-primary datacenter" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.datacenter=dc2' \ - --set 'global.federation.enabled=true' \ - --set 'global.federation.primaryDatacenter=dc1' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[] | select(.name == "api-gateway-controller")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.command | any(contains("consul-api-gateway server"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq -r '.command | any(contains("-primary-datacenter=dc1"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command when federation enabled in non-primary datacenter" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.datacenter=dc2' \ - --set 'global.federation.enabled=true' \ - --set 'global.federation.primaryDatacenter=dc1' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "api-gateway-controller-acl-init")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq -r '.command | any(contains("-auth-method-name=release-name-consul-k8s-component-auth-method-dc2"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '[.env[3].value] | any(contains("dc1"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command and environment with tls enabled and autoencrypt enabled" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "api-gateway-controller-acl-init")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "NAMESPACE") | [.valueFrom.fieldRef.fieldPath] | any(contains("metadata.namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "POD_NAME") | [.valueFrom.fieldRef.fieldPath] | any(contains("metadata.name"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_LOGIN_META") | [.value] | any(contains("component=api-gateway-controller,pod=$(NAMESPACE)/$(POD_NAME)"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_ADDRESSES") | [.value] | any(contains("release-name-consul-server.default.svc"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_GRPC_PORT") | [.value] | any(contains("8502"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_HTTP_PORT") | [.value] | any(contains("8501"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_DATACENTER") | [.value] | any(contains("dc1"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_API_TIMEOUT") | [.value] | any(contains("5s"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_USE_TLS") | [.value] | any(contains("true"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.env[] | select(.name == "CONSUL_CACERT_FILE") | [.value] | any(contains("/consul/tls/ca/tls.crt"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.volumeMounts[] | select(.name == "consul-ca-cert") | [.mountPath] | any(contains("/consul/tls/ca"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq '.volumeMounts[] | select(.name == "consul-data") | [.mountPath] | any(contains("/consul/login"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: init container for copy consul is created when global.acls.manageSystemACLs=true" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "copy-consul-bin")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.command | any(contains("cp"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | - yq -r '.volumeMounts[0] | any(contains("consul-bin"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: volumeMount for copy consul is created on container when global.acls.manageSystemACLs=true" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[0] | any(contains("consul-bin"))' | tee /dev/stderr) - - [ "${object}" = "true" ] -} - -@test "apiGateway/Deployment: volume for copy consul is created when global.acls.manageSystemACLs=true" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.volumes[0] | any(contains("consul-bin"))' | tee /dev/stderr) - - [ "${object}" = "true" ] -} - -@test "apiGateway/Deployment: auto-encrypt init container is created and is the first init-container when global.acls.manageSystemACLs=true and has correct command and environment with tls enabled and autoencrypt enabled" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1]' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.name' | tee /dev/stderr) - [ "${actual}" = "get-auto-encrypt-client-ca" ] -} - -#-------------------------------------------------------------------- -# resources - -@test "apiGateway/Deployment: resources has default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].resources' | tee /dev/stderr) - - [ $(echo "${actual}" | yq -r '.requests.memory') = "100Mi" ] - [ $(echo "${actual}" | yq -r '.requests.cpu') = "100m" ] - [ $(echo "${actual}" | yq -r '.limits.memory') = "100Mi" ] - [ $(echo "${actual}" | yq -r '.limits.cpu') = "100m" ] -} - -@test "apiGateway/Deployment: resources can be overridden" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.resources.foo=bar' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].resources.foo' | tee /dev/stderr) - [ "${actual}" = "bar" ] -} - -#-------------------------------------------------------------------- -# init container resources - -@test "apiGateway/Deployment: init container has default resources" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.initContainers[0].resources' | tee /dev/stderr) - - [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ] - [ $(echo "${actual}" | yq -r '.requests.cpu') = "50m" ] - [ $(echo "${actual}" | yq -r '.limits.memory') = "150Mi" ] - [ $(echo "${actual}" | yq -r '.limits.cpu') = "50m" ] -} - -@test "apiGateway/Deployment: init container resources can be set" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'apiGateway.initCopyConsulContainer.resources.requests.memory=memory' \ - --set 'apiGateway.initCopyConsulContainer.resources.requests.cpu=cpu' \ - --set 'apiGateway.initCopyConsulContainer.resources.limits.memory=memory2' \ - --set 'apiGateway.initCopyConsulContainer.resources.limits.cpu=cpu2' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.initContainers[0].resources' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) - [ "${actual}" = "memory" ] - - local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) - [ "${actual}" = "cpu" ] - - local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) - [ "${actual}" = "memory2" ] - - local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) - [ "${actual}" = "cpu2" ] -} - -#-------------------------------------------------------------------- -# priorityClassName - -@test "apiGateway/Deployment: no priorityClassName by default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.priorityClassName' | tee /dev/stderr) - - [ "${actual}" = "null" ] -} - -@test "apiGateway/Deployment: can set a priorityClassName" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.controller.priorityClassName=name' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.priorityClassName' | tee /dev/stderr) - - [ "${actual}" = "name" ] -} - -#-------------------------------------------------------------------- -# logLevel - -@test "apiGateway/Deployment: logLevel info by default from global" { - cd `chart_dir` - local cmd=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) - - local actual=$(echo "$cmd" | - yq 'any(contains("-log-level info"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: logLevel can be overridden" { - cd `chart_dir` - local cmd=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.logLevel=debug' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) - - local actual=$(echo "$cmd" | - yq 'any(contains("-log-level debug"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -#-------------------------------------------------------------------- -# replicas - -@test "apiGateway/Deployment: replicas defaults to 1" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.spec.replicas' | tee /dev/stderr) - - [ "${actual}" = "1" ] -} - -@test "apiGateway/Deployment: replicas can be set" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.controller.replicas=3' \ - . | tee /dev/stderr | - yq '.spec.replicas' | tee /dev/stderr) - - [ "${actual}" = "3" ] -} - - -#-------------------------------------------------------------------- -# get-auto-encrypt-client-ca - -@test "apiGateway/Deployment: get-auto-encrypt-client-ca uses server's stateful set address by default and passes ca cert" { - cd `chart_dir` - local command=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "get-auto-encrypt-client-ca").command | join(" ")' | tee /dev/stderr) - - # check server address - actual=$(echo $command | jq ' . | contains("-server-addr=release-name-consul-server")') - [ "${actual}" = "true" ] - - # check server port - actual=$(echo $command | jq ' . | contains("-server-port=8501")') - [ "${actual}" = "true" ] - - # check server's CA cert - actual=$(echo $command | jq ' . | contains("-ca-file=/consul/tls/ca/tls.crt")') - [ "${actual}" = "true" ] - - # check consul-api-timeout - actual=$(echo $command | jq ' . | contains("-consul-api-timeout=5s")') - [ "${actual}" = "true" ] -} - -#-------------------------------------------------------------------- -# Vault - -@test "apiGateway/Deployment: vault CA is not configured by default" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=test' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - . | tee /dev/stderr | - yq -r '.spec.template' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') - [ "${actual}" = "false" ] - local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') - [ "${actual}" = "false" ] -} - -@test "apiGateway/Deployment: vault CA is not configured when secretName is set but secretKey is not" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=test' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.ca.secretName=ca' \ - . | tee /dev/stderr | - yq -r '.spec.template' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') - [ "${actual}" = "false" ] - local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') - [ "${actual}" = "false" ] -} - -@test "apiGateway/Deployment: vault CA is not configured when secretKey is set but secretName is not" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=test' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ - . | tee /dev/stderr | - yq -r '.spec.template' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') - [ "${actual}" = "false" ] - local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') - [ "${actual}" = "false" ] -} - -@test "apiGateway/Deployment: vault CA is configured when both secretName and secretKey are set" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=test' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.ca.secretName=ca' \ - --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ - . | tee /dev/stderr | - yq -r '.spec.template' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-extra-secret"') - [ "${actual}" = "ca" ] - local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/ca-cert"') - [ "${actual}" = "/vault/custom/tls.crt" ] -} - -@test "apiGateway/Deployment: vault tls annotations are set when tls is enabled" { - cd `chart_dir` - local cmd=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=bar' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'server.serverCert.secretName=pki_int/issue/test' \ - --set 'global.tls.caCert.secretName=pki_int/cert/ca' \ - . | tee /dev/stderr | - yq -r '.spec.template.metadata' | tee /dev/stderr) - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/agent-inject-template-serverca.crt"]' | tee /dev/stderr)" - local expected=$'{{- with secret \"pki_int/cert/ca\" -}}\n{{- .Data.certificate -}}\n{{- end -}}' - [ "${actual}" = "${expected}" ] - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-serverca.crt"]' | tee /dev/stderr)" - [ "${actual}" = "pki_int/cert/ca" ] - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/agent-init-first"]' | tee /dev/stderr)" - [ "${actual}" = "true" ] - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr)" - [ "${actual}" = "true" ] - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr)" - [ "${actual}" = "test" ] -} - -@test "apiGateway/Deployment: vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set" { - cd `chart_dir` - local cmd=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=bar' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.vaultNamespace=vns' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.metadata' | tee /dev/stderr) - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" - [ "${actual}" = "vns" ] -} - -@test "apiGateway/Deployment: correct vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set and agentAnnotations are set without vaultNamespace annotation" { - cd `chart_dir` - local cmd=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=bar' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.vaultNamespace=vns' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.secretsBackend.vault.agentAnnotations=vault.hashicorp.com/agent-extra-secret: bar' \ - . | tee /dev/stderr | - yq -r '.spec.template.metadata' | tee /dev/stderr) - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" - [ "${actual}" = "vns" ] -} - -@test "apiGateway/Deployment: correct vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set and agentAnnotations are also set with vaultNamespace annotation" { - cd `chart_dir` - local cmd=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=foo' \ - --set 'global.secretsBackend.vault.consulServerRole=bar' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.vaultNamespace=vns' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.secretsBackend.vault.agentAnnotations="vault.hashicorp.com/namespace": bar' \ - . | tee /dev/stderr | - yq -r '.spec.template.metadata' | tee /dev/stderr) - - local actual="$(echo $cmd | - yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" - [ "${actual}" = "bar" ] -} - -@test "apiGateway/Deployment: vault agent annotations can be set" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulClientRole=test' \ - --set 'global.secretsBackend.vault.consulServerRole=foo' \ - --set 'global.secretsBackend.vault.consulCARole=test' \ - --set 'global.secretsBackend.vault.agentAnnotations=foo: bar' \ - . | tee /dev/stderr | - yq -r '.spec.template.metadata.annotations.foo' | tee /dev/stderr) - [ "${actual}" = "bar" ] -} - -#-------------------------------------------------------------------- -# global.cloud - -@test "apiGateway/Deployment: fails when global.cloud.enabled is true and global.cloud.clientId.secretName is not set but global.cloud.clientSecret.secretName and global.cloud.resourceId.secretName is set" { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientSecret.secretName=client-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-id-key' \ - --set 'global.cloud.resourceId.secretName=client-resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=client-resource-id-key' \ - . - [ "$status" -eq 1 ] - [[ "$output" =~ "When global.cloud.enabled is true, global.cloud.resourceId.secretName, global.cloud.clientId.secretName, and global.cloud.clientSecret.secretName must also be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.enabled is true and global.cloud.clientSecret.secretName is not set but global.cloud.clientId.secretName and global.cloud.resourceId.secretName is set" { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - . - [ "$status" -eq 1 ] - [[ "$output" =~ "When global.cloud.enabled is true, global.cloud.resourceId.secretName, global.cloud.clientId.secretName, and global.cloud.clientSecret.secretName must also be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.enabled is true and global.cloud.resourceId.secretName is not set but global.cloud.clientId.secretName and global.cloud.clientSecret.secretName is set" { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - . - [ "$status" -eq 1 ] - [[ "$output" =~ "When global.cloud.enabled is true, global.cloud.resourceId.secretName, global.cloud.clientId.secretName, and global.cloud.clientSecret.secretName must also be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.resourceId.secretName is set but global.cloud.resourceId.secretKey is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - . - [ "$status" -eq 1 ] - [[ "$output" =~ "When either global.cloud.resourceId.secretName or global.cloud.resourceId.secretKey is defined, both must be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.authURL.secretName is set but global.cloud.authURL.secretKey is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - --set 'global.cloud.authUrl.secretName=auth-url-name' \ - . - [ "$status" -eq 1 ] - - [[ "$output" =~ "When either global.cloud.authUrl.secretName or global.cloud.authUrl.secretKey is defined, both must be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.authURL.secretKey is set but global.cloud.authURL.secretName is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - --set 'global.cloud.authUrl.secretKey=auth-url-key' \ - . - [ "$status" -eq 1 ] - - [[ "$output" =~ "When either global.cloud.authUrl.secretName or global.cloud.authUrl.secretKey is defined, both must be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.apiHost.secretName is set but global.cloud.apiHost.secretKey is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - --set 'global.cloud.apiHost.secretName=auth-url-name' \ - . - [ "$status" -eq 1 ] - - [[ "$output" =~ "When either global.cloud.apiHost.secretName or global.cloud.apiHost.secretKey is defined, both must be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.apiHost.secretKey is set but global.cloud.apiHost.secretName is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - --set 'global.cloud.apiHost.secretKey=auth-url-key' \ - . - [ "$status" -eq 1 ] - - [[ "$output" =~ "When either global.cloud.apiHost.secretName or global.cloud.apiHost.secretKey is defined, both must be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.scadaAddress.secretName is set but global.cloud.scadaAddress.secretKey is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - --set 'global.cloud.scadaAddress.secretName=scada-address-name' \ - . - [ "$status" -eq 1 ] - - [[ "$output" =~ "When either global.cloud.scadaAddress.secretName or global.cloud.scadaAddress.secretKey is defined, both must be set." ]] -} - -@test "apiGateway/Deployment: fails when global.cloud.scadaAddress.secretKey is set but global.cloud.scadaAddress.secretName is not set." { - cd `chart_dir` - run helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.datacenter=dc-foo' \ - --set 'global.domain=bar' \ - --set 'global.cloud.enabled=true' \ - --set 'global.cloud.clientId.secretName=client-id-name' \ - --set 'global.cloud.clientId.secretKey=client-id-key' \ - --set 'global.cloud.clientSecret.secretName=client-secret-id-name' \ - --set 'global.cloud.clientSecret.secretKey=client-secret-id-key' \ - --set 'global.cloud.resourceId.secretName=resource-id-name' \ - --set 'global.cloud.resourceId.secretKey=resource-id-key' \ - --set 'global.cloud.scadaAddress.secretKey=scada-address-key' \ - . - [ "$status" -eq 1 ] - - [[ "$output" =~ "When either global.cloud.scadaAddress.secretName or global.cloud.scadaAddress.secretKey is defined, both must be set." ]] -} - -#-------------------------------------------------------------------- -# CONSUL_HTTP_SSL - -@test "apiGateway/Deployment: CONSUL_HTTP_SSL set correctly when not using TLS." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[2].value' | tee /dev/stderr) - [ "${actual}" = "\"false\"" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_SSL set correctly when using TLS." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[3].value' | tee /dev/stderr) - [ "${actual}" = "\"true\"" ] -} - -#-------------------------------------------------------------------- -# CONSUL_HTTP_ADDR - -@test "apiGateway/Deployment: CONSUL_HTTP_ADDR set correctly with external servers, TLS, and no clients." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.httpsPort=8501' \ - --set 'server.enabled=false' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[2].value] | any(contains("external-consul.host:8501"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_ADDR set correctly with external servers, no TLS, and no clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=false' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.httpsPort=8500' \ - --set 'server.enabled=false' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[1].value] | any(contains("external-consul.host:8500"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_ADDR set correctly with local servers, TLS, and clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[2].value] | any(contains("$(HOST_IP):8501"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_ADDR set correctly with local servers, no TLS, and clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=false' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[1].value] | any(contains("$(HOST_IP):8500"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_ADDR set correctly with local servers, TLS, and no clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[2].value] | any(contains("release-name-consul-server:8501"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_HTTP_ADDR set correctly with local servers, no TLS, and no clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=false' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[1].value] | any(contains("release-name-consul-server:8500"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -#-------------------------------------------------------------------- -# externalServers tlsServerName - -@test "apiGateway/Deployment: CONSUL_TLS_SERVER_NAME can be set for externalServers" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.httpsPort=8501' \ - --set 'externalServers.tlsServerName=hashi' \ - --set 'server.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[4].value == "hashi"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_TLS_SERVER_NAME will not be set for when clients are used" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.httpsPort=8501' \ - --set 'externalServers.tlsServerName=hashi' \ - --set 'client.enabled=true' \ - --set 'server.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[] | select (.name == "api-gateway-controller") | .env[] | select(.name == "CONSUL_TLS_SERVER_NAME")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -#-------------------------------------------------------------------- -# Admin Partitions - -@test "apiGateway/Deployment: CONSUL_PARTITION is set when using admin partitions" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'global.adminPartitions.enabled=true' \ - --set 'global.adminPartitions.name=hashi' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[3].value == "hashi"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_LOGIN_PARTITION is set when using admin partitions with ACLs" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'global.adminPartitions.enabled=true' \ - --set 'global.adminPartitions.name=hashi' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[6].value == "hashi"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_DYNAMIC_SERVER_DISCOVERY is set when not using clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[3].value == "true"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_DYNAMIC_SERVER_DISCOVERY is not set when using clients" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[3]' | tee /dev/stderr) - [ "${actual}" = "null" ] -} - -@test "apiGateway/Deployment: CONSUL_CACERT is set when using tls and clients even when useSystemRoots is true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=false' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.useSystemRoots=true' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[0].name == "CONSUL_CACERT"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_CACERT is set when using tls and internal servers" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[0].name == "CONSUL_CACERT"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_CACERT has correct path with Vault as secrets backend and client disabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.caCert.secretName=foo' \ - --set 'server.enabled=true' \ - --set 'client.enabled=false' \ - --set 'global.secretsBackend.vault.enabled=true' \ - --set 'global.secretsBackend.vault.consulServerRole=foo' \ - . | tee /dev/stderr| - yq '.spec.template.spec.containers[0].env[0].value == "/vault/secrets/serverca.crt"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Deployment: CONSUL_CACERT is not set when using tls and useSystemRoots" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=false' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.useSystemRoots=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].env[0].name == "CONSUL_CACERT"' | tee /dev/stderr) - [ "${actual}" = "false" ] -} - -@test "apiGateway/Deployment: consul-ca-cert volume mount is not set when using externalServers and useSystemRoots" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=false' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.useSystemRoots=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -@test "apiGateway/Deployment: consul-ca-cert volume mount is not set when using Vault as a secrets backend" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=true' \ - --set 'global.secretsBackend.vault.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -@test "apiGateway/Deployment: consul-ca-cert volume mount is not set on acl-init when using externalServers and useSystemRoots" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=false' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.useSystemRoots=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -@test "apiGateway/Deployment: consul-ca-cert volume mount is not set on acl-init when using Vault as secrets backend" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'server.enabled=true' \ - --set 'global.secretsBackend.vault.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -@test "apiGateway/Deployment: consul-auto-encrypt-ca-cert volume mount is set when tls.enabled, client.enabled, externalServers, useSystemRoots, and autoencrypt" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'client.enabled=true' \ - --set 'server.enabled=false' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.useSystemRoots=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-auto-encrypt-ca-cert") | .mountPath' | tee /dev/stderr) - [ "${actual}" = '"/consul/tls/ca"' ] -} - -#-------------------------------------------------------------------- -# extraLabels - -@test "apiGateway/Deployment: no extra labels defined by default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - . | tee /dev/stderr | - yq -r '.spec.template.metadata.labels | del(."app") | del(."chart") | del(."release") | del(."component")' | tee /dev/stderr) - [ "${actual}" = "{}" ] -} - -@test "apiGateway/Deployment: extra global labels can be set" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.extraLabels.foo=bar' \ - . | tee /dev/stderr) - local actualBar=$(echo "${actual}" | yq -r '.metadata.labels.foo' | tee /dev/stderr) - [ "${actualBar}" = "bar" ] - local actualTemplateBar=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) - [ "${actualTemplateBar}" = "bar" ] -} - -@test "apiGateway/Deployment: multiple global extra labels can be set" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-deployment.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=bar' \ - --set 'global.extraLabels.foo=bar' \ - --set 'global.extraLabels.baz=qux' \ - . | tee /dev/stderr) - local actualFoo=$(echo "${actual}" | yq -r '.metadata.labels.foo' | tee /dev/stderr) - local actualBaz=$(echo "${actual}" | yq -r '.metadata.labels.baz' | tee /dev/stderr) - [ "${actualFoo}" = "bar" ] - [ "${actualBaz}" = "qux" ] - local actualTemplateFoo=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) - local actualTemplateBaz=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.baz' | tee /dev/stderr) - [ "${actualTemplateFoo}" = "bar" ] - [ "${actualTemplateBaz}" = "qux" ] -} diff --git a/charts/consul/test/unit/api-gateway-controller-podsecuritypolicy.bats b/charts/consul/test/unit/api-gateway-controller-podsecuritypolicy.bats deleted file mode 100644 index dfd40c793f..0000000000 --- a/charts/consul/test/unit/api-gateway-controller-podsecuritypolicy.bats +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/PodSecurityPolicy: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-podsecuritypolicy.yaml \ - . -} - -@test "apiGateway/PodSecurityPolicy: enabled with apiGateway.enabled=true and global.enablePodSecurityPolicies=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-podsecuritypolicy.yaml \ - --set 'global.enablePodSecurityPolicies=true' \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} diff --git a/charts/consul/test/unit/api-gateway-controller-service.bats b/charts/consul/test/unit/api-gateway-controller-service.bats deleted file mode 100755 index 47cb7ff9aa..0000000000 --- a/charts/consul/test/unit/api-gateway-controller-service.bats +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/Service: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-service.yaml \ - . -} - -@test "apiGateway/Service: enable with apiGateway.enabled set to true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-service.yaml \ - --set 'global.enabled=false' \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/Service: disable with apiGateway.enabled" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-service.yaml \ - --set 'apiGateway.enabled=false' \ - . -} diff --git a/charts/consul/test/unit/api-gateway-controller-serviceaccount.bats b/charts/consul/test/unit/api-gateway-controller-serviceaccount.bats deleted file mode 100644 index 22486799b2..0000000000 --- a/charts/consul/test/unit/api-gateway-controller-serviceaccount.bats +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/ServiceAccount: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-serviceaccount.yaml \ - . -} - -@test "apiGateway/ServiceAccount: enabled with apiGateway.enabled true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-serviceaccount.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq -s 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/ServiceAccount: disabled with apiGateway.enabled false" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-controller-serviceaccount.yaml \ - --set 'apiGateway.enabled=false' \ - . -} -#-------------------------------------------------------------------- -# global.imagePullSecrets - -@test "apiGateway/ServiceAccount: can set image pull secrets" { - cd `chart_dir` - local object=$(helm template \ - -s templates/api-gateway-controller-serviceaccount.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.imagePullSecrets[0].name=my-secret' \ - --set 'global.imagePullSecrets[1].name=my-secret2' \ - . | tee /dev/stderr) - - local actual=$(echo "$object" | - yq -r '.imagePullSecrets[0].name' | tee /dev/stderr) - [ "${actual}" = "my-secret" ] - - local actual=$(echo "$object" | - yq -r '.imagePullSecrets[1].name' | tee /dev/stderr) - [ "${actual}" = "my-secret2" ] -} - -#-------------------------------------------------------------------- -# apiGateway.serviceAccount.annotations - -@test "apiGateway/ServiceAccount: no annotations by default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-serviceaccount.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.metadata.annotations | length > 0' | tee /dev/stderr) - [ "${actual}" = "false" ] -} - -@test "apiGateway/ServiceAccount: annotations when enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-controller-serviceaccount.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set "apiGateway.serviceAccount.annotations=foo: bar" \ - . | tee /dev/stderr | - yq -r '.metadata.annotations.foo' | tee /dev/stderr) - [ "${actual}" = "bar" ] -} diff --git a/charts/consul/test/unit/api-gateway-gatewayclass.bats b/charts/consul/test/unit/api-gateway-gatewayclass.bats deleted file mode 100755 index c79753c2f3..0000000000 --- a/charts/consul/test/unit/api-gateway-gatewayclass.bats +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/GatewayClass: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-gatewayclass.yaml \ - . -} - -@test "apiGateway/GatewayClass: enable with global.enabled false" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclass.yaml \ - --set 'global.enabled=false' \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClass: disable with apiGateway.enabled" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-gatewayclass.yaml \ - --set 'apiGateway.enabled=false' \ - . -} - -@test "apiGateway/GatewayClass: disable with global.enabled" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-gatewayclass.yaml \ - --set 'global.enabled=false' \ - . -} - -@test "apiGateway/GatewayClass: disable with apiGateway.managedGatewayClass.enabled" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-gatewayclass.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.managedGatewayClass.enabled=false' \ - . -} diff --git a/charts/consul/test/unit/api-gateway-gatewayclassconfig.bats b/charts/consul/test/unit/api-gateway-gatewayclassconfig.bats deleted file mode 100644 index 742f31afa0..0000000000 --- a/charts/consul/test/unit/api-gateway-gatewayclassconfig.bats +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bats - -load _helpers - -@test "apiGateway/GatewayClassConfig: disabled by default" { - cd `chart_dir` - assert_empty helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - . -} - -@test "apiGateway/GatewayClassConfig: enabled with apiGateway.enabled=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: deployment config disabled by default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - . | tee /dev/stderr | - yq '.spec | has("deployment") | not' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: deployment config enabled with defaultInstances=3" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.managedGatewayClass.deployment.defaultInstances=3' \ - . | tee /dev/stderr | - yq '.spec.deployment.defaultInstances == 3' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: deployment config enabled with maxInstances=3" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.managedGatewayClass.deployment.maxInstances=3' \ - . | tee /dev/stderr | - yq '.spec.deployment.maxInstances == 3' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: deployment config enabled with minInstances=3" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.managedGatewayClass.deployment.minInstances=3' \ - . | tee /dev/stderr | - yq '.spec.deployment.minInstances == 3' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: imageEnvoy can be set" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'apiGateway.imageEnvoy=bar' \ - . | tee /dev/stderr | - yq '.spec.image.envoy' | tee /dev/stderr) - [ "${actual}" = "\"bar\"" ] -} - -#-------------------------------------------------------------------- -# Consul server address - -@test "apiGateway/GatewayClassConfig: Consul server address set with external servers and no clients." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'server.enabled=false' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.consul.address == "external-consul.host"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: Consul server address set with external servers and clients." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'server.enabled=false' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.consul.address == "$(HOST_IP)"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: Consul server address set with local servers and no clients." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'client.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.consul.address == "release-name-consul-server.default.svc"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "apiGateway/GatewayClassConfig: Consul server address set with local servers and clients." { - cd `chart_dir` - local actual=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'client.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.consul.address == "$(HOST_IP)"' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -#-------------------------------------------------------------------- -# externalServers ports - -@test "apiGateway/GatewayClassConfig: ports for externalServers when not using TLS." { - cd `chart_dir` - local ports=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=false' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.grpcPort=1234' \ - --set 'externalServers.httpsPort=5678' \ - --set 'server.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.consul.ports' | tee /dev/stderr) - - local actual - actual=$(echo $ports | jq -r '.grpc' | tee /dev/stderr) - [ "${actual}" = "1234" ] - - actual=$(echo $ports | jq -r '.http' | tee /dev/stderr) - [ "${actual}" = "5678" ] -} - -@test "apiGateway/GatewayClassConfig: ports for externalServers when using TLS." { - cd `chart_dir` - local ports=$(helm template \ - -s templates/api-gateway-gatewayclassconfig.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ - --set 'global.tls.enabled=true' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=external-consul.host' \ - --set 'externalServers.grpcPort=1234' \ - --set 'externalServers.httpsPort=5678' \ - --set 'server.enabled=false' \ - . | tee /dev/stderr | - yq '.spec.consul.ports' | tee /dev/stderr) - - local actual - actual=$(echo $ports | jq -r '.grpc' | tee /dev/stderr) - [ "${actual}" = "1234" ] - - actual=$(echo $ports | jq -r '.http' | tee /dev/stderr) - [ "${actual}" = "5678" ] -} diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats index 5c42571265..0da6507e53 100755 --- a/charts/consul/test/unit/client-daemonset.bats +++ b/charts/consul/test/unit/client-daemonset.bats @@ -2811,8 +2811,6 @@ rollingUpdate: cd `chart_dir` run helm template \ -s templates/client-daemonset.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ --set 'global.tls.enabled=true' \ --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.datacenter=dc-foo' \ @@ -2831,8 +2829,6 @@ rollingUpdate: cd `chart_dir` run helm template \ -s templates/client-daemonset.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ --set 'global.tls.enabled=true' \ --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.datacenter=dc-foo' \ @@ -2852,8 +2848,6 @@ rollingUpdate: cd `chart_dir` run helm template \ -s templates/client-daemonset.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ --set 'global.tls.enabled=true' \ --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.datacenter=dc-foo' \ @@ -2876,8 +2870,6 @@ rollingUpdate: cd `chart_dir` run helm template \ -s templates/client-daemonset.yaml \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=foo' \ --set 'global.tls.enabled=true' \ --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.datacenter=dc-foo' \ diff --git a/charts/consul/test/unit/gateway-resources-job.bats b/charts/consul/test/unit/gateway-resources-job.bats index e38397231b..32173838fe 100644 --- a/charts/consul/test/unit/gateway-resources-job.bats +++ b/charts/consul/test/unit/gateway-resources-job.bats @@ -4,6 +4,15 @@ load _helpers target=templates/gateway-resources-job.yaml +@test "gatewayresources/Job: fails if .values.apiGateway is set" { + cd `chart_dir` + run helm template \ + -s templates/tests/test-runner.yaml \ + --set 'apiGateway.enabled=true' . + [ "$status" -eq 1 ] + [[ "$output" =~ "[DEPRECATED and REMOVED] the apiGateway stanza is no longer supported as of Consul 1.19.0. Use connectInject.apiGateway instead." ]] +} + @test "gatewayresources/Job: enabled by default" { cd `chart_dir` local actual=$(helm template \ @@ -31,33 +40,6 @@ target=templates/gateway-resources-job.yaml [ "$actual" = "true" ] } -#-------------------------------------------------------------------- -# fallback configuration -# to be removed in 1.17 (t-eckert 2023-05-23) - -@test "gatewayresources/Job: fallback configuration is used when apiGateway.enabled is true" { - cd `chart_dir` - local spec=$(helm template \ - -s $target \ - --set 'apiGateway.enabled=true' \ - --set 'apiGateway.image=testing' \ - --set 'apiGateway.managedGatewayClass.nodeSelector=foo: bar' \ - --set 'apiGateway.managedGatewayClass.tolerations=- key: bar' \ - --set 'apiGateway.managedGatewayClass.copyAnnotations.service.annotations=- bingo' \ - --set 'apiGateway.managedGatewayClass.serviceType=LoadBalancer' \ - . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].args' | tee /dev/stderr) - - local actual=$(echo "$spec" | jq '.[9] | ."-node-selector=foo"') - [ "${actual}" = "\"bar\"" ] - - local actual=$(echo "$spec" | jq '.[10] | ."-tolerations=- key"') - [ "${actual}" = "\"bar\"" ] - - local actual=$(echo "$spec" | jq '.[11]') - [ "${actual}" = "\"-service-annotations=- bingo\"" ] -} - #-------------------------------------------------------------------- # configuration diff --git a/charts/consul/test/unit/helpers.bats b/charts/consul/test/unit/helpers.bats index 20772788f8..4e33b91886 100644 --- a/charts/consul/test/unit/helpers.bats +++ b/charts/consul/test/unit/helpers.bats @@ -454,15 +454,3 @@ load _helpers [ "$status" -eq 1 ] [[ "$output" =~ "When the value global.experiments.resourceAPIs is set, terminatingGateways.enabled is currently unsupported." ]] } - -@test "connectInject/Deployment: fails if resource-apis is set and apiGateway is enabled" { - cd `chart_dir` - run helm template \ - -s templates/tests/test-runner.yaml \ - --set 'connectInject.enabled=true' \ - --set 'global.experiments[0]=resource-apis' \ - --set 'ui.enabled=false' \ - --set 'apiGateway.enabled=true' . - [ "$status" -eq 1 ] - [[ "$output" =~ "When the value global.experiments.resourceAPIs is set, apiGateway.enabled is currently unsupported." ]] -} diff --git a/charts/consul/test/unit/sync-catalog-clusterrole.bats b/charts/consul/test/unit/sync-catalog-clusterrole.bats index 17141e434f..afc3a42b45 100755 --- a/charts/consul/test/unit/sync-catalog-clusterrole.bats +++ b/charts/consul/test/unit/sync-catalog-clusterrole.bats @@ -56,7 +56,7 @@ load _helpers --set 'syncCatalog.enabled=true' \ --set 'global.enablePodSecurityPolicies=true' \ . | tee /dev/stderr | - yq -r '.rules[2].resources[0]' | tee /dev/stderr) + yq -r '.rules[3].resources[0]' | tee /dev/stderr) [ "${actual}" = "podsecuritypolicies" ] } @@ -83,4 +83,12 @@ load _helpers . | tee /dev/stderr | yq -c '.rules[0].verbs' | tee /dev/stderr) [ "${actual}" = '["get","list","watch","update","patch","delete","create"]' ] + + actual=$(helm template \ + -s templates/sync-catalog-clusterrole.yaml \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.toK8S=true' \ + . | tee /dev/stderr | + yq -c '.rules[0].verbs' | tee /dev/stderr) + [ "${actual}" = '["get","list","watch","update","patch","delete","create"]' ] } diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 2efbd96c68..7d4f00210c 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -2417,6 +2417,19 @@ connectInject: # @type: string service: null + # Metrics settings for gateways created with this gateway class configuration. + metrics: + # This value enables or disables metrics collection on a gateway, overriding the global gateway metrics collection settings. + # @type: boolean + enabled: "-" + # This value sets the port to use for scraping gateway metrics via prometheus, defaults to 20200 if not set. Must be in the port + # range of 1024-65535. + # @type: int + port: null + # This value sets the path to use for scraping gateway metrics via prometheus, defaults to /metrics if not set. + # @type: string + path: null + # The resource settings for Pods handling traffic for Gateway API. # @recurse: false # @type: map @@ -3427,175 +3440,6 @@ terminatingGateways: gateways: - name: terminating-gateway -# [DEPRECATED] Use connectInject.apiGateway instead. -# Configuration settings for the Consul API Gateway integration -apiGateway: - # When true the helm chart will install the Consul API Gateway controller - enabled: false - - # Image to use for the api-gateway-controller pods and gateway instances - # - # ~> **Note:** Using API Gateway <= 0.4 with external servers requires setting `client.enabled: true`. - # @type: string - image: null - - # The name (and tag) of the Envoy Docker image used for the - # apiGateway. For other Consul compoenents, imageEnvoy has been replaced with Consul Dataplane. - # @default: envoyproxy/envoy: - imageEnvoy: "envoyproxy/envoy:v1.25.11" - - # Override global log verbosity level for api-gateway-controller pods. One of "debug", "info", "warn", or "error". - # @type: string - logLevel: info - - # Configuration settings for the optional GatewayClass installed by consul-k8s (enabled by default) - managedGatewayClass: - # When true a GatewayClass is configured to automatically work with Consul as installed by helm. - enabled: true - - # This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) - # labels for gateway pod assignment, formatted as a multi-line string. - # - # Example: - # - # ```yaml - # nodeSelector: | - # beta.kubernetes.io/arch: amd64 - # ``` - # - # @type: string - nodeSelector: null - - # Toleration settings for gateway pods created with the managed gateway class. - # This should be a multi-line string matching the - # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. - # - # @type: string - tolerations: null - - # This value defines the type of service created for gateways (e.g. LoadBalancer, ClusterIP) - serviceType: LoadBalancer - - # This value toggles if the gateway ports should be mapped to host ports - useHostPorts: false - - # Configuration settings for annotations to be copied from the Gateway to other child resources. - copyAnnotations: - # This value defines a list of annotations to be copied from the Gateway to the Service created, formatted as a multi-line string. - # - # Example: - # - # ```yaml - # service: - # annotations: | - # - external-dns.alpha.kubernetes.io/hostname - # ``` - # - # @type: string - service: null - - # This value defines the number of pods to deploy for each Gateway as well as a min and max number of pods for all Gateways - # - # Example: - # - # ```yaml - # deployment: - # defaultInstances: 3 - # maxInstances: 8 - # minInstances: 1 - # ``` - # - # @type: map - deployment: null - - # Configuration for the ServiceAccount created for the api-gateway component - serviceAccount: - # This value defines additional annotations for the client service account. This should be formatted as a multi-line - # string. - # - # ```yaml - # annotations: | - # "sample/annotation1": "foo" - # "sample/annotation2": "bar" - # ``` - # - # @type: string - annotations: null - - # Configuration for the api-gateway controller component - controller: - # This value sets the number of controller replicas to deploy. - replicas: 1 - - # Annotations to apply to the api-gateway-controller pods. - # - # ```yaml - # annotations: | - # "annotation-key": "annotation-value" - # ``` - # - # @type: string - annotations: null - - # This value references an existing - # Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) - # that can be assigned to api-gateway-controller pods. - priorityClassName: "" - - # This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) - # labels for api-gateway-controller pod assignment, formatted as a multi-line string. - # - # Example: - # - # ```yaml - # nodeSelector: | - # beta.kubernetes.io/arch: amd64 - # ``` - # - # @type: string - nodeSelector: null - - # This value defines the tolerations for api-gateway-controller pod, this should be a multi-line string matching the - # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. - # - # @type: string - tolerations: null - - # Configuration for the Service created for the api-gateway-controller - service: - # Annotations to apply to the api-gateway-controller service. - # - # ```yaml - # annotations: | - # "annotation-key": "annotation-value" - # ``` - # - # @type: string - annotations: null - - # The resource settings for api gateway pods. - # @recurse: false - # @type: map - resources: - requests: - memory: "100Mi" - cpu: "100m" - limits: - memory: "100Mi" - cpu: "100m" - - # The resource settings for the `copy-consul-bin` init container. - # @recurse: false - # @type: map - initCopyConsulContainer: - resources: - requests: - memory: "25Mi" - cpu: "50m" - limits: - memory: "150Mi" - cpu: "50m" - # Configuration settings for the webhook-cert-manager # `webhook-cert-manager` ensures that cert bundles are up to date for the mutating webhook. webhookCertManager: diff --git a/control-plane/Dockerfile b/control-plane/Dockerfile index dda0ab6a96..b1752507fd 100644 --- a/control-plane/Dockerfile +++ b/control-plane/Dockerfile @@ -17,7 +17,7 @@ # go-discover builds the discover binary (which we don't currently publish # either). ARG GOLANG_VERSION -FROM golang:${GOLANG_VERSION}-alpine as go-discover +FROM golang:${GOLANG_VERSION}-alpine3.19 as go-discover RUN CGO_ENABLED=0 go install github.com/hashicorp/go-discover/cmd/discover@214571b6a5309addf3db7775f4ee8cf4d264fd5f # dev copies the binary from a local build diff --git a/control-plane/api-gateway/binding/binder.go b/control-plane/api-gateway/binding/binder.go index c6557985b5..c704a6b04e 100644 --- a/control-plane/api-gateway/binding/binder.go +++ b/control-plane/api-gateway/binding/binder.go @@ -67,6 +67,9 @@ type BinderConfig struct { // Policies is a list containing all GatewayPolicies that are part of the Gateway Deployment Policies []v1alpha1.GatewayPolicy + + // Configuration from helm. + HelmConfig common.HelmConfig } // Binder is used for generating a Snapshot of all operations that should occur both @@ -199,7 +202,8 @@ func (b *Binder) Snapshot() *Snapshot { OnUpdate: b.handleGatewaySyncStatus(snapshot, &b.config.Gateway, consulStatus), }) - registrations := registrationsForPods(entry.Namespace, b.config.Gateway, registrationPods) + metricsConfig := common.GatewayMetricsConfig(b.config.Gateway, *gatewayClassConfig, b.config.HelmConfig) + registrations := registrationsForPods(metricsConfig, entry.Namespace, b.config.Gateway, registrationPods) snapshot.Consul.Registrations = registrations // deregister any not explicitly registered service diff --git a/control-plane/api-gateway/binding/registration.go b/control-plane/api-gateway/binding/registration.go index ae26ab51f6..489e765c61 100644 --- a/control-plane/api-gateway/binding/registration.go +++ b/control-plane/api-gateway/binding/registration.go @@ -6,6 +6,7 @@ package binding import ( "fmt" + gatewaycommon "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" "github.com/hashicorp/consul/api" @@ -22,22 +23,34 @@ const ( // consulKubernetesCheckName is the name of health check in Consul for Kubernetes readiness status. consulKubernetesCheckName = "Kubernetes Readiness Check" + + // metricsConfiguration is the configuration key for binding a prometheus port to the envoy instance. + metricsConfiguration = "envoy_prometheus_bind_addr" ) -func registrationsForPods(namespace string, gateway gwv1beta1.Gateway, pods []corev1.Pod) []api.CatalogRegistration { +func registrationsForPods(metrics gatewaycommon.MetricsConfig, namespace string, gateway gwv1beta1.Gateway, pods []corev1.Pod) []api.CatalogRegistration { registrations := []api.CatalogRegistration{} for _, pod := range pods { - registrations = append(registrations, registrationForPod(namespace, gateway, pod)) + registrations = append(registrations, registrationForPod(metrics, namespace, gateway, pod)) } return registrations } -func registrationForPod(namespace string, gateway gwv1beta1.Gateway, pod corev1.Pod) api.CatalogRegistration { +func registrationForPod(metrics gatewaycommon.MetricsConfig, namespace string, gateway gwv1beta1.Gateway, pod corev1.Pod) api.CatalogRegistration { healthStatus := api.HealthCritical if isPodReady(pod) { healthStatus = api.HealthPassing } + var proxyConfigOverrides *api.AgentServiceConnectProxyConfig + if metrics.Enabled { + proxyConfigOverrides = &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + metricsConfiguration: fmt.Sprintf("%s:%d", pod.Status.PodIP, metrics.Port), + }, + } + } + return api.CatalogRegistration{ Node: common.ConsulNodeNameFromK8sNode(pod.Spec.NodeName), Address: pod.Status.HostIP, @@ -50,6 +63,7 @@ func registrationForPod(namespace string, gateway gwv1beta1.Gateway, pod corev1. Service: gateway.Name, Address: pod.Status.PodIP, Namespace: namespace, + Proxy: proxyConfigOverrides, Meta: map[string]string{ constants.MetaKeyPodName: pod.Name, constants.MetaKeyKubeNS: pod.Namespace, diff --git a/control-plane/api-gateway/binding/registration_test.go b/control-plane/api-gateway/binding/registration_test.go index 356915f9f7..6f60257112 100644 --- a/control-plane/api-gateway/binding/registration_test.go +++ b/control-plane/api-gateway/binding/registration_test.go @@ -6,6 +6,7 @@ package binding import ( "testing" + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" "github.com/hashicorp/consul/api" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -68,7 +69,7 @@ func TestRegistrationsForPods_Health(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - registrations := registrationsForPods(tt.consulNamespace, tt.gateway, tt.pods) + registrations := registrationsForPods(common.MetricsConfig{}, tt.consulNamespace, tt.gateway, tt.pods) require.Len(t, registrations, len(tt.expected)) for i := range registrations { diff --git a/control-plane/api-gateway/cache/consul.go b/control-plane/api-gateway/cache/consul.go index f2d6ec9bf9..0b0d067df7 100644 --- a/control-plane/api-gateway/cache/consul.go +++ b/control-plane/api-gateway/cache/consul.go @@ -83,6 +83,12 @@ type Cache struct { subscribers map[string][]*Subscription subscriberMutex *sync.Mutex + gatewayNameToPolicy map[string]*api.ACLPolicy + policyMutex *sync.Mutex + + gatewayNameToRole map[string]*api.ACLRole + aclRoleMutex *sync.Mutex + namespacesEnabled bool crossNamespaceACLPolicy string @@ -109,6 +115,10 @@ func New(config Config) *Cache { cacheMutex: &sync.Mutex{}, subscribers: make(map[string][]*Subscription), subscriberMutex: &sync.Mutex{}, + gatewayNameToPolicy: make(map[string]*api.ACLPolicy), + policyMutex: &sync.Mutex{}, + gatewayNameToRole: make(map[string]*api.ACLRole), + aclRoleMutex: &sync.Mutex{}, kinds: Kinds, synced: make(chan struct{}, len(Kinds)), logger: config.Logger, @@ -339,21 +349,47 @@ func (c *Cache) Write(ctx context.Context, entry api.ConfigEntry) error { } func (c *Cache) ensurePolicy(client *api.Client, gatewayName string) (string, error) { - policy := c.gatewayPolicy(gatewayName) + c.policyMutex.Lock() + defer c.policyMutex.Unlock() - created, _, err := client.ACL().PolicyCreate(&policy, &api.WriteOptions{}) + createPolicy := func() (string, error) { + policy := c.gatewayPolicy(gatewayName) + + created, _, err := client.ACL().PolicyCreate(&policy, &api.WriteOptions{}) + + if isPolicyExistsErr(err, policy.Name) { + existing, _, err := client.ACL().PolicyReadByName(policy.Name, &api.QueryOptions{}) + if err != nil { + return "", err + } + return existing.ID, nil + } - if isPolicyExistsErr(err, policy.Name) { - existing, _, err := client.ACL().PolicyReadByName(policy.Name, &api.QueryOptions{}) if err != nil { return "", err } - return existing.ID, nil + + c.gatewayNameToPolicy[gatewayName] = created + return created.ID, nil } + + cachedPolicy, found := c.gatewayNameToPolicy[gatewayName] + + if !found { + return createPolicy() + } + + existing, _, err := client.ACL().PolicyReadByName(cachedPolicy.Name, &api.QueryOptions{}) + + if existing == nil { + return createPolicy() + } + if err != nil { return "", err } - return created.ID, nil + + return existing.ID, nil } func (c *Cache) ensureRole(client *api.Client, gatewayName string) (string, error) { @@ -362,24 +398,41 @@ func (c *Cache) ensureRole(client *api.Client, gatewayName string) (string, erro return "", err } - aclRoleName := fmt.Sprint("managed-gateway-acl-role-", gatewayName) + c.aclRoleMutex.Lock() + defer c.aclRoleMutex.Unlock() + + createRole := func() (string, error) { + aclRoleName := fmt.Sprint("managed-gateway-acl-role-", gatewayName) + role := &api.ACLRole{ + Name: aclRoleName, + Description: "ACL Role for Managed API Gateways", + Policies: []*api.ACLLink{{ID: policyID}}, + } + + _, _, err = client.ACL().RoleCreate(role, &api.WriteOptions{}) + if err != nil { + return "", err + } + c.gatewayNameToRole[gatewayName] = role + return aclRoleName, err + } + + cachedRole, found := c.gatewayNameToRole[gatewayName] + + if !found { + return createRole() + } - aclRole, _, err := client.ACL().RoleReadByName(aclRoleName, &api.QueryOptions{}) + aclRole, _, err := client.ACL().RoleReadByName(cachedRole.Name, &api.QueryOptions{}) if err != nil { return "", err } - if aclRole != nil { - return aclRoleName, nil - } - role := &api.ACLRole{ - Name: aclRoleName, - Description: "ACL Role for Managed API Gateways", - Policies: []*api.ACLLink{{ID: policyID}}, + if aclRole != nil { + return cachedRole.Name, nil } - _, _, err = client.ACL().RoleCreate(role, &api.WriteOptions{}) - return aclRoleName, err + return createRole() } func (c *Cache) gatewayPolicy(gatewayName string) api.ACLPolicy { diff --git a/control-plane/api-gateway/common/helm_config.go b/control-plane/api-gateway/common/helm_config.go index 9b0ab1d7e8..ecf245c04c 100644 --- a/control-plane/api-gateway/common/helm_config.go +++ b/control-plane/api-gateway/common/helm_config.go @@ -43,6 +43,16 @@ type HelmConfig struct { // defined on a Gateway. MapPrivilegedServicePorts int + // EnableGatewayMetrics indicates whether or not gateway metrics should be enabled + // by default on a deployed gateway, passed from the helm chart via command-line flags to our controller. + EnableGatewayMetrics bool + + // The default path to use for scraping prometheus metrics, passed from the helm chart via command-line flags to our controller. + DefaultPrometheusScrapePath string + + // The default port to use for scraping prometheus metrics, passed from the helm chart via command-line flags to our controller. + DefaultPrometheusScrapePort string + InitContainerResources *v1.ResourceRequirements } diff --git a/control-plane/api-gateway/common/metrics.go b/control-plane/api-gateway/common/metrics.go new file mode 100644 index 0000000000..8ba582c8a5 --- /dev/null +++ b/control-plane/api-gateway/common/metrics.go @@ -0,0 +1,103 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package common + +import ( + "strconv" + + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +const ( + defaultScrapePort = 20200 + defaultScrapePath = "/metrics" +) + +type MetricsConfig struct { + Enabled bool + Path string + Port int +} + +func gatewayMetricsEnabled(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayClassConfig, config HelmConfig) bool { + // first check our annotations, if something is there, then it means we've explicitly + // annotated metrics collection + if scrape, isSet := gateway.Annotations[constants.AnnotationEnableMetrics]; isSet { + enabled, err := strconv.ParseBool(scrape) + if err == nil { + return enabled + } + // TODO: log an error + // we fall through to the other metrics enabled checks + } + + // if it's not set on the annotation, then we check to see if it's set on the GatewayClassConfig + if gcc.Spec.Metrics.Enabled != nil { + return *gcc.Spec.Metrics.Enabled + } + + // otherwise, fallback to the global helm setting + return config.EnableGatewayMetrics +} + +func fetchPortString(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayClassConfig, config HelmConfig) string { + // first check our annotations, if something is there, then it means we've explicitly + // annotated metrics collection + if portString, isSet := gateway.Annotations[constants.AnnotationPrometheusScrapePort]; isSet { + return portString + } + + // if it's not set on the annotation, then we check to see if it's set on the GatewayClassConfig + if gcc.Spec.Metrics.Port != nil { + return strconv.Itoa(int(*gcc.Spec.Metrics.Port)) + } + + // otherwise, fallback to the global helm setting + return config.DefaultPrometheusScrapePort +} + +func gatewayMetricsPort(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayClassConfig, config HelmConfig) int { + portString := fetchPortString(gateway, gcc, config) + + port, err := strconv.Atoi(portString) + if err != nil { + // if we can't parse the port string, just use the default + // TODO: log an error + return defaultScrapePort + } + + if port < 1024 || port > 65535 { + // if we requested a privileged port, use the default + // TODO: log an error + return defaultScrapePort + } + + return port +} + +func gatewayMetricsPath(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayClassConfig, config HelmConfig) string { + // first check our annotations, if something is there, then it means we've explicitly + // annotated metrics collection + if path, isSet := gateway.Annotations[constants.AnnotationPrometheusScrapePath]; isSet { + return path + } + + // if it's not set on the annotation, then we check to see if it's set on the GatewayClassConfig + if gcc.Spec.Metrics.Path != nil { + return *gcc.Spec.Metrics.Path + } + + // otherwise, fallback to the global helm setting + return config.DefaultPrometheusScrapePath +} + +func GatewayMetricsConfig(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayClassConfig, config HelmConfig) MetricsConfig { + return MetricsConfig{ + Enabled: gatewayMetricsEnabled(gateway, gcc, config), + Path: gatewayMetricsPath(gateway, gcc, config), + Port: gatewayMetricsPort(gateway, gcc, config), + } +} diff --git a/control-plane/api-gateway/controllers/gateway_controller.go b/control-plane/api-gateway/controllers/gateway_controller.go index 8447e69f64..537100fd70 100644 --- a/control-plane/api-gateway/controllers/gateway_controller.go +++ b/control-plane/api-gateway/controllers/gateway_controller.go @@ -204,6 +204,7 @@ func (r *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct ConsulGateway: consulGateway, ConsulGatewayServices: consulServices, Policies: policies, + HelmConfig: r.HelmConfig, }) updates := binder.Snapshot() diff --git a/control-plane/api-gateway/gatekeeper/dataplane.go b/control-plane/api-gateway/gatekeeper/dataplane.go index 090464e9c8..16839cbb09 100644 --- a/control-plane/api-gateway/gatekeeper/dataplane.go +++ b/control-plane/api-gateway/gatekeeper/dataplane.go @@ -22,12 +22,11 @@ const ( netBindCapability = "NET_BIND_SERVICE" consulDataplaneDNSBindHost = "127.0.0.1" consulDataplaneDNSBindPort = 8600 - defaultPrometheusScrapePath = "/metrics" defaultEnvoyProxyConcurrency = 1 volumeName = "consul-connect-inject-data" ) -func consulDataplaneContainer(config common.HelmConfig, gcc v1alpha1.GatewayClassConfig, name, namespace string) (corev1.Container, error) { +func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmConfig, gcc v1alpha1.GatewayClassConfig, name, namespace string) (corev1.Container, error) { // Extract the service account token's volume mount. var ( err error @@ -38,7 +37,7 @@ func consulDataplaneContainer(config common.HelmConfig, gcc v1alpha1.GatewayClas bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" } - args, err := getDataplaneArgs(namespace, config, bearerTokenFile, name) + args, err := getDataplaneArgs(metrics, namespace, config, bearerTokenFile, name) if err != nil { return corev1.Container{}, err } @@ -100,6 +99,15 @@ func consulDataplaneContainer(config common.HelmConfig, gcc v1alpha1.GatewayClas Name: "proxy-health", ContainerPort: int32(constants.ProxyDefaultHealthPort), }) + + if metrics.Enabled { + container.Ports = append(container.Ports, corev1.ContainerPort{ + Name: "prometheus", + ContainerPort: int32(metrics.Port), + Protocol: corev1.ProtocolTCP, + }) + } + // Configure the resource requests and limits for the proxy if they are set. if gcc.Spec.DeploymentSpec.Resources != nil { container.Resources = *gcc.Spec.DeploymentSpec.Resources @@ -122,7 +130,7 @@ func consulDataplaneContainer(config common.HelmConfig, gcc v1alpha1.GatewayClas return container, nil } -func getDataplaneArgs(namespace string, config common.HelmConfig, bearerTokenFile string, name string) ([]string, error) { +func getDataplaneArgs(metrics common.MetricsConfig, namespace string, config common.HelmConfig, bearerTokenFile string, name string) ([]string, error) { proxyIDFileName := "/consul/connect-inject/proxyid" envoyConcurrency := defaultEnvoyProxyConcurrency @@ -170,9 +178,10 @@ func getDataplaneArgs(namespace string, config common.HelmConfig, bearerTokenFil args = append(args, fmt.Sprintf("-envoy-admin-bind-port=%d", 19000)) - // Set a default scrape path that can be overwritten by the annotation. - prometheusScrapePath := defaultPrometheusScrapePath - args = append(args, "-telemetry-prom-scrape-path="+prometheusScrapePath) + if metrics.Enabled { + // Set up metrics collection. + args = append(args, "-telemetry-prom-scrape-path="+metrics.Path) + } return args, nil } diff --git a/control-plane/api-gateway/gatekeeper/deployment.go b/control-plane/api-gateway/gatekeeper/deployment.go index d85fbf8ef1..d3bb56a69f 100644 --- a/control-plane/api-gateway/gatekeeper/deployment.go +++ b/control-plane/api-gateway/gatekeeper/deployment.go @@ -5,6 +5,7 @@ package gatekeeper import ( "context" + "strconv" "github.com/google/go-cmp/cmp" appsv1 "k8s.io/api/apps/v1" @@ -92,7 +93,21 @@ func (g *Gatekeeper) deployment(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayC return nil, err } - container, err := consulDataplaneContainer(config, gcc, gateway.Name, gateway.Namespace) + annotations := map[string]string{ + "consul.hashicorp.com/connect-inject": "false", + constants.AnnotationGatewayConsulServiceName: gateway.Name, + constants.AnnotationGatewayKind: "api-gateway", + } + + metrics := common.GatewayMetricsConfig(gateway, gcc, config) + + if metrics.Enabled { + annotations[constants.AnnotationPrometheusScrape] = "true" + annotations[constants.AnnotationPrometheusPath] = metrics.Path + annotations[constants.AnnotationPrometheusPort] = strconv.Itoa(metrics.Port) + } + + container, err := consulDataplaneContainer(metrics, config, gcc, gateway.Name, gateway.Namespace) if err != nil { return nil, err } @@ -110,12 +125,8 @@ func (g *Gatekeeper) deployment(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayC }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: common.LabelsForGateway(&gateway), - Annotations: map[string]string{ - constants.AnnotationInject: "false", - constants.AnnotationGatewayConsulServiceName: gateway.Name, - constants.AnnotationGatewayKind: "api-gateway", - }, + Labels: common.LabelsForGateway(&gateway), + Annotations: annotations, }, Spec: corev1.PodSpec{ Volumes: []corev1.Volume{ diff --git a/control-plane/api/v1alpha1/api_gateway_types.go b/control-plane/api/v1alpha1/api_gateway_types.go index 90f6376d98..bc7b65fbe4 100644 --- a/control-plane/api/v1alpha1/api_gateway_types.go +++ b/control-plane/api/v1alpha1/api_gateway_types.go @@ -65,6 +65,9 @@ type GatewayClassConfigSpec struct { // The value to add to privileged ports ( ports < 1024) for gateway containers MapPrivilegedContainerPorts int32 `json:"mapPrivilegedContainerPorts,omitempty"` + + // Metrics defines how to configure the metrics for a gateway. + Metrics MetricsSpec `json:"metrics,omitempty"` } // +k8s:deepcopy-gen=true @@ -90,6 +93,22 @@ type DeploymentSpec struct { Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } +// +k8s:deepcopy-gen=true + +type MetricsSpec struct { + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:validation:Minimum=1024 + // The port used for metrics. + Port *int32 `json:"port,omitempty"` + + // The path used for metrics. + Path *string `json:"path,omitempty"` + + // Enable metrics for this class of gateways. If unspecified, will inherit + // behavior from the global Helm configuration. + Enabled *bool `json:"enabled,omitempty"` +} + //+kubebuilder:object:generate=true // CopyAnnotationsSpec defines the annotations that should be copied to the gateway service. diff --git a/control-plane/api/v1alpha1/terminatinggateway_types.go b/control-plane/api/v1alpha1/terminatinggateway_types.go index cf453160ff..d439e635fe 100644 --- a/control-plane/api/v1alpha1/terminatinggateway_types.go +++ b/control-plane/api/v1alpha1/terminatinggateway_types.go @@ -79,6 +79,9 @@ type LinkedService struct { // SNI is the optional name to specify during the TLS handshake with a linked service. SNI string `json:"sni,omitempty"` + + //DisableAutoHostRewrite disables terminating gateways auto host rewrite feature when set to true. + DisableAutoHostRewrite bool `json:"disableAutoHostRewrite,omitempty"` } func (in *TerminatingGateway) GetObjectMeta() metav1.ObjectMeta { @@ -218,12 +221,13 @@ func (in *TerminatingGateway) DefaultNamespaceFields(consulMeta common.ConsulMet func (in LinkedService) toConsul() capi.LinkedService { return capi.LinkedService{ - Namespace: in.Namespace, - Name: in.Name, - CAFile: in.CAFile, - CertFile: in.CertFile, - KeyFile: in.KeyFile, - SNI: in.SNI, + Namespace: in.Namespace, + Name: in.Name, + CAFile: in.CAFile, + CertFile: in.CertFile, + KeyFile: in.KeyFile, + SNI: in.SNI, + DisableAutoHostRewrite: in.DisableAutoHostRewrite, } } diff --git a/control-plane/api/v1alpha1/terminatinggateway_types_test.go b/control-plane/api/v1alpha1/terminatinggateway_types_test.go index 2daf93c6a4..02dcbc03c1 100644 --- a/control-plane/api/v1alpha1/terminatinggateway_types_test.go +++ b/control-plane/api/v1alpha1/terminatinggateway_types_test.go @@ -49,11 +49,12 @@ func TestTerminatingGateway_MatchesConsul(t *testing.T) { Spec: TerminatingGatewaySpec{ Services: []LinkedService{ { - Name: "name", - CAFile: "caFile", - CertFile: "certFile", - KeyFile: "keyFile", - SNI: "sni", + Name: "name", + CAFile: "caFile", + CertFile: "certFile", + KeyFile: "keyFile", + SNI: "sni", + DisableAutoHostRewrite: true, }, { Name: "*", @@ -71,11 +72,12 @@ func TestTerminatingGateway_MatchesConsul(t *testing.T) { }, Services: []capi.LinkedService{ { - Name: "name", - CAFile: "caFile", - CertFile: "certFile", - KeyFile: "keyFile", - SNI: "sni", + Name: "name", + CAFile: "caFile", + CertFile: "certFile", + KeyFile: "keyFile", + SNI: "sni", + DisableAutoHostRewrite: true, }, { Name: "*", diff --git a/control-plane/api/v1alpha1/zz_generated.deepcopy.go b/control-plane/api/v1alpha1/zz_generated.deepcopy.go index 320f05510f..2a1854d178 100644 --- a/control-plane/api/v1alpha1/zz_generated.deepcopy.go +++ b/control-plane/api/v1alpha1/zz_generated.deepcopy.go @@ -555,6 +555,7 @@ func (in *GatewayClassConfigSpec) DeepCopyInto(out *GatewayClassConfigSpec) { } in.DeploymentSpec.DeepCopyInto(&out.DeploymentSpec) in.CopyAnnotations.DeepCopyInto(&out.CopyAnnotations) + in.Metrics.DeepCopyInto(&out.Metrics) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayClassConfigSpec. @@ -1977,6 +1978,36 @@ func (in *MeshTLSConfig) DeepCopy() *MeshTLSConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } + if in.Path != nil { + in, out := &in.Path, &out.Path + *out = new(string) + **out = **in + } + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsSpec. +func (in *MetricsSpec) DeepCopy() *MetricsSpec { + if in == nil { + return nil + } + out := new(MetricsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PassiveHealthCheck) DeepCopyInto(out *PassiveHealthCheck) { *out = *in diff --git a/control-plane/catalog/to-consul/resource.go b/control-plane/catalog/to-consul/resource.go index 2d29d6c15a..879789d4b5 100644 --- a/control-plane/catalog/to-consul/resource.go +++ b/control-plane/catalog/to-consul/resource.go @@ -17,6 +17,7 @@ import ( consulapi "github.com/hashicorp/consul/api" "github.com/hashicorp/go-hclog" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,10 +34,11 @@ const ( // ConsulK8SNS is the key used in the meta to record the namespace // of the service/node registration. - ConsulK8SNS = "external-k8s-ns" - ConsulK8SRefKind = "external-k8s-ref-kind" - ConsulK8SRefValue = "external-k8s-ref-name" - ConsulK8SNodeName = "external-k8s-node-name" + ConsulK8SNS = "external-k8s-ns" + ConsulK8SRefKind = "external-k8s-ref-kind" + ConsulK8SRefValue = "external-k8s-ref-name" + ConsulK8SNodeName = "external-k8s-node-name" + ConsulK8STopologyZone = "external-k8s-topology-zone" // consulKubernetesCheckType is the type of health check in Consul for Kubernetes readiness status. consulKubernetesCheckType = "kubernetes-readiness" @@ -143,9 +145,11 @@ type ServiceResource struct { // in the form /. serviceMap map[string]*corev1.Service - // endpointsMap uses the same keys as serviceMap but maps to the endpoints - // of each service. - endpointsMap map[string]*corev1.Endpoints + // endpointSlicesMap tracks EndpointSlices associated with services that are being synced to Consul. + // The outer map's keys represent service identifiers in the same format as serviceMap and maps + // each service to its related EndpointSlices. The inner map's keys are EndpointSlice name keys + // the format "/". + endpointSlicesMap map[string]map[string]*discoveryv1.EndpointSlice // EnableIngress enables syncing of the hostname from an Ingress resource // to the service registration if an Ingress rule matches the service. @@ -225,22 +229,47 @@ func (t *ServiceResource) Upsert(key string, raw interface{}) error { t.serviceMap[key] = service t.Log.Debug("[ServiceResource.Upsert] adding service to serviceMap", "key", key, "service", service) - // If we care about endpoints, we should do the initial endpoints load. + // If we care about endpoints, we should load the associated endpoint slices. if t.shouldTrackEndpoints(key) { - endpoints, err := t.Client.CoreV1(). - Endpoints(service.Namespace). - Get(t.Ctx, service.Name, metav1.GetOptions{}) - if err != nil { - t.Log.Warn("error loading initial endpoints", - "key", key, - "err", err) - } else { - if t.endpointsMap == nil { - t.endpointsMap = make(map[string]*corev1.Endpoints) + allEndpointSlices := make(map[string]*discoveryv1.EndpointSlice) + labelSelector := fmt.Sprintf("%s=%s", discoveryv1.LabelServiceName, service.Name) + continueToken := "" + limit := int64(100) + + for { + opts := metav1.ListOptions{ + LabelSelector: labelSelector, + Limit: limit, + Continue: continueToken, + } + endpointSliceList, err := t.Client.DiscoveryV1(). + EndpointSlices(service.Namespace). + List(t.Ctx, opts) + + if err != nil { + t.Log.Warn("error loading endpoint slices list", + "key", key, + "err", err) + break + } + + for _, endpointSlice := range endpointSliceList.Items { + endptKey := service.Namespace + "/" + endpointSlice.Name + allEndpointSlices[endptKey] = &endpointSlice } - t.endpointsMap[key] = endpoints - t.Log.Debug("[ServiceResource.Upsert] adding service's endpoints to endpointsMap", "key", key, "service", service, "endpoints", endpoints) + + if endpointSliceList.Continue != "" { + continueToken = endpointSliceList.Continue + } else { + break + } + } + + if t.endpointSlicesMap == nil { + t.endpointSlicesMap = make(map[string]map[string]*discoveryv1.EndpointSlice) } + t.endpointSlicesMap[key] = allEndpointSlices + t.Log.Debug("[ServiceResource.Upsert] adding service's endpoint slices to endpointSlicesMap", "key", key, "service", service, "endpointSlices", allEndpointSlices) } // Update the registration and trigger a sync @@ -265,8 +294,8 @@ func (t *ServiceResource) Delete(key string, _ interface{}) error { func (t *ServiceResource) doDelete(key string) { delete(t.serviceMap, key) t.Log.Debug("[doDelete] deleting service from serviceMap", "key", key) - delete(t.endpointsMap, key) - t.Log.Debug("[doDelete] deleting endpoints from endpointsMap", "key", key) + delete(t.endpointSlicesMap, key) + t.Log.Debug("[doDelete] deleting endpoints from endpointSlicesMap", "key", key) // If there were registrations related to this service, then // delete them and sync. if _, ok := t.consulMap[key]; ok { @@ -582,27 +611,26 @@ func (t *ServiceResource) generateRegistrations(key string) { // pods are running on. This way we don't register _every_ K8S // node as part of the service. case corev1.ServiceTypeNodePort: - if t.endpointsMap == nil { + if t.endpointSlicesMap == nil { return } - endpoints := t.endpointsMap[key] - if endpoints == nil { + endpointSliceList := t.endpointSlicesMap[key] + if endpointSliceList == nil { return } - for _, subset := range endpoints.Subsets { - for _, subsetAddr := range subset.Addresses { + for _, endpointSlice := range endpointSliceList { + for _, endpoint := range endpointSlice.Endpoints { // Check that the node name exists // subsetAddr.NodeName is of type *string - if subsetAddr.NodeName == nil { + if endpoint.NodeName == nil { continue } - // Look up the node's ip address by getting node info - node, err := t.Client.CoreV1().Nodes().Get(t.Ctx, *subsetAddr.NodeName, metav1.GetOptions{}) + node, err := t.Client.CoreV1().Nodes().Get(t.Ctx, *endpoint.NodeName, metav1.GetOptions{}) if err != nil { - t.Log.Warn("error getting node info", "error", err) + t.Log.Error("error getting node info", "error", err) continue } @@ -614,37 +642,18 @@ func (t *ServiceResource) generateRegistrations(key string) { expectedType = corev1.NodeExternalIP } - // Find the ip address for the node and - // create the Consul service using it - var found bool - for _, address := range node.Status.Addresses { - if address.Type == expectedType { - found = true - r := baseNode - rs := baseService - r.Service = &rs - r.Service.ID = serviceID(r.Service.Service, subsetAddr.IP) - r.Service.Address = address.Address - - t.consulMap[key] = append(t.consulMap[key], &r) - // Only consider the first address that matches. In some cases - // there will be multiple addresses like when using AWS CNI. - // In those cases, Kubernetes will ensure eth0 is always the first - // address in the list. - // See https://github.com/kubernetes/kubernetes/blob/b559434c02f903dbcd46ee7d6c78b216d3f0aca0/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L1462-L1464 - break - } - } + for _, endpointAddr := range endpoint.Addresses { - // If an ExternalIP wasn't found, and ExternalFirst is set, - // use an InternalIP - if t.NodePortSync == ExternalFirst && !found { + // Find the ip address for the node and + // create the Consul service using it + var found bool for _, address := range node.Status.Addresses { - if address.Type == corev1.NodeInternalIP { + if address.Type == expectedType { + found = true r := baseNode rs := baseService r.Service = &rs - r.Service.ID = serviceID(r.Service.Service, subsetAddr.IP) + r.Service.ID = serviceID(r.Service.Service, endpointAddr) r.Service.Address = address.Address t.consulMap[key] = append(t.consulMap[key], &r) @@ -656,6 +665,29 @@ func (t *ServiceResource) generateRegistrations(key string) { break } } + + // If an ExternalIP wasn't found, and ExternalFirst is set, + // use an InternalIP + if t.NodePortSync == ExternalFirst && !found { + for _, address := range node.Status.Addresses { + if address.Type == corev1.NodeInternalIP { + r := baseNode + rs := baseService + r.Service = &rs + r.Service.ID = serviceID(r.Service.Service, endpointAddr) + r.Service.Address = address.Address + + t.consulMap[key] = append(t.consulMap[key], &r) + // Only consider the first address that matches. In some cases + // there will be multiple addresses like when using AWS CNI. + // In those cases, Kubernetes will ensure eth0 is always the first + // address in the list. + // See https://github.com/kubernetes/kubernetes/blob/b559434c02f903dbcd46ee7d6c78b216d3f0aca0/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L1462-L1464 + break + } + } + } + } } } @@ -675,94 +707,100 @@ func (t *ServiceResource) registerServiceInstance( overridePortNumber int, useHostname bool) { - if t.endpointsMap == nil { + if t.endpointSlicesMap == nil { return } - endpoints := t.endpointsMap[key] - if endpoints == nil { + endpointSliceList := t.endpointSlicesMap[key] + if endpointSliceList == nil { return } seen := map[string]struct{}{} - for _, subset := range endpoints.Subsets { + for _, endpointSlice := range endpointSliceList { // For ClusterIP services and if LoadBalancerEndpointsSync is true, we use the endpoint port instead // of the service port because we're registering each endpoint // as a separate service instance. epPort := baseService.Port if overridePortName != "" { // If we're supposed to use a specific named port, find it. - for _, p := range subset.Ports { - if overridePortName == p.Name { - epPort = int(p.Port) + for _, p := range endpointSlice.Ports { + if overridePortName == *p.Name { + epPort = int(*p.Port) break } } } else if overridePortNumber == 0 { // Otherwise we'll just use the first port in the list // (unless the port number was overridden by an annotation). - for _, p := range subset.Ports { - epPort = int(p.Port) + for _, p := range endpointSlice.Ports { + epPort = int(*p.Port) break } } - for _, subsetAddr := range subset.Addresses { - var addr string - // Use the address and port from the Ingress resource if - // ingress-sync is enabled and the service has an ingress - // resource that references it. - if t.EnableIngress && t.isIngressService(key) { - addr = t.serviceHostnameMap[key].hostName - epPort = int(t.serviceHostnameMap[key].port) - } else { - addr = subsetAddr.IP - if addr == "" && useHostname { - addr = subsetAddr.Hostname + for _, endpoint := range endpointSlice.Endpoints { + for _, endpointAddr := range endpoint.Addresses { + + var addr string + // Use the address and port from the Ingress resource if + // ingress-sync is enabled and the service has an ingress + // resource that references it. + if t.EnableIngress && t.isIngressService(key) { + addr = t.serviceHostnameMap[key].hostName + epPort = int(t.serviceHostnameMap[key].port) + } else { + addr = endpointAddr + if addr == "" && useHostname { + addr = *endpoint.Hostname + } + if addr == "" { + continue + } } - if addr == "" { + + // Its not clear whether K8S guarantees ready addresses to + // be unique so we maintain a set to prevent duplicates just + // in case. + if _, ok := seen[addr]; ok { continue } - } + seen[addr] = struct{}{} - // Its not clear whether K8S guarantees ready addresses to - // be unique so we maintain a set to prevent duplicates just - // in case. - if _, ok := seen[addr]; ok { - continue - } - seen[addr] = struct{}{} + r := baseNode + rs := baseService + r.Service = &rs + r.Service.ID = serviceID(r.Service.Service, addr) + r.Service.Address = addr + r.Service.Port = epPort + r.Service.Meta = make(map[string]string) + // Deepcopy baseService.Meta into r.Service.Meta as baseService is shared + // between all nodes of a service + for k, v := range baseService.Meta { + r.Service.Meta[k] = v + } + if endpoint.TargetRef != nil { + r.Service.Meta[ConsulK8SRefValue] = endpoint.TargetRef.Name + r.Service.Meta[ConsulK8SRefKind] = endpoint.TargetRef.Kind + } + if endpoint.NodeName != nil { + r.Service.Meta[ConsulK8SNodeName] = *endpoint.NodeName + } + if endpoint.Zone != nil { + r.Service.Meta[ConsulK8STopologyZone] = *endpoint.Zone + } - r := baseNode - rs := baseService - r.Service = &rs - r.Service.ID = serviceID(r.Service.Service, addr) - r.Service.Address = addr - r.Service.Port = epPort - r.Service.Meta = make(map[string]string) - // Deepcopy baseService.Meta into r.Service.Meta as baseService is shared - // between all nodes of a service - for k, v := range baseService.Meta { - r.Service.Meta[k] = v - } - if subsetAddr.TargetRef != nil { - r.Service.Meta[ConsulK8SRefValue] = subsetAddr.TargetRef.Name - r.Service.Meta[ConsulK8SRefKind] = subsetAddr.TargetRef.Kind - } - if subsetAddr.NodeName != nil { - r.Service.Meta[ConsulK8SNodeName] = *subsetAddr.NodeName - } + r.Check = &consulapi.AgentCheck{ + CheckID: consulHealthCheckID(endpointSlice.Namespace, serviceID(r.Service.Service, addr)), + Name: consulKubernetesCheckName, + Namespace: baseService.Namespace, + Type: consulKubernetesCheckType, + Status: consulapi.HealthPassing, + ServiceID: serviceID(r.Service.Service, addr), + Output: kubernetesSuccessReasonMsg, + } - r.Check = &consulapi.AgentCheck{ - CheckID: consulHealthCheckID(endpoints.Namespace, serviceID(r.Service.Service, addr)), - Name: consulKubernetesCheckName, - Namespace: baseService.Namespace, - Type: consulKubernetesCheckType, - Status: consulapi.HealthPassing, - ServiceID: serviceID(r.Service.Service, addr), - Output: kubernetesSuccessReasonMsg, + t.consulMap[key] = append(t.consulMap[key], &r) } - - t.consulMap[key] = append(t.consulMap[key], &r) } } } @@ -811,68 +849,88 @@ func (t *serviceEndpointsResource) Informer() cache.SharedIndexInformer { return cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return t.Service.Client.CoreV1(). - Endpoints(metav1.NamespaceAll). + return t.Service.Client.DiscoveryV1(). + EndpointSlices(metav1.NamespaceAll). List(t.Ctx, options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return t.Service.Client.CoreV1(). - Endpoints(metav1.NamespaceAll). + return t.Service.Client.DiscoveryV1(). + EndpointSlices(metav1.NamespaceAll). Watch(t.Ctx, options) }, }, - &corev1.Endpoints{}, + &discoveryv1.EndpointSlice{}, 0, cache.Indexers{}, ) } -func (t *serviceEndpointsResource) Upsert(key string, raw interface{}) error { +func (t *serviceEndpointsResource) Upsert(endptKey string, raw interface{}) error { svc := t.Service - endpoints, ok := raw.(*corev1.Endpoints) + + endpointSlice, ok := raw.(*discoveryv1.EndpointSlice) if !ok { - svc.Log.Warn("upsert got invalid type", "raw", raw) + svc.Log.Error("upsert got invalid type", "raw", raw) return nil } svc.serviceLock.Lock() defer svc.serviceLock.Unlock() + // Extract service name and format the service key + svcKey := endpointSlice.Namespace + "/" + endpointSlice.Labels[discoveryv1.LabelServiceName] + // Check if we care about endpoints for this service - if !svc.shouldTrackEndpoints(key) { + if !svc.shouldTrackEndpoints(svcKey) { return nil } // We are tracking this service so let's keep track of the endpoints - if svc.endpointsMap == nil { - svc.endpointsMap = make(map[string]*corev1.Endpoints) + if svc.endpointSlicesMap == nil { + svc.endpointSlicesMap = make(map[string]map[string]*discoveryv1.EndpointSlice) + } + if _, ok := svc.endpointSlicesMap[svcKey]; !ok { + svc.endpointSlicesMap[svcKey] = make(map[string]*discoveryv1.EndpointSlice) } - svc.endpointsMap[key] = endpoints + svc.endpointSlicesMap[svcKey][endptKey] = endpointSlice // Update the registration and trigger a sync - svc.generateRegistrations(key) + svc.generateRegistrations(svcKey) svc.sync() - svc.Log.Info("upsert endpoint", "key", key) + svc.Log.Info("upsert endpoint", "key", endptKey) return nil } -func (t *serviceEndpointsResource) Delete(key string, _ interface{}) error { +func (t *serviceEndpointsResource) Delete(endptKey string, raw interface{}) error { + + endpointSlice, ok := raw.(*discoveryv1.EndpointSlice) + if !ok { + t.Service.Log.Error("upsert got invalid type", "raw", raw) + return nil + } + t.Service.serviceLock.Lock() defer t.Service.serviceLock.Unlock() + // Extract service name and format key + svcName := endpointSlice.Labels[discoveryv1.LabelServiceName] + svcKey := endpointSlice.Namespace + "/" + svcName + // This is a bit of an optimization. We only want to force a resync // if we were tracking this endpoint to begin with and that endpoint // had associated registrations. - if _, ok := t.Service.endpointsMap[key]; ok { - delete(t.Service.endpointsMap, key) - if _, ok := t.Service.consulMap[key]; ok { - delete(t.Service.consulMap, key) - t.Service.sync() + if _, ok := t.Service.endpointSlicesMap[svcKey]; ok { + if _, ok := t.Service.endpointSlicesMap[svcKey][endptKey]; ok { + delete(t.Service.endpointSlicesMap[svcKey], endptKey) + if _, ok := t.Service.consulMap[svcKey]; ok { + delete(t.Service.consulMap, svcKey) + t.Service.sync() + } } } - t.Service.Log.Info("delete endpoint", "key", key) + t.Service.Log.Info("delete endpoint", "key", endptKey) return nil } diff --git a/control-plane/catalog/to-consul/resource_test.go b/control-plane/catalog/to-consul/resource_test.go index 3b8fb78497..3272849bd3 100644 --- a/control-plane/catalog/to-consul/resource_test.go +++ b/control-plane/catalog/to-consul/resource_test.go @@ -14,11 +14,14 @@ import ( "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/pointer" ) const nodeName1 = "ip-10-11-12-13.ec2.internal" @@ -44,6 +47,9 @@ func TestServiceResource_createDelete(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) + createNodes(t, client) + createEndpointSlice(t, client, svc.Name, metav1.NamespaceDefault) + // Delete require.NoError(t, client.CoreV1().Services(metav1.NamespaceDefault).Delete(context.Background(), "foo", metav1.DeleteOptions{})) @@ -759,23 +765,36 @@ func TestServiceResource_lbRegisterEndpoints(t *testing.T) { node1, _ := createNodes(t, client) - // Insert the endpoints - _, err := client.CoreV1().Endpoints(metav1.NamespaceDefault).Create( + // Insert the endpoint slice + _, err := client.DiscoveryV1().EndpointSlices(metav1.NamespaceDefault).Create( context.Background(), - &corev1.Endpoints{ + &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo", + GenerateName: "foo-", + Labels: map[string]string{discoveryv1.LabelServiceName: "foo"}, }, - - Subsets: []corev1.EndpointSubset{ + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: []discoveryv1.Endpoint{ { - Addresses: []corev1.EndpointAddress{ - {NodeName: &node1.Name, IP: "8.8.8.8"}, - }, - Ports: []corev1.EndpointPort{ - {Name: "http", Port: 8080}, - {Name: "rpc", Port: 2000}, + Addresses: []string{"8.8.8.8"}, + Conditions: discoveryv1.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(true), + Terminating: pointer.Bool(false), }, + TargetRef: &corev1.ObjectReference{Kind: "pod", Name: "foo", Namespace: metav1.NamespaceDefault}, + NodeName: &node1.Name, + Zone: pointer.String("us-west-2a"), + }, + }, + Ports: []discoveryv1.EndpointPort{ + { + Name: pointer.String("http"), + Port: pointer.Int32(8080), + }, + { + Name: pointer.String("rpc"), + Port: pointer.Int32(2000), }, }, }, @@ -814,7 +833,7 @@ func TestServiceResource_nodePort(t *testing.T) { createNodes(t, client) - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Insert the service svc := nodePortService("foo", metav1.NamespaceDefault) @@ -854,7 +873,7 @@ func TestServiceResource_nodePortPrefix(t *testing.T) { createNodes(t, client) - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Insert the service svc := nodePortService("foo", metav1.NamespaceDefault) @@ -894,23 +913,36 @@ func TestServiceResource_nodePort_singleEndpoint(t *testing.T) { node1, _ := createNodes(t, client) - // Insert the endpoints - _, err := client.CoreV1().Endpoints(metav1.NamespaceDefault).Create( + // Insert the endpoint slice + _, err := client.DiscoveryV1().EndpointSlices(metav1.NamespaceDefault).Create( context.Background(), - &corev1.Endpoints{ + &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo", + GenerateName: "foo-", + Labels: map[string]string{discoveryv1.LabelServiceName: "foo"}, }, - - Subsets: []corev1.EndpointSubset{ + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: []discoveryv1.Endpoint{ { - Addresses: []corev1.EndpointAddress{ - {NodeName: &node1.Name, IP: "1.2.3.4"}, - }, - Ports: []corev1.EndpointPort{ - {Name: "http", Port: 8080}, - {Name: "rpc", Port: 2000}, + Addresses: []string{"1.2.3.4"}, + Conditions: discoveryv1.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(true), + Terminating: pointer.Bool(false), }, + TargetRef: &corev1.ObjectReference{Kind: "pod", Name: "foo", Namespace: metav1.NamespaceDefault}, + NodeName: &node1.Name, + Zone: pointer.String("us-west-2a"), + }, + }, + Ports: []discoveryv1.EndpointPort{ + { + Name: pointer.String("http"), + Port: pointer.Int32(8080), + }, + { + Name: pointer.String("rpc"), + Port: pointer.Int32(2000), }, }, }, @@ -949,12 +981,12 @@ func TestServiceResource_nodePortAnnotatedPort(t *testing.T) { createNodes(t, client) - createEndpoints(t, client, "foo", metav1.NamespaceDefault) - // Insert the service svc := nodePortService("foo", metav1.NamespaceDefault) svc.Annotations = map[string]string{annotationServicePort: "rpc"} _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) + createEndpointSlice(t, client, svc.Name, metav1.NamespaceDefault) + require.NoError(t, err) // Verify what we got @@ -989,7 +1021,7 @@ func TestServiceResource_nodePortUnnamedPort(t *testing.T) { createNodes(t, client) - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Insert the service svc := nodePortService("foo", metav1.NamespaceDefault) @@ -1034,7 +1066,7 @@ func TestServiceResource_nodePort_internalOnlySync(t *testing.T) { createNodes(t, client) - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Insert the service svc := nodePortService("foo", metav1.NamespaceDefault) @@ -1082,7 +1114,7 @@ func TestServiceResource_nodePort_externalFirstSync(t *testing.T) { _, err := client.CoreV1().Nodes().UpdateStatus(context.Background(), node1, metav1.UpdateOptions{}) require.NoError(t, err) - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Insert the service svc := nodePortService("foo", metav1.NamespaceDefault) @@ -1124,8 +1156,10 @@ func TestServiceResource_clusterIP(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1139,6 +1173,8 @@ func TestServiceResource_clusterIP(t *testing.T) { require.Equal(r, "foo", actual[1].Service.Service) require.Equal(r, "2.2.2.2", actual[1].Service.Address) require.Equal(r, 8080, actual[1].Service.Port) + require.Equal(r, "us-west-2a", actual[0].Service.Meta["external-k8s-topology-zone"]) + require.Equal(r, "us-west-2b", actual[1].Service.Meta["external-k8s-topology-zone"]) require.NotEqual(r, actual[0].Service.ID, actual[1].Service.ID) }) } @@ -1160,8 +1196,10 @@ func TestServiceResource_clusterIP_healthCheck(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1198,8 +1236,10 @@ func TestServiceResource_clusterIPPrefix(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1236,8 +1276,10 @@ func TestServiceResource_clusterIPAnnotatedPortName(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1274,8 +1316,10 @@ func TestServiceResource_clusterIPAnnotatedPortNumber(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1314,8 +1358,10 @@ func TestServiceResource_clusterIPUnnamedPorts(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1351,8 +1397,10 @@ func TestServiceResource_clusterIPSyncDisabled(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1381,8 +1429,10 @@ func TestServiceResource_clusterIPAllNamespaces(t *testing.T) { _, err := client.CoreV1().Services(testNamespace).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", testNamespace) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", testNamespace) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1422,8 +1472,10 @@ func TestServiceResource_clusterIPTargetPortNamed(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1458,8 +1510,10 @@ func TestServiceResource_targetRefInMeta(t *testing.T) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc, metav1.CreateOptions{}) require.NoError(t, err) - // Insert the endpoints - createEndpoints(t, client, "foo", metav1.NamespaceDefault) + createNodes(t, client) + + // Insert the endpoint slice + createEndpointSlice(t, client, "foo", metav1.NamespaceDefault) // Verify what we got retry.Run(t, func(r *retry.R) { @@ -1945,7 +1999,10 @@ func TestServiceResource_addIngress(t *testing.T) { // Create the ingress _, err = client.NetworkingV1().Ingresses(metav1.NamespaceDefault).Create(context.Background(), test.ingress, metav1.CreateOptions{}) require.NoError(t, err) - createEndpoints(t, client, "test-service", metav1.NamespaceDefault) + + createNodes(t, client) + createEndpointSlice(t, client, "test-service", metav1.NamespaceDefault) + // Verify that the service name annotation is preferred retry.Run(t, func(r *retry.R) { syncer.Lock() @@ -2066,43 +2123,55 @@ func createNodes(t *testing.T, client *fake.Clientset) (*corev1.Node, *corev1.No return node1, node2 } -// createEndpoints calls the fake k8s client to create two endpoints on two nodes. -func createEndpoints(t *testing.T, client *fake.Clientset, serviceName string, namespace string) { +// createEndpointSlices calls the fake k8s client to create an endpoint slices with two endpoints on different nodes. +func createEndpointSlice(t *testing.T, client *fake.Clientset, serviceName string, namespace string) { node1 := nodeName1 node2 := nodeName2 targetRef := corev1.ObjectReference{Kind: "pod", Name: "foobar"} - _, err := client.CoreV1().Endpoints(namespace).Create( + + _, err := client.DiscoveryV1().EndpointSlices(namespace).Create( context.Background(), - &corev1.Endpoints{ + &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: namespace, + Labels: map[string]string{discoveryv1.LabelServiceName: serviceName}, + Name: serviceName + "-" + rand.String(5), }, - - Subsets: []corev1.EndpointSubset{ + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: []discoveryv1.Endpoint{ { - Addresses: []corev1.EndpointAddress{ - {NodeName: &node1, IP: "1.1.1.1", TargetRef: &targetRef}, - }, - Ports: []corev1.EndpointPort{ - {Name: "http", Port: 8080}, - {Name: "rpc", Port: 2000}, + Addresses: []string{"1.1.1.1"}, + Conditions: discoveryv1.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(true), + Terminating: pointer.Bool(false), }, + TargetRef: &targetRef, + NodeName: &node1, + Zone: pointer.String("us-west-2a"), }, - { - Addresses: []corev1.EndpointAddress{ - {NodeName: &node2, IP: "2.2.2.2"}, - }, - Ports: []corev1.EndpointPort{ - {Name: "http", Port: 8080}, - {Name: "rpc", Port: 2000}, + Addresses: []string{"2.2.2.2"}, + Conditions: discoveryv1.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(true), + Terminating: pointer.Bool(false), }, + NodeName: &node2, + Zone: pointer.String("us-west-2b"), + }, + }, + Ports: []discoveryv1.EndpointPort{ + { + Name: pointer.String("http"), + Port: pointer.Int32(8080), + }, + { + Name: pointer.String("rpc"), + Port: pointer.Int32(2000), }, }, }, metav1.CreateOptions{}) - require.NoError(t, err) } diff --git a/control-plane/cni/main_test.go b/control-plane/cni/main_test.go index dc929f1408..9af2e274b0 100644 --- a/control-plane/cni/main_test.go +++ b/control-plane/cni/main_test.go @@ -66,22 +66,6 @@ func Test_cmdAdd(t *testing.T) { expectedErr: fmt.Errorf("not running in a pod, namespace and pod should have values"), expectedRules: false, // Rules won't be applied because the command will throw an error first }, - { - name: "Missing prevResult in stdin data, should throw error", - cmd: &Command{ - client: fake.NewSimpleClientset(), - }, - podName: "missing-prev-result", - stdInData: missingPrevResultStdinData, - configuredPod: func(pod *corev1.Pod, cmd *Command) *corev1.Pod { - _, err := cmd.client.CoreV1().Pods(defaultNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) - require.NoError(t, err) - - return pod - }, - expectedErr: fmt.Errorf("must be called as final chained plugin"), - expectedRules: false, // Rules won't be applied because the command will throw an error first - }, { name: "Missing IPs in prevResult in stdin data, should throw error", cmd: &Command{ @@ -336,31 +320,6 @@ const goodStdinData = `{ "type": "consul-cni" }` -const missingPrevResultStdinData = `{ - "cniVersion": "0.3.1", - "name": "kindnet", - "type": "kindnet", - "capabilities": { - "testCapability": false - }, - "ipam": { - "type": "host-local" - }, - "dns": { - "nameservers": ["nameserver"], - "domain": "domain", - "search": ["search"], - "options": ["option"] - }, - "cni_bin_dir": "/opt/cni/bin", - "cni_net_dir": "/etc/cni/net.d", - "kubeconfig": "ZZZ-consul-cni-kubeconfig", - "log_level": "info", - "multus": false, - "name": "consul-cni", - "type": "consul-cni" -}` - const missingIPsStdinData = `{ "cniVersion": "0.3.1", "name": "kindnet", diff --git a/control-plane/config/crd/bases/consul.hashicorp.com_gatewayclassconfigs.yaml b/control-plane/config/crd/bases/consul.hashicorp.com_gatewayclassconfigs.yaml index ff3158f2a7..c2a857db34 100644 --- a/control-plane/config/crd/bases/consul.hashicorp.com_gatewayclassconfigs.yaml +++ b/control-plane/config/crd/bases/consul.hashicorp.com_gatewayclassconfigs.yaml @@ -127,6 +127,23 @@ spec: for gateway containers format: int32 type: integer + metrics: + description: Metrics defines how to configure the metrics for a gateway. + properties: + enabled: + description: Enable metrics for this class of gateways. If unspecified, + will inherit behavior from the global Helm configuration. + type: boolean + path: + description: The path used for metrics. + type: string + port: + description: The port used for metrics. + format: int32 + maximum: 65535 + minimum: 1024 + type: integer + type: object nodeSelector: additionalProperties: type: string diff --git a/control-plane/config/crd/bases/consul.hashicorp.com_terminatinggateways.yaml b/control-plane/config/crd/bases/consul.hashicorp.com_terminatinggateways.yaml index 7f22c65d09..1b8ab32cd6 100644 --- a/control-plane/config/crd/bases/consul.hashicorp.com_terminatinggateways.yaml +++ b/control-plane/config/crd/bases/consul.hashicorp.com_terminatinggateways.yaml @@ -69,6 +69,10 @@ spec: to use for TLS connections from the gateway to the linked service. type: string + disableAutoHostRewrite: + description: DisableAutoHostRewrite disables terminating gateways + auto host rewrite feature when set to true. + type: boolean keyFile: description: KeyFile is the optional path to a private key to use for TLS connections from the gateway to the linked service. diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go index 0094f40c76..fe6c5aed0b 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go @@ -182,9 +182,9 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{RequeueAfter: requeueAfter}, err } - // endpointAddressMap stores every IP that corresponds to a Pod in the Endpoints object. It is used to compare + // deregisterEndpointAddress stores every IP that corresponds to a Pod in the Endpoints object. It is used to compare // against service instances in Consul to deregister them if they are not in the map. - endpointAddressMap := map[string]bool{} + deregisterEndpointAddress := map[string]bool{} // Register all addresses of this Endpoints object as service instances in Consul. for _, subset := range serviceEndpoints.Subsets { @@ -193,16 +193,25 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu var pod corev1.Pod objectKey := types.NamespacedName{Name: address.TargetRef.Name, Namespace: address.TargetRef.Namespace} if err = r.Client.Get(ctx, objectKey, &pod); err != nil { - r.Log.Error(err, "failed to get pod", "name", address.TargetRef.Name) - errs = multierror.Append(errs, err) + // If the pod doesn't exist anymore, set up the deregisterEndpointAddress map to deregister it. + if k8serrors.IsNotFound(err) { + deregisterEndpointAddress[address.IP] = true + r.Log.Info("pod not found", "name", address.TargetRef.Name) + } else { + // If there was a different error fetching the pod, then log the error but don't deregister it + // since this could be a K8s API blip and we don't want to prematurely deregister. + deregisterEndpointAddress[address.IP] = false + r.Log.Error(err, "failed to get pod", "name", address.TargetRef.Name) + errs = multierror.Append(errs, err) + } continue } svcName, ok := pod.Annotations[constants.AnnotationKubernetesService] if ok && serviceEndpoints.Name != svcName { r.Log.Info("ignoring endpoint because it doesn't match explicit service annotation", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) - // deregistration for service instances that don't match the annotation happens - // later because we don't add this pod to the endpointAddressMap. + // Set up the deregisterEndpointAddress to deregister service instances that don't match the annotation. + deregisterEndpointAddress[address.IP] = true continue } @@ -219,8 +228,8 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.Log.Error(err, "failed to register services or health check", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) errs = multierror.Append(errs, err) } - // Build the endpointAddressMap up for deregistering service instances later. - endpointAddressMap[pod.Status.PodIP] = true + // Build the deregisterEndpointAddress map up for deregistering service instances later. + deregisterEndpointAddress[pod.Status.PodIP] = false } else { r.Log.Info("detected an update to pre-consul-dataplane service", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) nodeAgentClientCfg, err := r.consulClientCfgForNodeAgent(apiClient, pod, serverState) @@ -247,17 +256,17 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.Log.Error(err, "failed to register gateway or health check", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) errs = multierror.Append(errs, err) } - // Build the endpointAddressMap up for deregistering service instances later. - endpointAddressMap[pod.Status.PodIP] = true + // Build the deregisterEndpointAddress map up for deregistering service instances later. + deregisterEndpointAddress[pod.Status.PodIP] = false } } } } // Compare service instances in Consul with addresses in Endpoints. If an address is not in Endpoints, deregister - // from Consul. This uses endpointAddressMap which is populated with the addresses in the Endpoints object during - // the registration codepath. - requeueAfter, err := r.deregisterService(ctx, apiClient, serviceEndpoints.Name, serviceEndpoints.Namespace, endpointAddressMap) + // from Consul. This uses deregisterEndpointAddress which is populated with the addresses in the Endpoints object to + // either deregister or keep during the registration codepath. + requeueAfter, err := r.deregisterService(ctx, apiClient, serviceEndpoints.Name, serviceEndpoints.Namespace, deregisterEndpointAddress) if err != nil { r.Log.Error(err, "failed to deregister endpoints", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) errs = multierror.Append(errs, err) @@ -929,8 +938,8 @@ func getHealthCheckStatusReason(healthCheckStatus, podName, podNamespace string) // "k8s-service-name". So, we query Consul services by "k8s-service-name" metadata. // When querying by the k8s service name and namespace, the request will return service instances and // associated proxy service instances. -// The argument endpointsAddressesMap decides whether to deregister *all* service instances or selectively deregister -// them only if they are not in endpointsAddressesMap. If the map is nil, it will deregister all instances. If the map +// The argument deregisterEndpointAddress decides whether to deregister *all* service instances or selectively deregister +// them only if they are not in deregisterEndpointAddress. If the map is nil, it will deregister all instances. If the map // has addresses, it will only deregister instances not in the map. // If the pod backing a Consul service instance still exists and the graceful shutdown lifecycle mode is enabled, the instance // will not be deregistered. Instead, its health check will be updated to Critical in order to drain incoming traffic and @@ -941,7 +950,7 @@ func (r *Controller) deregisterService( apiClient *api.Client, k8sSvcName string, k8sSvcNamespace string, - endpointsAddressesMap map[string]bool) (time.Duration, error) { + deregisterEndpointAddress map[string]bool) (time.Duration, error) { // Get services matching metadata from Consul serviceInstances, err := r.serviceInstances(apiClient, k8sSvcName, k8sSvcNamespace) @@ -958,7 +967,7 @@ func (r *Controller) deregisterService( // every service instance. var serviceDeregistered bool - if addressIsMissingFromEndpointsMap(svc.ServiceAddress, endpointsAddressesMap) { + if deregister(svc.ServiceAddress, deregisterEndpointAddress) { // If graceful shutdown is enabled, continue to the next service instance and // mark that an event requeue is needed. We should requeue at the longest time interval // to prevent excessive re-queues. Also, updating the health status in Consul to Critical @@ -1621,10 +1630,15 @@ func getMultiPortIdx(pod corev1.Pod, serviceEndpoints corev1.Endpoints) int { return -1 } -func addressIsMissingFromEndpointsMap(address string, endpointsAddressesMap map[string]bool) bool { - if endpointsAddressesMap == nil { +// deregister returns that the address is marked for deregistration if the map is nil or if the address is explicitly +// marked in the map for deregistration. +func deregister(address string, deregisterEndpointAddress map[string]bool) bool { + if deregisterEndpointAddress == nil { return true } - _, ok := endpointsAddressesMap[address] - return !ok + deregister, ok := deregisterEndpointAddress[address] + if ok { + return deregister + } + return true } diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go index 7e4ea0d753..d22dfe0dac 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go @@ -5,6 +5,7 @@ package endpoints import ( "context" + "errors" "fmt" "strings" "testing" @@ -1766,10 +1767,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { }, }, { - // This test has 3 addresses, but only 2 are backed by pod resources. This will cause Reconcile to error - // on the invalid address but continue and process the other addresses. We check for error specific to - // pod3 being non-existant at the end, and validate the other 2 addresses have service instances. - name: "Endpoints with multiple addresses but one is invalid", + // This test has 3 addresses, but only 2 are backed by pod resources. This will cause Reconcile to + // deregister the instance associated with the non-existent pod and continue and process the other + // addresses. We validate the other 2 addresses have service instances. + name: "Endpoints with multiple addresses but one is deleted", svcName: "service-created", consulSvcName: "service-created", k8sObjects: func() []runtime.Object { @@ -1896,7 +1897,6 @@ func TestReconcileCreateEndpoint(t *testing.T) { Type: constants.ConsulKubernetesCheckType, }, }, - expErr: "1 error occurred:\n\t* pods \"pod3\" not found\n\n", }, { name: "Every configurable field set: port, different Consul service name, meta, tags, upstreams, metrics", @@ -2264,6 +2264,242 @@ func TestParseLocality(t *testing.T) { }) } +func TestReconcile_PodErrorPreservesToken(t *testing.T) { + t.Parallel() + cases := []struct { + name string + svcName string + consulSvcName string + k8sObjects func() []runtime.Object + expectedConsulSvcInstances []*api.CatalogService + expectedProxySvcInstances []*api.CatalogService + expectedHealthChecks []*api.HealthCheck + metricsEnabled bool + telemetryCollectorDisabled bool + nodeMeta map[string]string + pod1Err string + }{ + { + name: "Error when fetching pod results in not deregistering the service instances in consul", + svcName: "service-created", + consulSvcName: "service-created", + nodeMeta: map[string]string{ + "test-node": "true", + }, + pod1Err: "some fake error while fetching pod", + k8sObjects: func() []runtime.Object { + pod1 := createServicePod("pod1", "1.2.3.4", true, true) + endpoint := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-created", + Namespace: "default", + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.2.3.4", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "pod1", + Namespace: "default", + }, + }, + }, + }, + }, + } + return []runtime.Object{pod1, endpoint} + }, + expectedConsulSvcInstances: []*api.CatalogService{ + { + ServiceID: "pod1-service-created", + ServiceName: "service-created", + ServiceAddress: "1.2.3.4", + ServicePort: 0, + ServiceMeta: map[string]string{constants.MetaKeyPodName: "pod1", metaKeyKubeServiceName: "service-created", constants.MetaKeyKubeNS: "default", metaKeyManagedBy: constants.ManagedByValue, metaKeySyntheticNode: "true", constants.MetaKeyPodUID: ""}, + ServiceTags: []string{}, + ServiceProxy: &api.AgentServiceConnectProxyConfig{}, + NodeMeta: map[string]string{ + "synthetic-node": "true", + "test-node": "true", + }, + }, + }, + expectedProxySvcInstances: []*api.CatalogService{ + { + ServiceID: "pod1-service-created-sidecar-proxy", + ServiceName: "service-created-sidecar-proxy", + ServiceAddress: "1.2.3.4", + ServicePort: 20000, + ServiceProxy: &api.AgentServiceConnectProxyConfig{ + DestinationServiceName: "service-created", + DestinationServiceID: "pod1-service-created", + LocalServiceAddress: "", + LocalServicePort: 0, + Config: map[string]any{"envoy_telemetry_collector_bind_socket_dir": string("/consul/connect-inject")}, + }, + ServiceMeta: map[string]string{constants.MetaKeyPodName: "pod1", metaKeyKubeServiceName: "service-created", constants.MetaKeyKubeNS: "default", metaKeyManagedBy: constants.ManagedByValue, metaKeySyntheticNode: "true", constants.MetaKeyPodUID: ""}, + ServiceTags: []string{}, + NodeMeta: map[string]string{ + "synthetic-node": "true", + "test-node": "true", + }, + }, + }, + expectedHealthChecks: []*api.HealthCheck{ + { + CheckID: "default/pod1-service-created", + ServiceName: "service-created", + ServiceID: "pod1-service-created", + Name: constants.ConsulKubernetesCheckName, + Status: api.HealthPassing, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, + }, + { + CheckID: "default/pod1-service-created-sidecar-proxy", + ServiceName: "service-created-sidecar-proxy", + ServiceID: "pod1-service-created-sidecar-proxy", + Name: constants.ConsulKubernetesCheckName, + Status: api.HealthPassing, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, + }, + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + // Add the default namespace. + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}} + node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}} + // Create fake k8s client + k8sObjects := append(tt.k8sObjects(), &ns, &node) + + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(k8sObjects...).Build() + + customClient := fakeClientWithPodCustomization{fakeClient} + + // Create test consulServer server. + testClient := test.TestServerWithMockConnMgrWatcher(t, nil) + consulClient := testClient.APIClient + + // Create the endpoints controller. + ep := &Controller{ + Client: fakeClient, + Log: logrtest.New(t), + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + ReleaseName: "consulServer", + ReleaseNamespace: "default", + NodeMeta: tt.nodeMeta, + } + if tt.metricsEnabled { + ep.MetricsConfig = metrics.Config{ + DefaultEnableMetrics: true, + EnableGatewayMetrics: true, + } + } + + ep.EnableTelemetryCollector = !tt.telemetryCollectorDisabled + + namespacedName := types.NamespacedName{ + Namespace: "default", + Name: tt.svcName, + } + + // Do a first reconcile to setup the state in Consul with the instances and tokens. + resp, err := ep.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: namespacedName, + }) + require.NoError(t, err) + require.False(t, resp.Requeue) + + // Do a second reconcile while stubbing the k8s client to return an error for the pod. Since it's not a "not + // found" error, we should expect that the service instance does not get deregistered and that the acl token + // is not deleted, so we will assert after this that the state in Consul still exists. + ep.Client = customClient + resp, err = ep.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: namespacedName, + }) + require.Contains(t, err.Error(), tt.pod1Err) + require.False(t, resp.Requeue) + + // These are the same assertions in the Reconcile-Create test cases, ensuring the state in Consul is correct. + // After reconciliation, Consul should have the service with the correct number of instances + serviceInstances, _, err := consulClient.Catalog().Service(tt.consulSvcName, "", nil) + require.NoError(t, err) + require.Len(t, serviceInstances, len(tt.expectedConsulSvcInstances)) + for i, instance := range serviceInstances { + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceID, instance.ServiceID) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceName, instance.ServiceName) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceAddress, instance.ServiceAddress) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServicePort, instance.ServicePort) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceMeta, instance.ServiceMeta) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceTags, instance.ServiceTags) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceTaggedAddresses, instance.ServiceTaggedAddresses) + require.Equal(t, tt.expectedConsulSvcInstances[i].ServiceProxy, instance.ServiceProxy) + if tt.nodeMeta != nil { + require.Equal(t, tt.expectedConsulSvcInstances[i].NodeMeta, instance.NodeMeta) + } + } + proxyServiceInstances, _, err := consulClient.Catalog().Service(fmt.Sprintf("%s-sidecar-proxy", tt.consulSvcName), "", nil) + require.NoError(t, err) + require.Len(t, proxyServiceInstances, len(tt.expectedProxySvcInstances)) + for i, instance := range proxyServiceInstances { + require.Equal(t, tt.expectedProxySvcInstances[i].ServiceID, instance.ServiceID) + require.Equal(t, tt.expectedProxySvcInstances[i].ServiceName, instance.ServiceName) + require.Equal(t, tt.expectedProxySvcInstances[i].ServiceAddress, instance.ServiceAddress) + require.Equal(t, tt.expectedProxySvcInstances[i].ServicePort, instance.ServicePort) + require.Equal(t, tt.expectedProxySvcInstances[i].ServiceMeta, instance.ServiceMeta) + require.Equal(t, tt.expectedProxySvcInstances[i].ServiceTags, instance.ServiceTags) + if tt.nodeMeta != nil { + require.Equal(t, tt.expectedProxySvcInstances[i].NodeMeta, instance.NodeMeta) + } + // When comparing the ServiceProxy field we ignore the DestinationNamespace + // field within that struct because on Consul OSS it's set to "" but on Consul Enterprise + // it's set to "default" and we want to re-use this test for both OSS and Ent. + // This does mean that we don't test that field but that's okay because + // it's not getting set specifically in this test. + // To do the comparison that ignores that field we use go-cmp instead + // of the regular require.Equal call since it supports ignoring certain + // fields. + diff := cmp.Diff(tt.expectedProxySvcInstances[i].ServiceProxy, instance.ServiceProxy, + cmpopts.IgnoreFields(api.Upstream{}, "DestinationNamespace", "DestinationPartition")) + require.Empty(t, diff, "expected objects to be equal") + } + + // Check that the Consul health expectedCheck was created for the k8s pod. + for _, expectedCheck := range tt.expectedHealthChecks { + filter := fmt.Sprintf("ServiceID == %q", expectedCheck.ServiceID) + checks, _, err := consulClient.Health().Checks(expectedCheck.ServiceName, &api.QueryOptions{Filter: filter}) + require.NoError(t, err) + require.Equal(t, len(checks), 1) + // Ignoring Namespace because the response from ENT includes it and OSS does not. + var ignoredFields = []string{"Node", "Definition", "Namespace", "Partition", "CreateIndex", "ModifyIndex", "ServiceTags"} + require.True(t, cmp.Equal(checks[0], expectedCheck, cmpopts.IgnoreFields(api.HealthCheck{}, ignoredFields...))) + } + }) + } + +} + +type fakeClientWithPodCustomization struct { + client.WithWatch +} + +func (c fakeClientWithPodCustomization) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "pod1" { + return errors.New("some fake error while fetching pod") + } + err := c.WithWatch.Get(ctx, key, obj, opts...) + return err +} + // Tests updating an Endpoints object. // - Tests updates via the register codepath: // - When an address in an Endpoint is updated, that the corresponding service instance in Consul is updated. diff --git a/control-plane/go.mod b/control-plane/go.mod index 266a066b03..db780b5175 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -1,5 +1,7 @@ module github.com/hashicorp/consul-k8s/control-plane +replace github.com/hashicorp/consul/api => github.com/hashicorp/consul/api v1.10.1-0.20240312203720-262f4358003f + require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/containernetworking/cni v1.1.2 diff --git a/control-plane/go.sum b/control-plane/go.sum index 46188542dc..3c10aa0dbc 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -273,8 +273,8 @@ github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20240226161840-f3842c41 github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20240226161840-f3842c41cb2b/go.mod h1:9NKJHOcgmz/6P2y6MegNIOXhIKE/0ils/mHWd5sZgoU= github.com/hashicorp/consul-server-connection-manager v0.1.6 h1:ktj8Fi+dRXn9hhM+FXsfEJayhzzgTqfH08Ne5M6Fmug= github.com/hashicorp/consul-server-connection-manager v0.1.6/go.mod h1:HngMIv57MT+pqCVeRQMa1eTB5dqnyMm8uxjyv+Hn8cs= -github.com/hashicorp/consul/api v1.28.2 h1:mXfkRHrpHN4YY3RqL09nXU1eHKLNiuAN4kHvDQ16k/8= -github.com/hashicorp/consul/api v1.28.2/go.mod h1:KyzqzgMEya+IZPcD65YFoOVAgPpbfERu4I/tzG6/ueE= +github.com/hashicorp/consul/api v1.10.1-0.20240312203720-262f4358003f h1:8clIrMnJtO5ab5Kd1qF19s9s581cyGYhQxfPLVRaFZs= +github.com/hashicorp/consul/api v1.10.1-0.20240312203720-262f4358003f/go.mod h1:JnWx0qZd1Ffeoa42yVAxzv7/v7eaZyptkw0dG9F/gF4= github.com/hashicorp/consul/proto-public v0.6.0 h1:9qrBujmoTB5gQQ84kQO+YWvhjgYoYBNrOoHdo4cpHHM= github.com/hashicorp/consul/proto-public v0.6.0/go.mod h1:JF6983XNCzvw4wDNOLEwLqOq2IPw7iyT+pkswHSz08U= github.com/hashicorp/consul/sdk v0.16.0 h1:SE9m0W6DEfgIVCJX7xU+iv/hUl4m/nxqMTnCdMxDpJ8= diff --git a/control-plane/subcommand/gateway-resources/command.go b/control-plane/subcommand/gateway-resources/command.go index 911fb0afd9..946e2d2703 100644 --- a/control-plane/subcommand/gateway-resources/command.go +++ b/control-plane/subcommand/gateway-resources/command.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "os" + "strconv" "sync" "time" @@ -94,6 +95,10 @@ type Command struct { flagMapPrivilegedContainerPorts int + flagEnableMetrics string + flagMetricsPort string + flagMetricsPath string + k8sClient client.Client once sync.Once @@ -156,6 +161,10 @@ func (c *Command) init() { "gateway container.", ) + c.flags.StringVar(&c.flagEnableMetrics, "enable-metrics", "", "specify as 'true' or 'false' to enable or disable metrics collection") + c.flags.StringVar(&c.flagMetricsPath, "metrics-path", "", "specify to set the path used for metrics scraping") + c.flags.StringVar(&c.flagMetricsPort, "metrics-port", "", "specify to set the port used for metrics scraping") + c.flags.StringVar(&c.flagGatewayConfigLocation, "gateway-config-file-location", gatewayConfigFilename, "specify a different location for where the gateway config file is") @@ -262,6 +271,17 @@ func (c *Command) Run(args []string) int { }, } + if metricsEnabled, isSet := getMetricsEnabled(c.flagEnableMetrics); isSet { + classConfig.Spec.Metrics.Enabled = &metricsEnabled + if port, isValid := getScrapePort(c.flagMetricsPort); isValid { + port32 := int32(port) + classConfig.Spec.Metrics.Port = &port32 + } + if path, isSet := getScrapePath(c.flagMetricsPath); isSet { + classConfig.Spec.Metrics.Path = &path + } + } + class := &gwv1beta1.GatewayClass{ ObjectMeta: metav1.ObjectMeta{Name: c.flagGatewayClassName, Labels: labels}, Spec: gwv1beta1.GatewayClassSpec{ @@ -346,6 +366,18 @@ func (c *Command) validateFlags() error { } } + if c.flagEnableMetrics != "" { + if _, valid := getMetricsEnabled(c.flagEnableMetrics); !valid { + return errors.New("-enable-metrics must be either 'true' or 'false'") + } + } + + if c.flagMetricsPort != "" { + if _, valid := getScrapePort(c.flagMetricsPort); !valid { + return errors.New("-metrics-port must be a valid unprivileged port number") + } + } + return nil } @@ -604,6 +636,35 @@ func exponentialBackoffWithMaxIntervalAndTime() *backoff.ExponentialBackOff { return backoff } +func getScrapePort(v string) (int, bool) { + port, err := strconv.Atoi(v) + if err != nil { + // we only use the port if it's actually valid + return 0, false + } + if port < 1024 || port > 65535 { + return 0, false + } + return port, true +} + +func getScrapePath(v string) (string, bool) { + if v == "" { + return "", false + } + return v, true +} + +func getMetricsEnabled(v string) (bool, bool) { + if v == "true" { + return true, true + } + if v == "false" { + return false, true + } + return false, false +} + func nonZeroOrNil(v int) *int32 { if v == 0 { return nil diff --git a/control-plane/subcommand/inject-connect/v1controllers.go b/control-plane/subcommand/inject-connect/v1controllers.go index e923f6da9d..37ed344dba 100644 --- a/control-plane/subcommand/inject-connect/v1controllers.go +++ b/control-plane/subcommand/inject-connect/v1controllers.go @@ -114,22 +114,25 @@ func (c *Command) configureV1Controllers(ctx context.Context, mgr manager.Manage HTTPPort: consulConfig.HTTPPort, APITimeout: consulConfig.APITimeout, }, - ImageDataplane: c.flagConsulDataplaneImage, - ImageConsulK8S: c.flagConsulK8sImage, - ConsulDestinationNamespace: c.flagConsulDestinationNamespace, - NamespaceMirroringPrefix: c.flagK8SNSMirroringPrefix, - EnableNamespaces: c.flagEnableNamespaces, - PeeringEnabled: c.flagEnablePeering, - EnableOpenShift: c.flagEnableOpenShift, - EnableNamespaceMirroring: c.flagEnableK8SNSMirroring, - AuthMethod: c.consul.ConsulLogin.AuthMethod, - LogLevel: c.flagLogLevel, - LogJSON: c.flagLogJSON, - TLSEnabled: c.consul.UseTLS, - ConsulTLSServerName: c.consul.TLSServerName, - ConsulPartition: c.consul.Partition, - ConsulCACert: string(c.caCertPem), - InitContainerResources: &c.initContainerResources, + ImageDataplane: c.flagConsulDataplaneImage, + ImageConsulK8S: c.flagConsulK8sImage, + ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + NamespaceMirroringPrefix: c.flagK8SNSMirroringPrefix, + EnableNamespaces: c.flagEnableNamespaces, + PeeringEnabled: c.flagEnablePeering, + EnableOpenShift: c.flagEnableOpenShift, + EnableNamespaceMirroring: c.flagEnableK8SNSMirroring, + AuthMethod: c.consul.ConsulLogin.AuthMethod, + LogLevel: c.flagLogLevel, + LogJSON: c.flagLogJSON, + TLSEnabled: c.consul.UseTLS, + ConsulTLSServerName: c.consul.TLSServerName, + ConsulPartition: c.consul.Partition, + ConsulCACert: string(c.caCertPem), + EnableGatewayMetrics: c.flagEnableGatewayMetrics, + DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath, + DefaultPrometheusScrapePort: c.flagDefaultPrometheusScrapePort, + InitContainerResources: &c.initContainerResources, }, AllowK8sNamespacesSet: allowK8sNamespaces, DenyK8sNamespacesSet: denyK8sNamespaces, diff --git a/control-plane/subcommand/server-acl-init/command.go b/control-plane/subcommand/server-acl-init/command.go index 3d3c28c5ae..cf9283f531 100644 --- a/control-plane/subcommand/server-acl-init/command.go +++ b/control-plane/subcommand/server-acl-init/command.go @@ -68,8 +68,6 @@ type Command struct { flagIngressGatewayNames []string flagTerminatingGatewayNames []string - flagAPIGatewayController bool - // Flags to configure Consul connection. flagServerPort uint @@ -173,8 +171,6 @@ func (c *Command) init() { "Name of a terminating gateway that needs an acl token. May be specified multiple times. "+ "[Enterprise Only] If using Consul namespaces and registering the gateway outside of the "+ "default namespace, specify the value in the form ..") - c.flags.BoolVar(&c.flagAPIGatewayController, "api-gateway-controller", false, - "Toggle for configuring ACL login for the API gateway controller.") c.flags.UintVar(&c.flagServerPort, "server-port", 8500, "The HTTP or HTTPS port of the Consul server. Defaults to 8500.") @@ -589,28 +585,6 @@ func (c *Command) Run(args []string) int { } } - if c.flagAPIGatewayController { - rules, err := c.apiGatewayControllerRules() - if err != nil { - c.log.Error("Error templating api gateway rules", "err", err) - return 1 - } - serviceAccountName := c.withPrefix("api-gateway-controller") - - // API gateways require a global policy/token because they must - // create config-entry resources in the primary, even when deployed - // to a secondary datacenter - authMethodName := localComponentAuthMethodName - if !primary { - authMethodName = globalComponentAuthMethodName - } - err = c.createACLPolicyRoleAndBindingRule("api-gateway-controller", rules, consulDC, primaryDC, globalPolicy, primary, authMethodName, serviceAccountName, dynamicClient) - if err != nil { - c.log.Error(err.Error()) - return 1 - } - } - if c.flagMeshGateway { rules, err := c.meshGatewayRules() if err != nil { diff --git a/control-plane/subcommand/server-acl-init/command_test.go b/control-plane/subcommand/server-acl-init/command_test.go index 7c3b778f5e..c7bcb79384 100644 --- a/control-plane/subcommand/server-acl-init/command_test.go +++ b/control-plane/subcommand/server-acl-init/command_test.go @@ -2070,12 +2070,6 @@ func TestRun_PoliciesAndBindingRulesForACLLogin_PrimaryDatacenter(t *testing.T) PolicyNames: []string{"sync-catalog-policy"}, Roles: []string{resourcePrefix + "-sync-catalog-acl-role"}, }, - { - TestName: "API Gateway Controller", - TokenFlags: []string{"-api-gateway-controller"}, - PolicyNames: []string{"api-gateway-controller-policy"}, - Roles: []string{resourcePrefix + "-api-gateway-controller-acl-role"}, - }, { TestName: "Snapshot Agent", TokenFlags: []string{"-snapshot-agent"}, @@ -2223,13 +2217,6 @@ func TestRun_PoliciesAndBindingRulesACLLogin_SecondaryDatacenter(t *testing.T) { Roles: []string{resourcePrefix + "-sync-catalog-acl-role-" + secondaryDatacenter}, GlobalAuthMethod: false, }, - { - TestName: "API Gateway Controller", - TokenFlags: []string{"-api-gateway-controller"}, - PolicyNames: []string{"api-gateway-controller-policy-" + secondaryDatacenter}, - Roles: []string{resourcePrefix + "-api-gateway-controller-acl-role-" + secondaryDatacenter}, - GlobalAuthMethod: true, - }, { TestName: "Snapshot Agent", TokenFlags: []string{"-snapshot-agent"}, @@ -2381,12 +2368,6 @@ func TestRun_ValidateLoginToken_PrimaryDatacenter(t *testing.T) { Roles: []string{resourcePrefix + "-sync-catalog-acl-role"}, GlobalToken: false, }, - { - ComponentName: "api-gateway-controller", - TokenFlags: []string{"-api-gateway-controller"}, - Roles: []string{resourcePrefix + "-api-gateway-controller-acl-role"}, - GlobalToken: false, - }, { ComponentName: "snapshot-agent", TokenFlags: []string{"-snapshot-agent"}, @@ -2518,13 +2499,6 @@ func TestRun_ValidateLoginToken_SecondaryDatacenter(t *testing.T) { GlobalAuthMethod: false, GlobalToken: false, }, - { - ComponentName: "api-gateway-controller", - TokenFlags: []string{"-api-gateway-controller"}, - Roles: []string{resourcePrefix + "-api-gateway-controller-acl-role-dc2"}, - GlobalAuthMethod: true, - GlobalToken: true, - }, { ComponentName: "snapshot-agent", TokenFlags: []string{"-snapshot-agent"}, diff --git a/control-plane/subcommand/server-acl-init/rules.go b/control-plane/subcommand/server-acl-init/rules.go index f408037157..1f00e1019c 100644 --- a/control-plane/subcommand/server-acl-init/rules.go +++ b/control-plane/subcommand/server-acl-init/rules.go @@ -145,38 +145,6 @@ partition_prefix "" { return c.renderRules(anonTokenRulesTpl) } -func (c *Command) apiGatewayControllerRules() (string, error) { - apiGatewayRulesTpl := `{{- if .EnablePartitions }} -partition "{{ .PartitionName }}" { - mesh = "write" - acl = "write" -{{- else }} -operator = "write" -acl = "write" -{{- end }} - -{{- if .EnableNamespaces }} -namespace_prefix "" { - policy = "write" -{{- end }} - service_prefix "" { - policy = "write" - intentions = "write" - } - node_prefix "" { - policy = "read" - } -{{- if .EnableNamespaces }} -} -{{- end }} -{{- if .EnablePartitions }} -} -{{- end }} -` - - return c.renderRules(apiGatewayRulesTpl) -} - // This assumes users are using the default name for the service, i.e. // "mesh-gateway". func (c *Command) meshGatewayRules() (string, error) { diff --git a/control-plane/subcommand/server-acl-init/rules_test.go b/control-plane/subcommand/server-acl-init/rules_test.go index c1a02a2218..bb727968f3 100644 --- a/control-plane/subcommand/server-acl-init/rules_test.go +++ b/control-plane/subcommand/server-acl-init/rules_test.go @@ -5,7 +5,6 @@ package serveraclinit import ( "fmt" - "strings" "testing" "github.com/stretchr/testify/require" @@ -143,82 +142,6 @@ partition_prefix "" { } } -func TestAPIGatewayControllerRules(t *testing.T) { - cases := []struct { - Name string - EnableNamespaces bool - Partition string - Expected string - }{ - { - Name: "Namespaces are disabled", - Expected: ` -operator = "write" -acl = "write" - service_prefix "" { - policy = "write" - intentions = "write" - } - node_prefix "" { - policy = "read" - }`, - }, - { - Name: "Namespaces are enabled", - EnableNamespaces: true, - Expected: ` -operator = "write" -acl = "write" -namespace_prefix "" { - policy = "write" - service_prefix "" { - policy = "write" - intentions = "write" - } - node_prefix "" { - policy = "read" - } -}`, - }, - { - Name: "Namespaces are enabled, partitions enabled", - EnableNamespaces: true, - Partition: "Default", - Expected: ` -partition "Default" { - mesh = "write" - acl = "write" -namespace_prefix "" { - policy = "write" - service_prefix "" { - policy = "write" - intentions = "write" - } - node_prefix "" { - policy = "read" - } -} -}`, - }, - } - - for _, tt := range cases { - t.Run(tt.Name, func(t *testing.T) { - cmd := Command{ - flagEnableNamespaces: tt.EnableNamespaces, - consulFlags: &flags.ConsulFlags{ - Partition: tt.Partition, - }, - } - - meshGatewayRules, err := cmd.apiGatewayControllerRules() - - require.NoError(t, err) - require.Equal(t, tt.Expected, strings.Trim(meshGatewayRules, " ")) - }) - } -} - func TestMeshGatewayRules(t *testing.T) { cases := []struct { Name string diff --git a/scan.hcl b/scan.hcl index 402f81f950..d151d9c64c 100644 --- a/scan.hcl +++ b/scan.hcl @@ -39,6 +39,8 @@ repository { # NET-8174 (2024-02-26): Missing YAML Content Leads To Panic (requires malicious plugin) "GHSA-r53h-jv2g-vpx6", "CVE-2024-26147", # alias + "GHSA-jw44-4f3j-q396", # Tracked in NET-8174 + "CVE-2019-25210" # alias ] } }