Feature implementation from commits 19ca140..f12a82a

.changelog/3693.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+catalog: Topology zone and region information is now read from the Kubernetes endpoints and associated node and added to registered consul services under Metadata.
+```

.changelog/3718.txt

@@ -0,0 +1,4 @@
+```release-note:breaking-change
+api-gateway: The api-gateway stanza located under .Values.api-gateway was deprecated in
+1.16.0 of Consul and is being removed as of 1.19.0 in favor of connectInject.apiGateway.
+```

.changelog/3741.txt

@@ -0,0 +1,14 @@
+```release-note:security
+Upgrade to use Go `1.21.8`. This resolves CVEs
+[CVE-2024-24783](https://nvd.nist.gov/vuln/detail/CVE-2024-24783) (`crypto/x509`).
+[CVE-2023-45290](https://nvd.nist.gov/vuln/detail/CVE-2023-45290) (`net/http`).
+[CVE-2023-45289](https://nvd.nist.gov/vuln/detail/CVE-2023-45289) (`net/http`, `net/http/cookiejar`).
+[CVE-2024-24785](https://nvd.nist.gov/vuln/detail/CVE-2024-24785) (`html/template`).
+[CVE-2024-24784](https://nvd.nist.gov/vuln/detail/CVE-2024-24784) (`net/mail`).
+```
+
+```release-note:security
+Update the Consul Build Go base image to `alpine3.19`. This resolves CVEs
+[CVE-2023-52425](https://nvd.nist.gov/vuln/detail/CVE-2023-52425)
+[CVE-2023-52426⁠](https://nvd.nist.gov/vuln/detail/CVE-2023-52426)
+```

.changelog/3758.txt

@@ -0,0 +1,4 @@
+```release-note:bug
+control-plane: fix an issue where ACL tokens would prematurely be deleted and services would be deregistered if there
+was a K8s API error fetching the pod.
+```

.changelog/3779.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+api-gateway: Fix order of initialization for creating ACL role/policy to avoid error logs in consul.
+```

.changelog/3811.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+api-gateway: Expose prometheus scrape metrics on api-gateway pods.
+```

.go-version

@@ -1 +1 @@
-1.21.7
+1.21.8

.release/security-scan.hcl

@@ -39,6 +39,8 @@ binary {
         # NET-8174 (2024-02-26): Missing YAML Content Leads To Panic (requires malicious plugin)
         "GHSA-r53h-jv2g-vpx6", 
         "CVE-2024-26147", # alias
+        "GHSA-jw44-4f3j-q396", # Tracked in NET-8174
+        "CVE-2019-25210" # alias
       ]
     }
   }

CONTRIBUTING.md

@@ -14,7 +14,7 @@
     1. [Webhook](#webhook)
     1. [Update command.go](#update-commandgo)
     1. [Generating YAML](#generating-yaml)
-    1. [Updating consul-helm](#updating-consul-helm)
+    1. [Updating consul-helm](#updating-helm-chart)
     1. [Testing a new CRD](#testing-a-new-crd)
     1. [Update Consul K8s acceptance tests](#update-consul-k8s-acceptance-tests)
 1. [Adding a new ACL Token](#adding-a-new-acl-token)

acceptance/framework/consul/helm_cluster.go

@@ -130,6 +130,9 @@ func NewHelmCluster(
 func (h *HelmCluster) Create(t *testing.T) {
 	t.Helper()
 
+	// check and remove any CRDs with finalizers
+	helpers.GetCRDRemoveFinalizers(t, h.helmOptions.KubectlOptions)
+
 	// Make sure we delete the cluster if we receive an interrupt signal and
 	// register cleanup so that we delete the cluster when test finishes.
 	helpers.Cleanup(t, h.noCleanupOnFailure, h.noCleanup, func() {

acceptance/framework/helpers/helpers.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/helm"
+	"github.com/gruntwork-io/terratest/modules/k8s"
 	terratestLogger "github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
@@ -196,42 +197,55 @@ type Command struct {
 	Logger     *terratestLogger.Logger
 }
 
-func RunCommand(t testutil.TestingTB, command Command) (string, error) {
+type cmdResult struct {
+	output string
+	err    error
+}
+
+func RunCommand(t testutil.TestingTB, options *k8s.KubectlOptions, command Command) (string, error) {
 	t.Helper()
-	cmd, err := exec.Command(command.Command, command.Args...).CombinedOutput()
 
-	// Check and remove finalizers in crds in the namespace
+	resultCh := make(chan *cmdResult, 1)
+
+	go func() {
+		output, err := exec.Command(command.Command, command.Args...).CombinedOutput()
+		resultCh <- &cmdResult{output: string(output), err: err}
+	}()
+
+	// might not be needed
 	for _, arg := range command.Args {
 		if strings.Contains(arg, "delete") {
-			errCh := make(chan error)
 			go func() {
-				errCh <- getCRDRemoveFinalizers(t)
+				GetCRDRemoveFinalizers(t, options)
 			}()
-			if err := <-errCh; err != nil {
-				return "", err
-			}
 		}
 	}
 
-	return string(cmd), err
+	select {
+	case res := <-resultCh:
+		if res.err != nil {
+			logger.Logf(t, "Output: %v.", res.output)
+		}
+		return res.output, res.err
+		// Sometimes this func runs for too long handle timeout if needed.
+	case <-time.After(320 * time.Second):
+		GetCRDRemoveFinalizers(t, options)
+		logger.Logf(t, "RunCommand timed out")
+		return "", nil
+	}
 }
 
 // getCRDRemoveFinalizers gets CRDs with finalizers and removes them.
-func getCRDRemoveFinalizers(t testutil.TestingTB) error {
+func GetCRDRemoveFinalizers(t testutil.TestingTB, options *k8s.KubectlOptions) {
 	t.Helper()
-	// Get CRD names with finalizers
-	crdNames, err := getCRDsWithFinalizers()
+	crdNames, err := getCRDsWithFinalizers(options)
 	if err != nil {
-		return err
+		logger.Logf(t, "Unable to get CRDs with finalizers, %v.", err)
 	}
 
-	// Remove finalizers for each CRD with finalizers
 	if len(crdNames) > 0 {
-		if err := removeFinalizers(crdNames); err != nil {
-			return err
-		}
+		removeFinalizers(t, options, crdNames)
 	}
-	return nil
 }
 
 // CRD struct to parse CRD JSON output.
@@ -244,15 +258,19 @@ type CRD struct {
 	} `json:"items"`
 }
 
-// getCRDsWithFinalizers gets CRDs with finalizers.
-func getCRDsWithFinalizers() ([]string, error) {
-	cmd := exec.Command("kubectl", "get", "crd", "-o=json")
+func getCRDsWithFinalizers(options *k8s.KubectlOptions) ([]string, error) {
+	cmdArgs := createCmdArgs(options)
+	args := []string{"get", "crd", "-o=json"}
 
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		return nil, fmt.Errorf("error executing command: %v", err)
+	cmdArgs = append(cmdArgs, args...)
+	command := Command{
+		Command: "kubectl",
+		Args:    cmdArgs,
+		Env:     options.Env,
 	}
 
+	output, err := exec.Command(command.Command, command.Args...).CombinedOutput()
+
 	var crds CRD
 	if err := json.Unmarshal(output, &crds); err != nil {
 		return nil, fmt.Errorf("error parsing JSON: %v", err)
@@ -265,19 +283,40 @@ func getCRDsWithFinalizers() ([]string, error) {
 		}
 	}
 
-	return crdNames, nil
+	return crdNames, err
 }
 
 // removeFinalizers removes finalizers from CRDs.
-func removeFinalizers(crdNames []string) error {
+func removeFinalizers(t testutil.TestingTB, options *k8s.KubectlOptions, crdNames []string) {
+	cmdArgs := createCmdArgs(options)
 	for _, crd := range crdNames {
-		cmd := exec.Command("kubectl", "patch", "crd", crd, "--type=json", "-p=[{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}]")
+		args := []string{"patch", "crd", crd, "--type=json", "-p=[{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}]"}
 
-		err := cmd.Run()
+		cmdArgs = append(cmdArgs, args...)
+		command := Command{
+			Command: "kubectl",
+			Args:    cmdArgs,
+			Env:     options.Env,
+		}
+
+		_, err := exec.Command(command.Command, command.Args...).CombinedOutput()
 		if err != nil {
-			return fmt.Errorf("error removing finalizers from CRD %s: %v", crd, err)
+			logger.Logf(t, "Unable to remove finalizers, proceeding anyway: %v.", err)
 		}
 		fmt.Printf("Finalizers removed from CRD %s\n", crd)
 	}
-	return nil
+}
+
+func createCmdArgs(options *k8s.KubectlOptions) []string {
+	var cmdArgs []string
+	if options.ContextName != "" {
+		cmdArgs = append(cmdArgs, "--context", options.ContextName)
+	}
+	if options.ConfigPath != "" {
+		cmdArgs = append(cmdArgs, "--kubeconfig", options.ConfigPath)
+	}
+	if options.Namespace != "" {
+		cmdArgs = append(cmdArgs, "--namespace", options.Namespace)
+	}
+	return cmdArgs
 }

acceptance/framework/k8s/deploy.go

@@ -129,10 +129,7 @@ func CheckStaticServerConnectionMultipleFailureMessages(t *testing.T, options *k
 		expectedOutput = expectedSuccessOutput
 	}
 
-	retrier := &retry.Counter{
-		Count: 10,
-		Wait:  2 * time.Second,
-	}
+	retrier := &retry.Counter{Count: 30, Wait: 2 * time.Second}
 
 	args := []string{"exec", resourceType + sourceApp, "-c", sourceApp, "--", "curl", "-vvvsSf"}
 	args = append(args, curlArgs...)

acceptance/framework/k8s/kubectl.go

@@ -68,7 +68,7 @@ func RunKubectlAndGetOutputWithLoggerE(t testutil.TestingTB, options *k8s.Kubect
 	var output string
 	var err error
 	retry.RunWith(counter, t, func(r *retry.R) {
-		output, err = helpers.RunCommand(r, command)
+		output, err = helpers.RunCommand(r, options, command)
 		if err != nil {
 			// Want to retry on errors connecting to actual Kube API because
 			// these are intermittent.

acceptance/tests/api-gateway/api_gateway_test.go

@@ -107,11 +107,14 @@ func TestAPIGateway_Basic(t *testing.T) {
 			logger.Log(t, "creating static-client pod")
 			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client")
 
+			k8s.RunKubectl(t, ctx.KubectlOptions(t), "wait", "--for=condition=available", "--timeout=5m", fmt.Sprintf("deploy/%s", "static-server"))
+
 			logger.Log(t, "patching route to target http server")
 			k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"name":"static-server","port":80}]}]}}`, "--type=merge")
 
 			logger.Log(t, "creating target tcp server")
 			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server-tcp")
+			k8s.RunKubectl(t, ctx.KubectlOptions(t), "wait", "--for=condition=available", "--timeout=5m", fmt.Sprintf("deploy/%s", "static-server-tcp"))
 
 			logger.Log(t, "creating tcp-route")
 			k8s.RunKubectl(t, ctx.KubectlOptions(t), "apply", "-f", "../fixtures/cases/api-gateways/tcproute/route.yaml")

acceptance/tests/connect/connect_proxy_lifecycle_test.go

@@ -11,14 +11,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/sdk/testutil/retry"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/hashicorp/consul-k8s/acceptance/framework/connhelper"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
-	"github.com/hashicorp/consul/sdk/testutil/retry"
-	"github.com/stretchr/testify/require"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 type LifecycleShutdownConfig struct {
@@ -36,24 +37,22 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) {
 	cfg := suite.Config()
 	cfg.SkipWhenOpenshiftAndCNI(t)
 
-	t.Skipf("TODO(flaky-1.17): NET-XXXX")
-
 	for _, testCfg := range []LifecycleShutdownConfig{
 		{secure: false, helmValues: map[string]string{
 			helmDrainListenersKey:     "true",
-			helmGracePeriodSecondsKey: "15",
+			helmGracePeriodSecondsKey: "5",
 		}},
 		{secure: true, helmValues: map[string]string{
 			helmDrainListenersKey:     "true",
-			helmGracePeriodSecondsKey: "15",
+			helmGracePeriodSecondsKey: "5",
 		}},
 		{secure: false, helmValues: map[string]string{
 			helmDrainListenersKey:     "false",
-			helmGracePeriodSecondsKey: "15",
+			helmGracePeriodSecondsKey: "5",
 		}},
 		{secure: true, helmValues: map[string]string{
 			helmDrainListenersKey:     "false",
-			helmGracePeriodSecondsKey: "15",
+			helmGracePeriodSecondsKey: "5",
 		}},
 		{secure: false, helmValues: map[string]string{
 			helmDrainListenersKey:     "false",
@@ -80,8 +79,8 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) {
 			gracePeriodSeconds, err = strconv.ParseInt(val, 10, 64)
 			require.NoError(t, err)
 		} else {
-			// Half of the helm default to speed tests up
-			gracePeriodSeconds = 15
+			// 5s should be a good amount of time to confirm the pod doesn't terminate
+			gracePeriodSeconds = 5
 		}
 
 		name := fmt.Sprintf("secure: %t, drainListeners: %t, gracePeriodSeconds: %d", testCfg.secure, drainListenersEnabled, gracePeriodSeconds)
@@ -140,7 +139,8 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) {
 			require.Len(t, pods.Items, 1)
 			clientPodName := pods.Items[0].Name
 
-			var terminationGracePeriod int64 = 60
+			// We should terminate the pods shortly after envoy gracefully shuts down in our 5s test cases.
+			var terminationGracePeriod int64 = 6
 			logger.Logf(t, "killing the %q pod with %dseconds termination grace period", clientPodName, terminationGracePeriod)
 			err = ctx.KubernetesClient(t).CoreV1().Pods(ns).Delete(context.Background(), clientPodName, metav1.DeleteOptions{GracePeriodSeconds: &terminationGracePeriod})
 			require.NoError(t, err)
@@ -155,23 +155,29 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) {
 			}
 
 			if gracePeriodSeconds > 0 {
-				// Ensure outbound requests are still successful during grace
-				// period.
-				retry.RunWith(&retry.Timer{Timeout: time.Duration(gracePeriodSeconds) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) {
-					output, err := k8s.RunKubectlAndGetOutputE(r, ctx.KubectlOptions(r), args...)
-					require.NoError(r, err)
-					require.Condition(r, func() bool {
-						exists := false
-						if strings.Contains(output, "curl: (7) Failed to connect") {
-							exists = true
+				// Ensure outbound requests are still successful during grace period.
+				gracePeriodTimer := time.NewTimer(time.Duration(gracePeriodSeconds) * time.Second)
+			gracePeriodLoop:
+				for {
+					select {
+					case <-gracePeriodTimer.C:
+						break gracePeriodLoop
+					default:
+						output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...)
+						require.NoError(t, err)
+						require.True(t, !strings.Contains(output, "curl: (7) Failed to connect"))
+
+						// If listener draining is disabled, ensure inbound
+						// requests are accepted during grace period.
+						if !drainListenersEnabled {
+							connHelper.TestConnectionSuccess(t, connhelper.ConnHelperOpts{})
 						}
-						return !exists
-					})
-				})
+						// TODO: check that the connection is unsuccessful when drainListenersEnabled is true
+						// dans note: I found it isn't sufficient to use the existing TestConnectionFailureWithoutIntention
 
-				// If listener draining is enabled, ensure inbound
-				// requests are rejected during grace period.
-				// connHelper.TestConnectionSuccess(t)
+						time.Sleep(2 * time.Second)
+					}
+				}
 			} else {
 				// Ensure outbound requests fail because proxy has terminated
 				retry.RunWith(&retry.Timer{Timeout: time.Duration(terminationGracePeriod) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) {
@@ -188,7 +194,10 @@ func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) {
 			}
 
 			logger.Log(t, "ensuring pod is deregistered after termination")
-			retry.Run(t, func(r *retry.R) {
+			// We wait an arbitrarily long time here. With the deployment rollout creating additional endpoints reconciles,
+			// This can cause the re-queued reconcile used to come back and clean up the service registration to be re-re-queued at
+			// 2-3X the intended grace period.
+			retry.RunWith(&retry.Timer{Timeout: time.Duration(30) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) {
 				for _, name := range []string{
 					"static-client",
 					"static-client-sidecar-proxy",