Backport of Datadog Integration Acceptance Tests / Bug fixes into release/1.2.x

acceptance/framework/config/config.go

@@ -22,6 +22,9 @@ const (
 	HelmChartPath     = "../../../charts/consul"
 	LicenseSecretName = "license"
 	LicenseSecretKey  = "key"
+	DatadogSecretName = "datadog-secret"
+	DatadogAPIKey     = "api-key"
+	DatadogAppKey     = "app-key"
 )
 
 type KubeTestConfig struct {
@@ -70,6 +73,10 @@ type TestConfig struct {
 	EnableEnterprise  bool
 	EnterpriseLicense string
 
+	EnableDatadog bool
+	DatadogAPIKey string
+	DatadogAppKey string
+
 	EnableOpenshift bool
 
 	EnablePodSecurityPolicies bool

acceptance/framework/consul/helm_cluster.go

@@ -94,6 +94,15 @@ func NewHelmCluster(
 	valuesFromConfig, err := cfg.HelmValuesFromConfig()
 	require.NoError(t, err)
 
+	if cfg.EnableDatadog {
+		datadogNamespace := helmValues["global.metrics.datadog.namespace"]
+		configureNamespace(t, ctx.KubernetesClient(t), cfg, datadogNamespace)
+
+		if cfg.DatadogAPIKey != "" || cfg.DatadogAppKey != "" {
+			createOrUpdateDatadogSecret(t, ctx.KubernetesClient(t), cfg, datadogNamespace)
+		}
+	}
+
 	// Merge all helm values
 	helpers.MergeMaps(values, valuesFromConfig)
 	helpers.MergeMaps(values, helmValues)
@@ -159,6 +168,15 @@ func (h *HelmCluster) Create(t *testing.T) {
 		chartName = h.ChartPath
 	}
 
+	if strings.Contains(t.Name(), "Datadog") {
+		helm.AddRepo(t, h.helmOptions, "datadog", "https://helm.datadoghq.com")
+		// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
+		// asserting a chart version on Install which would fail in an obvious way should this not succeed.
+		_, err := helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
+		if err != nil {
+			logger.Logf(t, "Unable to update helm repository, proceeding anyway: %s.", err)
+		}
+	}
 	// Retry the install in case previous tests have not finished cleaning up.
 	retry.RunWith(&retry.Counter{Wait: 2 * time.Second, Count: 30}, t, func(r *retry.R) {
 		err := helm.InstallE(r, h.helmOptions, chartName, h.releaseName)
@@ -672,6 +690,14 @@ func createOrUpdateLicenseSecret(t *testing.T, client kubernetes.Interface, cfg
 	CreateK8sSecret(t, client, cfg, namespace, config.LicenseSecretName, config.LicenseSecretKey, cfg.EnterpriseLicense)
 }
 
+func createOrUpdateDatadogSecret(t *testing.T, client kubernetes.Interface, cfg *config.TestConfig, namespace string) {
+	secretMap := map[string]string{
+		config.DatadogAPIKey: cfg.DatadogAPIKey,
+		config.DatadogAppKey: cfg.DatadogAppKey,
+	}
+	CreateMultiKeyK8sSecret(t, client, cfg, namespace, config.DatadogSecretName, secretMap)
+}
+
 func configureNamespace(t *testing.T, client kubernetes.Interface, cfg *config.TestConfig, namespace string) {
 	ctx := context.Background()
 
@@ -783,3 +809,25 @@ func CreateK8sSecret(t *testing.T, client kubernetes.Interface, cfg *config.Test
 		_ = client.CoreV1().Secrets(namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
 	})
 }
+
+func CreateMultiKeyK8sSecret(t *testing.T, client kubernetes.Interface, cfg *config.TestConfig, namespace, secretName string, secretMap map[string]string) {
+	retry.RunWith(&retry.Counter{Wait: 2 * time.Second, Count: 15}, t, func(r *retry.R) {
+		_, err := client.CoreV1().Secrets(namespace).Get(context.Background(), secretName, metav1.GetOptions{})
+		if errors.IsNotFound(err) {
+			_, err := client.CoreV1().Secrets(namespace).Create(context.Background(), &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: secretName,
+				},
+				StringData: secretMap,
+				Type:       corev1.SecretTypeOpaque,
+			}, metav1.CreateOptions{})
+			require.NoError(r, err)
+		} else {
+			require.NoError(r, err)
+		}
+	})
+
+	helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() {
+		_ = client.CoreV1().Secrets(namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
+	})
+}

acceptance/framework/flags/flags.go

@@ -23,6 +23,10 @@ type TestFlags struct {
 	flagEnableEnterprise  bool
 	flagEnterpriseLicense string
 
+	flagEnableDatadog bool
+	flagDatadogAPIKey string
+	flagDatadogAppKey string
+
 	flagEnableOpenshift bool
 
 	flagEnablePodSecurityPolicies bool
@@ -109,6 +113,15 @@ func (t *TestFlags) init() {
 	flag.StringVar(&t.flagEnterpriseLicense, "enterprise-license", "",
 		"The enterprise license for Consul.")
 
+	flag.BoolVar(&t.flagEnableDatadog, "enable-datadog", false,
+		"If true, the test suite will run tests for datadog integration features. "+
+			"Note that some features will require setting the Datadog API and Application keys using the 'dd-api-key' and 'dd-app-key' flag below"+
+			"or the env vars DATADOG_API_KEY and DATADOG_APP_KEY")
+	flag.StringVar(&t.flagDatadogAPIKey, "dd-api-key", "",
+		"The Datadog Agent API Key used for datadog metrics tests.")
+	flag.StringVar(&t.flagDatadogAppKey, "dd-app-key", "",
+		"The Datadog Agent Application Key used for datadog metrics tests.")
+
 	flag.BoolVar(&t.flagEnableOpenshift, "enable-openshift", false,
 		"If true, the tests will automatically add Openshift Helm value for each Helm install.")
 
@@ -158,6 +171,14 @@ func (t *TestFlags) init() {
 	if t.flagEnterpriseLicense == "" {
 		t.flagEnterpriseLicense = os.Getenv("CONSUL_ENT_LICENSE")
 	}
+
+	if t.flagDatadogAPIKey == "" {
+		t.flagDatadogAPIKey = os.Getenv("DATADOG_API_KEY")
+	}
+
+	if t.flagDatadogAppKey == "" {
+		t.flagDatadogAppKey = os.Getenv("DATADOG_APP_KEY")
+	}
 }
 
 func (t *TestFlags) Validate() error {
@@ -205,6 +226,10 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig {
 		EnableEnterprise:  t.flagEnableEnterprise,
 		EnterpriseLicense: t.flagEnterpriseLicense,
 
+		EnableDatadog: t.flagEnableDatadog,
+		DatadogAPIKey: t.flagDatadogAPIKey,
+		DatadogAppKey: t.flagDatadogAppKey,
+
 		KubeEnvs:           kubeEnvs,
 		EnableMultiCluster: t.flagEnableMultiCluster,
 

acceptance/framework/k8s/deploy.go

@@ -69,8 +69,10 @@ func DeployKustomize(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailu
 		KubectlDeleteK(t, options, kustomizeDir)
 	})
 
-	// The timeout to allow for connect-init to wait for services to be registered by the endpoints controller.
-	RunKubectl(t, options, "wait", "--for=condition=available", "--timeout=5m", fmt.Sprintf("deploy/%s", deployment.Name))
+	if !strings.Contains(t.Name(), "Datadog") {
+		// The timeout to allow for connect-init to wait for services to be registered by the endpoints controller.
+		RunKubectl(t, options, "wait", "--for=condition=available", "--timeout=5m", fmt.Sprintf("deploy/%s", deployment.Name))
+	}
 }
 
 func DeployJob(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, noCleanup bool, debugDirectory, kustomizeDir string) {

acceptance/tests/datadog/README.md

@@ -0,0 +1,20 @@
+## Datadog Acceptance Testing
+
+Tests helm chart override automation configurations for expected operation.
+
+
+```yaml
+global:
+  metrics:
+    enabled: true
+    enableAgentMetrics: true
+    disableAgentHostName: true
+    enableHostMetrics: true
+    datadog:
+      enabled: true
+      dogstatsd:
+        enabled: true
+        socketTransportType: "UDS"
+        dogstatsdAddr: "/var/run/datadog/dsd.socket"
+        dogstatsdTags: [ "source:consul","consul_service:consul-server" ]
+```

acceptance/tests/datadog/datadog_test.go

@@ -0,0 +1,76 @@
+package datadog
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/datadog"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+// Test that prometheus metrics, when enabled, are accessible from the
+// endpoints that have been exposed on the server, client and gateways.
+func TestDatadogDogstatsDUnixDomainSocket(t *testing.T) {
+	env := suite.Environment()
+	cfg := suite.Config()
+	ctx := env.DefaultContext(t)
+	// ns := ctx.KubectlOptions(t).Namespace
+
+	helmValues := map[string]string{
+		"global.datacenter":                                    "dc1",
+		"global.metrics.enabled":                               "true",
+		"global.metrics.enableAgentMetrics":                    "true",
+		"global.metrics.disableAgentHostName":                  "true",
+		"global.metrics.enableHostMetrics":                     "true",
+		"global.metrics.datadog.enabled":                       "true",
+		"global.metrics.datadog.namespace":                     "datadog",
+		"global.metrics.datadog.dogstatsd.enabled":             "true",
+		"global.metrics.datadog.dogstatsd.socketTransportType": "UDS",
+	}
+
+	datadogOperatorHelmValues := map[string]string{
+		"replicaCount":     "1",
+		"image.tag":        datadog.DefaultHelmChartVersion,
+		"image.repository": "gcr.io/datadoghq/operator",
+	}
+
+	releaseName := helpers.RandomName()
+	datadogOperatorRelease := datadog.DatadogOperatorReleaseName
+
+	// Install the consul cluster in the default kubernetes ctx.
+	consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
+	consulCluster.Create(t)
+
+	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay
+	datadogNamespace := helmValues["global.metrics.datadog.namespace"]
+	logger.Log(t, fmt.Sprintf("deploying datadog-operator via helm | namespace: %s | release-name: %s", datadogNamespace, datadogOperatorRelease))
+	datadogCluster := datadog.NewDatadogCluster(t, ctx, cfg, datadogOperatorRelease, datadogNamespace, datadogOperatorHelmValues)
+	datadogCluster.Create(t)
+	//k8s.DeployKustomize(t, ctx.KubectlOptionsForNamespace(datadogNamespace), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/datadog-operator")
+	//k8s.WaitForAllPodsToBeReady(t, ctx.KubernetesClient(t), datadogNamespace, "app.kubernetes.io/name=datadog-operator")
+
+	logger.Log(t, fmt.Sprintf("deploying datadog-agent | namespace: %s", datadogNamespace))
+	k8s.DeployKustomize(t, ctx.KubectlOptionsForNamespace(datadogNamespace), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/datadog")
+	k8s.WaitForAllPodsToBeReady(t, ctx.KubernetesClient(t), datadogNamespace, "agent.datadoghq.com/component=agent")
+
+	logger.Log(t, fmt.Sprintf("applying dogstatd over unix domain sockets patch to datadog-agent | namespace: %s", datadogNamespace))
+	k8s.DeployKustomize(t, ctx.KubectlOptionsForNamespace(datadogNamespace), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/datadog-dogstatsd-uds")
+	k8s.WaitForAllPodsToBeReady(t, ctx.KubernetesClient(t), datadogNamespace, "agent.datadoghq.com/component=agent")
+
+	datadogAPIClient := datadogCluster.DatadogClient(t)
+	api := datadogV1.NewMetricsApi(datadogAPIClient.ApiClient)
+
+	response, fullResponse, err := api.ListMetrics(datadogAPIClient.Ctx, "consul.acl")
+	if err != nil {
+		logger.Logf(t, "Error when calling MetricsApi.ListMetris: %v", err)
+		logger.Logf(t, "Full Response: %v", fullResponse)
+	}
+	content, _ := json.MarshalIndent(response, "", " ")
+	logger.Logf(t, "Full Response: %v", string(content))
+	require.Contains(t, string(content), `consul.acl.ResolveToken.50percentile`)
+}

acceptance/tests/datadog/main_test.go

@@ -0,0 +1,16 @@
+package datadog
+
+import (
+	"os"
+	"testing"
+
+	testsuite "github.com/hashicorp/consul-k8s/acceptance/framework/suite"
+)
+
+var suite testsuite.Suite
+
+func TestMain(m *testing.M) {
+	suite = testsuite.NewSuite(m)
+	os.Exit(suite.Run())
+
+}