tests: add exnternal-prometheus integration test

.github/workflows/integration_tests.yml

@@ -102,6 +102,7 @@ jobs:
         - cluster-domain
         - deep
         - external-issuer
+        - external-prometheus-deep
         - helm-deep
         - helm-upgrade
         - multicluster

.github/workflows/release.yml

@@ -101,6 +101,7 @@ jobs:
         - cluster-domain
         - deep
         - external-issuer
+        - external-prometheus-deep
         - helm-deep
         - helm-upgrade
         - uninstall

bin/_test-helpers.sh

@@ -6,7 +6,7 @@ set +e
 
 ##### Test setup helpers #####
 
-export default_test_names=(deep external-issuer helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable)
+export default_test_names=(deep external-issuer external-prometheus-deep helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable)
 export all_test_names=(cluster-domain cni-calico-deep multicluster "${default_test_names[*]}")
 
 tests_usage() {
@@ -502,6 +502,14 @@ run_external-issuer_test() {
   run_test "$test_directory/externalissuer/external_issuer_test.go" --external-issuer=true
 }
 
+run_external-prometheus-deep_test() {
+  run_test "$test_directory/install_test.go" --external-prometheus=true
+  while IFS= read -r line; do tests+=("$line"); done <<< "$(go list "$test_directory"/.../...)"
+  for test in "${tests[@]}"; do
+    run_test "$test" --external-prometheus=true
+  done
+}
+
 run_cluster-domain_test() {
   run_test "$test_directory/install_test.go" --cluster-domain='custom.domain'
 }

test/cli/cli_install_static_test.go

@@ -34,7 +34,7 @@ func TestMain(m *testing.M) {
 		exit(1, "-linkerd flag is required")
 	}
 
-	TestHelper = testutil.NewGenericTestHelper(*linkerd, "", "l5d", "linkerd-viz", "", "", "", "", "", "", "", "", false, false, false, false, false, *http.DefaultClient, testutil.KubernetesHelper{})
+	TestHelper = testutil.NewGenericTestHelper(*linkerd, "", "l5d", "linkerd-viz", "", "", "", "", "", "", "", "", false, false, false, false, false, false, *http.DefaultClient, testutil.KubernetesHelper{})
 	os.Exit(m.Run())
 }
 

test/integration/edges/edges_test.go

@@ -135,13 +135,17 @@ func TestDirectEdges(t *testing.T) {
 
 		// check edges
 		timeout := 50 * time.Second
+		testDataPath := "testdata"
+		if TestHelper.ExternalPrometheus() {
+			testDataPath += "/external_prometheus"
+		}
 		err = TestHelper.RetryFor(timeout, func() error {
 			out, err = TestHelper.LinkerdRun("-n", testNamespace, "-o", "json", "viz", "edges", "deploy")
 			if err != nil {
 				return err
 			}
 
-			tpl := template.Must(template.ParseFiles("testdata/direct_edges.golden"))
+			tpl := template.Must(template.ParseFiles(testDataPath + "/direct_edges.golden"))
 			vars := struct {
 				Ns    string
 				VizNs string

test/integration/edges/testdata/external_prometheus/direct_edges.golden

@@ -0,0 +1,29 @@
+\[
+  \{
+    "src": "prometheus",
+    "src_namespace": "default",
+    "dst": "slow-cooker",
+    "dst_namespace": "{{.Ns}}",
+    "client_id": "prometheus.default",
+    "server_id": "default.{{.Ns}}",
+    "no_tls_reason": ""
+  \},
+  \{
+    "src": "prometheus",
+    "src_namespace": "default",
+    "dst": "terminus",
+    "dst_namespace": "{{.Ns}}",
+    "client_id": "prometheus.default",
+    "server_id": "default.{{.Ns}}",
+    "no_tls_reason": ""
+  \},
+  \{
+    "src": "slow-cooker",
+    "src_namespace": "{{.Ns}}",
+    "dst": "terminus",
+    "dst_namespace": "{{.Ns}}",
+    "client_id": "default.{{.Ns}}",
+    "server_id": "default.{{.Ns}}",
+    "no_tls_reason": ""
+  \}
+\]

test/integration/endpoints/endpoints_test.go

@@ -22,25 +22,33 @@ func TestMain(m *testing.M) {
 func TestGoodEndpoints(t *testing.T) {
 	ns := TestHelper.GetLinkerdNamespace()
 	vizNs := TestHelper.GetVizNamespace()
+	testDataPath := "testdata"
 	cmd := []string{
 		"endpoints",
 		fmt.Sprintf("linkerd-controller-api.%s.svc.cluster.local:8085", ns),
 		fmt.Sprintf("linkerd-dst.%s.svc.cluster.local:8086", ns),
 		fmt.Sprintf("linkerd-grafana.%s.svc.cluster.local:3000", vizNs),
 		fmt.Sprintf("linkerd-identity.%s.svc.cluster.local:8080", ns),
-		fmt.Sprintf("linkerd-prometheus.%s.svc.cluster.local:9090", vizNs),
 		fmt.Sprintf("linkerd-proxy-injector.%s.svc.cluster.local:443", ns),
 		fmt.Sprintf("linkerd-sp-validator.%s.svc.cluster.local:443", ns),
 		fmt.Sprintf("linkerd-tap.%s.svc.cluster.local:8088", vizNs),
 		fmt.Sprintf("linkerd-web.%s.svc.cluster.local:8084", vizNs),
-		"-ojson",
 	}
+
+	if !TestHelper.ExternalPrometheus() {
+		cmd = append(cmd, fmt.Sprintf("linkerd-prometheus.%s.svc.cluster.local:9090", vizNs))
+	} else {
+		cmd = append(cmd, "prometheus.default.svc.cluster.local:9090")
+		testDataPath += "/external_prometheus"
+	}
+
+	cmd = append(cmd, "-ojson")
 	out, err := TestHelper.LinkerdRun(cmd...)
 	if err != nil {
 		testutil.AnnotatedFatal(t, "unexpected error", err)
 	}
 
-	tpl := template.Must(template.ParseFiles("testdata/linkerd_endpoints.golden"))
+	tpl := template.Must(template.ParseFiles(testDataPath + "/linkerd_endpoints.golden"))
 	vars := struct {
 		Ns    string
 		VizNs string

test/integration/endpoints/testdata/external_prometheus/linkerd_endpoints.golden

@@ -0,0 +1,65 @@
+\[
+  \{
+    "namespace": "default",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 9090,
+    "pod": "prometheus\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "prometheus.default"
+  \},
+  \{
+    "namespace": "{{.Ns}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8085,
+    "pod": "linkerd\-controller\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-controller\-api\.{{.Ns}}"
+  \},
+  \{
+    "namespace": "{{.Ns}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8086,
+    "pod": "linkerd\-destination\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-dst\.{{.Ns}}"
+  \},
+  \{
+    "namespace": "{{.Ns}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8080,
+    "pod": "linkerd\-identity\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-identity\.{{.Ns}}"
+  \},
+  \{
+    "namespace": "{{.Ns}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8443,
+    "pod": "linkerd\-proxy\-injector-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-proxy\-injector\.{{.Ns}}"
+  \},
+  \{
+    "namespace": "{{.Ns}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8443,
+    "pod": "linkerd\-sp\-validator-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-sp\-validator\.{{.Ns}}"
+  \},
+  \{
+    "namespace": "{{.VizNs}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 3000,
+    "pod": "linkerd\-grafana\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-grafana\.{{.VizNs}}"
+  \},
+  \{
+    "namespace": "{{.VizNs}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8088,
+    "pod": "linkerd\-tap\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-tap\.{{.VizNs}}"
+  \},
+  \{
+    "namespace": "{{.VizNs}}",
+    "ip": "\d+\.\d+\.\d+\.\d+",
+    "port": 8084,
+    "pod": "linkerd\-web\-[a-f0-9]+\-[a-z0-9]+",
+    "service": "linkerd\-web\.{{.VizNs}}"
+  \}
+\]

test/integration/install_test.go

@@ -49,7 +49,6 @@ var (
 		{Namespace: "linkerd", Name: "linkerd-dst"},
 		{Namespace: "linkerd-viz", Name: "linkerd-grafana"},
 		{Namespace: "linkerd", Name: "linkerd-identity"},
-		{Namespace: "linkerd-viz", Name: "linkerd-prometheus"},
 		{Namespace: "linkerd-viz", Name: "linkerd-web"},
 		{Namespace: "linkerd-viz", Name: "linkerd-tap"},
 		{Namespace: "linkerd", Name: "linkerd-dst-headless"},
@@ -415,6 +414,24 @@ func TestInstallOrUpgradeCli(t *testing.T) {
 			"failed to wait for condition=available for deploy/%s in namespace %s: %s: %s", name, ns, err, o)
 	}
 
+	if TestHelper.ExternalPrometheus() {
+
+		// Install external prometheus
+		out, err := TestHelper.LinkerdRun("inject", "testdata/external_prometheus.yaml")
+		if err != nil {
+			testutil.AnnotatedFatalf(t, "'linkerd inject' command failed", "'linkerd inject' command failed: %s", err)
+		}
+
+		out, err = TestHelper.KubectlApply(out, "")
+		if err != nil {
+			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
+				"kubectl apply command failed\n%s", out)
+		}
+
+		// Update args to use external proemtheus
+		vizArgs = append(vizArgs, "--set", "prometheusUrl=http://prometheus.default.svc.cluster.local:9090", "--set", "prometheus.enabled=false")
+	}
+
 	// Install Linkerd Viz Extension
 	exec = append(vizCmd, vizArgs...)
 	out, err = TestHelper.LinkerdRun(exec...)
@@ -521,6 +538,11 @@ func TestInstallHelm(t *testing.T) {
 func TestControlPlaneResourcesPostInstall(t *testing.T) {
 	expectedServices := linkerdSvcEdge
 	expectedDeployments := testutil.LinkerdDeployReplicasEdge
+	if !TestHelper.ExternalPrometheus() {
+		expectedServices = append(expectedServices, testutil.Service{Namespace: "linkerd-viz", Name: "linkerd-prometheus"})
+		expectedDeployments["linkerd-prometheus"] = testutil.DeploySpec{Namespace: "linkerd-viz", Replicas: 1, Containers: []string{}}
+	}
+
 	// Upgrade Case
 	if TestHelper.UpgradeHelmFromVersion() != "" {
 		expectedServices = linkerdSvcStable
@@ -824,6 +846,10 @@ func TestCheckPostInstall(t *testing.T) {
 func TestCheckViz(t *testing.T) {
 	cmd := []string{"viz", "check", "--wait=0"}
 	golden := "check.viz.golden"
+	if TestHelper.ExternalPrometheus() {
+		golden = "check.viz.external-prometheus.golden"
+	}
+
 	timeout := time.Minute
 	err := TestHelper.RetryFor(timeout, func() error {
 		out, err := TestHelper.LinkerdRun(cmd...)
@@ -1024,7 +1050,11 @@ func TestCheckProxy(t *testing.T) {
 }
 
 func TestRestarts(t *testing.T) {
-	for deploy, spec := range testutil.LinkerdDeployReplicasEdge {
+	expectedDeployments := testutil.LinkerdDeployReplicasEdge
+	if !TestHelper.ExternalPrometheus() {
+		expectedDeployments["linkerd-prometheus"] = testutil.DeploySpec{Namespace: "linkerd-viz", Replicas: 1, Containers: []string{}}
+	}
+	for deploy, spec := range expectedDeployments {
 		if err := TestHelper.CheckPods(context.Background(), spec.Namespace, deploy, spec.Replicas); err != nil {
 			if rce, ok := err.(*testutil.RestartCountError); ok {
 				testutil.AnnotatedWarn(t, "CheckPods timed-out", rce)