diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 974ce1d753ffc..ca29d9a48773f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -102,6 +102,7 @@ jobs: - cluster-domain - deep - external-issuer + - external-prometheus-deep - helm-deep - helm-upgrade - multicluster diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee4852e3f9230..1701bc1124376 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,6 +101,7 @@ jobs: - cluster-domain - deep - external-issuer + - external-prometheus-deep - helm-deep - helm-upgrade - uninstall diff --git a/bin/_test-helpers.sh b/bin/_test-helpers.sh index d6d91f993e968..d7ffb7f57431b 100644 --- a/bin/_test-helpers.sh +++ b/bin/_test-helpers.sh @@ -6,7 +6,7 @@ set +e ##### Test setup helpers ##### -export default_test_names=(deep external-issuer helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable) +export default_test_names=(deep external-issuer external-prometheus-deep helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable) export all_test_names=(cluster-domain cni-calico-deep multicluster "${default_test_names[*]}") tests_usage() { @@ -502,6 +502,14 @@ run_external-issuer_test() { run_test "$test_directory/externalissuer/external_issuer_test.go" --external-issuer=true } +run_external-prometheus-deep_test() { + run_test "$test_directory/install_test.go" --external-prometheus=true + while IFS= read -r line; do tests+=("$line"); done <<< "$(go list "$test_directory"/.../...)" + for test in "${tests[@]}"; do + run_test "$test" --external-prometheus=true + done +} + run_cluster-domain_test() { run_test "$test_directory/install_test.go" --cluster-domain='custom.domain' } diff --git a/test/cli/cli_install_static_test.go b/test/cli/cli_install_static_test.go index 8b9285ff6c211..fe6c255cab894 100644 --- a/test/cli/cli_install_static_test.go +++ b/test/cli/cli_install_static_test.go @@ -34,7 +34,7 @@ func TestMain(m *testing.M) { exit(1, "-linkerd flag is required") } - TestHelper = testutil.NewGenericTestHelper(*linkerd, "", "l5d", "linkerd-viz", "", "", "", "", "", "", "", "", false, false, false, false, false, *http.DefaultClient, testutil.KubernetesHelper{}) + TestHelper = testutil.NewGenericTestHelper(*linkerd, "", "l5d", "linkerd-viz", "", "", "", "", "", "", "", "", false, false, false, false, false, false, *http.DefaultClient, testutil.KubernetesHelper{}) os.Exit(m.Run()) } diff --git a/test/integration/edges/edges_test.go b/test/integration/edges/edges_test.go index 61a46cadd64f4..132187dc61676 100644 --- a/test/integration/edges/edges_test.go +++ b/test/integration/edges/edges_test.go @@ -135,13 +135,17 @@ func TestDirectEdges(t *testing.T) { // check edges timeout := 50 * time.Second + testDataPath := "testdata" + if TestHelper.ExternalPrometheus() { + testDataPath += "/external_prometheus" + } err = TestHelper.RetryFor(timeout, func() error { out, err = TestHelper.LinkerdRun("-n", testNamespace, "-o", "json", "viz", "edges", "deploy") if err != nil { return err } - tpl := template.Must(template.ParseFiles("testdata/direct_edges.golden")) + tpl := template.Must(template.ParseFiles(testDataPath + "/direct_edges.golden")) vars := struct { Ns string VizNs string diff --git a/test/integration/edges/testdata/external_prometheus/direct_edges.golden b/test/integration/edges/testdata/external_prometheus/direct_edges.golden new file mode 100644 index 0000000000000..cd6dc02036080 --- /dev/null +++ b/test/integration/edges/testdata/external_prometheus/direct_edges.golden @@ -0,0 +1,29 @@ +\[ + \{ + "src": "prometheus", + "src_namespace": "default", + "dst": "slow-cooker", + "dst_namespace": "{{.Ns}}", + "client_id": "prometheus.default", + "server_id": "default.{{.Ns}}", + "no_tls_reason": "" + \}, + \{ + "src": "prometheus", + "src_namespace": "default", + "dst": "terminus", + "dst_namespace": "{{.Ns}}", + "client_id": "prometheus.default", + "server_id": "default.{{.Ns}}", + "no_tls_reason": "" + \}, + \{ + "src": "slow-cooker", + "src_namespace": "{{.Ns}}", + "dst": "terminus", + "dst_namespace": "{{.Ns}}", + "client_id": "default.{{.Ns}}", + "server_id": "default.{{.Ns}}", + "no_tls_reason": "" + \} +\] diff --git a/test/integration/endpoints/endpoints_test.go b/test/integration/endpoints/endpoints_test.go index c0ba41eb01e15..1f5a488b8b475 100644 --- a/test/integration/endpoints/endpoints_test.go +++ b/test/integration/endpoints/endpoints_test.go @@ -22,25 +22,33 @@ func TestMain(m *testing.M) { func TestGoodEndpoints(t *testing.T) { ns := TestHelper.GetLinkerdNamespace() vizNs := TestHelper.GetVizNamespace() + testDataPath := "testdata" cmd := []string{ "endpoints", fmt.Sprintf("linkerd-controller-api.%s.svc.cluster.local:8085", ns), fmt.Sprintf("linkerd-dst.%s.svc.cluster.local:8086", ns), fmt.Sprintf("linkerd-grafana.%s.svc.cluster.local:3000", vizNs), fmt.Sprintf("linkerd-identity.%s.svc.cluster.local:8080", ns), - fmt.Sprintf("linkerd-prometheus.%s.svc.cluster.local:9090", vizNs), fmt.Sprintf("linkerd-proxy-injector.%s.svc.cluster.local:443", ns), fmt.Sprintf("linkerd-sp-validator.%s.svc.cluster.local:443", ns), fmt.Sprintf("linkerd-tap.%s.svc.cluster.local:8088", vizNs), fmt.Sprintf("linkerd-web.%s.svc.cluster.local:8084", vizNs), - "-ojson", } + + if !TestHelper.ExternalPrometheus() { + cmd = append(cmd, fmt.Sprintf("linkerd-prometheus.%s.svc.cluster.local:9090", vizNs)) + } else { + cmd = append(cmd, "prometheus.default.svc.cluster.local:9090") + testDataPath += "/external_prometheus" + } + + cmd = append(cmd, "-ojson") out, err := TestHelper.LinkerdRun(cmd...) if err != nil { testutil.AnnotatedFatal(t, "unexpected error", err) } - tpl := template.Must(template.ParseFiles("testdata/linkerd_endpoints.golden")) + tpl := template.Must(template.ParseFiles(testDataPath + "/linkerd_endpoints.golden")) vars := struct { Ns string VizNs string diff --git a/test/integration/endpoints/testdata/external_prometheus/linkerd_endpoints.golden b/test/integration/endpoints/testdata/external_prometheus/linkerd_endpoints.golden new file mode 100644 index 0000000000000..e01314e2d4065 --- /dev/null +++ b/test/integration/endpoints/testdata/external_prometheus/linkerd_endpoints.golden @@ -0,0 +1,65 @@ +\[ + \{ + "namespace": "default", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 9090, + "pod": "prometheus\-[a-f0-9]+\-[a-z0-9]+", + "service": "prometheus.default" + \}, + \{ + "namespace": "{{.Ns}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8085, + "pod": "linkerd\-controller\-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-controller\-api\.{{.Ns}}" + \}, + \{ + "namespace": "{{.Ns}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8086, + "pod": "linkerd\-destination\-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-dst\.{{.Ns}}" + \}, + \{ + "namespace": "{{.Ns}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8080, + "pod": "linkerd\-identity\-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-identity\.{{.Ns}}" + \}, + \{ + "namespace": "{{.Ns}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8443, + "pod": "linkerd\-proxy\-injector-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-proxy\-injector\.{{.Ns}}" + \}, + \{ + "namespace": "{{.Ns}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8443, + "pod": "linkerd\-sp\-validator-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-sp\-validator\.{{.Ns}}" + \}, + \{ + "namespace": "{{.VizNs}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 3000, + "pod": "linkerd\-grafana\-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-grafana\.{{.VizNs}}" + \}, + \{ + "namespace": "{{.VizNs}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8088, + "pod": "linkerd\-tap\-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-tap\.{{.VizNs}}" + \}, + \{ + "namespace": "{{.VizNs}}", + "ip": "\d+\.\d+\.\d+\.\d+", + "port": 8084, + "pod": "linkerd\-web\-[a-f0-9]+\-[a-z0-9]+", + "service": "linkerd\-web\.{{.VizNs}}" + \} +\] diff --git a/test/integration/install_test.go b/test/integration/install_test.go index 2ab2a11e4e4bb..b2c92897e6093 100644 --- a/test/integration/install_test.go +++ b/test/integration/install_test.go @@ -49,7 +49,6 @@ var ( {Namespace: "linkerd", Name: "linkerd-dst"}, {Namespace: "linkerd-viz", Name: "linkerd-grafana"}, {Namespace: "linkerd", Name: "linkerd-identity"}, - {Namespace: "linkerd-viz", Name: "linkerd-prometheus"}, {Namespace: "linkerd-viz", Name: "linkerd-web"}, {Namespace: "linkerd-viz", Name: "linkerd-tap"}, {Namespace: "linkerd", Name: "linkerd-dst-headless"}, @@ -415,6 +414,24 @@ func TestInstallOrUpgradeCli(t *testing.T) { "failed to wait for condition=available for deploy/%s in namespace %s: %s: %s", name, ns, err, o) } + if TestHelper.ExternalPrometheus() { + + // Install external prometheus + out, err := TestHelper.LinkerdRun("inject", "testdata/external_prometheus.yaml") + if err != nil { + testutil.AnnotatedFatalf(t, "'linkerd inject' command failed", "'linkerd inject' command failed: %s", err) + } + + out, err = TestHelper.KubectlApply(out, "") + if err != nil { + testutil.AnnotatedFatalf(t, "'kubectl apply' command failed", + "kubectl apply command failed\n%s", out) + } + + // Update args to use external proemtheus + vizArgs = append(vizArgs, "--set", "prometheusUrl=http://prometheus.default.svc.cluster.local:9090", "--set", "prometheus.enabled=false") + } + // Install Linkerd Viz Extension exec = append(vizCmd, vizArgs...) out, err = TestHelper.LinkerdRun(exec...) @@ -521,6 +538,11 @@ func TestInstallHelm(t *testing.T) { func TestControlPlaneResourcesPostInstall(t *testing.T) { expectedServices := linkerdSvcEdge expectedDeployments := testutil.LinkerdDeployReplicasEdge + if !TestHelper.ExternalPrometheus() { + expectedServices = append(expectedServices, testutil.Service{Namespace: "linkerd-viz", Name: "linkerd-prometheus"}) + expectedDeployments["linkerd-prometheus"] = testutil.DeploySpec{Namespace: "linkerd-viz", Replicas: 1, Containers: []string{}} + } + // Upgrade Case if TestHelper.UpgradeHelmFromVersion() != "" { expectedServices = linkerdSvcStable @@ -824,6 +846,10 @@ func TestCheckPostInstall(t *testing.T) { func TestCheckViz(t *testing.T) { cmd := []string{"viz", "check", "--wait=0"} golden := "check.viz.golden" + if TestHelper.ExternalPrometheus() { + golden = "check.viz.external-prometheus.golden" + } + timeout := time.Minute err := TestHelper.RetryFor(timeout, func() error { out, err := TestHelper.LinkerdRun(cmd...) @@ -1024,7 +1050,11 @@ func TestCheckProxy(t *testing.T) { } func TestRestarts(t *testing.T) { - for deploy, spec := range testutil.LinkerdDeployReplicasEdge { + expectedDeployments := testutil.LinkerdDeployReplicasEdge + if !TestHelper.ExternalPrometheus() { + expectedDeployments["linkerd-prometheus"] = testutil.DeploySpec{Namespace: "linkerd-viz", Replicas: 1, Containers: []string{}} + } + for deploy, spec := range expectedDeployments { if err := TestHelper.CheckPods(context.Background(), spec.Namespace, deploy, spec.Replicas); err != nil { if rce, ok := err.(*testutil.RestartCountError); ok { testutil.AnnotatedWarn(t, "CheckPods timed-out", rce) diff --git a/test/integration/stat/stat_test.go b/test/integration/stat/stat_test.go index 1c231daf26c8d..4a2d71088b001 100644 --- a/test/integration/stat/stat_test.go +++ b/test/integration/stat/stat_test.go @@ -35,30 +35,39 @@ func TestMain(m *testing.M) { // requesting, and the test will pass. func TestCliStatForLinkerdNamespace(t *testing.T) { ctx := context.Background() - - pods, err := TestHelper.GetPodNamesForDeployment(ctx, TestHelper.GetVizNamespace(), "linkerd-prometheus") + var prometheusPod, prometheusAuthority, prometheusNamespace, prometheusDeployment, metricsPod string + // Get Metrics Pod + pods, err := TestHelper.GetPodNamesForDeployment(ctx, TestHelper.GetVizNamespace(), "linkerd-metrics-api") if err != nil { - testutil.AnnotatedFatalf(t, "failed to get pods for prometheus", - "failed to get pods for prometheus: %s", err) + testutil.AnnotatedFatalf(t, "failed to get pods for metrics-api", + "failed to get pods for metrics-api: %s", err) } if len(pods) != 1 { - testutil.Fatalf(t, "expected 1 pod for prometheus, got %d", len(pods)) + testutil.Fatalf(t, "expected 1 pod for metrics-api, got %d", len(pods)) + } + metricsPod = pods[0] + + // Retrieve Prometheus pod details + if TestHelper.ExternalPrometheus() { + prometheusNamespace = "default" + prometheusDeployment = "prometheus" + } else { + prometheusNamespace = TestHelper.GetVizNamespace() + prometheusDeployment = "linkerd-prometheus" } - prometheusPod := pods[0] - pods, err = TestHelper.GetPodNamesForDeployment(ctx, TestHelper.GetVizNamespace(), "linkerd-metrics-api") + pods, err = TestHelper.GetPodNamesForDeployment(ctx, prometheusNamespace, prometheusDeployment) if err != nil { - testutil.AnnotatedFatalf(t, "failed to get pods for metrics-api", - "failed to get pods for metrics-api: %s", err) + testutil.AnnotatedFatalf(t, "failed to get pods for prometheus", + "failed to get pods for prometheus: %s", err) } if len(pods) != 1 { - testutil.Fatalf(t, "expected 1 pod for metrics-api, got %d", len(pods)) + testutil.Fatalf(t, "expected 1 pod for prometheus, got %d", len(pods)) } - metricsPod := pods[0] - - prometheusAuthority := "linkerd-prometheus." + TestHelper.GetVizNamespace() + ".svc.cluster.local:9090" + prometheusPod = pods[0] + prometheusAuthority = prometheusDeployment + "." + prometheusNamespace + ".svc.cluster.local:9090" - for _, tt := range []struct { + testCases := []struct { args []string expectedRows map[string]string status string @@ -74,67 +83,103 @@ func TestCliStatForLinkerdNamespace(t *testing.T) { }, }, { - args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace()}, + args: []string{"viz", "stat", "ns", TestHelper.GetLinkerdNamespace()}, expectedRows: map[string]string{ - "linkerd-metrics-api": "1/1", - "linkerd-grafana": "1/1", - "linkerd-prometheus": "1/1", - "linkerd-tap": "1/1", - "linkerd-web": "1/1", - "tap-injector": "1/1", + TestHelper.GetLinkerdNamespace(): "5/5", }, }, { - args: []string{"viz", "stat", fmt.Sprintf("po/%s", prometheusPod), "-n", TestHelper.GetVizNamespace(), "--from", fmt.Sprintf("po/%s", metricsPod), "--from-namespace", TestHelper.GetVizNamespace()}, + args: []string{"viz", "stat", fmt.Sprintf("po/%s", prometheusPod), "-n", prometheusNamespace, "--from", fmt.Sprintf("po/%s", metricsPod), "--from-namespace", TestHelper.GetVizNamespace()}, expectedRows: map[string]string{ prometheusPod: "1/1", }, status: "Running", }, { - args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", TestHelper.GetVizNamespace()}, + args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", prometheusNamespace}, expectedRows: map[string]string{ "linkerd-metrics-api": "1/1", }, }, { - args: []string{"viz", "stat", "svc", "linkerd-prometheus", "-n", TestHelper.GetVizNamespace(), "--from", "deploy/linkerd-metrics-api", "--from-namespace", TestHelper.GetVizNamespace()}, - expectedRows: map[string]string{ - "linkerd-prometheus": "1/1", - }, - }, - { - args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace(), "--to", "svc/linkerd-prometheus", "--to-namespace", TestHelper.GetVizNamespace()}, + args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("svc/%s", prometheusDeployment), "--to-namespace", prometheusNamespace}, expectedRows: map[string]string{ "linkerd-metrics-api": "1/1", }, }, { - args: []string{"viz", "stat", "ns", TestHelper.GetLinkerdNamespace()}, - expectedRows: map[string]string{ - TestHelper.GetLinkerdNamespace(): "5/5", - }, - }, - { - args: []string{"viz", "stat", "ns", TestHelper.GetVizNamespace()}, - expectedRows: map[string]string{ - TestHelper.GetVizNamespace(): "6/6", - }, - }, - { - args: []string{"viz", "stat", "po", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("au/%s", prometheusAuthority), "--to-namespace", TestHelper.GetVizNamespace()}, + args: []string{"viz", "stat", "po", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("au/%s", prometheusAuthority), "--to-namespace", prometheusNamespace}, expectedRows: map[string]string{ metricsPod: "1/1", }, status: "Running", }, { - args: []string{"viz", "stat", "au", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", TestHelper.GetVizNamespace()}, + args: []string{"viz", "stat", "au", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", prometheusNamespace}, expectedRows: map[string]string{ prometheusAuthority: "-", }, }, - } { + } + + if !TestHelper.ExternalPrometheus() { + testCases = append(testCases, []struct { + args []string + expectedRows map[string]string + status string + }{ + { + args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace()}, + expectedRows: map[string]string{ + "linkerd-metrics-api": "1/1", + "linkerd-grafana": "1/1", + "linkerd-prometheus": "1/1", + "linkerd-tap": "1/1", + "linkerd-web": "1/1", + "tap-injector": "1/1", + }, + }, + { + args: []string{"viz", "stat", "ns", TestHelper.GetVizNamespace()}, + expectedRows: map[string]string{ + TestHelper.GetVizNamespace(): "6/6", + }, + }, + { + args: []string{"viz", "stat", "svc", "linkerd-prometheus", "-n", TestHelper.GetVizNamespace(), "--from", "deploy/linkerd-metrics-api", "--from-namespace", TestHelper.GetVizNamespace()}, + expectedRows: map[string]string{ + "linkerd-prometheus": "1/1", + }, + }, + }..., + ) + } else { + testCases = append(testCases, []struct { + args []string + expectedRows map[string]string + status string + }{ + { + args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace()}, + expectedRows: map[string]string{ + "linkerd-metrics-api": "1/1", + "linkerd-grafana": "1/1", + "linkerd-tap": "1/1", + "linkerd-web": "1/1", + "tap-injector": "1/1", + }, + }, + { + args: []string{"viz", "stat", "ns", TestHelper.GetVizNamespace()}, + expectedRows: map[string]string{ + TestHelper.GetVizNamespace(): "5/5", + }, + }, + }..., + ) + } + + for _, tt := range testCases { tt := tt // pin timeout := 20 * time.Second t.Run("linkerd "+strings.Join(tt.args, " "), func(t *testing.T) { diff --git a/test/integration/testdata/check.viz.external-prometheus.golden b/test/integration/testdata/check.viz.external-prometheus.golden new file mode 100644 index 0000000000000..df58c53ac11f3 --- /dev/null +++ b/test/integration/testdata/check.viz.external-prometheus.golden @@ -0,0 +1,14 @@ +linkerd-viz +----------- +√ linkerd-viz Namespace exists +√ linkerd-viz ClusterRoles exist +√ linkerd-viz ClusterRoleBindings exist +√ tap API server has valid cert +√ tap API server cert is valid for at least 60 days +√ tap API service is running +√ linkerd-viz pods are injected +√ viz extension pods are running +√ can initialize the client +√ viz extension self-check + +Status check results are √ diff --git a/test/integration/testdata/external_prometheus.yaml b/test/integration/testdata/external_prometheus.yaml new file mode 100644 index 0000000000000..727646fafb536 --- /dev/null +++ b/test/integration/testdata/external_prometheus.yaml @@ -0,0 +1,241 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: ["nodes", "nodes/proxy", "pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: default +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: prometheus + namespace: default +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: prometheus-config + namespace: default +data: + prometheus.yml: |- + global: + evaluation_interval: 10s + scrape_interval: 10s + scrape_timeout: 10s + + rule_files: + - /etc/prometheus/*_rules.yml + - /etc/prometheus/*_rules.yaml + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'grafana' + kubernetes_sd_configs: + - role: pod + namespaces: + names: ['linkerd'] + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_container_name + action: keep + regex: ^grafana$ + + # Required for: https://grafana.com/grafana/dashboards/315 + - job_name: 'kubernetes-nodes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + metric_relabel_configs: + - source_labels: [__name__] + regex: '(container|machine)_(cpu|memory|network|fs)_(.+)' + action: keep + - source_labels: [__name__] + regex: 'container_memory_failures_total' # unneeded large metric + action: drop + + - job_name: 'linkerd-controller' + kubernetes_sd_configs: + - role: pod + namespaces: + names: ['linkerd'] + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_linkerd_io_control_plane_component + - __meta_kubernetes_pod_container_port_name + action: keep + regex: (.*);admin-http$ + - source_labels: [__meta_kubernetes_pod_container_name] + action: replace + target_label: component + + - job_name: 'linkerd-service-mirror' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_linkerd_io_control_plane_component + - __meta_kubernetes_pod_container_port_name + action: keep + regex: linkerd-service-mirror;admin-http$ + - source_labels: [__meta_kubernetes_pod_container_name] + action: replace + target_label: component + + - job_name: 'linkerd-proxy' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_container_name + - __meta_kubernetes_pod_container_port_name + - __meta_kubernetes_pod_label_linkerd_io_control_plane_ns + action: keep + regex: ^linkerd-proxy;linkerd-admin;linkerd$ + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod + # special case k8s' "job" label, to not interfere with prometheus' "job" + # label + # __meta_kubernetes_pod_label_linkerd_io_proxy_job=foo => + # k8s_job=foo + - source_labels: [__meta_kubernetes_pod_label_linkerd_io_proxy_job] + action: replace + target_label: k8s_job + # drop __meta_kubernetes_pod_label_linkerd_io_proxy_job + - action: labeldrop + regex: __meta_kubernetes_pod_label_linkerd_io_proxy_job + # __meta_kubernetes_pod_label_linkerd_io_proxy_deployment=foo => + # deployment=foo + - action: labelmap + regex: __meta_kubernetes_pod_label_linkerd_io_proxy_(.+) + # drop all labels that we just made copies of in the previous labelmap + - action: labeldrop + regex: __meta_kubernetes_pod_label_linkerd_io_proxy_(.+) + # __meta_kubernetes_pod_label_linkerd_io_foo=bar => + # foo=bar + - action: labelmap + regex: __meta_kubernetes_pod_label_linkerd_io_(.+) + # Copy all pod labels to tmp labels + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + replacement: __tmp_pod_label_$1 + # Take `linkerd_io_` prefixed labels and copy them without the prefix + - action: labelmap + regex: __tmp_pod_label_linkerd_io_(.+) + replacement: __tmp_pod_label_$1 + # Drop the `linkerd_io_` originals + - action: labeldrop + regex: __tmp_pod_label_linkerd_io_(.+) + # Copy tmp labels into real labels + - action: labelmap + regex: __tmp_pod_label_(.+) +--- +kind: Service +apiVersion: v1 +metadata: + name: prometheus + namespace: default +spec: + type: ClusterIP + selector: + app: prometheus + ports: + - name: admin-http + port: 9090 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + nodeSelector: + beta.kubernetes.io/os: linux + securityContext: + fsGroup: 65534 + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yml + - --log.level=info + - --storage.tsdb.path=/data + - --storage.tsdb.retention.time=6h + image: prom/prometheus:v2.19.3 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 + name: prometheus + ports: + - containerPort: 9090 + name: admin-http + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + volumeMounts: + - mountPath: /data + name: data + - mountPath: /etc/prometheus/prometheus.yml + name: prometheus-config + subPath: prometheus.yml + readOnly: true + serviceAccountName: prometheus + volumes: + - name: data + emptyDir: {} + - configMap: + name: prometheus-config + name: prometheus-config diff --git a/test/integration/uninstall/uninstall_test.go b/test/integration/uninstall/uninstall_test.go index 68543985af4cb..1b6b35b43bf3b 100644 --- a/test/integration/uninstall/uninstall_test.go +++ b/test/integration/uninstall/uninstall_test.go @@ -72,6 +72,9 @@ func TestResourcesPostInstall(t *testing.T) { // Tests Pods and Deployments expectedDeployments := testutil.LinkerdDeployReplicasEdge + if !TestHelper.ExternalPrometheus() { + expectedDeployments["linkerd-prometheus"] = testutil.DeploySpec{Namespace: "linkerd-viz", Replicas: 1, Containers: []string{}} + } // Upgrade Case if TestHelper.UpgradeHelmFromVersion() != "" { expectedDeployments = testutil.LinkerdDeployReplicasStable diff --git a/testutil/test_helper.go b/testutil/test_helper.go index 6d08f92793fda..5eefd7e9d0788 100644 --- a/testutil/test_helper.go +++ b/testutil/test_helper.go @@ -30,6 +30,7 @@ type TestHelper struct { upgradeFromVersion string clusterDomain string externalIssuer bool + externalPrometheus bool multicluster bool uninstall bool cni bool @@ -87,7 +88,6 @@ var LinkerdDeployReplicasEdge = map[string]DeploySpec{ "linkerd-tap": {"linkerd-viz", 1, []string{"tap"}}, "linkerd-grafana": {"linkerd-viz", 1, []string{}}, "linkerd-identity": {"linkerd", 1, []string{"identity"}}, - "linkerd-prometheus": {"linkerd-viz", 1, []string{}}, "linkerd-sp-validator": {"linkerd", 1, []string{"sp-validator"}}, "linkerd-web": {"linkerd-viz", 1, []string{"web"}}, "linkerd-proxy-injector": {"linkerd", 1, []string{"proxy-injector"}}, @@ -111,6 +111,7 @@ func NewGenericTestHelper( helmMulticlusterReleaseName, helmMulticlusterChart string, externalIssuer, + externalPrometheus, multicluster, cni, calico, @@ -133,14 +134,15 @@ func NewGenericTestHelper( releaseName: helmReleaseName, upgradeFromVersion: upgradeFromVersion, }, - clusterDomain: clusterDomain, - externalIssuer: externalIssuer, - uninstall: uninstall, - cni: cni, - calico: calico, - httpClient: httpClient, - multicluster: multicluster, - KubernetesHelper: kubernetesHelper, + clusterDomain: clusterDomain, + externalIssuer: externalIssuer, + externalPrometheus: externalPrometheus, + uninstall: uninstall, + cni: cni, + calico: calico, + httpClient: httpClient, + multicluster: multicluster, + KubernetesHelper: kubernetesHelper, } } @@ -173,6 +175,7 @@ func NewTestHelper() *TestHelper { upgradeFromVersion := flag.String("upgrade-from-version", "", "when specified, the upgrade test uses it as the base version of the upgrade") clusterDomain := flag.String("cluster-domain", "cluster.local", "when specified, the install test uses a custom cluster domain") externalIssuer := flag.Bool("external-issuer", false, "when specified, the install test uses it to install linkerd with --identity-external-issuer=true") + externalPrometheus := flag.Bool("external-prometheus", false, "when specified, the install test uses an external prometheus") runTests := flag.Bool("integration-tests", false, "must be provided to run the integration tests") verbose := flag.Bool("verbose", false, "turn on debug logging") upgradeHelmFromVersion := flag.String("upgrade-helm-from-version", "", "Indicate a version of the Linkerd helm chart from which the helm installation is being upgraded") @@ -221,12 +224,13 @@ func NewTestHelper() *TestHelper { multiclusterReleaseName: *multiclusterHelmReleaseName, upgradeFromVersion: *upgradeHelmFromVersion, }, - clusterDomain: *clusterDomain, - externalIssuer: *externalIssuer, - cni: *cni, - calico: *calico, - uninstall: *uninstall, - certsPath: *certsPath, + clusterDomain: *clusterDomain, + externalIssuer: *externalIssuer, + externalPrometheus: *externalPrometheus, + cni: *cni, + calico: *calico, + uninstall: *uninstall, + certsPath: *certsPath, } version, err := testHelper.LinkerdRun("version", "--client", "--short") @@ -319,6 +323,11 @@ func (h *TestHelper) ExternalIssuer() bool { return h.externalIssuer } +// ExternalPrometheus determines whether linkerd should be installed with --set prometheusUrl +func (h *TestHelper) ExternalPrometheus() bool { + return h.externalPrometheus +} + // Multicluster determines whether multicluster components should be installed func (h *TestHelper) Multicluster() bool { return h.multicluster