diff --git a/test/extended/router/metrics.go b/test/extended/router/metrics.go index 3d992a200023..a33a18aea208 100644 --- a/test/extended/router/metrics.go +++ b/test/extended/router/metrics.go @@ -16,15 +16,16 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" + clientset "k8s.io/client-go/kubernetes" + watchtools "k8s.io/client-go/tools/watch" + "k8s.io/kubernetes/pkg/client/conditions" + e2e "k8s.io/kubernetes/test/e2e/framework" + corev1 "k8s.io/api/core/v1" kapierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - watchtools "k8s.io/client-go/tools/watch" - "k8s.io/kubernetes/pkg/client/conditions" - e2e "k8s.io/kubernetes/test/e2e/framework" exutil "github.com/openshift/origin/test/extended/util" ) @@ -34,44 +35,41 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { var ( oc = exutil.NewCLI("router-metrics", exutil.KubeConfigPath()) - username, password, execPodName, ns, host string - statsPort int - routerNamespace, serviceIP, bearerToken string + username, password, bearerToken string + metricsPort int32 + execPodName, ns, host string ) g.BeforeEach(func() { - var err error - var template *corev1.PodTemplateSpec - template, routerNamespace, err = exutil.GetRouterPodTemplate(oc) - if kapierrs.IsNotFound(err) { - g.Skip("no router installed on the cluster") - return - } - o.Expect(err).NotTo(o.HaveOccurred()) - env := template.Spec.Containers[0].Env - - if len(findEnvVar(env, "ROUTER_METRICS_TYPE")) == 0 { - g.Skip("router does not have ROUTER_METRICS_TYPE set") - return - } + // This test needs to make assertions against a single router pod, so all access + // to the router should happen through a single endpoint. - username, password, err = findStatsUsernameAndPassword(oc, routerNamespace, env) + // Discover the endpoint. + endpoint, err := oc.AdminKubeClient().CoreV1().Endpoints("openshift-ingress").Get("router-internal-default", metav1.GetOptions{}) o.Expect(err).NotTo(o.HaveOccurred()) - - statsPort = 1936 - statsPortString := findEnvVar(env, "STATS_PORT") - if len(statsPortString) > 0 { - if port, err := strconv.Atoi(statsPortString); err == nil { - statsPort = port + o.Expect(endpoint.Subsets).NotTo(o.BeEmpty()) + subset := endpoint.Subsets[0] + o.Expect(subset.Addresses).NotTo(o.BeEmpty()) + + // Extract the metrics port by name. + for _, port := range subset.Ports { + if port.Name == "metrics" { + metricsPort = port.Port + break } } + o.Expect(metricsPort).NotTo(o.BeZero()) - host, err = exutil.WaitForRouterInternalIP(oc) - o.Expect(err).NotTo(o.HaveOccurred()) + // Extract the IP of a single router pod. + host = subset.Addresses[0].IP - serviceIP, err = exutil.WaitForRouterServiceIP(oc) + // Extract the router pod's stats credentials. + statsSecret, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-ingress").Get("router-stats-default", metav1.GetOptions{}) o.Expect(err).NotTo(o.HaveOccurred()) + username, password = string(statsSecret.Data["statsUsername"]), string(statsSecret.Data["statsPassword"]) + // Extract a bearer token from Prometheus authorized to access + // the router metrics URL. bearerToken, err = findMetricsBearerToken(oc) o.Expect(err).NotTo(o.HaveOccurred()) @@ -90,11 +88,11 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { defer func() { oc.AdminKubeClient().CoreV1().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }() g.By("listening on the health port") - err := expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:%d/healthz", host, statsPort), 200) + err := expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:%d/healthz", host, metricsPort), 200) o.Expect(err).NotTo(o.HaveOccurred()) }) - g.It("[Flaky] should expose prometheus metrics for a route", func() { + g.It("should expose prometheus metrics for a route", func() { g.By("when a route exists") configPath := exutil.FixturePath("testdata", "router", "router-metrics.yaml") err := oc.Run("create").Args("-f", configPath).Execute() @@ -104,11 +102,11 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { defer func() { oc.AdminKubeClient().CoreV1().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }() g.By("preventing access without a username and password") - err = expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, statsPort), 401, 403) + err = expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, metricsPort), 401, 403) o.Expect(err).NotTo(o.HaveOccurred()) g.By("validate access using username and password") - _, err = getAuthenticatedURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, statsPort), username, password) + _, err = getAuthenticatedURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, metricsPort), username, password) o.Expect(err).NotTo(o.HaveOccurred()) g.By("checking for the expected metrics") @@ -121,12 +119,11 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { p := expfmt.TextParser{} err = wait.PollImmediate(2*time.Second, 240*time.Second, func() (bool, error) { - results, err = getBearerTokenURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, statsPort), bearerToken) + results, err = getBearerTokenURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, metricsPort), bearerToken) o.Expect(err).NotTo(o.HaveOccurred()) metrics, err = p.TextToMetricFamilies(bytes.NewBufferString(results)) o.Expect(err).NotTo(o.HaveOccurred()) - //e2e.Logf("Metrics:\n%s", results) if len(findGaugesWithLabels(metrics["haproxy_server_up"], serverLabels)) == 2 { if findGaugesWithLabels(metrics["haproxy_backend_connections_total"], routeLabels)[0] >= float64(times) { @@ -134,7 +131,7 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { } // send a burst of traffic to the router g.By("sending traffic to a weighted route") - err = expectRouteStatusCodeRepeatedExec(ns, execPodName, fmt.Sprintf("http://%s", serviceIP), "weighted.metrics.example.com", http.StatusOK, times) + err = expectRouteStatusCodeRepeatedExec(ns, execPodName, fmt.Sprintf("http://%s", host), "weighted.metrics.example.com", http.StatusOK, times) o.Expect(err).NotTo(o.HaveOccurred()) } g.By("retrying metrics until all backend servers appear") @@ -196,7 +193,7 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { time.Sleep(15 * time.Second) g.By("checking that some metrics are not reset to 0 after router restart") - updatedResults, err := getBearerTokenURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, statsPort), bearerToken) + updatedResults, err := getBearerTokenURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/metrics", host, metricsPort), bearerToken) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { e2e.Logf("final metrics:\n%s", updatedResults) }() @@ -211,11 +208,11 @@ var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { defer func() { oc.AdminKubeClient().CoreV1().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }() g.By("preventing access without a username and password") - err := expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:%d/debug/pprof/heap", host, statsPort), 401, 403) + err := expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:%d/debug/pprof/heap", host, metricsPort), 401, 403) o.Expect(err).NotTo(o.HaveOccurred()) g.By("at /debug/pprof") - results, err := getAuthenticatedURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/debug/pprof/heap?debug=1", host, statsPort), username, password) + results, err := getAuthenticatedURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:%d/debug/pprof/heap?debug=1", host, metricsPort), username, password) o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(results).To(o.ContainSubstring("# runtime.MemStats")) }) @@ -336,15 +333,6 @@ func locatePrometheus(oc *exutil.CLI) (url, bearerToken string, ok bool) { return "https://prometheus-k8s.openshift-monitoring.svc:9091", bearerToken, true } -func findEnvVar(vars []corev1.EnvVar, key string) string { - for _, v := range vars { - if v.Name == key { - return v.Value - } - } - return "" -} - func findMetricsWithLabels(f *dto.MetricFamily, promLabels map[string]string) []*dto.Metric { var result []*dto.Metric if f == nil { @@ -429,39 +417,6 @@ func getBearerTokenURLViaPod(ns, execPodName, url, bearer string) (string, error return output, nil } -func findEnvVarSecret(vars []corev1.EnvVar, key string) (string, string) { - for _, v := range vars { - if v.Name == key { - if v.ValueFrom != nil && v.ValueFrom.SecretKeyRef != nil { - ref := v.ValueFrom.SecretKeyRef - return ref.Name, ref.Key - } - } - } - return "", "" -} - -func findStatsUsernameAndPassword(oc *exutil.CLI, ns string, env []corev1.EnvVar) (string, string, error) { - secretName, userKey := findEnvVarSecret(env, "STATS_USERNAME") - _, passKey := findEnvVarSecret(env, "STATS_PASSWORD") - - if len(secretName) == 0 || len(userKey) == 0 || len(passKey) == 0 { - return "", "", fmt.Errorf("stats username and password not found, env: %v", env) - } - - secret, err := oc.AdminKubeClient().CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) - if err != nil { - return "", "", err - } - - username, ok1 := secret.Data[userKey] - password, ok2 := secret.Data[passKey] - if !ok1 || !ok2 { - return "", "", fmt.Errorf("secret '%s/%s' does not contain key %q or %q", ns, secretName, userKey, passKey) - } - return string(username), string(password), nil -} - func findMetricsBearerToken(oc *exutil.CLI) (string, error) { sa, err := oc.AdminKubeClient().CoreV1().ServiceAccounts("openshift-monitoring").Get("prometheus-k8s", metav1.GetOptions{}) if err != nil {