Merge pull request #1299 from raptorsun/feature/MON-1695

[MON-1695] expose /api/v1/labels end point for Thanos query.

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Note: This CHANGELOG is only for the monitoring team to track all monitoring related changes. Please see OpenShift release notes for official changes.
 
+## 4.10
+
+- [#1299](https://github.com/openshift/cluster-monitoring-operator/pull/1299) Expose expose /api/v1/labels endpoint for Thanos query.
+
 ## 4.9
 
 - [#1312](https://github.com/openshift/cluster-monitoring-operator/pull/1312) Support label to exclude namespaces from user-workload monitoring.

assets/thanos-querier/deployment.yaml

@@ -140,7 +140,7 @@ spec:
         - --tls-private-key-file=/etc/tls/private/tls.key
         - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
         - --logtostderr=true
-        - --allow-paths=/api/v1/query,/api/v1/query_range
+        - --allow-paths=/api/v1/query,/api/v1/query_range,/api/v1/labels,/api/v1/label/*/values
         image: quay.io/brancz/kube-rbac-proxy:v0.11.0
         name: kube-rbac-proxy
         ports:
@@ -160,6 +160,7 @@ spec:
         - --insecure-listen-address=127.0.0.1:9095
         - --upstream=http://127.0.0.1:9090
         - --label=namespace
+        - --enable-label-apis
         image: quay.io/prometheuscommunity/prom-label-proxy:v0.3.0
         name: prom-label-proxy
         resources:

jsonnet/components/thanos-querier.libsonnet

@@ -484,7 +484,7 @@ function(params)
                   '--tls-private-key-file=/etc/tls/private/tls.key',
                   '--tls-cipher-suites=' + cfg.tlsCipherSuites,
                   '--logtostderr=true',
-                  '--allow-paths=/api/v1/query,/api/v1/query_range',
+                  '--allow-paths=/api/v1/query,/api/v1/query_range,/api/v1/labels,/api/v1/label/*/values',
                 ],
                 terminationMessagePolicy: 'FallbackToLogsOnError',
                 volumeMounts: [
@@ -505,6 +505,7 @@ function(params)
                   '--insecure-listen-address=127.0.0.1:9095',
                   '--upstream=http://127.0.0.1:9090',
                   '--label=namespace',
+                  '--enable-label-apis',
                 ],
                 resources: {
                   requests: {

test/e2e/framework/client.go

@@ -211,6 +211,27 @@ func (c *PrometheusClient) PrometheusRules() ([]byte, error) {
 	return body, nil
 }
 
+// PrometheusRules runs an HTTP GET request against the Prometheus label API and returns
+// the response body.
+func (c *PrometheusClient) PrometheusLabel(label string) ([]byte, error) {
+	resp, err := c.Do("GET", fmt.Sprintf("/api/v1/label/%s/values", url.QueryEscape(label)), nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code response, want %d, got %d (%q)", http.StatusOK, resp.StatusCode, ClampMax(body))
+	}
+
+	return body, nil
+}
+
 // AlertmanagerQueryAlerts runs an HTTP GET request against the Alertmanager
 // /api/v2/alerts endpoint and returns the response body.
 func (c *PrometheusClient) AlertmanagerQueryAlerts(kvs ...string) ([]byte, error) {

test/e2e/user_workload_monitoring_test.go

@@ -87,6 +87,10 @@ func TestUserWorkloadMonitoringMetrics(t *testing.T) {
 			name: "assert tenancy model is enforced for metrics",
 			f:    assertTenancyForMetrics,
 		},
+		{
+			name: "assert tenancy model is enforced for labels",
+			f:    assertTenancyForLabels,
+		},
 		{
 			name: "assert prometheus is not deployed in user namespace",
 			f:    f.AssertStatefulsetDoesNotExist("prometheus-not-to-be-reconciled", userWorkloadTestNs),
@@ -247,13 +251,13 @@ func TestUserWorkloadMonitoringWithAdditionalAlertmanagerConfigs(t *testing.T) {
     timeout: "30s"
     apiVersion: v1
     tlsConfig:
-      key: 
+      key:
         name: alertmanager-tls
         key: tls.key
-      cert: 
+      cert:
         name: alertmanager-tls
         key: tls.crt
-      ca: 
+      ca:
         name: alertmanager-tls
         key: tls.ca
     staticConfigs: ["127.0.0.1", "127.0.0.2"]
@@ -860,6 +864,143 @@ func assertTenancyForRules(t *testing.T) {
 	}
 }
 
+func assertTenancyForLabels(t *testing.T) {
+	const testAccount = "test-labels"
+
+	_, err := f.CreateServiceAccount(userWorkloadTestNs, testAccount)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Grant enough permissions to read labels.
+	_, err = f.CreateRoleBindingFromClusterRole(userWorkloadTestNs, testAccount, "admin")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var token string
+	err = framework.Poll(5*time.Second, 5*time.Minute, func() error {
+		token, err = f.GetServiceAccountToken(userWorkloadTestNs, testAccount)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The tenancy port (9092) is only exposed in-cluster so we need to use
+	// port forwarding to access kube-rbac-proxy.
+	host, cleanUp, err := f.ForwardPort(t, "thanos-querier", 9092)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanUp()
+
+	client := framework.NewPrometheusClient(
+		host,
+		token,
+		&framework.QueryParameterInjector{
+			Name:  "namespace",
+			Value: userWorkloadTestNs,
+		},
+	)
+
+	t.Logf("Checking all labels")
+
+	// check /api/v1/labels endpoint
+	err = framework.Poll(5*time.Second, time.Minute, func() error {
+		resp, err := client.Do("GET", "/api/v1/labels", nil)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("unexpected status code response, want %d, got %d (%s)", http.StatusOK, resp.StatusCode, framework.ClampMax(b))
+		}
+
+		res, err := gabs.ParseJSON(b)
+		if err != nil {
+			return err
+		}
+
+		labels, err := res.Path("data").Children()
+		if err != nil {
+			return err
+		}
+
+		for _, label := range labels {
+			t.Logf("label %q", label.Data().(string))
+		}
+
+		if len(labels) == 0 {
+			return errors.Errorf("expecting a label list with at least one item.")
+		}
+
+		return nil
+
+	})
+	if err != nil {
+		t.Fatalf("failed to query labels from Thanos querier: %v", err)
+	}
+
+	// check /api/v1/label/namespace/values has a single value
+	t.Logf("Checking Label namespace having a single value")
+	const label = "namespace"
+
+	err = framework.Poll(5*time.Second, time.Minute, func() error {
+		// The tenancy port (9092) is only exposed in-cluster so we need to use
+		// port forwarding to access kube-rbac-proxy.
+		host, cleanUp, err := f.ForwardPort(t, "thanos-querier", 9092)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer cleanUp()
+
+		client := framework.NewPrometheusClient(
+			host,
+			token,
+			&framework.QueryParameterInjector{
+				Name:  "namespace",
+				Value: userWorkloadTestNs,
+			},
+		)
+
+		b, err := client.PrometheusLabel(label)
+		if err != nil {
+			return err
+		}
+
+		res, err := gabs.ParseJSON(b)
+		if err != nil {
+			return err
+		}
+
+		values, err := res.Path("data").Children()
+		if err != nil {
+			return err
+		}
+
+		if len(values) != 1 {
+			return errors.Errorf("expecting for label %q value list with exact one item.", label)
+		}
+
+		if values[0].Data().(string) != userWorkloadTestNs {
+			return errors.Errorf("expecting for label %q having value %q, but got %q .", label, userWorkloadTestNs, values[0].Data().(string))
+		}
+
+		return nil
+	})
+
+}
+
 func assertGRPCTLSRotation(t *testing.T) {
 	ctx := context.Background()
 	countGRPCSecrets := func(ns string) int {