OCPBUGS-1265: test/extended/prometheus: Consider telemeterClient.enabled

cmd/openshift-tests/minimal.go

@@ -2572,7 +2572,7 @@ var (
 		"[sig-builds][Feature:Builds] verify /run filesystem contents  are writeable using a simple Docker Strategy Build [Skipped:Disconnected] [Suite:openshift/conformance/parallel]":                                                                  {},
 		"[sig-builds][Feature:Builds] build have source revision metadata  started build should contain source revision information [Skipped:Disconnected] [Suite:openshift/conformance/parallel]":                                                        {},
 		"[sig-network] multicast when using one of the OpenshiftSDN modes 'redhat/openshift-ovs-multitenant, redhat/openshift-ovs-networkpolicy' should allow multicast traffic in namespaces where it is enabled [Suite:openshift/conformance/parallel]": {},
-		"[sig-instrumentation] Prometheus when installed on the cluster should report telemetry if a cloud.openshift.com token is present [Late] [Skipped:Disconnected] [Suite:openshift/conformance/parallel]":                                           {},
+		"[sig-instrumentation] Prometheus when installed on the cluster should report telemetry [Late] [Skipped:Disconnected] [Suite:openshift/conformance/parallel]":                                                                                     {},
 		"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes":                                                                                                                                                                        {},
 		"[sig-imageregistry][Feature:ImageTriggers] Image change build triggers TestSimpleImageChangeBuildTriggerFromImageStreamTagCustomWithConfigChange [Suite:openshift/conformance/parallel]":                                                         {},
 		"[sig-auth][Feature:OAuthServer] [Token Expiration] Using a OAuth client with a non-default token max age to generate tokens that do not expire works as expected when using a code authorization flow [Suite:openshift/conformance/parallel]":    {},

test/extended/prometheus/prometheus.go

@@ -29,6 +29,7 @@ import (
 	e2e "k8s.io/kubernetes/test/e2e/framework"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	admissionapi "k8s.io/pod-security-admission/api"
+	"sigs.k8s.io/yaml"
 
 	configv1 "github.com/openshift/api/config/v1"
 
@@ -39,6 +40,16 @@ import (
 	helper "github.com/openshift/origin/test/extended/util/prometheus"
 )
 
+// ClusterMonitoringConfiguration is a subset of https://github.com/openshift/cluster-monitoring-operator/blob/8d331d78b22948d36c20da0552763ddd8a4e2093/pkg/manifests/config.go#L124-L136
+type ClusterMonitoringConfiguration struct {
+	TelemeterClientConfig *TelemeterClientConfig `json:"telemeterClient"`
+}
+
+// TelemeterClientConfig is a subset of https://github.com/openshift/cluster-monitoring-operator/blob/8d331d78b22948d36c20da0552763ddd8a4e2093/pkg/manifests/config.go#L335-L342
+type TelemeterClientConfig struct {
+	Enabled *bool `json:"enabled"`
+}
+
 var _ = g.Describe("[sig-instrumentation][Late] OpenShift alerting rules [apigroup:image.openshift.io]", func() {
 	defer g.GinkgoRecover()
 
@@ -186,6 +197,7 @@ var _ = g.Describe("[sig-instrumentation][Late] OpenShift alerting rules [apigro
 
 var _ = g.Describe("[sig-instrumentation][Late] Alerts", func() {
 	defer g.GinkgoRecover()
+	ctx := context.TODO()
 	var (
 		oc = exutil.NewCLIWithoutNamespace("prometheus")
 	)
@@ -382,8 +394,10 @@ sort_desc(
 	})
 
 	g.It("shouldn't exceed the 650 series limit of total series sent via telemetry from each cluster", func() {
-		if !hasPullSecret(oc.AdminKubeClient(), "cloud.openshift.com") {
-			e2eskipper.Skipf("Telemetry is disabled")
+		if enabled, err := telemetryIsEnabled(ctx, oc.AdminKubeClient()); err != nil {
+			e2e.Failf("could not determine if Telemetry is enabled: %v", err)
+		} else {
+			e2eskipper.Skipf("Telemetry is disabled: %v", enabled)
 		}
 
 		// we only consider series sent since the beginning of the test
@@ -419,6 +433,7 @@ sort_desc(
 
 var _ = g.Describe("[sig-instrumentation] Prometheus", func() {
 	defer g.GinkgoRecover()
+	ctx := context.TODO()
 	var (
 		oc = exutil.NewCLIWithPodSecurityLevel("prometheus", admissionapi.LevelBaseline)
 
@@ -439,9 +454,11 @@ var _ = g.Describe("[sig-instrumentation] Prometheus", func() {
 	})
 
 	g.Describe("when installed on the cluster", func() {
-		g.It("should report telemetry if a cloud.openshift.com token is present [Late]", func() {
-			if !hasPullSecret(oc.AdminKubeClient(), "cloud.openshift.com") {
-				e2eskipper.Skipf("Telemetry is disabled")
+		g.It("should report telemetry [Late]", func() {
+			if enabled, err := telemetryIsEnabled(ctx, oc.AdminKubeClient()); err != nil {
+				e2e.Failf("could not determine if Telemetry is enabled: %v", err)
+			} else {
+				e2eskipper.Skipf("Telemetry is disabled: %v", enabled)
 			}
 
 			tests := map[string]bool{}
@@ -900,17 +917,26 @@ func getBearerTokenURLViaPod(ns, execPodName, url, bearer string) (string, error
 	return output, nil
 }
 
-func hasPullSecret(client clientset.Interface, name string) bool {
-	scrt, err := client.CoreV1().Secrets("openshift-config").Get(context.Background(), "pull-secret", metav1.GetOptions{})
+// telemetryIsEnabled returns (nil, nil) if Telemetry is enabled,
+// (error, nil) if Telemetry is not enabled, and (_, error) if it fails
+// to determine whether or not Telemetry is enabled.
+func telemetryIsEnabled(ctx context.Context, client clientset.Interface) (enabled error, err error) {
+	domain := "cloud.openshift.com"
+	if hasSecret, err := hasPullSecret(ctx, client, domain); err != nil || hasSecret != nil {
+		return hasSecret, err
+	}
+
+	return isTelemeterClientEnabled(ctx, client)
+}
+
+func hasPullSecret(ctx context.Context, client clientset.Interface, name string) (enabled error, err error) {
+	scrt, err := client.CoreV1().Secrets("openshift-config").Get(ctx, "pull-secret", metav1.GetOptions{})
 	if err != nil {
-		if kapierrs.IsNotFound(err) {
-			return false
-		}
-		e2e.Failf("could not retrieve pull-secret: %v", err)
+		return nil, fmt.Errorf("could not retrieve pull-secret: %w", err)
 	}
 
 	if scrt.Type != v1.SecretTypeDockerConfigJson {
-		e2e.Failf("error expecting secret type %s got %s", v1.SecretTypeDockerConfigJson, scrt.Type)
+		return nil, fmt.Errorf("error expecting openshift-config/pull-secret type %s got %s", v1.SecretTypeDockerConfigJson, scrt.Type)
 	}
 
 	ps := struct {
@@ -920,9 +946,37 @@ func hasPullSecret(client clientset.Interface, name string) bool {
 	}{}
 
 	if err := json.Unmarshal(scrt.Data[v1.DockerConfigJsonKey], &ps); err != nil {
-		e2e.Failf("could not unmarshal pullSecret from openshift-config/pull-secret: %v", err)
+		return nil, fmt.Errorf("could not unmarshal pullSecret from openshift-config/pull-secret: %w", err)
+	}
+
+	if len(ps.Auths[name].Auth) == 0 {
+		return fmt.Errorf("openshift-config/pull-secret does not contain auth for %s", name), nil
+	}
+
+	return nil, nil
+}
+
+func isTelemeterClientEnabled(ctx context.Context, client clientset.Interface) (enabled error, err error) {
+	config, err := client.CoreV1().ConfigMaps("openshift-monitoring").Get(ctx, "cluster-monitoring-config", metav1.GetOptions{})
+	if err != nil {
+		if kapierrs.IsNotFound(err) {
+			return nil, nil // Telemetry is enabled by default
+		}
+		return nil, fmt.Errorf("could not retrieve monitoring configuration: %w", err)
+	}
+	var structuredConfig ClusterMonitoringConfiguration
+	if yamlConfig, ok := config.Data["config.yaml"]; !ok {
+		return nil, fmt.Errorf("openshift-monitoring/cluster-monitoring-config data lacks a config.yaml key: %v", config.Data)
+	} else if err := yaml.Unmarshal([]byte(yamlConfig), &structuredConfig); err != nil {
+		return nil, fmt.Errorf("error unmarshalling openshift-monitoring/cluster-monitoring-config config.yaml: %w", err)
+	}
+	if structuredConfig.TelemeterClientConfig == nil || structuredConfig.TelemeterClientConfig.Enabled == nil {
+		return nil, nil // Telemetry is enabled by default
+	}
+	if !*structuredConfig.TelemeterClientConfig.Enabled {
+		return fmt.Errorf("openshift-monitoring/cluster-monitoring-config telemeterClient enabled is: %t", *structuredConfig.TelemeterClientConfig.Enabled), nil
 	}
-	return len(ps.Auths[name].Auth) > 0
+	return nil, nil
 }
 
 func isTechPreviewCluster(oc *exutil.CLI) bool {

test/extended/util/annotate/rules.go

@@ -244,7 +244,7 @@ var (
 			`\[sig-instrumentation\] Prometheus when installed on the cluster should have non-Pod host cAdvisor metrics`,
 			`\[sig-instrumentation\] Prometheus when installed on the cluster should provide ingress metrics`,
 			`\[sig-instrumentation\] Prometheus when installed on the cluster should provide named network metrics`,
-			`\[sig-instrumentation\] Prometheus when installed on the cluster should report telemetry if a cloud.openshift.com token is present \[Late\]`,
+			`\[sig-instrumentation\] Prometheus when installed on the cluster should report telemetry \[Late\]`,
 			`\[sig-instrumentation\] Prometheus when installed on the cluster should start and expose a secured proxy and unsecured metrics`,
 			`\[sig-instrumentation\] Prometheus when installed on the cluster shouldn't have failing rules evaluation`,
 			`\[sig-instrumentation\] Prometheus when installed on the cluster shouldn't report any alerts in firing state apart from Watchdog and AlertmanagerReceiversNotConfigured \[Early\]`,