Bug 1780170: Expose metrics

manifests/0000_90_console-operator_01_prometheusrbac.yaml

@@ -0,0 +1,32 @@
+# Role for accessing metrics exposed by the console-operator
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: openshift-console-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Grant cluster-monitoring access to console-operator metrics
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: openshift-console-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: openshift-monitoring

manifests/0000_90_console-operator_02_servicemonitor.yaml

@@ -0,0 +1,21 @@
+# Configure cluster-monitoring for console-operator
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: console-operator
+  namespace: openshift-console-operator
+spec:
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 30s
+      path: /metrics
+      port: https
+      scheme: https
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: metrics.openshift-console-operator.svc
+  jobLabel: component
+  selector:
+    matchLabels:
+      name: console-operator
+

manifests/02-namespace.yaml

@@ -11,3 +11,5 @@ metadata:
   name: openshift-console-operator
   annotations:
     openshift.io/node-selector: ""
+  labels:
+    openshift.io/cluster-monitoring: "true"

manifests/04-rbac-rolebinding-cluster.yaml

@@ -1,3 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: console-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: console-operator
+subjects:
+  - kind: ServiceAccount
+    name: console-operator
+    namespace: openshift-console-operator
+---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -10,3 +23,19 @@ subjects:
 - kind: Group
   name: system:authenticated
   apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: console-operator-auth-delegator
+roleRef:
+  # for protected endpoints like /metrics, the operator must perform
+  # authentication (tokenreview) & authorization (subjectaccessreview)
+  # which are granted by this ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: console-operator
+    namespace: openshift-console-operator

manifests/04-rbac-rolebinding.yaml

@@ -25,19 +25,6 @@ subjects:
 - kind: ServiceAccount
   name: console-operator
 ---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: console-operator
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: console-operator
-subjects:
-- kind: ServiceAccount
-  name: console-operator
-  namespace: openshift-console-operator
----
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -79,3 +66,18 @@ subjects:
   - kind: ServiceAccount
     name: console-operator
     namespace: openshift-console-operator
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: console-operator
+  namespace: kube-system
+roleRef:
+  kind: Role
+  name: extension-apiserver-authentication-reader
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: console-operator
+    namespace: openshift-console-operator
+

manifests/05-config.yaml

@@ -9,7 +9,3 @@ data:
     kind: GenericOperatorConfig
     leaderElection:
       namespace: openshift-console-operator
-    authentication:
-      disabled: true
-    authorization:
-      disabled: true

manifests/05-service.yaml

@@ -0,0 +1,20 @@
+# Expose operator metrics
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.alpha.openshift.io/serving-cert-secret-name: serving-cert
+  labels:
+    name: console-operator
+  name: metrics
+  namespace: openshift-console-operator
+spec:
+  ports:
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: 8443
+  selector:
+    name: console-operator
+  sessionAffinity: None
+  type: ClusterIP

manifests/07-operator.yaml

@@ -46,6 +46,8 @@ spec:
         volumeMounts:
         - mountPath: /var/run/configmaps/config
           name: config
+        - mountPath: /var/run/secrets/serving-cert
+          name: serving-cert
         env:
         - name: IMAGE
           value: registry.svc.ci.openshift.org/openshift:console
@@ -65,3 +67,7 @@ spec:
       - name: config
         configMap:
           name: console-operator-config
+      - name: serving-cert
+        secret:
+          secretName: serving-cert
+          optional: true

pkg/console/operator/sync_v400.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	// kube
 	oauthv1 "github.com/openshift/api/oauth/v1"
 	routev1 "github.com/openshift/api/route/v1"
@@ -35,6 +37,19 @@ import (
 	servicesub "github.com/openshift/console-operator/pkg/console/subresource/service"
 )
 
+var (
+	// metric: console_url{url="https://<url>"} 1
+	consoleURLMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "console_url",
+		Help: "URL of the console exposed on the cluster",
+		// one label
+	}, []string{"url"})
+)
+
+func init() {
+	prometheus.MustRegister(consoleURLMetric)
+}
+
 // The sync loop starts from zero and works its way through the requirements for a running console.
 // If at any point something is missing, it creates/updates that piece and immediately dies.
 // The next loop will pick up where they previous left off and move the process forward one step.
@@ -151,6 +166,7 @@ func (co *consoleOperator) sync_v400(updatedOperatorConfig *operatorv1.Console,
 	// public hostname so that the world can know the console is ready to roll
 	klog.V(4).Infoln("sync_v400: updating console status")
 	consoleURL := getConsoleURL(rt)
+
 	if consoleURL == "" {
 		err := customerrors.NewSyncError("waiting on route host")
 		klog.Errorf("%q: %v", "route", err)
@@ -197,6 +213,17 @@ func (co *consoleOperator) sync_v400(updatedOperatorConfig *operatorv1.Console,
 
 func (co *consoleOperator) SyncConsoleConfig(consoleConfig *configv1.Console, consoleURL string) (*configv1.Console, error) {
 	updated := consoleConfig.DeepCopy()
+
+	// track the URL state in prometheus before we update it
+	if consoleConfig.Status.ConsoleURL != consoleURL {
+		// not using this URL anymore
+		consoleURLMetric.WithLabelValues(consoleConfig.Status.ConsoleURL).Set(0)
+	}
+	if len(consoleURL) != 0 {
+		// only update to new if we have a url
+		consoleURLMetric.WithLabelValues(consoleURL).Set(1)
+	}
+
 	if updated.Status.ConsoleURL != consoleURL {
 		klog.V(4).Infof("updating console.config.openshift.io with url: %v", consoleURL)
 		updated.Status.ConsoleURL = consoleURL