Change kube-controllers targets to separated scheduler and controller-manager

assets/prometheus-k8s/role-binding-specific-namespaces.yaml

@@ -65,4 +65,30 @@ items:
   - kind: ServiceAccount
     name: prometheus-k8s
     namespace: openshift-monitoring
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: prometheus-k8s
+    namespace: openshift-kube-scheduler
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: prometheus-k8s
+  subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: openshift-monitoring
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: prometheus-k8s
+    namespace: openshift-kube-controller-manager
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: prometheus-k8s
+  subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: openshift-monitoring
 kind: RoleBindingList

assets/prometheus-k8s/role-specific-namespaces.yaml

@@ -85,4 +85,38 @@ items:
     - get
     - list
     - watch
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: prometheus-k8s
+    namespace: openshift-kube-scheduler
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - nodes
+    - services
+    - endpoints
+    - pods
+    verbs:
+    - get
+    - list
+    - watch
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: prometheus-k8s
+    namespace: openshift-kube-controller-manager
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - nodes
+    - services
+    - endpoints
+    - pods
+    verbs:
+    - get
+    - list
+    - watch
 kind: RoleList

assets/prometheus-k8s/service-monitor-kube-controller-manager.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-controller-manager
+  name: kube-controller-manager
+  namespace: openshift-monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    metricRelabelings:
+    - action: drop
+      regex: etcd_(debugging|disk|request|server).*
+      sourceLabels:
+      - __name__
+    port: https
+    relabelings:
+    - action: replace
+      regex: (.+)(?::\d+)
+      replacement: $1:10252
+      sourceLabels:
+      - __address__
+      targetLabel: __address__
+  jobLabel: null
+  namespaceSelector:
+    matchNames:
+    - openshift-kube-controller-manager
+  selector: {}

assets/prometheus-k8s/service-monitor-kube-scheduler.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-scheduler
+  name: kube-scheduler
+  namespace: openshift-monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: https
+    relabelings:
+    - action: replace
+      regex: (.+)(?::\d+)
+      replacement: $1:10251
+      sourceLabels:
+      - __address__
+      targetLabel: __address__
+  jobLabel: null
+  namespaceSelector:
+    matchNames:
+    - openshift-kube-scheduler
+  selector: {}

hack/build-jsonnet.sh

@@ -39,16 +39,10 @@ done
 #
 # * CoreDNS is not used in OpenShift
 #
-# * kube-controller-manager and kube-scheduler don't exist in OpenShift, but
-#   instead are grouped into the kube-scheduler component, which is separately
-#   handled in `jsonnet/prometheus.jsonnet`.
 
 rm -rf "assets/prometheus-operator/0alertmanager-custom-resource-definition.yaml"
 rm -rf "assets/prometheus-operator/0prometheus-custom-resource-definition.yaml"
 rm -rf "assets/prometheus-operator/0prometheusrule-custom-resource-definition.yaml"
 rm -rf "assets/prometheus-operator/0servicemonitor-custom-resource-definition.yaml"
 rm -rf "assets/prometheus-k8s/service-monitor-core-d-n-s.yaml"
-rm -rf "assets/prometheus-k8s/service-monitor-kube-controller-manager.yaml"
-rm -rf "assets/prometheus-k8s/service-monitor-kube-scheduler.yaml"
-rm -rf "assets/prometheus-k8s/service-monitor-kube-scheduler.yaml"
 

jsonnet/main.jsonnet

@@ -49,6 +49,8 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
                  namespaces+: [
                    'openshift-cluster-version',
                    'openshift-apiserver',
+                   'openshift-kube-scheduler',
+                   'openshift-kube-controller-manager',
                  ],
                },
              },

jsonnet/prometheus.jsonnet

@@ -104,59 +104,8 @@ local namespacesRole =
     clusterRole+:
       clusterRole.withRulesMixin([authenticationRole, authorizationRole, namespacesRole]),
 
-    // OpenShift currently has the kube-controller-manager and
-    // kube-scheduler combined in one component called the
-    // kube-controllers. This Service and ServiceMonitor enable scraping
-    // its metrics.
-
-    kubeControllersService:
-      local service = k.core.v1.service;
-      local servicePort = k.core.v1.service.mixin.spec.portsType;
-
-      local kubeControllersPort = servicePort.newNamed('http-metrics', 8444, 8444);
-
-      service.new('kube-controllers', {
-        'openshift.io/component': 'controllers',
-        'openshift.io/control-plane': 'true',
-      }, kubeControllersPort) +
-      service.mixin.metadata.withNamespace('kube-system') +
-      service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controllers' }) +
-      service.mixin.spec.withClusterIp('None'),
-
-    serviceMonitorKubeControllers:
-      {
-        apiVersion: 'monitoring.coreos.com/v1',
-        kind: 'ServiceMonitor',
-        metadata: {
-          labels: {
-            'k8s-app': 'kube-controllers',
-          },
-          name: 'kube-controllers',
-        },
-        spec: {
-          endpoints: [
-            {
-              bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
-              interval: '30s',
-              port: 'http-metrics',
-              scheme: 'https',
-              tlsConfig: {
-                caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
-              },
-            },
-          ],
-          jobLabel: 'k8s-app',
-          namespaceSelector: {
-            matchNames: ['kube-system'],
-          },
-          selector: {
-            matchLabels: {
-              'k8s-app': 'kube-controllers',
-            },
-          },
-        },
-      },
-
+    // OpenShift has the kube-apiserver as well as an aggregated API called
+    // OpenShift apiserver, containing all the extended APIs.
     serviceMonitorClusterVersionOperator:
       {
         apiVersion: 'monitoring.coreos.com/v1',
@@ -331,6 +280,92 @@ local namespacesRole =
         },
       },
 
+    // In OpenShift the kube-scheduler runs in its own namespace, and has a TLS
+    // cert from the serving certs controller.
+
+    serviceMonitorKubeScheduler+:
+      {
+        spec+: {
+          jobLabel: null,
+          namespaceSelector: {
+            matchNames: [
+              'openshift-kube-scheduler',
+            ],
+          },
+          selector: {},
+          endpoints:
+            std.map(
+              function(a) a {
+
+                //TODO(brancz): Once OpenShift is based on Kubernetes 1.12 the
+                //scheduler will serve metrics on a secure port, then the below
+                //commented out code is what we will need without the relabel
+                //configs.
+
+                //bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+                interval: '30s',
+                port: 'https',
+                //scheme: 'https',
+                //tlsConfig: {
+                //  caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
+                //  serverName: 'scheduler.openshift-kube-scheduler.svc',
+                //},
+                relabelings: [{
+                  sourceLabels: ['__address__'],
+                  action: 'replace',
+                  targetLabel: '__address__',
+                  regex: '(.+)(?::\\d+)',
+                  replacement: '$1:10251',
+                }],
+              },
+              super.endpoints,
+            ),
+        },
+      },
+
+    // In OpenShift the kube-controller-manager runs in its own namespace, and
+    // has a TLS cert from the serving certs controller.
+
+    serviceMonitorKubeControllerManager+:
+      {
+        spec+: {
+          jobLabel: null,
+          namespaceSelector: {
+            matchNames: [
+              'openshift-kube-controller-manager',
+            ],
+          },
+          selector: {},
+          endpoints:
+            std.map(
+              function(a) a {
+
+                //TODO(brancz): Once OpenShift is based on Kubernetes 1.12 the
+                //controller-manager will serve metrics on a secure port, then
+                //the below commented out code is what we will need without the
+                //relabel configs.
+
+                //bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+                interval: '30s',
+                port: 'https',
+                //scheme: 'https',
+                //tlsConfig: {
+                //  caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
+                //  serverName: 'controller-manager.openshift-kube-controller-manager.svc',
+                //},
+                relabelings: [{
+                  sourceLabels: ['__address__'],
+                  action: 'replace',
+                  targetLabel: '__address__',
+                  regex: '(.+)(?::\\d+)',
+                  replacement: '$1:10252',
+                }],
+              },
+              super.endpoints,
+            ),
+        },
+      },
+
     // These patches inject the oauth proxy as a sidecar and configures it with
     // TLS. Additionally as the Alertmanager is protected with TLS, authN and
     // authZ it requires some additonal configuration.