MON-4386: Introduce telemetry profile

rexagod · rexagod · commit d9bb6e4641b3 · 2025-10-05T20:28:33.000+05:30
This profile only caters to metrics that telemetry rules rely upon.

Signed-off-by: Pranshu Srivastava &lt;rexagod@gmail.com&gt;
diff --git a/assets/control-plane/telemetry-service-monitor-kubelet.yaml b/assets/control-plane/telemetry-service-monitor-kubelet.yaml
@@ -0,0 +1,118 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: kubernetes
+    app.kubernetes.io/managed-by: cluster-monitoring-operator
+    app.kubernetes.io/name: kubelet
+    app.kubernetes.io/part-of: openshift-monitoring
+    k8s-app: kubelet
+    monitoring.openshift.io/collection-profile: telemetry
+  name: kubelet-telemetry
+  namespace: openshift-monitoring
+spec:
+  attachMetadata:
+    node: true
+  endpoints:
+  - bearerTokenFile: ""
+    honorLabels: true
+    interval: 30s
+    metricRelabelings:
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    scrapeTimeout: 30s
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/kubelet-serving-ca-bundle/ca-bundle.crt
+      insecureSkipVerify: false
+  - bearerTokenFile: ""
+    honorLabels: true
+    honorTimestamps: true
+    interval: 30s
+    metricRelabelings:
+    - action: labeldrop
+      regex: __tmp_keep_metric
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    path: /metrics/cadvisor
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    scrapeTimeout: 30s
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/kubelet-serving-ca-bundle/ca-bundle.crt
+      insecureSkipVerify: false
+    trackTimestampsStaleness: true
+  - bearerTokenFile: ""
+    honorLabels: true
+    interval: 30s
+    metricRelabelings:
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    path: /metrics/probes
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    scrapeTimeout: 30s
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/kubelet-serving-ca-bundle/ca-bundle.crt
+      insecureSkipVerify: false
+  - bearerTokenFile: ""
+    interval: 30s
+    metricRelabelings:
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    port: https-metrics
+    relabelings:
+    - action: keep
+      regex: (linux|)
+      sourceLabels:
+      - __meta_kubernetes_node_label_kubernetes_io_os
+    - action: replace
+      regex: (.+)(?::\d+)
+      replacement: $1:9637
+      sourceLabels:
+      - __address__
+      targetLabel: __address__
+    - action: replace
+      replacement: crio
+      sourceLabels:
+      - endpoint
+      targetLabel: endpoint
+    - action: replace
+      replacement: crio
+      targetLabel: job
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/kubelet-serving-ca-bundle/ca-bundle.crt
+      insecureSkipVerify: false
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  scrapeClass: tls-client-certificate-auth
+  selector:
+    matchLabels:
+      k8s-app: kubelet
diff --git a/assets/kube-state-metrics/telemetry-service-monitor.yaml b/assets/kube-state-metrics/telemetry-service-monitor.yaml
@@ -0,0 +1,53 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/managed-by: cluster-monitoring-operator
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/part-of: openshift-monitoring
+    app.kubernetes.io/version: 2.17.0
+    monitoring.openshift.io/collection-profile: telemetry
+  name: kube-state-metrics-telemetry
+  namespace: openshift-monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: ""
+    honorLabels: true
+    interval: 1m
+    metricRelabelings:
+    - action: labeldrop
+      regex: instance
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    port: https-main
+    relabelings:
+    - action: labeldrop
+      regex: pod
+    scheme: https
+    scrapeTimeout: 1m
+    tlsConfig:
+      insecureSkipVerify: false
+      serverName: kube-state-metrics.openshift-monitoring.svc
+  - bearerTokenFile: ""
+    interval: 1m
+    metricRelabelings:
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    port: https-self
+    scheme: https
+    scrapeTimeout: 1m
+    tlsConfig:
+      insecureSkipVerify: false
+      serverName: kube-state-metrics.openshift-monitoring.svc
+  jobLabel: app.kubernetes.io/name
+  scrapeClass: tls-client-certificate-auth
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: exporter
+      app.kubernetes.io/name: kube-state-metrics
+      app.kubernetes.io/part-of: openshift-monitoring
diff --git a/assets/node-exporter/telemetry-service-monitor.yaml b/assets/node-exporter/telemetry-service-monitor.yaml
@@ -0,0 +1,48 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/managed-by: cluster-monitoring-operator
+    app.kubernetes.io/name: node-exporter
+    app.kubernetes.io/part-of: openshift-monitoring
+    app.kubernetes.io/version: 1.9.1
+    monitoring.openshift.io/collection-profile: telemetry
+  name: node-exporter-telemetry
+  namespace: openshift-monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: ""
+    interval: 15s
+    metricRelabelings:
+    - action: replace
+      regex: (node_mountstats_nfs_read_bytes_total|node_mountstats_nfs_write_bytes_total|node_mountstats_nfs_operations_requests_total)
+      replacement: "true"
+      sourceLabels:
+      - __name__
+      targetLabel: __tmp_keep
+    - action: labeldrop
+      regex: __tmp_keep
+    - action: keep
+      regex: ()
+      sourceLabels:
+      - __name__
+    port: https
+    relabelings:
+    - action: replace
+      regex: (.*)
+      replacement: $1
+      sourceLabels:
+      - __meta_kubernetes_pod_node_name
+      targetLabel: instance
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: false
+      serverName: node-expo.openshift-monitoring.svc
+  jobLabel: app.kubernetes.io/name
+  scrapeClass: tls-client-certificate-auth
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: exporter
+      app.kubernetes.io/name: node-exporter
+      app.kubernetes.io/part-of: openshift-monitoring
diff --git a/jsonnet/components/control-plane.libsonnet b/jsonnet/components/control-plane.libsonnet
@@ -167,6 +167,11 @@ function(params)
                                            ])
     ),
 
+    telemetryServiceMonitorKubelet: generateServiceMonitor.telemetry(
+      self.serviceMonitorKubelet, std.join('|', [])
+    ),
+
+
     // This avoids creating service monitors which are already managed by the respective operators.
     serviceMonitorApiserver:: {},
     serviceMonitorKubeScheduler:: {},
diff --git a/jsonnet/components/kube-state-metrics.libsonnet b/jsonnet/components/kube-state-metrics.libsonnet
@@ -170,6 +170,10 @@ function(params)
                                     ])
     ),
 
+    telemetryServiceMonitor: generateServiceMonitor.telemetry(
+      self.serviceMonitor, std.join('|', [])
+    ),
+
     kubeRbacProxySecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-state-metrics-kube-rbac-proxy-config'),
 
     // This removes the upstream addon-resizer and all resource requests and
diff --git a/jsonnet/components/node-exporter.libsonnet b/jsonnet/components/node-exporter.libsonnet
@@ -223,6 +223,10 @@ function(params)
                                      ])
     ),
 
+    telemetryServiceMonitor: generateServiceMonitor.telemetry(
+      self.serviceMonitor, std.join('|', [])
+    ),
+
     securityContextConstraints: {
       allowHostDirVolumePlugin: true,
       allowHostNetwork: true,
diff --git a/jsonnet/utils/generate-service-monitors.libsonnet b/jsonnet/utils/generate-service-monitors.libsonnet
@@ -2,13 +2,16 @@
   local minimalLabel = {
     'monitoring.openshift.io/collection-profile': 'minimal',
   },
-  // 1. Add the prefix minimal to the ServiceMonitor name
-  // 2. Add the minimal label "monitoring.openshift.io/collection-profile: minimal"
+  local telemetryLabel = {
+    'monitoring.openshift.io/collection-profile': 'telemetry',
+  },
+  // 1. Add the profile prefix to the ServiceMonitor name
+  // 2. Add the profile label "monitoring.openshift.io/collection-profile: <profile>"
   // 3. Add a metricRelabelings with action keep and regex equal to metrics
-  local minimal(sm, metrics) = sm {
+  local run(sm, metrics, label) = sm {
     metadata+: {
-      name+: '-minimal',
-      labels+: minimalLabel,
+      name+: '-' + label['monitoring.openshift.io/collection-profile'],
+      labels+: label,
     },
     spec+: {
       endpoints: std.map(
@@ -39,5 +42,6 @@
     },
   },
 
-  minimal(sm, metrics): minimal(removeDrop(sm), metrics),
+  minimal(sm, metrics): run(removeDrop(sm), metrics, minimalLabel),
+  telemetry(sm, metrics): run(removeDrop(sm), metrics, telemetryLabel),
 }
diff --git a/pkg/manifests/types.go b/pkg/manifests/types.go
@@ -24,11 +24,16 @@ type CollectionProfiles []CollectionProfile
 type ExternalLabels map[string]string
 
 const (
-	FullCollectionProfile    = "full"
-	MinimalCollectionProfile = "minimal"
+	FullCollectionProfile      = "full"
+	MinimalCollectionProfile   = "minimal"
+	TelemetryCollectionProfile = "telemetry"
 )
 
-var SupportedCollectionProfiles = CollectionProfiles{FullCollectionProfile, MinimalCollectionProfile}
+var SupportedCollectionProfiles = CollectionProfiles{
+	FullCollectionProfile,
+	MinimalCollectionProfile,
+	TelemetryCollectionProfile,
+}
 
 // The `ClusterMonitoringConfiguration` resource defines settings that
 // customize the default platform monitoring stack through the
