feat: set up alerts for a smooth Prom3 upgrade

Author: machine424

assets/cluster-monitoring-operator/prometheus-rule.yaml

@@ -30,6 +30,22 @@ spec:
       for: 15m
       labels:
         severity: warning
+    - alert: TargetInvalidContentType
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} could not determine a valid content type for some scrape targets.
+        summary: A valid content type could not be determined for some scrape targets.
+      expr: increase(prometheus_target_scrape_pool_invalid_content_type_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: critical
+    - alert: NarrowLeQuantileSelectors
+      annotations:
+        description: XXX.
+        summary: XXX.
+      expr: increase(prometheus_target_scrape_pool_invalid_content_type_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: critical
   - name: openshift-kubernetes.rules
     rules:
     - expr: sum(rate(container_cpu_usage_seconds_total{container="",pod!=""}[5m])) BY (pod, namespace)

jsonnet/rules.libsonnet

@@ -27,6 +27,32 @@ function(params) {
             severity: 'warning',
           },
         },
+        {
+          expr: 'increase(prometheus_target_scrape_pool_invalid_content_type_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0',
+          alert: 'TargetInvalidContentType',
+          'for': '15m',
+          annotations: {
+            description: 'Prometheus {{$labels.namespace}}/{{$labels.pod}} could not determine a valid content type for some scrape targets.',
+            summary: 'A valid content type could not be determined for some scrape targets.',
+          },
+          labels: {
+            # severity: 'warning', (before the cut, to take advantage of critical alerts treatment)
+            severity: 'critical',
+          },
+        },
+        {
+          expr: 'increase(prometheus_target_scrape_pool_invalid_content_type_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0',
+          alert: 'NarrowLeQuantileSelectors',
+          'for': '15m',
+          annotations: {
+            description: 'XXX.',
+            summary: 'XXX.',
+          },
+          labels: {
+            # severity: 'warning', (before the cut, to take advantage of critical alerts treatment)
+            severity: 'critical',
+          },
+        },
       ],
     },
     {