feat(observability): hcp kube-apiserver error rate and latency alerts to SRE

dev-infrastructure/modules/metrics/rp-hcp-rules.bicep

@@ -0,0 +1,18 @@
+// Alerts for HCP KubeAPIServer availability, routed to the RP ICM queue.
+
+@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for hosted control planes)')
+param azureMonitoringWorkspaceId string
+
+param actionGroups array
+
+@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.')
+param severityCeiling int = 0
+
+module generatedAlerts 'rules/generatedRPHCPPrometheusAlertingRules.bicep' = {
+  name: 'generatedRPHCPPrometheusAlertingRules'
+  params: {
+    azureMonitoring: azureMonitoringWorkspaceId
+    actionGroups: actionGroups
+    severityCeiling: severityCeiling
+  }
+}

dev-infrastructure/templates/monitoring.bicep

@@ -206,6 +206,15 @@ module rpAlerts '../modules/metrics/rp-rules.bicep' = {
   }
 }
 
+module rpHcpAlerts '../modules/metrics/rp-hcp-rules.bicep' = {
+  name: 'rpHcpAlerts'
+  params: {
+    azureMonitoringWorkspaceId: hcpAzureMonitoringWorkspaceId
+    actionGroups: rpActionGroups
+    severityCeiling: alertSeverityCeiling
+  }
+}
+
 module msftAlerts '../modules/metrics/msft-rules.bicep' = {
   name: 'msftAlerts'
   params: {

observability/alerts-rp-hcps.yaml

@@ -0,0 +1,9 @@
+prometheusRules:
+  rulesFolders:
+  - alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml
+  - alerts/UserJourneyKubeApiserverErrorsMonitor-prometheusRule.yaml
+  - alerts/UserJourneyKubeApiserverLatencyMonitor-prometheusRule.yaml
+  untestedRules: []
+  testDependencies:
+  - recording-rules-hcps.yaml
+  outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep

observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml

@@ -13,131 +13,157 @@ spec:
     # Long-window averages for slow-burn alerts
     - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_30d
       expr: |
-        avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30d]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30d]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_7d
       expr: |
-        avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[7d]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[7d]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_1d
       expr: |
-        avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_3d
       expr: |
-        avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     # Short-window sum_over_time for burn-rate calculations
     - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_30m
       expr: |
-        sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_1h
       expr: |
-        sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_2h
       expr: |
-        sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_6h
       expr: |
-        sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     # Short-window count_over_time for burn-rate calculations
     - record: hostedClusterAPI_kubeapiserver_available:count_over_time_30m
       expr: |
-        sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:count_over_time_1h
       expr: |
-        sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:count_over_time_2h
       expr: |
-        sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:count_over_time_6h
       expr: |
-        sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     # Long-window count_over_time for sample-count guards
     - record: hostedClusterAPI_kubeapiserver_available:count_over_time_1d
       expr: |
-        sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:count_over_time_3d
       expr: |
-        sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d]))
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d]))
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     # SLI rules using 15m offset blackout to exclude newly-provisioned clusters.
     # The offset blackout ensures a cluster existed at least 15 minutes ago before
     # contributing to SLI calculations, preventing provisioning noise from skewing
     # availability metrics. Numerator counts status="True" samples; denominator counts
     # all samples (any status via max by) so downtime is counted as 0, not skipped.
     - record: hostedClusterAPI_kubeapiserver_available:sli_sum_15m
       expr: |
-        sum by (name, namespace, _id, cluster) (
+        sum by (name, namespace, _id, resource_id, cluster) (
           sum_over_time(
             (
               hostedClusterAPI_kubeapiserver_available{status="True"}
-              and on (name, namespace, _id, cluster)
-              max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
+              and on (name, namespace, _id, resource_id, cluster)
+              max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
             )[15m:1m]
           )
         )
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:sli_count_15m
       expr: |
-        sum by (name, namespace, _id, cluster) (
+        sum by (name, namespace, _id, resource_id, cluster) (
           count_over_time(
             (
-              max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
-              and on (name, namespace, _id, cluster)
-              max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
+              max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
+              and on (name, namespace, _id, resource_id, cluster)
+              max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
             )[15m:1m]
           )
         )
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:sli_sum_6h
       expr: |
-        sum by (name, namespace, _id, cluster) (
+        sum by (name, namespace, _id, resource_id, cluster) (
           sum_over_time(
             (
               hostedClusterAPI_kubeapiserver_available{status="True"}
-              and on (name, namespace, _id, cluster)
-              max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
+              and on (name, namespace, _id, resource_id, cluster)
+              max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
             )[6h:5m]
           )
         )
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
     - record: hostedClusterAPI_kubeapiserver_available:sli_count_6h
       expr: |
-        sum by (name, namespace, _id, cluster) (
+        sum by (name, namespace, _id, resource_id, cluster) (
           count_over_time(
             (
-              max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
-              and on (name, namespace, _id, cluster)
-              max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
+              max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
+              and on (name, namespace, _id, resource_id, cluster)
+              max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
             )[6h:5m]
           )
         )
-        and on (name, namespace, _id, cluster)
-        count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
+    - record: hostedClusterAPI_kubeapiserver_available:sli_sum_3d
+      expr: |
+        sum by (name, namespace, _id, resource_id, cluster) (
+          sum_over_time(
+            (
+              hostedClusterAPI_kubeapiserver_available{status="True"}
+              and on (name, namespace, _id, resource_id, cluster)
+              max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
+            )[3d:30m]
+          )
+        )
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
+    - record: hostedClusterAPI_kubeapiserver_available:sli_count_3d
+      expr: |
+        sum by (name, namespace, _id, resource_id, cluster) (
+          count_over_time(
+            (
+              max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)
+              and on (name, namespace, _id, resource_id, cluster)
+              max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0)
+            )[3d:30m]
+          )
+        )
+        and on (name, namespace, _id, resource_id, cluster)
+        count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)