diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep index 57dd23e379d..88b9d8fc2c6 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep @@ -262,7 +262,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule correlationId: 'UJNodePoolErrors1h5m/{{ $labels.cluster }}' description: 'More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours.' info: 'More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool operation error rate critically high (>72%)' title: '{{ $labels.cluster }}: Node Pool operation error rate critically high (>72%)' } @@ -292,7 +292,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule correlationId: 'UJNodePoolErrors6h30m/{{ $labels.cluster }}' description: 'More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.' info: 'More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool operation error rate elevated (>30%) for 30+ minutes' title: '{{ $labels.cluster }}: Node Pool operation error rate elevated (>30%) for 30+ minutes' } @@ -321,7 +321,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule correlationId: 'UJNodePoolErrors3d/{{ $labels.cluster }}' description: 'More than 5% of node pool operations are in failed state sustained over 6 hours, indicating persistent degradation at the 95% SLO boundary that would exhaust the error budget in ~7 days.' info: 'More than 5% of node pool operations are in failed state sustained over 6 hours, indicating persistent degradation at the 95% SLO boundary that would exhaust the error budget in ~7 days.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool operation error rate exceeds SLO target (>5%) for 6+ hours' title: '{{ $labels.cluster }}: Node Pool operation error rate exceeds SLO target (>5%) for 6+ hours' } @@ -349,7 +349,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule correlationId: 'UJNodePoolErrorsDegradation/{{ $labels.cluster }}' description: 'The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire.' info: 'The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool operation failure rate exceeds 15% for 30 minutes' title: '{{ $labels.cluster }}: Node Pool operation failure rate exceeds 15% for 30 minutes' } @@ -376,7 +376,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule correlationId: 'UJNodePoolStuckOperation/{{ $labels.cluster }}/{{ $labels.resource_id }}/{{ $labels.phase }}' description: 'Node pool operation for {{ $labels.resource_id }} has been in {{ $labels.phase }} phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation.' info: 'Node pool operation for {{ $labels.resource_id }} has been in {{ $labels.phase }} phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool operation stuck in {{ $labels.phase }} for over 2 hours' title: '{{ $labels.cluster }}: Node Pool operation stuck in {{ $labels.phase }} for over 2 hours resource_id:{{ $labels.resource_id }}' } @@ -416,7 +416,7 @@ resource arohcpNodepoolSaturationAlerts 'Microsoft.AlertsManagement/prometheusRu correlationId: 'UJNodePoolSaturationQueueDepth/{{ $labels.cluster }}/{{ $labels.name }}' description: 'Node pool controller workqueue {{ $labels.name }} has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed.' info: 'Node pool controller workqueue {{ $labels.name }} has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} depth is high' title: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} depth is high' } @@ -443,7 +443,7 @@ resource arohcpNodepoolSaturationAlerts 'Microsoft.AlertsManagement/prometheusRu correlationId: 'UJNodePoolSaturationRetryHotLoop/{{ $labels.cluster }}/{{ $labels.name }}' description: 'Node pool controller workqueue {{ $labels.name }} has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.' info: 'Node pool controller workqueue {{ $labels.name }} has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.' - runbook_url: 'aka.ms/arohcp-runbook-nodepool' + runbook_url: 'https://aka.ms/arohcp-runbook-nodepool' summary: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} retry hot loop' title: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} retry hot loop' } diff --git a/observability/alerts/nodepool-slo-prometheusRule.yaml b/observability/alerts/nodepool-slo-prometheusRule.yaml index eaa8d81baf8..0fa3e1fe25f 100644 --- a/observability/alerts/nodepool-slo-prometheusRule.yaml +++ b/observability/alerts/nodepool-slo-prometheusRule.yaml @@ -104,7 +104,7 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool operation error rate critically high (>72%)" description: "More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ---------------------------------------- # Medium Burn Alert (non-paging for now per approver request) # ---------------------------------------- @@ -122,7 +122,7 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool operation error rate elevated (>30%) for 30+ minutes" description: "More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ---------------------------------------- # Slow Burn Alert (Sev 4 — ticket) # ---------------------------------------- @@ -140,7 +140,7 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool operation error rate exceeds SLO target (>5%) for 6+ hours" description: "More than 5% of node pool operations are in failed state sustained over 6 hours, indicating persistent degradation at the 95% SLO boundary that would exhaust the error budget in ~7 days." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ---------------------------------------- # Degradation Alert (Sev 4 — ticket) # ---------------------------------------- @@ -157,7 +157,7 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool operation failure rate exceeds 15% for 30 minutes" description: "The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ---------------------------------------- # Stuck Operations Alert (non-paging for now per approver request) # ---------------------------------------- @@ -177,7 +177,7 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool operation stuck in {{ $labels.phase }} for over 2 hours" description: "Node pool operation for {{ $labels.resource_id }} has been in {{ $labels.phase }} phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ======================================== # Node Pool User Journey Alerts — Saturation # ======================================== @@ -199,7 +199,7 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} depth is high" description: "Node pool controller workqueue {{ $labels.name }} has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # Retry hot loop: high retry ratio on node pool workqueue - alert: UJNodePoolSaturationRetryHotLoop expr: | @@ -222,4 +222,4 @@ spec: annotations: summary: "{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} retry hot loop" description: "Node pool controller workqueue {{ $labels.name }} has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" diff --git a/observability/alerts/nodepool-slo-prometheusRule_test.yaml b/observability/alerts/nodepool-slo-prometheusRule_test.yaml index 4f672c849b2..0c4bd3afb33 100644 --- a/observability/alerts/nodepool-slo-prometheusRule_test.yaml +++ b/observability/alerts/nodepool-slo-prometheusRule_test.yaml @@ -104,7 +104,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool operation error rate critically high (>72%)" description: "More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # Degradation also fires (15% threshold met at 80%) - eval_time: 35m alertname: UJNodePoolErrorsDegradation @@ -116,7 +116,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool operation failure rate exceeds 15% for 30 minutes" description: "The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ======================================== # Test 3: Degradation — 20% failure rate, only degradation alert fires # ======================================== @@ -167,7 +167,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool operation failure rate exceeds 15% for 30 minutes" description: "The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ======================================== # Test 4: Stuck operation — operation in "updating" for > 2 hours # ======================================== @@ -196,7 +196,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool operation stuck in updating for over 2 hours" description: "Node pool operation for /sub/3/np/stuck has been in updating phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ======================================== # Test 5: No stuck operation — operation in "updating" for < 2 hours # ======================================== @@ -253,7 +253,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool controller workqueue OperationNodePoolUpdate depth is high" description: "Node pool controller workqueue OperationNodePoolUpdate has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ======================================== # Test 8: Queue depth normal — below threshold, no alert # ======================================== @@ -284,7 +284,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool controller workqueue OperationNodePoolUpdate retry hot loop" description: "Node pool controller workqueue OperationNodePoolUpdate has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # ======================================== # Test 10: Retry hot loop — low retry ratio, no alert # ======================================== @@ -359,7 +359,7 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool operation error rate elevated (>30%) for 30+ minutes" description: "More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool" # 50% < 72% → fast burn should NOT fire - eval_time: 35m alertname: UJNodePoolErrors1h5m @@ -388,4 +388,4 @@ tests: exp_annotations: summary: "mgmt-1: Node Pool operation stuck in deleting for over 2 hours" description: "Node pool operation for /sub/7/np/stuck-del has been in deleting phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation." - runbook_url: "aka.ms/arohcp-runbook-nodepool" + runbook_url: "https://aka.ms/arohcp-runbook-nodepool"