Skip to content

fix(observability): add https:// scheme to nodepool alert runbook_url #6055

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Jul 14, 2026
Merged

fix(observability): add https:// scheme to nodepool alert runbook_url #6055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule
correlationId: 'UJNodePoolErrors1h5m/{{ $labels.cluster }}'
description: 'More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours.'
info: 'More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool operation error rate critically high (>72%)'
Comment thread
swiencki marked this conversation as resolved.
title: '{{ $labels.cluster }}: Node Pool operation error rate critically high (>72%)'
}
Expand Down Expand Up @@ -292,7 +292,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule
correlationId: 'UJNodePoolErrors6h30m/{{ $labels.cluster }}'
description: 'More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.'
info: 'More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool operation error rate elevated (>30%) for 30+ minutes'
title: '{{ $labels.cluster }}: Node Pool operation error rate elevated (>30%) for 30+ minutes'
}
Expand Down Expand Up @@ -321,7 +321,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule
correlationId: 'UJNodePoolErrors3d/{{ $labels.cluster }}'
description: 'More than 5% of node pool operations are in failed state sustained over 6 hours, indicating persistent degradation at the 95% SLO boundary that would exhaust the error budget in ~7 days.'
info: 'More than 5% of node pool operations are in failed state sustained over 6 hours, indicating persistent degradation at the 95% SLO boundary that would exhaust the error budget in ~7 days.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool operation error rate exceeds SLO target (>5%) for 6+ hours'
title: '{{ $labels.cluster }}: Node Pool operation error rate exceeds SLO target (>5%) for 6+ hours'
}
Expand Down Expand Up @@ -349,7 +349,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule
correlationId: 'UJNodePoolErrorsDegradation/{{ $labels.cluster }}'
description: 'The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire.'
info: 'The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool operation failure rate exceeds 15% for 30 minutes'
title: '{{ $labels.cluster }}: Node Pool operation failure rate exceeds 15% for 30 minutes'
}
Expand All @@ -376,7 +376,7 @@ resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRule
correlationId: 'UJNodePoolStuckOperation/{{ $labels.cluster }}/{{ $labels.resource_id }}/{{ $labels.phase }}'
description: 'Node pool operation for {{ $labels.resource_id }} has been in {{ $labels.phase }} phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation.'
info: 'Node pool operation for {{ $labels.resource_id }} has been in {{ $labels.phase }} phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool operation stuck in {{ $labels.phase }} for over 2 hours'
title: '{{ $labels.cluster }}: Node Pool operation stuck in {{ $labels.phase }} for over 2 hours resource_id:{{ $labels.resource_id }}'
}
Expand Down Expand Up @@ -416,7 +416,7 @@ resource arohcpNodepoolSaturationAlerts 'Microsoft.AlertsManagement/prometheusRu
correlationId: 'UJNodePoolSaturationQueueDepth/{{ $labels.cluster }}/{{ $labels.name }}'
description: 'Node pool controller workqueue {{ $labels.name }} has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed.'
info: 'Node pool controller workqueue {{ $labels.name }} has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} depth is high'
title: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} depth is high'
}
Expand All @@ -443,7 +443,7 @@ resource arohcpNodepoolSaturationAlerts 'Microsoft.AlertsManagement/prometheusRu
correlationId: 'UJNodePoolSaturationRetryHotLoop/{{ $labels.cluster }}/{{ $labels.name }}'
description: 'Node pool controller workqueue {{ $labels.name }} has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.'
info: 'Node pool controller workqueue {{ $labels.name }} has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.'
runbook_url: 'aka.ms/arohcp-runbook-nodepool'
runbook_url: 'https://aka.ms/arohcp-runbook-nodepool'
summary: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} retry hot loop'
title: '{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} retry hot loop'
}
Expand Down
14 changes: 7 additions & 7 deletions observability/alerts/nodepool-slo-prometheusRule.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool operation error rate critically high (>72%)"
description: "More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
Comment thread
swiencki marked this conversation as resolved.
# ----------------------------------------
# Medium Burn Alert (non-paging for now per approver request)
# ----------------------------------------
Expand All @@ -122,7 +122,7 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool operation error rate elevated (>30%) for 30+ minutes"
description: "More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ----------------------------------------
# Slow Burn Alert (Sev 4 — ticket)
# ----------------------------------------
Expand All @@ -140,7 +140,7 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool operation error rate exceeds SLO target (>5%) for 6+ hours"
description: "More than 5% of node pool operations are in failed state sustained over 6 hours, indicating persistent degradation at the 95% SLO boundary that would exhaust the error budget in ~7 days."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ----------------------------------------
# Degradation Alert (Sev 4 — ticket)
# ----------------------------------------
Expand All @@ -157,7 +157,7 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool operation failure rate exceeds 15% for 30 minutes"
description: "The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ----------------------------------------
# Stuck Operations Alert (non-paging for now per approver request)
# ----------------------------------------
Expand All @@ -177,7 +177,7 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool operation stuck in {{ $labels.phase }} for over 2 hours"
description: "Node pool operation for {{ $labels.resource_id }} has been in {{ $labels.phase }} phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ========================================
# Node Pool User Journey Alerts — Saturation
# ========================================
Expand All @@ -199,7 +199,7 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} depth is high"
description: "Node pool controller workqueue {{ $labels.name }} has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# Retry hot loop: high retry ratio on node pool workqueue
- alert: UJNodePoolSaturationRetryHotLoop
expr: |
Expand All @@ -222,4 +222,4 @@ spec:
annotations:
summary: "{{ $labels.cluster }}: Node Pool controller workqueue {{ $labels.name }} retry hot loop"
description: "Node pool controller workqueue {{ $labels.name }} has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
16 changes: 8 additions & 8 deletions observability/alerts/nodepool-slo-prometheusRule_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool operation error rate critically high (>72%)"
description: "More than 72% of node pool operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# Degradation also fires (15% threshold met at 80%)
- eval_time: 35m
alertname: UJNodePoolErrorsDegradation
Expand All @@ -116,7 +116,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool operation failure rate exceeds 15% for 30 minutes"
description: "The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ========================================
# Test 3: Degradation — 20% failure rate, only degradation alert fires
# ========================================
Expand Down Expand Up @@ -167,7 +167,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool operation failure rate exceeds 15% for 30 minutes"
description: "The node pool operation failure rate has been above 15% for 30 minutes. This provides early warning of degradation before SLO-based burn rate alerts fire."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ========================================
# Test 4: Stuck operation — operation in "updating" for > 2 hours
# ========================================
Expand Down Expand Up @@ -196,7 +196,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool operation stuck in updating for over 2 hours"
description: "Node pool operation for /sub/3/np/stuck has been in updating phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ========================================
# Test 5: No stuck operation — operation in "updating" for < 2 hours
# ========================================
Expand Down Expand Up @@ -253,7 +253,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool controller workqueue OperationNodePoolUpdate depth is high"
description: "Node pool controller workqueue OperationNodePoolUpdate has had a depth > 10 for more than 5 minutes, indicating work is accumulating faster than it can be processed."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ========================================
# Test 8: Queue depth normal — below threshold, no alert
# ========================================
Expand Down Expand Up @@ -284,7 +284,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool controller workqueue OperationNodePoolUpdate retry hot loop"
description: "Node pool controller workqueue OperationNodePoolUpdate has a retry ratio > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# ========================================
# Test 10: Retry hot loop — low retry ratio, no alert
# ========================================
Expand Down Expand Up @@ -359,7 +359,7 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool operation error rate elevated (>30%) for 30+ minutes"
description: "More than 30% of node pool operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"
# 50% < 72% → fast burn should NOT fire
- eval_time: 35m
alertname: UJNodePoolErrors1h5m
Expand Down Expand Up @@ -388,4 +388,4 @@ tests:
exp_annotations:
summary: "mgmt-1: Node Pool operation stuck in deleting for over 2 hours"
description: "Node pool operation for /sub/7/np/stuck-del has been in deleting phase for over 2 hours. Stuck operations are invisible to success/failure SLIs and require investigation."
runbook_url: "aka.ms/arohcp-runbook-nodepool"
runbook_url: "https://aka.ms/arohcp-runbook-nodepool"