diff --git a/examples/azuremonitor_runbooks.yaml b/examples/azuremonitor_runbooks.yaml index 4f3cb03078..64b640604c 100644 --- a/examples/azuremonitor_runbooks.yaml +++ b/examples/azuremonitor_runbooks.yaml @@ -6,23 +6,14 @@ runbooks: Perform comprehensive diagnostic analysis for this Azure Monitor alert using a systematic approach: 1. ALERT CONTEXT ANALYSIS: - - Extract and analyze the alert details: metric name, **PromQL query**, **rule description**, threshold, severity, and current state + - Extract and analyze the alert details: metric name, query, threshold, severity, and current state - Identify the timeframe when the alert fired and duration - - **CRITICAL: Note the extracted PromQL query from the alert - this is the exact query that triggered the alert** - - **ESSENTIAL: Read the rule description to understand what the alert is monitoring and why it's important** - Determine the affected resources (pods, nodes, services, namespaces) from the alert - - Understand what the alert is measuring and why it triggered based on both query and description + - Understand what the alert is measuring and why it triggered - 2. EXECUTE ALERT'S ORIGINAL QUERY FOR TIMELINE ANALYSIS: - - **MANDATORY: Use `execute_alert_promql_query` tool with the alert ID to run the exact query that triggered the alert** - - Start with 1-2 hour time range to see recent trends: `execute_alert_promql_query` with time_range "2h" - - Extend to longer periods (6h, 1d) if needed to identify patterns - - This shows you the exact metric behavior that caused the alert to fire - - Compare the timeline with the alert's fired time to understand the progression - - 3. CURRENT STATE ASSESSMENT: + 2. CURRENT STATE ASSESSMENT: - Use kubectl commands to check the current status of affected resources - - Use `execute_azuremonitor_prometheus_query` for instant/current values of the alert metric + - Query current values of the alert metric using Azure Monitor Prometheus queries - Compare current values with the alert threshold to see if issue persists - Check if the alert is still active or has resolved @@ -33,22 +24,10 @@ runbooks: - Identify any resource constraints or scheduling issues 4. METRIC CORRELATION AND TRENDS: - - Use `execute_azuremonitor_prometheus_range_query` for related metrics around the alert timeframe - - **Analyze the alert's query timeline first**, then expand to related metrics - - Query CPU: `rate(container_cpu_usage_seconds_total[5m])` with range queries - - Query Memory: `container_memory_working_set_bytes` with range queries - - Query Pod Status: `kube_pod_status_phase` to check pod states over time - - Query Deployment Status: `kube_deployment_status_replicas` vs `kube_deployment_spec_replicas` - - Use time ranges that cover the alert period plus some buffer (2h, 4h, 6h) + - Query related Azure Monitor Prometheus metrics around the alert timeframe + - Analyze trends for the last 1-2 hours to understand the pattern leading to the alert + - Correlate with other important metrics (CPU, memory, network, disk) to find relationships - Look for sudden spikes, gradual increases, or cyclical patterns - - **Timeline Correlation**: Compare all metric timelines with the alert fired time - - 4.1. SPECIFIC QUERY ANALYSIS TECHNIQUES: - - **For deployment alerts**: Query replica mismatches and pod creation/deletion patterns - - **For resource alerts**: Compare requests vs limits vs actual usage over time - - **For node alerts**: Check node conditions and resource pressure metrics - - **For application alerts**: Correlate with request rates, error rates, response times - - **Use step intervals** appropriate for the time range (60s for detailed, 300s for longer periods) 5. EVENT TIMELINE ANALYSIS: - Check Kubernetes events around the alert firing time using kubectl @@ -168,88 +147,3 @@ runbooks: - Look for configuration map or secret mounting issues Follow the standard diagnostic steps but emphasize pod lifecycle and scheduling analysis. - - - match: - issue_name: ".*[Dd]eployment.*[Rr]eplica.*" - source_type: "azuremonitoralerts" - instructions: > - This is a deployment replica mismatch alert (like KubeDeploymentReplicasMismatch). Focus your analysis on: - - 1. DEPLOYMENT REPLICA ANALYSIS: - - **FIRST: Use `execute_alert_promql_query` with the alert ID to see the replica mismatch timeline** - - Query `kube_deployment_status_replicas` vs `kube_deployment_spec_replicas` with range queries - - Query `kube_deployment_status_ready_replicas` to see how many replicas are actually ready - - Query `kube_deployment_status_available_replicas` to check availability - - Use 2-6 hour time ranges to understand the pattern of replica mismatches - - 2. POD CREATION AND LIFECYCLE: - - Query `kube_pod_status_phase` for the affected deployment to see pod states over time - - Check for pod creation/deletion patterns with `rate(kube_pod_created[5m])` - - Look for pods stuck in Pending, ContainerCreating, or other waiting states - - Use kubectl to check current pod status and recent events - - 3. RESOURCE AND SCALING ANALYSIS: - - Check if resource constraints are preventing pod creation - - Query node capacity: `kube_node_status_allocatable` for CPU/memory - - Query resource requests vs availability - - Check if HPA (Horizontal Pod Autoscaler) is affecting replica counts - - Look for node pressure or taints preventing scheduling - - 4. DEPLOYMENT CONFIGURATION: - - Check deployment strategy (RollingUpdate vs Recreate) - - Verify resource requests and limits are reasonable - - Check for pod disruption budgets that might limit scaling - - Look for node selectors or affinity rules affecting placement - - 5. TIMELINE CORRELATION: - - Compare replica mismatch timing with deployment rollouts or scaling events - - Look for infrastructure changes or node issues at the same time - - Check if the mismatch is temporary (during deployments) or persistent - - Correlate with resource pressure or cluster capacity issues - - 6. ROOT CAUSE IDENTIFICATION: - - Distinguish between temporary scaling delays vs persistent issues - - Identify if pods can't start due to resources, images, or configuration - - Check if the deployment is stuck in a rollout or experiencing constant restarts - - Determine if this is a capacity planning issue or a configuration problem - - Follow the standard diagnostic steps but emphasize deployment scaling and pod lifecycle analysis. - - - match: - issue_name: ".*[Cc]ontainer.*[Ww]aiting.*" - source_type: "azuremonitoralerts" - instructions: > - This is a container waiting alert (like KubeContainerWaiting). Focus your analysis on: - - 1. CONTAINER STATE ANALYSIS: - - **FIRST: Use `execute_alert_promql_query` with the alert ID to see the waiting container timeline** - - Query `kube_pod_container_status_waiting_reason` to identify specific waiting reasons - - Query `kube_pod_container_status_waiting` over time to see patterns - - Use kubectl describe pod to get detailed container status and events - - 2. WAITING REASON INVESTIGATION: - - **ImagePullBackOff**: Check image availability, registry access, pull secrets - - **CrashLoopBackOff**: Examine container logs for startup failures - - **CreateContainerConfigError**: Check ConfigMap/Secret references - - **InvalidImageName**: Verify image names and tags - - **ContainerCreating**: Check resource availability and volume mounts - - 3. IMAGE AND REGISTRY ANALYSIS: - - Verify image exists in the specified registry - - Check image pull secrets and registry authentication - - Test registry connectivity from nodes - - Check for image size issues or registry rate limiting - - 4. RESOURCE AND CONFIGURATION: - - Check resource requests vs node availability - - Verify volume mounts and persistent volume claims - - Check security contexts and admission controllers - - Look for init container dependencies - - 5. TIMELINE AND PATTERN ANALYSIS: - - Check if waiting is consistent or intermittent - - Correlate with deployment rollouts or configuration changes - - Look for node-specific issues (does it happen on specific nodes?) - - Check for cluster-wide vs application-specific problems - - Follow the standard diagnostic steps but emphasize container startup and configuration analysis. diff --git a/holmes/plugins/runbooks/catalog.json b/holmes/plugins/runbooks/catalog.json index 7550db5592..a88043535b 100644 --- a/holmes/plugins/runbooks/catalog.json +++ b/holmes/plugins/runbooks/catalog.json @@ -4,6 +4,21 @@ "update_date": "2025-06-17", "description": "Runbook to investigate DNS resolution issue on Kubernetes cluster", "link": "networking/dns_troubleshooting_instructions.md" + }, + { + "update_date": "2025-07-16", + "description": "Runbook to investigate DNS resolution issue on Kubernetes cluster using Azure Monitor and Prometheus", + "link": "networking/dns_troubleshooting_prometheus_instructions.md" + }, + { + "update_date": "2025-07-16", + "description": "Runbook to investigate Node readiness issues on Kubernetes cluster using Azure Monitor and Prometheus", + "link": "node_troubleshooting_prometheus_instructions.md" + }, + { + "update_date": "2025-07-16", + "description": "Runbook to investigate pod scheduling issue on Kubernetes cluster using Azure Monitor and Prometheus", + "link": "pod_troubleshooting_prometheus_instructions.md" } ] } diff --git a/holmes/plugins/runbooks/networking/dns_troubleshooting_prometheus_instructions.md b/holmes/plugins/runbooks/networking/dns_troubleshooting_prometheus_instructions.md new file mode 100644 index 0000000000..c40ffae48f --- /dev/null +++ b/holmes/plugins/runbooks/networking/dns_troubleshooting_prometheus_instructions.md @@ -0,0 +1,86 @@ +# DNS Resolution Failures – Troubleshooting Runbook (Kubernetes) + +## Goal + +Diagnose and remediate DNS resolution failures within a Kubernetes cluster. Focus on identifying misconfigurations, upstream resolver issues, or blocked traffic leading to DNS errors (e.g., `SERVFAIL`, `NXDOMAIN`, `REFUSED`, `NotImplemented`, `DNSSEC/EDNS` errors). + +## Workflow + +### 1. **Detect and Quantify DNS Failures** + +* Use Prometheus metrics from the AzureMonitorMetrics toolset. + +**PromQL Queries** + +Node level metrics + +| Metric name | Description | Extra labels | +|--------------------------------|------------------------------|------------------------| +| **cilium_forward_count_total** | Total forwarded packet count | `direction` | +| **cilium_forward_bytes_total** | Total forwarded byte count | `direction` | +| **cilium_drop_count_total** | Total dropped packet count | `direction`, `reason` | +| **cilium_drop_bytes_total** | Total dropped byte count | `direction`, `reason` | + +Pod level metrics + +| Metric name | Description | Extra Labels | +|----------------------------------|---------------------------------------------|------------------------------------------------------------------------------| +| **hubble_dns_queries_total** | Total DNS requests by query | `source` or `destination`, `query`, `qtypes` (query type) | +| **hubble_dns_responses_total** | Total DNS responses by query/response | `source` or `destination`, `query`, `qtypes` (query type), `rcode`, `ips_returned` | +| **hubble_drop_total** | Total dropped packet count | `source` or `destination`, `protocol`, `reason` | +| **hubble_tcp_flags_total** | Total TCP packets count by flag | `source` or `destination`, `flag` | +| **hubble_flows_processed_total** | Total network flows processed (L4/L7 traffic) | `source` or `destination`, `protocol`, `verdict`, `type`, `subtype` | + + +Query the metrics to determine if there are spikes in errors and where the errors are associated based on node or pod. + +--- + + +## Synthesize Findings + +Use the combination of metrics and logs to clearly state: + +> **"Pods in `X` namespace are experiencing NXDOMAIN errors due to misconfigured `nameserver` entries in `/etc/resolv.conf`."** + +> **"CoreDNS is returning `SERVFAIL` for upstream lookups—logs show timeout errors; likely due to unreachable Azure DNS servers."** + +> **"High latency and spike in `REFUSED` errors from debug pods in `team-a-ns`, combined with recent NetworkPolicy changes."** + +--- + +## Remediation Actions + +### Immediate Fixes + +| Symptom | Action | +| ------------------------ | ---------------------------------------------------------------------- | +| CoreDNS plugin/misconfig | Revert ConfigMap: `kubectl rollout undo deploy coredns -n kube-system` | +| CoreDNS crashloop | Check logs → Restart pod → Scale replicas | +| NetworkPolicy blocks | Inspect and patch rules allowing DNS (port 53 UDP/TCP) | +| Upstream DNS unreachable | Update `forward` plugin in CoreDNS to fallback DNS (e.g., 8.8.8.8) | +| DNS saturation | Scale CoreDNS, reduce noisy traffic | +| NXDOMAIN/typos | Check domain spelling, app DNS caching behavior | + +--- + +### Mid-Range Mitigation + +| Risk Area | Recommended Action | +| -------------------- | ------------------------------------------------------ | +| CoreDNS config drift | Use GitOps, `kubectl diff`, or webhook validation | +| Low observability | Enable alerts on error spikes (rcode ≠ 0) | +| Noisy apps | Rate-limit retries in app logic | +| No ownership | Tag CoreDNS with owner, set alerts for team escalation | + +--- + +## Prevention and Best Practices + +| Domain | Strategy | +| -------------------- | ------------------------------------------------------------------ | +| **Testing** | Add DNS checks in CI smoke tests | +| **Alerting** | Alerts on high `rcode≠0` error rate, p95 latency | +| **Runbooks** | Document common failures and resolution queries | +| **Automation** | Auto-notify teams on risky changes via Logic Apps/MCP | +| **Infra Resilience** | Deploy multiple DNS replicas, validate cross-zone DNS reachability | diff --git a/holmes/plugins/runbooks/node_troubleshooting_prometheus_instructions.md b/holmes/plugins/runbooks/node_troubleshooting_prometheus_instructions.md new file mode 100644 index 0000000000..e593533731 --- /dev/null +++ b/holmes/plugins/runbooks/node_troubleshooting_prometheus_instructions.md @@ -0,0 +1,108 @@ +# Node Not Ready – Troubleshooting Runbook (Kubernetes) + +## Goal + +Diagnose and remediate scenarios where one or more Kubernetes nodes report as `NotReady` or `Unknown`, resulting in pod scheduling failures and application downtime. + +--- + +## Workflow + +### 1. **Detect Node Failures** + +* Use Prometheus metrics and alerts from the AzureMonitorMetrics toolset. +* Typical symptoms include: + * Node in `NotReady` or `Unknown` status + * Unschedulable pods + * Firing alerts such as `KubeNodeUnreachable` + +**Core signals** + +| **Metric** | **Use** | **Extra Labels** | +|---------------------------------------------------------|-------------------------|----------------------------------| +| **kube_node_status_condition** | Detect NotReady status | `node`, `condition`, `status` | +| **kube_node_status_condition** | Detect Unknown status | `node`, `condition`, `status` | +| **container_memory_working_set_bytes** | Spot OOM conditions | `container`, `pod`, `namespace` | +| **node_filesystem_usage / node_filesystem_free_bytes** | Detect disk pressure | `device`, `mountpoint`, `fstype` | +| **node_disk_inode_utilization** | Detect inode pressure | `device`, `instance`, `job` | +| **container_cpu_usage_seconds_total** | Diagnose CPU starvation | `container`, `pod`, `namespace` | + + +--- + +### 2. **Contextual Signals and Metadata** + +Use metadata from kube-state-metrics and node-exporter to understand node specs and conditions. + +**Kube-State Metrics** + +| **Metric name** | **Description** | **Extra Labels** | +|--------------------------------|----------------------------|------------------------------------------| +| **kube_node_status_capacity** | Node resource capacity | `node`, `resource`, `unit` | +| **kube_node_status_condition** | Node status conditions | `node`, `condition`, `status` | +| **kube_node_status_allocatable** | Allocatable node resources | `node`, `resource`, `unit` | +| **kube_node_info** | Node OS/kernel/arch info | `node`, `architecture`, `kernel_version` | +| **kube_node_spec_taint** | Node taints | `node`, `key`, `effect` | + + +**Node Exporter Metrics** + +| **Metric name** | **Description** | **Extra Labels** | +|---------------------------------------|------------------------------------|--------------------------------------| +| **node_cpu_seconds_total** | CPU usage by mode and core | `cpu`, `mode`, `instance`, `job` | +| **node_memory_MemAvailable_bytes** | Available system memory | `instance`, `job` | +| **node_memory_Cached_bytes** | Cached memory | `instance`, `job` | +| **node_memory_MemFree_bytes** | Free memory | `instance`, `job` | +| **node_memory_Slab_bytes** | Slab allocator usage | `instance`, `job` | +| **node_memory_MemTotal_bytes** | Total system memory | `instance`, `job` | +| **node_netstat_Tcp_RetransSegs** | TCP retransmissions | `instance`, `job` | +| **node_load1 / load5 / load15** | System load averages | `instance`, `job` | +| **node_disk_read_bytes_total** | Disk read throughput | `device`, `instance`, `job` | +| **node_disk_written_bytes_total** | Disk write throughput | `device`, `instance`, `job` | +| **node_disk_io_time_seconds_total** | Disk I/O wait time | `device`, `instance`, `job` | +| **node_filesystem_size_bytes** | Total filesystem capacity | `device`, `mountpoint`, `fstype` | +| **node_filesystem_avail_bytes** | Available filesystem capacity | `device`, `mountpoint`, `fstype` | +| **node_filesystem_readonly** | Read-only filesystem flags | `device`, `mountpoint`, `fstype` | +| **node_network_receive_bytes_total** | Network ingress throughput | `device`, `instance`, `job` | +| **node_network_transmit_bytes_total** | Network egress throughput | `device`, `instance`, `job` | +| **node_network_receive_drop_total** | Dropped inbound packets | `device`, `instance`, `job` | +| **node_network_transmit_drop_total** | Dropped outbound packets | `device`, `instance`, `job` | +| **node_vmstat_pgmajfault** | Major page faults | `instance`, `job` | +| **node_exporter_build_info** | Exporter build/version metadata | `version`, `instance`, `job` | +| **node_time_seconds** | System time | `instance`, `job` | +| **node_uname_info** | Host system name and version info | `nodename`, `machine`, `release` | + +--- + +## Synthesize Findings + +> **"Node `aks-nodepool1-xyz` is reporting `NotReady` due to memory pressure."** + +> **"Multiple nodes exhibit inode saturation, impacting pod scheduling."** + +> **"CNI and containerd processes are consuming high CPU; node capacity is insufficient for current workload."** + +--- + +## Remediation Actions + +### Immediate Fixes + +| Symptom | Remediation | +| ------------------------------ | ---------------------------------------------------------------------------------------- | +| Disk pressure on `/` or `/var` | SSH/DaemonSet cleanup, check for noisy logging pods, resize disks or enable log rotation | +| Memory pressure / kubelet OOM | Reduce pod memory requests, taint & evict, restart kubelet or reimage VMSS instance | +| High CPU usage | Identify and tune sidecars/CNIs, upgrade VM size, enable autoscaling | +| Network unreachable | Validate NSG/UdR, verify MTU/routing for CNI, reimage or force update via VMSS | + +--- + +## Prevention and Best Practices + +| Domain | Strategy | +| ------------------------- | ----------------------------------------------------------------- | +| **Capacity Mgmt** | Use VPA or autoscaler, align node sizes to workload profiles | +| **Alerting** | Alerts on node NotReady, disk pressure, memory usage thresholds | +| **Logging Hygiene** | Prevent excessive logs from apps; enforce log rotation policies | +| **Deployment Guardrails** | Use policies to prevent pod overcommit and noisy containers | +| **Node Upkeep** | Periodically rotate nodes via rolling upgrade or nodepool reimage | diff --git a/holmes/plugins/runbooks/pod_troubleshooting_prometheus_instructions.md b/holmes/plugins/runbooks/pod_troubleshooting_prometheus_instructions.md new file mode 100644 index 0000000000..0711f9fefa --- /dev/null +++ b/holmes/plugins/runbooks/pod_troubleshooting_prometheus_instructions.md @@ -0,0 +1,110 @@ +# Pod Scheduling Issues – Troubleshooting Runbook (Kubernetes) + +## Goal + +Diagnose and remediate pod scheduling issues in Kubernetes clusters. These issues typically manifest as pods stuck in `Pending`, or failing with `CrashLoopBackOff`, `ImagePullBackOff`, or `CreateContainerConfigError`. These are often silent until escalated by users or backlog pressure in deployments. + +--- + +## Workflow + +### 1. **Detect Pod Scheduling Failures** + +Use Prometheus metrics and kube-state metrics to detect stalled or failing pods. + +#### Pod-level kube-state metrics + +| **Metric name** | **Description** | **Extra Labels** | +| ---------------------------------------------------------- | ------------------------------------------ | ----------------------------------------- | +| **kube_pod_container_status_last_terminated_reason** | Last container termination reason | `pod`, `container`, `reason` | +| **kube_pod_container_status_restarts_total** | Total container restarts | `pod`, `container`, `namespace` | +| **kube_pod_container_resource_requests** | Requested resources per container | `pod`, `container`, `resource` | +| **kube_pod_status_phase** | Current pod phase (Pending, Running, etc.) | `pod`, `phase`, `namespace` | +| **kube_pod_container_resource_limits** | Resource limits per container | `pod`, `container`, `resource` | +| **kube_pod_info** | Static pod info including node & IP | `pod`, `node`, `namespace` | +| **kube_pod_owner** | Pod owner reference | `pod`, `owner_kind`, `owner_name` | +| **kube_pod_labels** | Pod labels | `pod`, `label_*`, `namespace` | +| **kube_pod_annotations** | Pod annotations | `pod`, `annotation_*`, `namespace` | +| **kube_pod_container_status_waiting_reason** | Waiting status reasons (e.g. ErrImagePull) | `pod`, `container`, `reason`, `namespace` | +| **kube_pod_container_info** | Co**_** | | + + +#### Container cAdvisor metrics + +| **Metric name** | **Description** | **Extra Labels** | +| --------------------------------------------------------- | ------------------------------------------- | -------------------------------------------- | +| **container_spec_cpu_period** | CPU period for CFS scheduler | `container`, `pod`, `namespace`, `image` | +| **container_spec_cpu_quota** | CPU quota for CFS scheduler | `container`, `pod`, `namespace`, `image` | +| **container_cpu_usage_seconds_total** | Total CPU time used by the container | `container`, `pod`, `namespace`, `mode` | +| **container_memory_rss** | Resident memory used | `container`, `pod`, `namespace` | +| **container_memory_working_set_bytes** | Working set memory (RSS + cache - inactive) | `container`, `pod`, `namespace` | +| **container_memory_cache** | Page cache memory | `container`, `pod`, `namespace` | +| **container_memory_swap** | Swap memory usage | `container`, `pod`, `namespace` | +| **container_memory_usage_bytes** | Total memory usage | `container`, `pod`, `namespace` | +| **container_cpu_cfs_throttled_periods_total** | CPU throttling count | `container`, `pod`, `namespace` | +| **container_cpu_cfs_periods_total** | Total CFS scheduling periods | `container`, `pod`, `namespace` | +| **container_network_receive_bytes_total** | Total network bytes received | `container`, `interface`, `namespace`, `pod` | +| **container_network_transmit_bytes_total** | Total network bytes sent | `container`, `interface`, `namespace`, `pod` | +| **container_network_receive_packets_total** | Network packets received | `container`, `interface`, `namespace`, `pod` | +| **container_network_transmit_packets_total** | Network packets sent | `container`, `interface`, `namespace`, `pod` | +| **container_network_receive_packets_dropped_total** | Dropped received packets | `container`, `interface`, `namespace`, `pod` | +| **container_network_transmit_packets_dropped_total** | Dropped sent packets | `container`, `interface`, `namespace`, `pod` | +| **container_fs_reads_total** | Filesystem read ops | `container`, `device`, `namespace`, `pod` | +| **container_fs_writes_total** | Filesystem write ops | `container`, `device`, `namespace`, `pod` | +| **container_fs_reads_bytes_total** | Bytes read from filesystem | `container`, `device`, `namespace`, `pod` | +| **container_fs_writes_bytes_total** | Bytes written to filesystem | `container`, `device`, `namespace`, `pod` | + +--- + +## Synthesize Findings + +The agent workflow includes: + +* Correlating metrics to cluster time range, pod, and node. +* Validating scheduling constraints: `requests`, `limits`, `nodeSelector`, `affinity`, `taints`, `zones`. +* Checking node labels and taints via `kube_node_labels` and `kube_node_taint`. +* Highlighting failed scheduling messages from Kubernetes Events. + +### Common Root Cause Patterns + +| Symptom | Likely Root Cause | +| --------------------------------------- | ---------------------------------------- | +| High retries + "insufficient cpu" | Resource over-request | +| "Taint not tolerated" | Missing toleration | +| "Didn't match selector" + 0 valid nodes | Node selector or affinity misalignment | +| Scheduling success + repeated crashes | Likely application bug or resource issue | + +--- + +## Remediation Actions + +### Immediate Fixes + +| Scenario | Action | +| ------------------- | ---------------------------------------------------------------- | +| Resource too high | Reduce CPU/memory requests or use larger node pool | +| Taint mismatch | Add toleration or remove taint from nodes | +| Affinity too strict | Loosen `requiredDuringScheduling` to `preferredDuringScheduling` | +| Zonal pinning issue | Remove zone constraints or redeploy in a healthy zone | +| Image pull issue | Fix image reference or credentials; retry | + +### Mid-Range Strategy + +| Fix Type | Strategy | +| ------------------------- | ----------------------------------------------------------------------- | +| Capacity planning | Prometheus dashboards for scheduler retries, pressure | +| Proactive taint mapping | Define node pool conventions for taint/toleration logic | +| Deployment best practices | Enforce CI checks for affinity, selector, and resource config | +| Priority class usage | Use `PriorityClass` and preemption for critical workloads | +| Retina visualizations | Analyze scheduler latency and constraints with Retina dashboard views | +| Auto-remediation | Logic App / Azure Function to auto-tag frequent failures (e.g. Pending) | + +--- + +## Prevention and Best Practices + +* Define standard limits/requests across namespaces. +* Periodically validate node inventory against workload requirements. +* Integrate scheduling error detection into GitOps and CI pipelines. +* Alert on prolonged Pending pods or spike in scheduler retry metrics. +* Ensure each node pool has clear workload purpose, label, and taint logic. \ No newline at end of file diff --git a/holmes/plugins/sources/azuremonitoralerts/__init__.py b/holmes/plugins/sources/azuremonitoralerts/__init__.py index 7689012d4a..73eba5e80f 100644 --- a/holmes/plugins/sources/azuremonitoralerts/__init__.py +++ b/holmes/plugins/sources/azuremonitoralerts/__init__.py @@ -14,9 +14,6 @@ class AzureMonitorAlertsSource(SourcePlugin): """Source plugin for Azure Monitor Prometheus metric alerts.""" - # Class-level flag to prevent repeated console output across instances - _console_output_shown = False - def __init__(self, cluster_resource_id: Optional[str] = None): self.cluster_resource_id = cluster_resource_id or get_aks_cluster_resource_id() self.access_token = None @@ -212,37 +209,13 @@ def convert_to_issue(self, source_alert: dict, rule_details: dict) -> Issue: alert_props = source_alert.get("properties", {}) essentials = alert_props.get("essentials", {}) - # Set the current alert name for Prometheus query extraction - # Try multiple possible fields for alert name - alert_name = (essentials.get("alertName") or - essentials.get("name") or - alert_props.get("name") or - source_alert.get("name", "Unknown")) - - logging.debug(f"Alert name sources: alertName='{essentials.get('alertName')}', " - f"essentials.name='{essentials.get('name')}', " - f"properties.name='{alert_props.get('name')}', " - f"root.name='{source_alert.get('name')}', " - f"final='{alert_name}'") - - self._current_alert_name = alert_name - - # Extract query and description from rule details + # Extract query from rule details query = self._extract_query_from_rule(rule_details) - rule_description = self._extract_description_from_rule(rule_details) - - # Output extracted information to console for verification (only once globally) - if not AzureMonitorAlertsSource._console_output_shown: - print(f"[Azure Monitor Alert] Alert: {alert_name}") - print(f"[Azure Monitor Alert] Query: {query}") - print(f"[Azure Monitor Alert] Rule Description: {rule_description}") - AzureMonitorAlertsSource._console_output_shown = True # Create formatted description description_parts = [ f"Alert: {essentials.get('alertName', 'Unknown')}", f"Description: {essentials.get('description', 'No description')}", - f"Rule Description: {rule_description}", f"Severity: {essentials.get('severity', 'Unknown')}", f"State: {essentials.get('alertState', 'Unknown')}", f"Fired Time: {essentials.get('firedDateTime', 'Unknown')}", @@ -259,7 +232,6 @@ def convert_to_issue(self, source_alert: dict, rule_details: dict) -> Issue: "cluster_resource_id": self.cluster_resource_id, "cluster_name": self.cluster_name, "extracted_query": query, - "extracted_description": rule_description, } return Issue( @@ -341,11 +313,6 @@ def _is_prometheus_alert_rule(self, rule_details: dict) -> bool: def _extract_query_from_rule(self, rule_details: dict) -> str: """Extract the Prometheus query from alert rule details.""" try: - # Check if this is a Prometheus rule group - if "prometheusRuleGroups" in rule_details.get("id", ""): - return self._extract_prometheus_query_from_rule_group(rule_details) - - # Fallback to original Azure Monitor metric alert parsing properties = rule_details.get("properties", {}) criteria = properties.get("criteria", {}) all_of = criteria.get("allOf", []) @@ -354,7 +321,9 @@ def _extract_query_from_rule(self, rule_details: dict) -> str: condition = all_of[0] # Take the first condition metric_name = condition.get("metricName", "") - # For Azure Monitor metric alerts, return the metric name as the query + # For now, return the metric name as the query + # In practice, Azure Monitor metric alerts may not have the full PromQL query + # but rather use Azure's metric query format if metric_name: return metric_name @@ -368,439 +337,6 @@ def _extract_query_from_rule(self, rule_details: dict) -> str: logging.warning(f"Failed to extract query from rule: {e}") return "Query extraction failed" - def _extract_description_from_rule(self, rule_details: dict) -> str: - """Extract the description from alert rule details.""" - try: - # Check if this is a Prometheus rule group - if "prometheusRuleGroups" in rule_details.get("id", ""): - return self._extract_prometheus_description_from_rule_group(rule_details) - - # Fallback to Azure Monitor metric alert description - properties = rule_details.get("properties", {}) - - # Try to get description from different possible fields - description = (properties.get("description") or - properties.get("summary") or - properties.get("displayName")) - - if description: - return description - - return "No description available" - - except Exception as e: - logging.warning(f"Failed to extract description from rule: {e}") - return "Description extraction failed" - - def _extract_prometheus_query_from_rule_group(self, rule_details: dict) -> str: - """Extract PromQL query from Prometheus rule group for a specific alert.""" - try: - # Get the alert name from the current alert context - alert_name = getattr(self, '_current_alert_name', '') - - if not alert_name or alert_name == "Unknown": - logging.warning(f"No valid alert name available for Prometheus query extraction: '{alert_name}'") - # Try to extract from the rule details directly - return self._extract_query_from_rule_group_directly(rule_details) - - # Parse alert name to extract rule group name and alert rule name - rule_group_name, alert_rule_name = self._parse_alert_name(alert_name) - - if not rule_group_name or not alert_rule_name: - logging.warning(f"Could not parse alert name: '{alert_name}', trying direct extraction") - # Fallback to direct extraction from rule details - return self._extract_query_from_rule_group_directly(rule_details) - - logging.info(f"Extracting PromQL query for rule group: '{rule_group_name}', alert rule: '{alert_rule_name}'") - - # Fetch the complete Prometheus rule group - prometheus_rule_group = self._fetch_prometheus_rule_group(rule_group_name) - - if not prometheus_rule_group: - logging.warning(f"Could not fetch Prometheus rule group: '{rule_group_name}', trying direct extraction") - return self._extract_query_from_rule_group_directly(rule_details) - - # Extract the specific alert rule and its query - promql_query = self._find_alert_rule_query(prometheus_rule_group, alert_rule_name) - - if not promql_query: - logging.warning(f"Could not find query for alert rule: '{alert_rule_name}', trying direct extraction") - return self._extract_query_from_rule_group_directly(rule_details) - - # Apply cluster filtering to the query - from holmes.plugins.toolsets.azuremonitor_metrics.utils import enhance_promql_with_cluster_filter - enhanced_query = enhance_promql_with_cluster_filter(promql_query, self.cluster_name) - - logging.info(f"Successfully extracted and enhanced PromQL query: {enhanced_query}") - return enhanced_query - - except Exception as e: - logging.error(f"Failed to extract Prometheus query from rule group: {e}") - # Final fallback - return self._extract_query_from_rule_group_directly(rule_details) - - def _extract_prometheus_description_from_rule_group(self, rule_details: dict) -> str: - """Extract description from Prometheus rule group for a specific alert.""" - try: - # Get the alert name from the current alert context - alert_name = getattr(self, '_current_alert_name', '') - - if not alert_name or alert_name == "Unknown": - logging.warning(f"No valid alert name available for Prometheus description extraction: '{alert_name}'") - # Try to extract from the rule details directly - return self._extract_description_from_rule_group_directly(rule_details) - - # Parse alert name to extract rule group name and alert rule name - rule_group_name, alert_rule_name = self._parse_alert_name(alert_name) - - if not rule_group_name or not alert_rule_name: - logging.warning(f"Could not parse alert name: '{alert_name}', trying direct extraction") - # Fallback to direct extraction from rule details - return self._extract_description_from_rule_group_directly(rule_details) - - logging.info(f"Extracting description for rule group: '{rule_group_name}', alert rule: '{alert_rule_name}'") - - # Fetch the complete Prometheus rule group - prometheus_rule_group = self._fetch_prometheus_rule_group(rule_group_name) - - if not prometheus_rule_group: - logging.warning(f"Could not fetch Prometheus rule group: '{rule_group_name}', trying direct extraction") - return self._extract_description_from_rule_group_directly(rule_details) - - # Extract the specific alert rule and its description - rule_description = self._find_alert_rule_description(prometheus_rule_group, alert_rule_name) - - if not rule_description: - logging.warning(f"Could not find description for alert rule: '{alert_rule_name}', trying direct extraction") - return self._extract_description_from_rule_group_directly(rule_details) - - logging.info(f"Successfully extracted rule description: {rule_description}") - return rule_description - - except Exception as e: - logging.error(f"Failed to extract Prometheus description from rule group: {e}") - # Final fallback - return self._extract_description_from_rule_group_directly(rule_details) - - def _extract_description_from_rule_group_directly(self, rule_details: dict) -> str: - """ - Fallback method to extract description information directly from rule details. - Used when alert name parsing fails or specific rule lookup fails. - """ - try: - properties = rule_details.get("properties", {}) - - # Try to find any useful description information in the rule details - if "rules" in properties: - rules = properties["rules"] - logging.info(f"Attempting direct description extraction from {len(rules)} rules in group") - - # Try to find any rule with a description - for i, rule in enumerate(rules): - rule_name = rule.get("alert", "") or rule.get("record", "") or f"rule_{i}" - - # Try different possible description fields - description = (rule.get("annotations", {}).get("description") or - rule.get("annotations", {}).get("summary") or - rule.get("description") or - rule.get("summary")) - - if description: - logging.info(f"Found description in rule '{rule_name}': {description}") - return description - - # If no descriptions found, provide generic info - return f"Prometheus rule group with {len(rules)} rules (no descriptions available)" - - # If no rules found, look for other indicators - rule_name = properties.get("name", "Unknown Rule Group") - return f"Prometheus rule group: {rule_name} (no rules found for description extraction)" - - except Exception as e: - logging.warning(f"Direct description extraction failed: {e}") - return f"Description extraction failed: {str(e)}" - - def _find_alert_rule_description(self, rule_group: dict, alert_rule_name: str) -> Optional[str]: - """ - Find the description for a specific alert rule within a Prometheus rule group. - - Args: - rule_group: Complete Prometheus rule group details - alert_rule_name: Name of the specific alert rule to find - - Returns: - str: Rule description if found, None otherwise - """ - try: - properties = rule_group.get("properties", {}) - rules = properties.get("rules", []) - - logging.info(f"Searching for alert rule description '{alert_rule_name}' in {len(rules)} rules") - - # Find the exact matching rule - for rule in rules: - rule_name = rule.get("alert", "") or rule.get("record", "") - - if rule_name == alert_rule_name: - # Found the matching rule, extract description - annotations = rule.get("annotations", {}) - - # Try different description fields in order of preference - description = (annotations.get("description") if annotations else None) or \ - (annotations.get("summary") if annotations else None) or \ - rule.get("description") or \ - rule.get("summary") - - if description: - logging.info(f"Found description for '{alert_rule_name}': {description}") - return description - else: - # Rule found but no description - available_fields = list(annotations.keys()) if annotations else [] - logging.warning(f"Alert rule '{alert_rule_name}' found but has no description. Available annotation fields: {available_fields}") - return "No description available for this rule" - - # If we get here, the alert rule was not found - available_rules = [rule.get("alert", "") or rule.get("record", "") for rule in rules] - logging.warning(f"Alert rule '{alert_rule_name}' not found for description extraction. Available rules: {available_rules}") - return None - - except Exception as e: - logging.error(f"Exception while finding alert rule description for '{alert_rule_name}': {e}") - return None - - def _extract_query_from_rule_group_directly(self, rule_details: dict) -> str: - """ - Fallback method to extract query information directly from rule details. - Used when alert name parsing fails or specific rule lookup fails. - """ - try: - properties = rule_details.get("properties", {}) - - # Try to find any useful query information in the rule details - if "rules" in properties: - rules = properties["rules"] - logging.info(f"Attempting direct extraction from {len(rules)} rules in group") - - # Try to find any rule with a query - for i, rule in enumerate(rules): - # Log rule structure for debugging - rule_name = rule.get("alert", "") or rule.get("record", "") or f"rule_{i}" - rule_type = "alert" if rule.get("alert") else "record" if rule.get("record") else "unknown" - - logging.debug(f"Examining rule '{rule_name}' (type: {rule_type})") - - # Try different possible query fields - query = (rule.get("expr") or - rule.get("expression") or - rule.get("query")) - - if query: - logging.info(f"Found query in rule '{rule_name}': {query}") - # Apply cluster filtering - from holmes.plugins.toolsets.azuremonitor_metrics.utils import enhance_promql_with_cluster_filter - enhanced_query = enhance_promql_with_cluster_filter(query, self.cluster_name) - return enhanced_query - else: - # Log available fields in the rule - available_fields = list(rule.keys()) - logging.debug(f"Rule '{rule_name}' has no query field. Available fields: {available_fields}") - - # If no queries found, provide info about available rules - rule_info = [] - for rule in rules: - rule_name = rule.get("alert", "") or rule.get("record", "") or "unnamed" - rule_type = "alert" if rule.get("alert") else "record" if rule.get("record") else "unknown" - rule_info.append(f"{rule_name} ({rule_type})") - - logging.warning(f"No queries found in any rules. Available rules: {rule_info}") - return f"Prometheus rule group with {len(rules)} rules but no extractable queries" - - # If no rules found, look for other indicators - rule_name = properties.get("name", "Unknown Rule Group") - logging.warning(f"No rules found in rule group properties") - return f"Prometheus rule group: {rule_name} (no rules found for query extraction)" - - except Exception as e: - logging.warning(f"Direct query extraction failed: {e}") - return f"Query extraction failed: {str(e)}" - - def _parse_alert_name(self, alert_name: str) -> tuple[str, str]: - """ - Parse alert name to extract rule group name and alert rule name. - - Args: - alert_name: Full alert name like "Prometheus Recommended Cluster level Alerts - vishwa-tme-1/KubeContainerWaiting" - - Returns: - tuple: (rule_group_name, alert_rule_name) - """ - try: - if "/" in alert_name: - # Split on the last "/" to separate rule group from alert rule - rule_group_name = alert_name.rsplit("/", 1)[0] # "Prometheus Recommended Cluster level Alerts - vishwa-tme-1" - alert_rule_name = alert_name.rsplit("/", 1)[1] # "KubeContainerWaiting" - return rule_group_name.strip(), alert_rule_name.strip() - - # If no "/" found, assume the entire name is the alert rule name - return "", alert_name.strip() - - except Exception as e: - logging.warning(f"Failed to parse alert name '{alert_name}': {e}") - return "", "" - - def _fetch_prometheus_rule_group(self, rule_group_name: str) -> Optional[dict]: - """ - Fetch Prometheus rule group details from Azure Monitor. - - Args: - rule_group_name: Name of the Prometheus rule group - - Returns: - dict: Rule group details if found, None otherwise - """ - try: - # Get access token for Azure Management API - access_token = self._get_azure_management_token() - if not access_token: - logging.error("Could not obtain Azure access token for management API") - return None - - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json", - "Accept": "application/json", - } - - # We need to find the rule group by name across all resource groups - # First, try to list all Prometheus rule groups in the subscription - list_url = f"https://management.azure.com/subscriptions/{self.subscription_id}/providers/Microsoft.AlertsManagement/prometheusRuleGroups" - - list_params = { - "api-version": "2023-03-01" - } - - response = requests.get( - url=list_url, - headers=headers, - params=list_params, - timeout=60 - ) - - if response.status_code != 200: - logging.error(f"Failed to list Prometheus rule groups: HTTP {response.status_code} - {response.text}") - return None - - rule_groups_list = response.json().get("value", []) - - # Find the rule group with matching name - target_rule_group = None - for rule_group in rule_groups_list: - rg_name = rule_group.get("name", "") - if rg_name == rule_group_name: - target_rule_group = rule_group - break - - if not target_rule_group: - logging.warning(f"Prometheus rule group '{rule_group_name}' not found in subscription") - return None - - # Get the full details of the specific rule group - rule_group_id = target_rule_group.get("id", "") - if not rule_group_id: - logging.error("Rule group ID not found") - return None - - detail_url = f"https://management.azure.com{rule_group_id}" - - detail_params = { - "api-version": "2023-03-01" - } - - detail_response = requests.get( - url=detail_url, - headers=headers, - params=detail_params, - timeout=60 - ) - - if detail_response.status_code == 200: - rule_group_details = detail_response.json() - logging.info(f"Successfully fetched Prometheus rule group: {rule_group_name}") - return rule_group_details - else: - logging.error(f"Failed to fetch rule group details: HTTP {detail_response.status_code} - {detail_response.text}") - return None - - except Exception as e: - logging.error(f"Exception while fetching Prometheus rule group '{rule_group_name}': {e}") - return None - - def _find_alert_rule_query(self, rule_group: dict, alert_rule_name: str) -> Optional[str]: - """ - Find the PromQL query for a specific alert rule within a Prometheus rule group. - - Args: - rule_group: Complete Prometheus rule group details - alert_rule_name: Name of the specific alert rule to find - - Returns: - str: PromQL query if found, None otherwise - """ - try: - properties = rule_group.get("properties", {}) - rules = properties.get("rules", []) - - logging.info(f"Searching for alert rule '{alert_rule_name}' in {len(rules)} rules") - - # First, try to find the exact matching rule - matching_rule = None - for rule in rules: - rule_name = rule.get("alert", "") or rule.get("record", "") - - if rule_name == alert_rule_name: - matching_rule = rule - break - - if matching_rule: - # Log the full rule structure for debugging - logging.debug(f"Found matching rule: {json.dumps(matching_rule, indent=2)}") - - # Try different possible query fields - promql_query = (matching_rule.get("expr") or - matching_rule.get("expression") or - matching_rule.get("query")) - - if promql_query: - logging.info(f"Found PromQL query for '{alert_rule_name}': {promql_query}") - return promql_query - else: - # Rule found but no query - log all available fields - available_fields = list(matching_rule.keys()) - logging.warning(f"Alert rule '{alert_rule_name}' found but has no query field. Available fields: {available_fields}") - - # Try to extract any query-like content - for field in ["expr", "expression", "query", "condition", "criteria"]: - if field in matching_rule and matching_rule[field]: - logging.info(f"Found potential query in '{field}' field: {matching_rule[field]}") - return str(matching_rule[field]) - - return None - - # If we get here, the alert rule was not found - available_rules = [] - for rule in rules: - rule_name = rule.get("alert", "") or rule.get("record", "") - rule_type = "alert" if rule.get("alert") else "record" if rule.get("record") else "unknown" - available_rules.append(f"{rule_name} ({rule_type})") - - logging.warning(f"Alert rule '{alert_rule_name}' not found. Available rules: {available_rules}") - return None - - except Exception as e: - logging.error(f"Exception while finding alert rule query for '{alert_rule_name}': {e}") - return None - def _get_azure_management_token(self) -> Optional[str]: """Get Azure access token for Azure Management API.""" try: diff --git a/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_instructions.jinja2 b/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_instructions.jinja2 index 35d8b90b11..586ed0992d 100644 --- a/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_instructions.jinja2 +++ b/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_instructions.jinja2 @@ -60,8 +60,8 @@ When investigating AKS issues, consider querying: - Infrastructure metrics - Network metrics -### 7. Alert Investigation Workflow with Query Analysis: -**IMPORTANT**: When users ask about Azure Monitor alerts, use this comprehensive approach: +### 7. Alert Investigation Workflow: +**IMPORTANT**: When users ask about Azure Monitor alerts, use this two-step approach: **Step 1 - List Active Alerts:** - **MANDATORY: Use ONLY the `get_active_prometheus_alerts` tool for Azure Monitor alerts** @@ -70,62 +70,23 @@ When investigating AKS issues, consider querying: - **DO NOT create your own alert summary - show the tool output verbatim** - **Show the complete formatted output with icons, markdown, and styling exactly as returned** - The tool output includes: - - Beautiful formatting with icons and professional layout (🔔 🚨 📋 ⚡ 📖 etc.) + - Beautiful formatting with icons and professional layout (🔔 🚨 📋 ⚡ etc.) - Alert names and full Alert IDs (complete Azure resource paths in code blocks) - - **PromQL queries that triggered each alert** - these are crucial for investigation - - **Rule descriptions explaining what each alert monitors** - essential context for understanding alerts - - Alert descriptions, rule IDs, severity, status, fired times - - The exact command instructions for investigation - -**Step 2 - Execute Alert's Original Query for Timeline Analysis:** -When investigating a specific alert, **ALWAYS use the `execute_alert_promql_query` tool** to: -- Execute the exact PromQL query that triggered the alert -- Analyze the timeline of the metric that caused the alert -- Use different time ranges (1h, 2h, 6h, 1d) to see trends -- Example: `execute_alert_promql_query` with alert_id and time_range "2h" - -**Step 3 - Deep Investigation with Custom Queries:** -Based on the alert's query and timeline, create related queries: -- **Trend analysis**: Use `execute_azuremonitor_prometheus_range_query` to analyze trends -- **Current state**: Use `execute_azuremonitor_prometheus_query` for instant values -- **Related metrics**: Query related metrics to understand root cause -- **Resource analysis**: Query CPU, memory, disk, network metrics around the alert time - -**Step 4 - Timeline Correlation:** -- Compare the alert's fired time with metric trends -- Look for patterns before and after the alert triggered -- Identify if the issue is ongoing or resolved -- Find correlations with other system metrics - -**Query Investigation Best Practices:** -1. **Start with the alert's query**: Always execute the original PromQL query first -2. **Expand time range**: Look at longer periods (6h, 1d) to see patterns -3. **Investigate related metrics**: Query CPU, memory, pod status, deployment status -4. **Use range queries for trends**: Historical data shows patterns better than instant values -5. **Cross-reference timing**: Compare alert fired time with metric spikes/drops - -**Example Investigation Flow:** -``` -1. List alerts → get_active_prometheus_alerts -2. See alert query: kube_deployment_status_replicas{cluster="myaks"} != kube_deployment_spec_replicas{cluster="myaks"} -3. Execute alert query → execute_alert_promql_query (alert_id, time_range="2h") -4. Analyze deployment status → execute_azuremonitor_prometheus_range_query with deployment queries -5. Check pod status → query kube_pod_status_phase for related pods -6. Investigate resources → query CPU/memory metrics for the timeframe -``` - -**Timeline Analysis Questions to Answer:** -- When did the metric first breach the threshold? -- Is the issue ongoing or resolved? -- What other metrics show anomalies at the same time? -- Are there patterns or recurring issues? -- What was the system state before and after the alert? + - Rule IDs, queries, descriptions, severity, status, fired times + - The exact command instructions for Step 2 +- **If the tool returns formatted output, copy it exactly - do not paraphrase or summarize** + +**Step 2 - Investigate Specific Alert:** +- Tell the user to copy the Alert ID from the detailed list +- Explain they should run: `holmes investigate azuremonitormetrics ` +- This provides focused AI-powered root cause analysis for the selected alert + +**Do NOT suggest investigating all alerts at once** - the workflow is designed for targeted investigation. **Display Format Requirements:** - Always show the complete tool output with full Alert IDs -- Include the instruction text with exact tool names -- Present all alert details (severity, status, fired time, **query**, etc.) -- Highlight the PromQL queries as they're essential for investigation +- Include the instruction text with the exact holmes investigate command +- Present all alert details (severity, status, fired time, query, etc.) - Do not abbreviate or summarize the alert information ### 8. Time Range Considerations: diff --git a/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_toolset.py b/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_toolset.py index a106198cd4..36fec5a4da 100644 --- a/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_toolset.py +++ b/holmes/plugins/toolsets/azuremonitor_metrics/azuremonitor_metrics_toolset.py @@ -519,7 +519,6 @@ def _invoke(self, params: Any) -> StructuredToolResult: issue = issues[i-1] # Get the corresponding issue alert_raw_data = issue.raw if hasattr(issue, 'raw') else {} query = alert_raw_data.get("extracted_query", "Not available") - rule_description = alert_raw_data.get("extracted_description", "Not available") # Choose icon based on severity severity = alert['severity'] @@ -565,8 +564,7 @@ def _invoke(self, params: Any) -> StructuredToolResult: summary_lines.append(f" 📋 **Alert ID:** `{alert['alert_id']}`") summary_lines.append(f" 🔬 **Type:** Prometheus Metric Alert") summary_lines.append(f" ⚡ **Query:** `{query}`") - summary_lines.append(f" 📖 **Rule Description:** {rule_description}") - summary_lines.append(f" 📝 **Alert Description:** {alert['description']}") + summary_lines.append(f" 📝 **Description:** {alert['description']}") summary_lines.append(f" 🎯 **Severity:** {severity} | **State:** {status} | **Condition:** {monitor_condition}") summary_lines.append(f" 🕒 **Fired Time:** {time_str}") @@ -660,228 +658,6 @@ def get_parameterized_one_liner(self, params) -> str: return f"Get specific Prometheus alert {alert_id} for cluster: {cluster_id}" return f"Get active Prometheus alerts for cluster: {cluster_id}" -class ExecuteAlertPromQLQuery(BaseAzureMonitorMetricsTool): - """Tool to execute the original PromQL query from an Azure Monitor alert for investigation.""" - - def __init__(self, toolset: "AzureMonitorMetricsToolset"): - super().__init__( - name="execute_alert_promql_query", - description="Execute the original PromQL query from an Azure Monitor alert to investigate alert conditions. This tool extracts the exact query that triggered the alert and executes it to help with root cause analysis.", - parameters={ - "alert_id": ToolParameter( - description="Alert ID to extract and execute the PromQL query from", - type="string", - required=True, - ), - "time_range": ToolParameter( - description="Time range for the query execution (e.g., '1h', '2h', '6h', '1d'). Defaults to 1 hour if not specified.", - type="string", - required=False, - ), - "cluster_resource_id": ToolParameter( - description="Azure resource ID of the AKS cluster (optional, will use configured cluster if not provided)", - type="string", - required=False, - ), - }, - toolset=toolset, - ) - - def _invoke(self, params: Any) -> StructuredToolResult: - try: - alert_id = get_param_or_raise(params, "alert_id") - time_range = params.get("time_range", "1h") - cluster_resource_id = params.get("cluster_resource_id") - - # Use configured cluster resource ID if not provided as parameter - if not cluster_resource_id and self.toolset.config: - cluster_resource_id = self.toolset.config.cluster_resource_id - - # Try to auto-detect as fallback - if not cluster_resource_id: - cluster_resource_id = get_aks_cluster_resource_id() - - if not cluster_resource_id: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error="No AKS cluster specified. Please provide cluster_resource_id parameter or configure it in your config.yaml file.", - params=params, - ) - - # Fetch the specific alert to get its query - from holmes.plugins.sources.azuremonitoralerts import AzureMonitorAlertsSource - - try: - source = AzureMonitorAlertsSource(cluster_resource_id=cluster_resource_id) - issue = source.fetch_issue(alert_id) - - if not issue: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error=f"Alert {alert_id} not found or is not a Prometheus metric alert for the cluster.", - params=params, - ) - - # Extract the PromQL query from the alert - raw_data = issue.raw if hasattr(issue, 'raw') else {} - extracted_query = raw_data.get("extracted_query", "") - - if not extracted_query or extracted_query in ["Query not available", "Query extraction failed"]: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error=f"Could not extract PromQL query from alert {alert_id}. Query: {extracted_query}", - params=params, - ) - - # Check if Azure Monitor workspace is configured - if not self.toolset.config or not self.toolset.config.azure_monitor_workspace_endpoint: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error="Azure Monitor workspace is not configured. Run check_azure_monitor_prometheus_enabled first.", - params=params, - ) - - # Convert time range to seconds for range query - time_seconds = self._parse_time_range(time_range) - if not time_seconds: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error=f"Invalid time range format: {time_range}. Use formats like '1h', '2h', '6h', '1d'.", - params=params, - ) - - # Execute the query as a range query - cluster_name = self._ensure_cluster_name_available() - - # Print information about what we're doing - print(f"[Azure Monitor] Executing alert's original PromQL query: {extracted_query}") - print(f"[Azure Monitor] Time range: {time_range} ({time_seconds} seconds)") - print(f"[Azure Monitor] Alert: {issue.name}") - - # Calculate start and end times - end_time = datetime.now() - start_time = end_time - dateutil.relativedelta.relativedelta(seconds=time_seconds) - - start_rfc3339 = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') - end_rfc3339 = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') - - # Calculate appropriate step size - step = max(time_seconds // 100, 60) # At least 1 minute, roughly 100 data points - - url = urljoin(self.toolset.config.azure_monitor_workspace_endpoint, "api/v1/query_range") - - payload = { - "query": extracted_query, - "start": start_rfc3339, - "end": end_rfc3339, - "step": step, - } - - # Get authenticated headers - headers = self.toolset._get_authenticated_headers() - - response = requests.post( - url=url, - headers=headers, - data=payload, - timeout=120 - ) - - if response.status_code == 200: - data = response.json() - status = data.get("status") - error_message = None - - if status == "success": - result_data = data.get("data", {}) - if not result_data.get("result"): - status = "no_data" - error_message = "The alert's query returned no results. This might indicate the issue has been resolved or the query needs different parameters." - else: - error_message = data.get("error", "Unknown error from Prometheus endpoint") - - response_data = { - "status": status, - "error_message": error_message, - "tool_name": self.name, - "alert_id": alert_id, - "alert_name": issue.name, - "extracted_query": extracted_query, - "time_range": time_range, - "start": start_rfc3339, - "end": end_rfc3339, - "step": step, - "cluster_name": cluster_name, - "description": f"Executed original PromQL query from alert '{issue.name}' over {time_range} time range" - } - - if self.toolset.config.tool_calls_return_data: - response_data["data"] = data.get("data") - - result_status = ToolResultStatus.SUCCESS - if status == "no_data": - result_status = ToolResultStatus.NO_DATA - elif status != "success": - result_status = ToolResultStatus.ERROR - - return StructuredToolResult( - status=result_status, - data=json.dumps(response_data, indent=2), - params=params, - ) - else: - error_msg = f"HTTP {response.status_code}: {response.text}" - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error=f"Azure Monitor Prometheus query failed: {error_msg}", - params=params, - ) - - except Exception as source_error: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error=f"Failed to fetch alert or execute query: {str(source_error)}", - params=params, - ) - - except Exception as e: - return StructuredToolResult( - status=ToolResultStatus.ERROR, - error=f"Failed to execute alert PromQL query: {str(e)}", - params=params, - ) - - def _parse_time_range(self, time_range: str) -> Optional[int]: - """Parse time range string to seconds.""" - try: - time_range = time_range.lower().strip() - - # Extract number and unit - import re - match = re.match(r'^(\d+)([smhd])$', time_range) - if not match: - return None - - value = int(match.group(1)) - unit = match.group(2) - - multipliers = { - 's': 1, - 'm': 60, - 'h': 3600, - 'd': 86400 - } - - return value * multipliers.get(unit, 1) - - except Exception: - return None - - def get_parameterized_one_liner(self, params) -> str: - alert_id = params.get("alert_id", "") - time_range = params.get("time_range", "1h") - return f"Execute alert's PromQL query: alert_id={alert_id}, time_range={time_range}" - class ExecuteAzureMonitorPrometheusRangeQuery(BaseAzureMonitorMetricsTool): """Tool to execute range PromQL queries against Azure Monitor workspace. ALWAYS display the EXACT query in the result to the user."""