changing latency slo format to match sloth and adding unti tests and …

…make target
Kuadrant · May 3, 2024 · 6b77aaa · 6b77aaa
1 parent 65c22e6
commit 6b77aaa
Show file tree

Hide file tree

Showing 8 changed files with 491 additions and 395 deletions.
diff --git a/examples/alerts/README.md b/examples/alerts/README.md
@@ -0,0 +1,35 @@
+## SLO Multi burn rate multi window alerts
+Kuadrant have created two example SLO alerts to help give ideas on the types of SLO alerts that could be used with the operator. We have created one alert for latency and one for availability, both are Multiwindow, Multi-Burn-Rate Alerts. The alerts show a scenario where a 28d rolling window is used and a uptime of 99.95% i.e only .1% error margin is desired. This in real world time would be downtime of around. These values can be changed to suit different scenarios
+
+| Time Frame | Duration   |
+|------------|------------|
+| Daily:     | 43s        |
+| Weekly:    | 5m 2.4s    |
+| Monthly:   | 21m 44s    |
+| Quarterly: | 1h 5m 12s  |
+| Yearly:    | 4h 20m 49s |
+
+### Sloth
+Sloth is a tool to aid in the creation of multi burn rate and multi window SLO alerts and was used to create both the availability and latency alerts. It follows the common standard set out by [Google's SRE book](https://sre.google/workbook/implementing-slos/). Sloth generates alerts based on specific specs given. The specs for our example alerts can be found in the example/sloth folder.
+
+#### Metrics used for the alerts
+
+#### Availability
+For the availability SLO alerts the Istio metric `istio_requests_total` was used as its a counter type metric meaning the values can only increase as well as it gives information on all requests handled by the Istio proxy.
+
+#### Latency
+For the availability SLO alerts the Istio metric 'istio_requests_total' was used as its a Distribution type metric meaning values are mapped to different ranges of frequency's as well as it gives information duration of requests.
+
+### Sloth generation
+You can modify the examples Sloth specs we have and regenerate the prometheus rules using the Sloth CLI and the generate command. For more information please the [Sloth website](https://sloth.dev/usage/cli/)
+
+```
+generate -i examples/alerts/sloth/latency.yaml --default-slo-period=28d
+```
+
+### Prometheus unit tests
+There are also two matching unit tests to verify and test the alerts that Sloth has generated. These can be run using the make target:
+
+```
+make alerts-tests
+```
diff --git a/examples/alerts/slo-availability.yaml b/examples/alerts/slo-availability.yaml
diff --git a/examples/alerts/slo-latency.yaml b/examples/alerts/slo-latency.yaml
diff --git a/examples/alerts/sloth/availability.yaml b/examples/alerts/sloth/availability.yaml
@@ -23,4 +23,4 @@ slos:
       ticket_alert:
         labels:
           severity: warning
-          
+
diff --git a/examples/alerts/sloth/latency.yaml b/examples/alerts/sloth/latency.yaml
@@ -8,18 +8,22 @@ slos:
     description: "Multi window multi burn rate SLO based on latency for HTTP request responses."
     sli:
       events:
-        error_query: sum(rate(istio_request_duration_milliseconds_bucket{le="250", job="ingress-metrics-proxy", response_code="200" }[{{.window}}]) )by (request_host) 
+        error_query:  (
+                      sum(rate(istio_request_duration_milliseconds_count{job="ingress-metrics-proxy",response_code="200"}[{{.window}}]))by (request_host) 
+                      - 
+                      sum(rate(istio_request_duration_milliseconds_bucket{le="250", job="ingress-metrics-proxy", response_code="200" }[{{.window}}]) )by (request_host)
+                      )
         total_query: sum(rate(istio_request_duration_milliseconds_count{job="ingress-metrics-proxy",response_code="200"}[{{.window}}]))by (request_host)
     alerting:
       name: KuadrantlatencyHighErrorRate
       labels:
         category: "latency"
       annotations:
         # Overwrite default Sloth SLO alert summmary on ticket and page alerts.
-        summary: "High latency on HTTPRoute requests responses {{$labels.request_host}}"
+        summary: "High latency on HTTPRoute requests responses"
       page_alert:
         labels:
           severity: critical
       ticket_alert:
         labels:
-          severity: warning
+          severity: warning
diff --git a/examples/alerts/availability_test.yaml → examples/alerts/tests/availability_test.yaml b/examples/alerts/availability_test.yaml → examples/alerts/tests/availability_test.yaml
@@ -1,5 +1,5 @@
 rule_files:
-  - slo-availability.yaml
+  - /prometheus/availability-rules.yaml
 
 evaluation_interval: 1m
 
@@ -9,7 +9,7 @@ tests:
       - series: istio_requests_total{job="ingress-metrics-proxy",response_code="500"}
         values: "0+0x30 0+10x30"
       - series: istio_requests_total{job="ingress-metrics-proxy"}
-        values: "0+1x30 31+90x30"
+        values: "0+1x30 31+100x30"
     alert_rule_test:
       - eval_time: 30m
         alertname: KuadrantAvailabilityHighErrorRate
@@ -40,5 +40,6 @@ tests:
                   sloth_slo: requests-availability
               exp_annotations:
                   summary: High error rate on HTTPRoute requests responses
-                  title: (ticket) kuadrant requests-availability SLO error budget burn rate is too fast.          
+                  title: (ticket) kuadrant requests-availability SLO error budget burn rate is too fast.  
+
 
diff --git a/examples/alerts/latency_test.yaml → examples/alerts/tests/latency_test.yaml b/examples/alerts/latency_test.yaml → examples/alerts/tests/latency_test.yaml
@@ -1,15 +1,15 @@
 rule_files:
-  - slo-latency.yaml
+  - /prometheus/latency-rules.yaml
 
 evaluation_interval: 1m
 
 tests:
   - interval: 1m
     input_series:
       - series: istio_request_duration_milliseconds_bucket{le="250",job="ingress-metrics-proxy",response_code="200"}
-        values: "0+0x30 31+10x30"
+        values: "0+1x30 31+10x30"
       - series: istio_request_duration_milliseconds_count{job="ingress-metrics-proxy",response_code="200"}
-        values: "0+1x30 31+90x30"
+        values: "0+1x30 31+100x30"
     alert_rule_test:
       - eval_time: 30m
         alertname: KuadrantlatencyHighErrorRate
@@ -20,22 +20,25 @@ tests:
             - exp_labels:
                   alertname: KuadrantlatencyHighErrorRate
                   category: latency
+                  owner: kuadrant-org
                   severity: critical
-                  sloth_id: kuadrant-kuadrant-requests-latency
+                  sloth_id: kuadrant-requests-latency
                   sloth_service: kuadrant
                   sloth_severity: page
-                  sloth_slo: kuadrant-requests-latency
+                  sloth_slo: requests-latency
               exp_annotations:
                   summary: High latency on HTTPRoute requests responses 
-                  title: (page) kuadrant kuadrant-requests-latency SLO error budget burn rate is too fast.
+                  title: (page) kuadrant requests-latency SLO error budget burn rate is too fast.
             - exp_labels:
                   alertname: KuadrantlatencyHighErrorRate
                   category: latency
+                  owner: kuadrant-org
                   severity: warning
-                  sloth_id: kuadrant-kuadrant-requests-latency
+                  sloth_id: kuadrant-requests-latency
                   sloth_service: kuadrant
                   sloth_severity: ticket
-                  sloth_slo: kuadrant-requests-latency
+                  sloth_slo: requests-latency
               exp_annotations:
                   summary: High latency on HTTPRoute requests responses 
-                  title: (ticket) kuadrant kuadrant-requests-latency SLO error budget burn rate is too fast.         
+                  title: (ticket) kuadrant requests-latency SLO error budget burn rate is too fast.       
+
diff --git a/make/alerts.mk b/make/alerts.mk
@@ -0,0 +1,25 @@
+export WORKDIR ?= $(shell pwd)
+export IMAGE ?= quay.io/prometheus/prometheus
+export AVAILABILITY_SLO_RULES ?= ${WORKDIR}/examples/alerts/slo-availability.yaml
+export LATENCY_SLO_RULES ?= ${WORKDIR}/examples/alerts/slo-latency.yaml
+export UNIT_TEST_DIR ?= ${WORKDIR}/examples/alerts/tests
+
+
+
+container-runtime-tool:
+	$(eval CONTAINER_RUNTIME_BIN := $(shell if command -v docker &>/dev/null; then \
+                                            echo "docker"; \
+                                        elif command -v podman &>/dev/null; then \
+                                            echo "podman"; \
+                                        else \
+                                            echo "Neither Docker nor Podman is installed. Exiting..."; \
+                                            exit 1; \
+                                        fi))
+
+
+alerts-tests: container-runtime-tool
+	$(CONTAINER_RUNTIME_BIN) run --rm -t \
+	-v $(AVAILABILITY_SLO_RULES):/prometheus/slo-availability.yaml \
+	-v $(LATENCY_SLO_RULES):/prometheus/slo-latency.yaml \
+    -v $(UNIT_TEST_DIR):/prometheus/tests --entrypoint=/bin/sh \
+$(IMAGE) -c 'tail -n +7 slo-latency.yaml > latency-rules.yaml  && tail -n +7 slo-availability.yaml > availability-rules.yaml && cd tests && promtool test rules *'