Merge pull request #546 from paulfantom/promscale-alerts

charts/promscale: updated alerting rules
timescale · Jan 18, 2023 · b15520e · b15520e
2 parents f69e597 + 02f799c
commit b15520e
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 14 deletions.
diff --git a/charts/promscale/Chart.yaml b/charts/promscale/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: promscale
 description: Promscale Connector deployment
 
-version: 14.6.1
+version: 14.7.0
 appVersion: 0.16.0
 
 home: https://github.com/timescale/promscale

diff --git a/charts/promscale/templates/prometheus-rule.yaml b/charts/promscale/templates/prometheus-rule.yaml
@@ -102,38 +102,55 @@ spec:
         runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleIngestHighLatency.md
     - alert: PromscaleIngestHighDataDuplication
       expr: |
-        rate(promscale_ingest_duplicates_total{kind="sample"}[5m]) > 0
-      for: 5m
+        sum by (job, namespace) (rate(promscale_ingest_duplicates_total{kind="sample"}[15m])) > 0
+      for: 30m
       labels:
         severity: warning
       annotations:
         summary: Duplicate data being inserted.
-        description: "More than {{ $value }} samples/sec are rejected as duplicates by promscale."
+        description: "More than {{ $value | humanize }} samples/sec are rejected as duplicates by promscale."
         runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleIngestHighDataDuplication.md
   - name: promscale-query
     rules:
     - alert: PromscaleQueryHighErrorRate
       expr: |
         (
-          sum by (job, instance, namespace, type) (
-            rate(promscale_query_requests_total{code=~"5.."}[5m])
+          sum by (job, instance, namespace, type, handler) (
+            rate(promscale_query_requests_total{code=~"5..",handler!="/api/v1/query_range",err!="canceled"}[5m])
           )
         /
-          sum by (job, instance, namespace, type) (
-            rate(promscale_query_requests_total[5m])
+          sum by (job, instance, namespace, type, handler) (
+            rate(promscale_query_requests_total{handler!="/api/v1/query_range",err!="canceled"}[5m])
           )
         ) > 0.05
       labels:
         severity: warning
       annotations:
         summary: High error rate in querying Promscale.
-        description: "Evaluating queries via Promscale has {{ $value | humanizePercentage }} error rate."
+        description: "Evaluating queries via Promscale {{ $labels.handler }} endpoint has {{ $value | humanizePercentage }} error rate."
+        runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleQueryHighErrorRate.md
+    - alert: PromscaleQueryHighErrorRate
+      expr: |
+        (
+          sum by (job, instance, namespace, type, handler) (
+            rate(promscale_query_requests_total{code=~"5..",handler="/api/v1/query_range",err!="canceled"}[5m])
+          )
+        /
+          sum by (job, instance, namespace, type, handler) (
+            rate(promscale_query_requests_total{handler="/api/v1/query_range",err!="canceled"}[5m])
+          )
+        ) > 0.1
+      labels:
+        severity: warning
+      annotations:
+        summary: High error rate in querying Promscale.
+        description: "Evaluating queries via Promscale {{ $labels.handler }} endpoint has {{ $value | humanizePercentage }} error rate."
         runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleQueryHighErrorRate.md
     - alert: PromscaleQueryHighErrorRate
       expr: |
         (
           sum by (job, instance, namespace, type) (
-            rate(promscale_query_requests_total{code=~"5.."}[5m])
+            rate(promscale_query_requests_total{code=~"5..",err!="canceled"}[5m])
           )
         /
           sum by (job, instance, namespace, type) (
@@ -356,8 +373,8 @@ spec:
     - alert: PromscalePostgreSQLSharedBuffersLow
       expr: |
         (
-          ((promscale_sql_database_open_chunks_total_table_size + promscale_sql_database_open_chunks_total_index_size) 
-          / 
+          ((promscale_sql_database_open_chunks_total_table_size + promscale_sql_database_open_chunks_total_index_size)
+          /
           promscale_sql_database_shared_buffers_size)
           > 1 )
       for: 10m
@@ -367,6 +384,5 @@ spec:
         summary: Promscale database performance will be affected.
         description: "Currently open chunks are {{ $value | humanizePercentage }} of PostgreSQL shared_buffers. This will impact database performance."
         runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscalePostgreSQLSharedBuffersLow.md
-
 `}}
 {{- end }}
diff --git a/charts/timescaledb-single/Chart.yaml b/charts/timescaledb-single/Chart.yaml
@@ -4,7 +4,7 @@
 apiVersion: v1
 name: timescaledb-single
 description: 'TimescaleDB HA Deployment.'
-version: 0.28.0
+version: 0.28.1
 # appVersion specifies the version of the software, which can vary wildly,
 # e.g. TimescaleDB 1.4.1 on PostgreSQL 11 or TimescaleDB 1.5.0 on PostgreSQL 12.
 # https://github.com/helm/helm/blob/master/docs/charts.md#the-appversion-field