From 7ad686e14e69b0c50edad0fb1f8b233cad3eff47 Mon Sep 17 00:00:00 2001
From: Jaanus Sellin <sellinjaanus@gmail.com>
Date: Wed, 28 Aug 2024 12:50:36 +0300
Subject: [PATCH] fix: change .inc calls to .increment (#8000)

We are observing incorrect data in Prometheus, which is consistently
non-reproducible. After a restart, the issue does not occur, but if the
pods run for an extended period, they seem to enter a strange state
where the counters become entangled and start sharing arbitrary values
that are added to the counters.

For example, the `feature_lifecycle_stage_entered` counter gets an
arbitrary value, such as 12, added when `inc()` is called. The
`exceedsLimitErrorCounter` shows the same behavior, and the code
implementation is identical.

We also tested some existing `increase()` counters, and they do not
suffer from this issue.

All calls to `counter.labels(labels).inc(`) will be replaced by
`counter.increment()` to try to mitigate the issue.
---
 src/lib/metrics.ts | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts
index 6654d07afefb..c4fd4aa5cc71 100644
--- a/src/lib/metrics.ts
+++ b/src/lib/metrics.ts
@@ -428,9 +428,9 @@ export default class MetricsMonitor {
                 eventBus.on(
                     events.STAGE_ENTERED,
                     (entered: { stage: string; feature: string }) => {
-                        featureLifecycleStageEnteredCounter
-                            .labels({ stage: entered.stage })
-                            .inc();
+                        featureLifecycleStageEnteredCounter.increment({
+                            stage: entered.stage,
+                        });
                     },
                 );
 
@@ -440,9 +440,7 @@ export default class MetricsMonitor {
                         resource,
                         limit,
                     }: { resource: string; limit: number }) => {
-                        exceedsLimitErrorCounter
-                            .labels({ resource, limit })
-                            .inc();
+                        exceedsLimitErrorCounter.increment({ resource, limit });
                     },
                 );
 
@@ -862,16 +860,16 @@ export default class MetricsMonitor {
             });
         });
         eventStore.on(PROJECT_CREATED, () => {
-            projectActionsCounter.labels({ action: PROJECT_CREATED }).inc();
+            projectActionsCounter.increment({ action: PROJECT_CREATED });
         });
         eventStore.on(PROJECT_ARCHIVED, () => {
-            projectActionsCounter.labels({ action: PROJECT_ARCHIVED }).inc();
+            projectActionsCounter.increment({ action: PROJECT_ARCHIVED });
         });
         eventStore.on(PROJECT_REVIVED, () => {
-            projectActionsCounter.labels({ action: PROJECT_REVIVED }).inc();
+            projectActionsCounter.increment({ action: PROJECT_REVIVED });
         });
         eventStore.on(PROJECT_DELETED, () => {
-            projectActionsCounter.labels({ action: PROJECT_DELETED }).inc();
+            projectActionsCounter.increment({ action: PROJECT_DELETED });
         });
 
         const logger = config.getLogger('metrics.ts');