feat: E2E for prometheus

.github/workflows/test-pr-e2e.yml

@@ -5,6 +5,8 @@ on:
   pull_request:
     paths:
       - 'keep/**'
+      - 'keep-ui/**'
+      - 'tests/**'
 
 env:
   PYTHON_VERSION: 3.11
@@ -142,8 +144,8 @@ jobs:
         with:
           name: test-artifacts
           path: |
-            page_source_test1.html
-            page_source_test2.html
+            playwright_dump_*.html
+            playwright_dump_*.png
             backend_logs-${{ matrix.db_type }}.txt
             frontend_logs-${{ matrix.db_type }}.txt
         continue-on-error: true

.gitignore

@@ -197,4 +197,6 @@ docs/node_modules/
 
 
 scripts/automatic_extraction_rules.py
-page_source_test*.html
+
+playwright_dump_*.html
+playwright_dump_*.png

keep-ui/app/alerts/alert-menu.tsx

@@ -158,6 +158,7 @@ export default function AlertMenu({
             icon={EllipsisHorizontalIcon}
             className="hover:bg-gray-100"
             color="gray"
+            title="Alert actions"
           />
         </Menu.Button>
         {isMenuOpen && (

keep/providers/prometheus_provider/prometheus_provider.py

@@ -160,11 +160,15 @@ def get_status(event: dict) -> AlertStatus:
 
     @staticmethod
     def _format_alert(
-        event: dict, provider_instance: Optional["PrometheusProvider"] = None
+        event: dict | list[AlertDto], provider_instance: Optional["PrometheusProvider"] = None
     ) -> list[AlertDto]:
         # TODO: need to support more than 1 alert per event
         alert_dtos = []
-        alerts = event.get("alerts", [event])
+        if isinstance(event, list):
+            return event
+        else:
+            alerts = event.get("alerts", [event])
+
         for alert in alerts:
             alert_id = alert.get("id", alert.get("labels", {}).get("alertname"))
             description = alert.get("annotations", {}).pop(

tests/e2e_tests/docker-compose-e2e-mysql.yml

@@ -20,7 +20,7 @@ services:
       file: docker-compose.common.yml
       service: keep-frontend-common
     build:
-      context: ./keep-ui
+      context: ./keep-ui/
       dockerfile: ../docker/Dockerfile.ui
     environment:
       - AUTH_TYPE=NO_AUTH
@@ -50,5 +50,13 @@ services:
       file: docker-compose.common.yml
       service: keep-websocket-server-common
 
+  prometheus-server-for-test-target:
+    image: prom/prometheus
+    volumes:
+      - ./tests/e2e_tests/test_pushing_prometheus_config.yaml:/etc/prometheus/prometheus.yml
+      - ./tests/e2e_tests/test_pushing_prometheus_rules.yaml:/etc/prometheus/test_pushing_prometheus_rules.yaml
+    ports:
+      - "9090:9090"
+
 volumes:
   mysql-data:

tests/e2e_tests/docker-compose-e2e-postgres.yml

@@ -17,8 +17,9 @@ services:
     extends:
       file: docker-compose.common.yml
       service: keep-frontend-common
-    image:
-      us-central1-docker.pkg.dev/keephq/keep/keep-ui
+    build:
+      context: ./keep-ui/
+      dockerfile: ../docker/Dockerfile.ui
     environment:
       - AUTH_TYPE=NO_AUTH
       - API_URL=http://keep-backend:8080
@@ -46,5 +47,13 @@ services:
       file: docker-compose.common.yml
       service: keep-websocket-server-common
 
+  prometheus-server-for-test-target:
+    image: prom/prometheus
+    volumes:
+      - ./tests/e2e_tests/test_pushing_prometheus_config.yaml:/etc/prometheus/prometheus.yml
+      - ./tests/e2e_tests/test_pushing_prometheus_rules.yaml:/etc/prometheus/test_pushing_prometheus_rules.yaml
+    ports:
+      - "9090:9090"
+
 volumes:
   postgres-data:

tests/e2e_tests/test_end_to_end.py

@@ -28,7 +28,7 @@
 #    - Spin up the environment using docker-compose.
 #    - Run "playwright codegen localhost:3000"
 #    - Copy the generated code to a new test function.
-import re
+import re, os, sys
 import string
 
 # Running the tests in GitHub Actions:
@@ -64,11 +64,14 @@ def test_insert_new_alert(browser):
         browser.reload()
         browser.get_by_text("1", exact=True).click()
     except Exception:
-        # Capture a screenshot on failure
-        # browser.screenshot(path="screenshot-test1.png")
-
-        # Save the page source on failure
-        with open("page_source_test1.html", "w") as f:
+        # Current file + test name for unique html and png dump.
+        current_test_name = \
+            "playwright_dump_" + \
+            os.path.basename(__file__)[:-3] + \
+            "_" + sys._getframe().f_code.co_name
+
+        browser.screenshot(path=current_test_name + ".png")
+        with open(current_test_name + ".html", "w") as f:
             f.write(browser.content())
         raise
 
@@ -105,10 +108,13 @@ def test_providers_page_is_accessible(browser):
         # make sure the provider is connected
         browser.get_by_text(f"resend id: {random_provider_name}").click()
     except Exception:
-        # Capture a screenshot on failure
-        # browser.screenshot(path="screenshot-test2.png")
-
-        # Save the page source on failure
-        with open("page_source_test2.html", "w") as f:
+        # Current file + test name for unique html and png dump.
+        current_test_name = \
+            "playwright_dump_" + \
+            os.path.basename(__file__)[:-3] + \
+            "_" + sys._getframe().f_code.co_name
+
+        browser.screenshot(path=current_test_name + ".png")
+        with open(current_test_name + ".html", "w") as f:
             f.write(browser.content())
         raise

tests/e2e_tests/test_pushing_prometheus_alerts.py

@@ -0,0 +1,83 @@
+import re, os, sys, time
+import requests
+
+from datetime import datetime
+from playwright.sync_api import expect
+
+# Dear developer, thank you for checking E2E tests!
+# For instructions, please check test_end_to_end.py. 
+
+os.environ["PLAYWRIGHT_HEADLESS"] = "false"
+
+def test_pulling_prometheus_alerts_to_provider(browser):
+    try: 
+        provider_name = "playwright_test_" + datetime.now().strftime("%Y%m%d%H%M%S")
+
+        # Wait for prometheus to wake up and evaluate alert rule as "firing"
+        alerts = None
+        while alerts is None or \
+                len(alerts["data"]["alerts"]) == 0 or \
+                alerts["data"]["alerts"][0]['state'] != "firing":        
+            print("Waiting for prometheus to fire an alert...")
+            time.sleep(1)
+            alerts = requests.get("http://localhost:9090/api/v1/alerts").json()
+            print(alerts)
+
+        # Create prometheus provider 
+        browser.goto("http://localhost:3000/providers")
+        browser.get_by_placeholder("Filter providers...").click()
+        browser.get_by_placeholder("Filter providers...").fill("prometheus")
+        browser.get_by_placeholder("Filter providers...").press("Enter")
+        browser.get_by_text("Available Providers").hover()
+        browser.locator("div").filter(has_text=re.compile(r"^prometheus dataalertConnect$")).nth(1).hover()
+
+        browser.get_by_role("button", name="Connect").click()
+        browser.get_by_placeholder("Enter provider name").click()
+        browser.get_by_placeholder("Enter provider name").fill(provider_name)
+        browser.get_by_placeholder("Enter url").click()
+
+        if os.getenv("GITHUB_ACTIONS") == "true":
+            browser.get_by_placeholder("Enter url").fill("http://prometheus-server-for-test-target:9090/")
+        else:
+            browser.get_by_placeholder("Enter url").fill("http://localhost:9090/")
+
+        browser.mouse.wheel(1000, 10000)  # Scroll down.
+        browser.get_by_role("button", name="Connect").click()
+
+
+        # Validate provider is created
+        expect(browser.locator("div").filter(has_text=re.compile(re.escape(provider_name))).first).to_be_visible()
+
+        browser.reload()
+
+        # Check if alerts were pulled
+        for i in range(0, 5):
+            browser.get_by_role("link", name="Feed").click()
+            browser.wait_for_timeout(5000) # Wait for alerts to be loaded
+
+        browser.reload()
+
+        # Make sure we pulled multiple instances of the alert
+        browser.get_by_text("AlwaysFiringAlert").click()
+
+        # Delete provider 
+        browser.get_by_role("link", name="Providers").click()
+        browser.locator("div").filter(has_text=re.compile(re.escape(provider_name))).first.hover()
+        browser.locator(".tile-basis").first.click()
+        browser.once("dialog", lambda dialog: dialog.accept())
+        browser.get_by_role("button", name="Delete").click()
+
+        # Assert provider was deleted
+        expect(browser.locator("div").filter(has_text=re.compile(re.escape(provider_name))).first).not_to_be_visible()
+    except Exception:
+        # Current file + test name for unique html and png dump.
+        current_test_name = \
+            "playwright_dump_" + \
+            os.path.basename(__file__)[:-3] + \
+            "_" + sys._getframe().f_code.co_name
+
+        browser.screenshot(path=current_test_name + ".png")
+        with open(current_test_name + ".html", "w") as f:
+            f.write(browser.content())
+
+        raise

tests/e2e_tests/test_pushing_prometheus_config.yaml

@@ -0,0 +1,12 @@
+# my global config
+global:
+  scrape_interval: 1s
+  evaluation_interval: 1s
+
+rule_files:
+  - "test_pushing_prometheus_rules.yaml"
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]

tests/e2e_tests/test_pushing_prometheus_rules.yaml

@@ -0,0 +1,10 @@
+groups:
+- name: example
+  rules:
+  - alert: AlwaysFiringAlert
+    expr: rate(prometheus_http_requests_total{}[1m]) > 0
+    for: 1s
+    labels:
+      severity: page
+    annotations:
+      summary: Unless mathematics has changed, this alert should always fire.