Integration tests in python

.github/workflows/test.yml

@@ -74,6 +74,21 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Setup dev environment
+        run: |
+          pip install -r requirements-dev.txt
+          ### pytest-sentry configuration ###
+          if [ "$GITHUB_REPOSITORY" = "getsentry/self-hosted" ]; then
+            echo "PYTEST_SENTRY_DSN=https://[email protected]/6627632" >> $GITHUB_ENV
+            echo "PYTEST_SENTRY_TRACES_SAMPLE_RATE=0" >> $GITHUB_ENV
+
+            # This records failures on master to sentry in order to detect flakey tests, as it's
+            # expected that people have failing tests on their PRs
+            if [ "$GITHUB_REF" = "refs/heads/master" ]; then
+              echo "PYTEST_SENTRY_ALWAYS_REPORT=1" >> $GITHUB_ENV
+            fi
+          fi
+
       - name: Get Compose
         run: |
           # Always remove `docker compose` support as that's the newer version

_integration-test/custom-ca-roots/setup.sh

@@ -25,8 +25,11 @@ openssl req -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/self.test
 
 # openssl req -in nginx/self.test.req -text -noout
 
+# Store extension details in a temporary file since python subprocess can't use process substitution
+printf "subjectAltName=DNS:self.test" >/tmp/extension_details
+
 openssl x509 -req -in $TEST_NGINX_CONF_PATH/self.test.req -CA $TEST_NGINX_CONF_PATH/ca.crt -CAkey $TEST_NGINX_CONF_PATH/ca.key \
-  -extfile <(printf "subjectAltName=DNS:self.test") \
+  -extfile /tmp/extension_details \
   -CAcreateserial -out $TEST_NGINX_CONF_PATH/self.test.crt -days 1 -sha256
 
 # openssl x509 -in nginx/self.test.crt -text -noout
@@ -42,4 +45,4 @@ openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout $TEST_NGINX_CONF_PATH/
 
 cp _integration-test/custom-ca-roots/test.py sentry/test-custom-ca-roots.py
 
-$dc up -d fixture-custom-ca-roots
+docker compose --ansi never up -d fixture-custom-ca-roots

_integration-test/run.py

@@ -0,0 +1,159 @@
+import subprocess
+import os
+from functools import lru_cache
+import requests
+import unittest
+import sentry_sdk
+import time
+import json
+
+SENTRY_CONFIG_PY = "sentry/sentry.conf.py"
+SENTRY_TEST_HOST = os.getenv("SENTRY_TEST_HOST", "http://localhost:9000")
+TEST_USER = "[email protected]"
+TEST_PASS = "test123TEST"
+TIMEOUT_SECONDS = 60
+
+
+def run_shell_command(cmd, **kwargs) -> subprocess.CompletedProcess:
+    return subprocess.run(cmd, shell=True, check=True, **kwargs)
+
+
+class IntegrationTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.session = requests.Session()
+        response = None
+        run_shell_command("docker compose --ansi never up -d")
+        for i in range(TIMEOUT_SECONDS):
+            try:
+                response = requests.get(SENTRY_TEST_HOST)
+            except requests.ConnectionError:
+                pass
+            if response is not None and response.status_code == 200:
+                break
+            time.sleep(1)
+        assert response.status_code == 200
+
+        # Create test user
+        run_shell_command(
+            f"echo y | docker compose exec web sentry createuser --force-update --superuser --email {TEST_USER} --password {TEST_PASS}"
+        )
+
+    def poll_for_response(self, request: str) -> requests.Response:
+        for i in range(TIMEOUT_SECONDS):
+            response = self.session.get(request, headers={"Referer": SENTRY_TEST_HOST})
+            if response.status_code == 200:
+                break
+            time.sleep(1)
+        assert response.status_code == 200
+        return response
+
+    @lru_cache
+    def get_sentry_dsn(self) -> str:
+        for i in range(TIMEOUT_SECONDS):
+            response = self.session.get(
+                f"{SENTRY_TEST_HOST}/api/0/projects/sentry/internal/keys/",
+                headers={"Referer": SENTRY_TEST_HOST},
+            )
+            if response.status_code == 200:
+                sentry_dsn = json.loads(response.text)[0]["dsn"]["public"]
+                if len(sentry_dsn) > 0:
+                    break
+            time.sleep(1)
+        sentry_dsn = json.loads(response.text)[0]["dsn"]["public"]
+        assert len(sentry_dsn) > 0
+        return sentry_dsn
+
+    def test_initial_redirect(self):
+        initial_auth_redirect = self.session.get(SENTRY_TEST_HOST)
+        assert initial_auth_redirect.url == f"{SENTRY_TEST_HOST}/auth/login/sentry/"
+
+    def test_login(self):
+        login_csrf_token = (
+            self.session.get(SENTRY_TEST_HOST)
+            .text.split('"csrfmiddlewaretoken" value="')[1]
+            .split('"')[0]
+        )
+        login_response = self.session.post(
+            f"{SENTRY_TEST_HOST}/auth/login/sentry/",
+            data={
+                "op": "login",
+                "username": TEST_USER,
+                "password": TEST_PASS,
+                "csrfmiddlewaretoken": login_csrf_token,
+            },
+            headers={"Referer": f"{SENTRY_TEST_HOST}/auth/login/sentry/"},
+        )
+        assert '"isAuthenticated":true' in login_response.text
+        assert '"username":"[email protected]"' in login_response.text
+        assert '"isSuperuser":true' in login_response.text
+        assert login_response.cookies["sc"] is not None
+        # Set up initial/required settings (InstallWizard request)
+        self.session.headers.update({"X-CSRFToken": login_response.cookies["sc"]})
+        response = self.session.put(
+            f"{SENTRY_TEST_HOST}/api/0/internal/options/?query=is:required",
+            headers={"Referer": SENTRY_TEST_HOST},
+            data={
+                "mail.use-tls": False,
+                "mail.username": "",
+                "mail.port": 25,
+                "system.admin-email": "[email protected]",
+                "mail.password": "",
+                "system.url-prefix": SENTRY_TEST_HOST,
+                "auth.allow-registration": False,
+                "beacon.anonymous": True,
+            },
+        )
+        assert response.status_code == 200
+
+    def test_receive_event(self):
+        event_id = None
+        with sentry_sdk.init(dsn=self.get_sentry_dsn()):
+            event_id = sentry_sdk.capture_exception(Exception("a failure"))
+        assert event_id is not None
+        response = self.poll_for_response(
+            f"{SENTRY_TEST_HOST}/api/0/projects/sentry/internal/events/{event_id}/"
+        )
+        response_json = json.loads(response.text)
+        assert response_json["eventID"] == event_id
+        assert response_json["metadata"]["value"] == "a failure"
+
+    def test_cleanup_crons_running(self):
+        cleanup_crons = run_shell_command(
+            "docker compose --ansi never ps -a | tee debug.log | grep -E -e '\\-cleanup\\s+running\\s+' -e '\\-cleanup[_-].+\\s+Up\\s+'",
+            capture_output=True,
+        ).stdout
+        assert len(cleanup_crons) > 0
+
+    def test_custom_cas(self):
+        run_shell_command(". _integration-test/custom-ca-roots/setup.sh")
+        run_shell_command(
+            "docker compose --ansi never run --no-deps web python3 /etc/sentry/test-custom-ca-roots.py"
+        )
+        run_shell_command(". _integration-test/custom-ca-roots/teardown.sh")
+
+    def test_receive_transaction_events(self):
+        with sentry_sdk.init(
+            dsn=self.get_sentry_dsn(), profiles_sample_rate=1.0, traces_sample_rate=1.0
+        ):
+
+            def dummy_func():
+                sum = 0
+                for i in range(5):
+                    sum += i
+                    time.sleep(0.25)
+
+            with sentry_sdk.start_transaction(op="task", name="Test Transactions"):
+                dummy_func()
+        profiles_response = self.poll_for_response(
+            f"{SENTRY_TEST_HOST}/api/0/organizations/sentry/events/?dataset=profiles&field=profile.id&project=1&statsPeriod=1h"
+        )
+        assert profiles_response.status_code == 200
+        profiles_response_json = json.loads(profiles_response.text)
+        assert len(profiles_response_json) > 0
+        spans_response = self.poll_for_response(
+            f"{SENTRY_TEST_HOST}/api/0/organizations/sentry/events/?dataset=spansIndexed&field=id&project=1&statsPeriod=1h"
+        )
+        assert spans_response.status_code == 200
+        spans_response_json = json.loads(spans_response.text)
+        assert len(spans_response_json) > 0

integration-test.sh

@@ -15,7 +15,7 @@ export MINIMIZE_DOWNTIME=0
 
 if [[ "$test_option" == "--initial-install" ]]; then
   echo "Testing initial install"
-  source _integration-test/run.sh
+  pytest --reruns 5 _integration-test/run.py
   source _integration-test/ensure-customizations-not-present.sh
   source _integration-test/ensure-backup-restore-works.sh
 elif [[ "$test_option" == "--customizations" ]]; then
@@ -34,7 +34,7 @@ EOT
   echo "Testing in-place upgrade and customizations"
   export MINIMIZE_DOWNTIME=1
   ./install.sh
-  source _integration-test/run.sh
+  pytest --reruns 5 _integration-test/run.py
   source _integration-test/ensure-customizations-work.sh
   source _integration-test/ensure-backup-restore-works.sh
 fi

requirements-dev.txt

@@ -0,0 +1,5 @@
+sentry-sdk>=1.39.2
+requests>=2.31.0
+pytest>=8.0.0
+pytest-rerunfailures>=11.0
+pytest-sentry>=0.1.11