[#1570] Speed up CI

.github/workflows/ci.yml

@@ -14,6 +14,9 @@ env:
   IMAGE_NAME: maykinmedia/open-inwoner
 
 jobs:
+  #
+  # Unit and integration tests
+  # 
   tests:
     name: Run the Django test suite
     runs-on: ubuntu-latest
@@ -26,7 +29,9 @@ jobs:
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        options: 
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+          --name postgres
 
       elasticsearch:
         image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
@@ -38,50 +43,105 @@ jobs:
           - 9300:9300
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - name: Set up backend environment
+        uses: maykinmedia/setup-django-backend@v1
         with:
+          apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client libgdal-dev gdal-bin'
           python-version: '3.9'
-      - uses: actions/setup-node@v2-beta
-        with:
-          node-version: '12'
-
-      - name: Install system packages
-        run: |
-          sudo apt-get update \
-          && sudo apt-get install -y --no-install-recommends \
-            libgdal-dev \
-            gdal-bin
-
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-          pip install -r requirements/dev.txt
-          playwright install
-          playwright install msedge
-          playwright install-deps
-
-      - name: Build frontend
-        run: |
-          npm ci
-          npm run build
+          optimize-postgres: 'yes'
+          pg-service: 'postgres'
+          setup-node: 'yes'
+          npm-ci-flags: '--legacy-peer-deps'
 
       - name: Run tests
         run: |
           python src/manage.py collectstatic --noinput --link
-          coverage run src/manage.py test src
+          coverage run \
+            --concurrency=multiprocessing \
+            --parallel-mode \
+            src/manage.py test src \
+            --parallel \
+            --exclude-tag=e2e \
+            --exclude-tag=elastic
+          coverage run src/manage.py test src --tag=elastic
+          coverage combine
+
         env:
           DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci
           SECRET_KEY: dummy
           DB_USER: postgres
           DB_PASSWORD: ''
 
       - name: Publish coverage report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  #
+  # End-to-end tests
+  #
+  e2etests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        browser:
+          - chromium
+          - firefox
+          - webkit
+          - msedge
+
+    name: End-to-end tests, ${{ matrix.browser }}
+
+    services:
+      postgres:
+        image: postgis/postgis:10-2.5
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # Needed because the postgres container does not provide a healthcheck
+        options:
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+          --name postgres
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up backend environment
+        uses: maykinmedia/setup-django-backend@v1
+        with:
+          apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client libgdal-dev gdal-bin'
+          python-version: '3.9'
+          optimize-postgres: 'yes'
+          pg-service: 'postgres'
+          setup-node: 'yes'
+          npm-ci-flags: '--legacy-peer-deps'
+
+      # See https://playwright.dev/python/docs/ci#caching-browsers
+      - name: Cache Playwright browser
+        id: cache-browser
+        uses: actions/cache@v3
+        with:
+          path: /home/runner/.cache/ms-playwright
+          key:
+            ${{ runner.os }}-${{ matrix.browser }}-playwright-${{ hashFiles('requirements/ci.txt') }}
+
+      - name: Install playwright deps
+        run: playwright install --with-deps ${{ matrix.browser }}
+
+      - name: Run testsuite
+        run: |
+          python src/manage.py collectstatic --noinput --link
+          src/manage.py test src --tag=e2e
+        env:
+          DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci
+          SECRET_KEY: dummy
+          DB_USER: postgres
+          DB_PASSWORD: ''
+          E2E_DRIVER: ${{ matrix.browser }}
+
+  #
+  # Docker
+  #
   docker:
     needs: tests
 

.nvmrc

@@ -0,0 +1 @@
+12

src/open_inwoner/accounts/tests/test_auth_2fa_sms.py

@@ -17,7 +17,6 @@
 
 from open_inwoner.configurations.models import SiteConfiguration
 from open_inwoner.utils.logentry import LOG_ACTIONS
-from open_inwoner.utils.validators import format_phone_number
 
 from ..gateways import GatewayError
 from .factories import UserFactory

src/open_inwoner/conf/ci.py

@@ -70,3 +70,7 @@
 )
 
 PLAYWRIGHT_MULTI_ONLY_DEFAULT = False
+
+PASSWORD_HASHERS = [
+    "django.contrib.auth.hashers.MD5PasswordHasher",
+]

src/open_inwoner/conf/test.py

@@ -20,3 +20,4 @@
 ]
 
 CSP_REPORTS_SAVE = True
+PLAYWRIGHT_MULTI_ONLY_DEFAULT = False

src/open_inwoner/haalcentraal/tests/test_signal.py

@@ -91,15 +91,10 @@ def test_user_is_not_updated_without_defining_service(self, m):
         )
         user.bsn = "999993847"
 
-        with self.assertLogs() as captured:
-            user.save()
+        user.save()
 
         user.refresh_from_db()
 
-        self.assertEqual(
-            captured.records[1].getMessage(),
-            "no service defined for Haal Centraal",
-        )
         self.assertEqual(user.first_name, "")
         self.assertEqual(user.infix, "")
         self.assertEqual(user.last_name, "")

src/open_inwoner/plans/tests/test_views.py

@@ -675,6 +675,7 @@ def test_plan_action_status_not_your_action(self):
         self.assertEqual(response.status_code, 404)
 
 
+@tag("e2e")
 @override_settings(ROOT_URLCONF="open_inwoner.cms.tests.urls")
 class PlanBegeleiderListViewTests(WebTest):
     def setUp(self):

src/open_inwoner/search/tests/test_feedback.py

@@ -1,6 +1,7 @@
 import urllib
 
 from django.contrib import messages
+from django.test import tag
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
@@ -15,6 +16,7 @@
 from .utils import ESMixin
 
 
+@tag("elastic")
 class TestFeedbackFunctionality(ESMixin, WebTest):
     def setUp(self):
         self.user = UserFactory()

src/open_inwoner/search/tests/test_logging.py

@@ -1,3 +1,4 @@
+from django.test import tag
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
@@ -11,6 +12,7 @@
 from .utils import ESMixin
 
 
+@tag("elastic")
 @freeze_time("2021-10-18 13:00:00")
 class TestLogging(ESMixin, WebTest):
     def test_search_query_of_logged_in_user_is_logged(self):