Manage and check dependencies (#589)

* manage dependencies * manage dependencies * manage dependencies * manage dependencies * fix dependencies * fix tests * fix tests
makinacorpus · Oct 8, 2024 · 6fe52aa · 6fe52aa
1 parent f831b05
commit 6fe52aa
Show file tree

Hide file tree

Showing 9 changed files with 96 additions and 41 deletions.
diff --git a/.docker/entrypoint.sh b/.docker/entrypoint.sh
@@ -3,7 +3,7 @@
 cd /app/src || exit
 
 # Activate venv
-. /app/venv/bin/activate
+. /opt/venv/bin/activate
 
 if [ "$COLLECTSTATIC" == "1" ]
 then

diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
@@ -0,0 +1,52 @@
+name: Check deps
+
+on:
+  pull_request:
+    paths:
+      - setup.py
+      - requirements.txt
+      - requirements-dev.txt
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  quality:
+    name: Checking dependency graph
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ['ubuntu-20.04']
+        python-version: ['3.8']
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v5
+        with:
+          cache: 'pip'
+          python-version: '3.8.18'
+
+      - name: Install dependencies
+        run: |
+          pip3 install -c requirements-dev.txt pip-tools
+
+      - name: Check dependency graph
+        run: |
+          pip-compile -q --strip-extras
+          pip-compile -q --strip-extras requirements-dev.in
+
+      - name: Verify dependency graph is ok
+        uses: tj-actions/verify-changed-files@v20
+        id: verify-changed-files
+        with:
+          files: |
+            requirements.txt
+            requirements-dev.txt
+
+      - name: Validating graph
+        if: steps.verify-changed-files.outputs.files_changed == 'true'
+        run: |
+          echo "Dependency file(s) changed: ${{ steps.verify-changed-files.outputs.changed_files }}"
+          git diff
+          core.setFailed('Please add your new dependencies in setup.py and/or dev-requirements.in then run pip-compile to add them in requirements. (see docs/contribute/development)')
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -77,6 +77,8 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: pip
+
     - name: Install dependencies
       run: |
         sudo apt-get update -q
@@ -85,11 +87,13 @@ jobs:
         pip install -r requirements.txt -U
         pip install -r requirements-dev.txt
         npm ci
+
     - name: Test with coverage
       run: |
         cd src
         coverage run ./manage.py test
         coverage run -a ./manage.py test --settings screamshotter.settings.test_timeout screenshotter.tests.CaptureTestCase.test_timeout_screenshot
+
     - name: Coverage upload
       run: |
         pip install codecov
@@ -101,14 +105,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Build Docker image
         run: |
           docker build -t screamshotter_ci:latest .
+
       - name: Upload image
-        uses: ishworkh/docker-image-artifact-upload@v1
+        uses: ishworkh/docker-image-artifact-upload@v2.1.0
         with:
           image: "screamshotter_ci:latest"
 
@@ -123,6 +130,7 @@ jobs:
                   version: '20.04'
                 - os: jammy
                   version: '22.04'
+
     env:
       DISTRO: 'ubuntu:${{ matrix.os }}'
       CODE: ${{ matrix.version }}
@@ -150,9 +158,10 @@ jobs:
     name: Tests E2E docker
     runs-on: ubuntu-latest
     needs: [build_docker_image]
+
     steps:
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/docker-image-artifact-download@v2.1.0
         with:
           image: "screamshotter_ci:latest"
 
@@ -248,7 +257,7 @@ jobs:
           name: 'debian-20.04'
 
       - name: Download docker image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/docker-image-artifact-download@v2.1.0
         with:
           image: "screamshotter_ci:latest"
 

diff --git a/Dockerfile b/Dockerfile
@@ -13,8 +13,8 @@ ENV MAX_REQUESTS=250
 ENV PUPPETEER_CACHE_DIR=/app/puppeteer/
 
 RUN useradd -ms /bin/bash django
-RUN mkdir -p /app/static
-RUN chown django:django /app
+RUN mkdir -p /app/static /opt
+RUN chown django:django /app /opt
 
 RUN apt-get -qq update && apt-get install -qq -y \
     gconf-service \
@@ -81,28 +81,28 @@ RUN apt-get -qq update && apt-get install -qq -y \
 RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py
 
 USER django
-RUN python3 -m venv /app/venv
-RUN /app/venv/bin/pip3 install --no-cache-dir pip setuptools wheel -U
+RUN python3 -m venv /opt/venv
+RUN /opt/venv/bin/pip3 install --no-cache-dir pip setuptools wheel -U
 
 COPY requirements.txt /app/
-RUN /app/venv/bin/pip3 install --no-cache-dir -r /app/requirements.txt -U && rm /app/requirements.txt
-RUN /app/venv/bin/nodeenv /app/venv/ -C '' -p -n 20.9.0
+RUN /opt/venv/bin/pip3 install --no-cache-dir -r /app/requirements.txt -U && rm /app/requirements.txt
+RUN /opt/venv/bin/nodeenv /app/venv/ -C '' -p -n 20.9.0
 
 # upgrade npm & requirements
 COPY package.json /app/package.json
 COPY package-lock.json /app/package-lock.json
-RUN . /app/venv/bin/activate && npm ci && rm /app/*.json
+RUN . /opt/venv/bin/activate && npm ci && rm /app/*.json
 
 FROM build AS dev
 
 COPY requirements-dev.txt /app/
-RUN /app/venv/bin/pip3 install --no-cache-dir -r /app/requirements-dev.txt -U && rm /app/requirements-dev.txt
+RUN /opt/venv/bin/pip3 install --no-cache-dir -r /app/requirements-dev.txt -U && rm /app/requirements-dev.txt
 
 CMD ["./manage.py", "runserver", "0.0.0.0:8000"]
 
 FROM base
 
-COPY --from=build /app/venv /app/venv
+COPY --from=build /opt/venv /opt/venv
 COPY --from=build /app/node_modules /app/node_modules
 COPY --from=build /app/puppeteer /app/puppeteer
 COPY src /app/src

diff --git a/Makefile b/Makefile
@@ -4,4 +4,7 @@ build_deb:
 	docker run --name screamshotter_deb_run -t screamshotter_deb bash -c "exit"
 	docker cp screamshotter_deb_run:/dpkg ./
 	docker stop screamshotter_deb_run
-	docker rm screamshotter_deb_run
+	docker rm screamshotter_deb_run
+
+deps:
+	docker compose run --rm  web bash -c "cd /app && pip-compile -q --strip-extras && pip-compile -q --strip-extras requirements-dev.in"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -8,7 +8,7 @@ services:
         BASE_IMAGE: focal
 
     volumes:
-      - ./src/:/app/src/
+      - ./:/app/
     ports:
       - "8000:8000"
     environment:

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile requirements-dev.in
+#    pip-compile --strip-extras requirements-dev.in
 #
 asgiref==3.8.1
     # via
@@ -18,7 +18,7 @@ click==8.1.7
     # via pip-tools
 coverage==7.6.1
     # via -r requirements-dev.in
-django==4.2.13
+django==4.2.16
     # via
     #   -c ./requirements.txt
     #   django-debug-toolbar

diff --git a/requirements.txt b/requirements.txt
@@ -2,13 +2,14 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile
+#    pip-compile --strip-extras
 #
 asgiref==3.8.1
     # via django
 backports-zoneinfo==0.2.1 ; python_version < "3.9"
     # via
     #   django
+    #   djangorestframework
     #   screamshotter (setup.py)
 certifi==2024.8.30
     # via
@@ -30,7 +31,7 @@ gevent==24.2.1
     # via gunicorn
 greenlet==3.1.1
     # via gevent
-gunicorn[gevent]==23.0.0
+gunicorn==23.0.0
     # via screamshotter (setup.py)
 idna==3.10
     # via requests
@@ -44,8 +45,6 @@ nodeenv==1.9.1
     # via screamshotter (setup.py)
 packaging==24.1
     # via gunicorn
-pytz==2024.2
-    # via djangorestframework
 requests==2.32.3
     # via coreapi
 sentry-sdk==2.15.0

diff --git a/src/screenshotter/tests.py b/src/screenshotter/tests.py
@@ -1,4 +1,3 @@
-import os
 from tempfile import TemporaryDirectory
 from unittest import skipIf
 
@@ -10,7 +9,7 @@
 from django.test import SimpleTestCase, override_settings
 from django.urls import reverse
 from rest_framework.serializers import Serializer
-from rest_framework.test import APIClient
+from rest_framework.test import APISimpleTestCase
 
 from .exceptions import ScreenshotterException
 from .puppeteer import take_screenshot
@@ -24,26 +23,22 @@ class CaptureTestCase(SimpleTestCase):
     @override_settings(MEDIA_ROOT=temp_dir.name)
     def test_capture_mime(self):
         png = take_screenshot('https://www.google.fr')
-        png_path = os.path.join(temp_dir.name, 'test.png')
 
         cfile = ContentFile(content=png)
-        default_storage.save(name=png_path, content=cfile)
+        default_storage.save(name='test_capture_mime.png', content=cfile)
 
-        self.assertTrue(os.path.exists(png_path))
-        self.assertNotEqual(os.path.getsize(png_path), 0)
-
-        mime = magic.from_file(png_path, mime=True)
+        self.assertNotEqual(cfile.size, 0)
+        mime = magic.from_buffer(default_storage.open('test_capture_mime.png').read(), mime=True)
         self.assertEqual(mime, "image/png", )
 
     @override_settings(MEDIA_ROOT=temp_dir.name)
     def test_capture_size(self):
         png = take_screenshot('https://www.google.fr', width=1280, height=720)
 
-        png_path = os.path.join(temp_dir.name, 'test2.png')
         cfile = ContentFile(content=png)
-        default_storage.save(name=png_path, content=cfile)
+        default_storage.save(name='test_capture_size.png', content=cfile)
 
-        image = Image.open(png_path)
+        image = Image.open(default_storage.path('test_capture_size.png'))
 
         self.assertEqual(image.width, 1280)
         self.assertEqual(image.height, 720)
@@ -78,23 +73,20 @@ def test_bad_settings(self):
             bool(app_settings.BAD_SETTINGS)
 
 
-class CaptureApiTestCase(SimpleTestCase):
-    def setUp(self):
-        self.api_client = APIClient()
-
+class CaptureApiTestCase(APISimpleTestCase):
     def test_api_good_request_json(self):
         serializer = ScreenshotSerializer()
         data = serializer.data
         data['url'] = "https://www.google.fr"
 
-        response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=json', data=data)
+        response = self.client.post(reverse('screenshotter:screenshot') + '?format=json', data=data)
         data = response.json()
         self.assertEqual(response.status_code, 200, data)
         self.assertIn('base64', data)
 
     def test_api_bad_request(self):
         serializer = ScreenshotSerializer()
-        response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=json', data=serializer.data)
+        response = self.client.post(reverse('screenshotter:screenshot') + '?format=json', data=serializer.data)
         data = response.json()
         self.assertEqual(response.status_code, 400)
         self.assertIn('url', data)
@@ -104,15 +96,15 @@ def test_api_wrong_response(self):
         serializer = ScreenshotSerializer()
         data = serializer.data
         data['url'] = "https://kikou.com"
-        response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=json', data=data)
+        response = self.client.post(reverse('screenshotter:screenshot') + '?format=json', data=data)
         self.assertEqual(response.status_code, 500, response.json())
 
     def test_api_browsable(self):
         serializer = ScreenshotSerializer()
         data = serializer.data
         data['url'] = "https://www.google.fr"
 
-        response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=api', data=data)
+        response = self.client.post(reverse('screenshotter:screenshot') + '?format=api', data=data)
         self.assertEqual(response.status_code, 200)
         self.assertIn(b'<html>', response.content)
         # check default renderer is json in browsable api (response in "base64": "xxx" format)
@@ -123,7 +115,7 @@ def test_png_default(self):
         data = serializer.data
         data['url'] = "https://www.google.fr"
 
-        response = self.api_client.post(reverse('screenshotter:screenshot'), data=data)
+        response = self.client.post(reverse('screenshotter:screenshot'), data=data)
         self.assertEqual(response.status_code, 200)
         mime = magic.from_buffer(response.content, mime=True)
         self.assertEqual(mime, "image/png")