lets just try dockah (#15585)

* lets just try dockah * here * hm? * ghcr? * login * use da cache * hehe * f * perms * big dumb * oh lala * hipshot * ? * that's no kubes * oh yeah * connect up da services * oh that is simpler * paths? * restore da matrix, neo * hmmmmm * don't locale me * maybe bash * omfg * straws! * restore matrix, neo * hmmmmmmmmmm * ooo * hold on docs 4 now * split up docs stuff * GHA doesn't have git anymore * i hate this uid/gidshit * 🤦 * just skip for now * whoami * pro * allup * CI!!! * cache it all * Revert "GHA doesn't have git anymore" This reverts commit 157441b. * mount the whole shebang * env in container * wront spot * argumentsssss * am fog * hm * no? * when in doubt * ? * ? * set it aside for now * okay now to fight git? * hold * ref: actions/checkout#1487 * allup * byapss * validate * No, I will not be squashing all that, witness my garbage. * not sure if neccessary * too slow and doesn't capture RUN caches anyway * nose clean * no, we wanted that :) * oh, right this is what enables the gha docker cache * more caching cannot hurt * test cachin * oh, right * Revert "test cachin" This reverts commit f03fd35. * test all the caches * test the caches * test cache * test caching and validate deps task * test test test * reverts * test code only change * does caching work in container runs? * revert test change
pypi · Mar 14, 2024 · cca230b · cca230b
1 parent 9611ff4
commit cca230b
Show file tree

Hide file tree

Showing 5 changed files with 85 additions and 68 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,60 +9,102 @@ on:
   workflow_dispatch:  # generally only for the "combine-prs" workflow
 permissions:
   contents: read
+  packages: read
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 jobs:
-  deps:
+  build:
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-      - name: Install platform dependencies
-        run: |
-          sudo apt -y update
-          sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v5
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Cache RUN --mount=type=cache /var/cache/apt
+        uses: actions/cache@v3
         with:
-          python-version-file: '.python-version'
-      - name: pip cache
-        uses: actions/cache@v4
+          path: var-cache-apt
+          key: var-cache-apt-${{ hashFiles('Dockerfile') }}
+          restore-keys: |
+            var-cache-apt-
+      - name: Cache RUN --mount=type=cache /var/lib/apt
+        uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: pip-${{ runner.os }}
-      - name: Cache built Python environment
-        uses: actions/cache@v4
+          path: var-lib-apt
+          key: var-lib-apt-${{ hashFiles('Dockerfile') }}
+          restore-keys: |
+            var-lib-apt-
+      - name: Cache RUN --mount=type=cache /root/.cache/pip
+        uses: actions/cache@v3
+        with:
+          path: root-cache-pip
+          key: root-cache-pip-${{ hashFiles('requirements/*.txt') }}
+          restore-keys: |
+            root-cache-pip-
+      - name: Cache RUN --mount=type=cache /root/.npm
+        uses: actions/cache@v3
+        with:
+          path: root-npm
+          key: root-npm-${{ hashFiles('package.json', 'package-lock.json') }}
+          restore-keys: |
+            root-npm-
+      - name: inject var-cache-apt into docker
+        uses: reproducible-containers/[email protected]
         with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
-      - name: Install Python dependencies
+          cache-source: var-cache-apt
+          cache-target: /var/cache/apt
+          skip-extraction: ${{ steps.cache-var-cache-apt.outputs.cache-hit }}
+      - name: inject var-lib-apt into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: var-lib-apt
+          cache-target: /var/lib/apt
+          skip-extraction: ${{ steps.cache-var-lib-apt.outputs.cache-hit }}
+      - name: inject root-cache-pip into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: root-cache-pip
+          cache-target: /root/.cache/pip
+          skip-extraction: ${{ steps.cache-root-cache-pip.outputs.cache-hit }}
+      - name: inject root-npm into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: root-npm
+          cache-target: /root/.npm
+          skip-extraction: ${{ steps.cache-root-npm.outputs.cache-hit }}
+      - name: Build Docker Image
         run: |
-          pip install -U setuptools wheel pip
-          pip install -r requirements.txt --no-deps
-          pip install -r requirements/dev.txt
-          pip check
+            echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+            docker buildx build --build-arg="DEVEL=yes" --build-arg="CI=yes" --cache-to type=gha,mode=max --cache-from type=gha --push -t ghcr.io/pypi/warehouse:ci-${{ github.run_id }} .
   test:
-    # Time out if our test suite has gotten hung
-    timeout-minutes: 15
-    needs: deps
+    needs: build
     strategy:
       matrix:
         include:
           - name: Tests
-            command: bin/tests --postgresql-host localhost
+            command: bin/tests --postgresql-host postgres
           - name: Lint
             command: bin/lint
-          - name: User Documentation
-            command: bin/user-docs
-          - name: Developer Documentation
-            command: bin/dev-docs
           - name: Dependencies
-            command: bin/github-actions-deps
+            command: bin/deps
           - name: Licenses
             command: bin/licenses
           - name: Translations
             command: bin/translations
+          - name: User Documentation
+            command: bin/user-docs
+          - name: Developer Documentation
+            command: bin/dev-docs
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/pypi/warehouse:ci-${{ github.run_id }}
+      env:
+        BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://stripe:12111 api_version=2020-08-27
     services:
       postgres:
         image: ${{ (matrix.name == 'Tests') && 'postgres:14.11' || '' }}
@@ -77,31 +119,15 @@ jobs:
         ports:
           - 12111:12111
     name: ${{ matrix.name }}
-    env:
-      BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://localhost:12111 api_version=2020-08-27
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-      - name: Install platform dependencies
-        run: |
-          sudo apt -y update
-          sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v5
-        with:
-          python-version-file: '.python-version'
       - name: Cache mypy results
         if: ${{ (matrix.name == 'Lint') }}
         uses: actions/cache@v4
         with:
           path: |
               dev/.mypy_cache
-          key: ${{ runner.os }}-mypy-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
-      - name: Restore built Python environment from deps
-        uses: actions/cache/restore@v4
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
-          # Since we don't install deps again, we fail if we can't restore the cache (timeout, etc)
-          fail-on-cache-miss: true
+          key: ${{ runner.os }}-mypy-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
       - name: Run ${{ matrix.name }}
         run: ${{ matrix.command }}
diff --git a/Dockerfile b/Dockerfile
@@ -112,6 +112,10 @@ FROM python:3.11.8-slim-bookworm as build
 # test dependencies.
 ARG DEVEL=no
 
+# Define whether we're building a CI image. This will include all the docs stuff
+# as well for the matrix!
+ARG CI=no
+
 # To enable Ipython in the development environment set to yes (for using ipython
 # as the warehouse shell interpreter,
 # i.e. 'docker compose run --rm web python -m warehouse shell --type=ipython')
@@ -176,6 +180,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
                     -r /tmp/requirements/deploy.txt \
                     -r /tmp/requirements/main.txt \
                     $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
+                    $(if [ "$CI" = "yes" ]; then echo '-r /tmp/requirements/docs-dev.txt -r /tmp/requirements/docs-user.txt -r /tmp/requirements/docs-blog.txt'; fi ) \
     && pip check \
     && find /opt/warehouse -name '*.pyc' -delete
 
@@ -198,6 +203,10 @@ WORKDIR /opt/warehouse/src/
 # test dependencies.
 ARG DEVEL=no
 
+# Define whether we're building a CI image. This will include all the docs stuff
+# as well for the matrix!
+ARG CI=no
+
 # This is a work around because otherwise postgresql-client bombs out trying
 # to create symlinks to these directories.
 RUN set -x \
@@ -211,8 +220,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     set -x \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
-        libpq5 libxml2 libxslt1.1 libcurl4  \
+        libpq5 libxml2 libxslt1.1 libcurl4 \
         $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev vim'; fi) \
+        $(if [ "$CI" = "yes" ]; then echo 'git'; fi) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

diff --git a/bin/github-actions-deps b/bin/github-actions-deps
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-# This is only used for dependabot and Github Actions CI
+# This is only used for dependabot
 -r requirements/main.txt
 -r requirements/deploy.txt
 -r requirements/docs-dev.txt

diff --git a/warehouse/locale/Makefile b/warehouse/locale/Makefile
@@ -39,7 +39,7 @@ build-mos: compile-pot
 translations: compile-pot
 ifneq ($(GITHUB_ACTIONS), false)
 	cd ../../; \
-	git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
+	cmp -s ./warehouse/locale/messages.pot /opt/warehouse/src/warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
 else
 endif