Merge pull request #878 from GitGuardian/agateau/standalone-linux-pac…

…kages

Standalone linux packages

.github/workflows/build_release_assets.yml

@@ -1,43 +1,46 @@
 name: Build release assets
 
-# This workflow is used by the tag workflow to build all release assets. It
-# can also be triggered manually.
+# This workflow is used by the ci and tag workflows to build all release
+# assets. It can also be triggered manually.
 
 on:
   workflow_call:
+    inputs:
+      release_mode:
+        description: 'Release mode (signed binaries, no commit sha in version number)'
+        type: boolean
+        default: false
   workflow_dispatch:
+    inputs:
+      release_mode:
+        description: 'Release mode (signed binaries, no commit sha in version number)'
+        type: boolean
+        default: false
 
 jobs:
-  build_packages:
+  build_wheel_sdist:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
 
       - name: Install packaging tools
-        shell: bash
         run: |
-          curl -L https://github.com/goreleaser/nfpm/releases/download/v2.15.0/nfpm_amd64.deb -o nfpm_amd64.deb
-          sudo dpkg -i nfpm_amd64.deb
-
-          pip install shiv==1.0.1 build
+          pip install build
 
       - name: Create packages
-        shell: bash
-        run: scripts/build-packages/build-packages
+        run: |
+          python -m build
 
       - name: Upload packages
         uses: actions/upload-artifact@v4
         with:
-          name: packages
+          name: dist
           path: |
             dist
-            packages
 
-  build_standalone_binaries:
-    name: Build signed, standalone binaries
+  build_os_packages:
+    name: Build packages
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -103,21 +106,33 @@ jobs:
           # Make it available
           cp rcodesign /usr/local/bin
 
-      - name: Install normal dependencies
+      - name: Install dependencies
+        shell: bash
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade pipenv==2023.10.3
+          python -m pip install --upgrade \
+            pipenv==2023.10.3 \
+            pyinstaller==6.5.0
           pipenv install --system --dev
         env:
           # Disable lock otherwise Windows-only dependencies like colorama are not installed
           PIPENV_SKIP_LOCK: 1
 
-      - name: Install standalone-specific dependencies
+      - name: Install Linux specific dependencies
+        if: matrix.os == 'ubuntu-22.04'
         run: |
-          python -m pip install pyinstaller==6.5.0
+          NFPM_VERSION=2.36.1
+          NFPM_CHECKSUM=05c17a1e09c470807b149fdd7bcd8f600eea044f459fc3ce81aa230103c0baf5
+
+          scripts/download \
+            https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_amd64.deb \
+            nfpm.deb \
+            $NFPM_CHECKSUM
+
+          sudo dpkg -i nfpm.deb
 
       - name: Prepare macOS secrets
-        if: startsWith(matrix.os, 'macos-')
+        if: startsWith(matrix.os, 'macos-') && inputs.release_mode
         run: |
           set -euo pipefail
           SECRETS_DIR=$TMPDIR/secrets
@@ -142,7 +157,12 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          scripts/build-standalone-exe --sign
+          if [ "${{ inputs.release_mode }}" = "true" ] ; then
+            args="--sign"
+          else
+            args="--git-version"
+          fi
+          scripts/build-os-packages/build-os-packages $args
 
       - name: Override base Docker image used for functional tests on Windows
         if: matrix.os == 'windows-2022'
@@ -157,7 +177,7 @@ jobs:
         # See note about steps requiring the GITGUARDIAN_API at the top of this file
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          scripts/build-standalone-exe functests
+          scripts/build-os-packages/build-os-packages functests
         env:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
@@ -166,8 +186,10 @@ jobs:
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: standalone-binaries-${{ matrix.os }}
+          name: os-packages-${{ matrix.os }}
           path: |
-            dist/ggshield-*.gz
-            dist/ggshield-*.pkg
-            dist/ggshield-*.zip
+            packages/ggshield-*.gz
+            packages/ggshield-*.pkg
+            packages/ggshield-*.zip
+            packages/ggshield-*.rpm
+            packages/ggshield_*.deb

.github/workflows/ci.yml

@@ -153,113 +153,9 @@ jobs:
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
           TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }}
 
-  build-standalone:
-    name: Standalone exe
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu-22.04
-          - macos-13
-          - windows-2022
-          - macos-14
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # Get enough commits to run `ggshield secret scan commit-range` on ourselves
-          fetch-depth: 10
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Install normal dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade pipenv==2023.10.3
-          pipenv install --system --dev
-        env:
-          # Disable lock otherwise Windows-only dependencies like colorama are not installed
-          PIPENV_SKIP_LOCK: 1
-
-      - name: Install standalone-specific dependencies
-        run: |
-          python -m pip install pyinstaller==6.5.0
-
-      - name: Build
-        shell: bash
-        run: |
-          scripts/build-standalone-exe
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: ggshield-${{ matrix.os }}
-          path: |
-            dist/ggshield-*.gz
-            dist/ggshield-*.pkg
-            dist/ggshield-*.zip
-
-      - name: Override base Docker image used for functional tests on Windows
-        if: matrix.os == 'windows-2022'
-        # This is required because GitHub Windows runner is not configured to
-        # run Linux-based Docker images
-        shell: bash
-        run: |
-          echo "GGTEST_DOCKER_IMAGE=mcr.microsoft.com/windows/nanoserver:ltsc2022" >> $GITHUB_ENV
-
-      - name: Functional tests
-        shell: bash
-        # See note about steps requiring the GITGUARDIAN_API at the top of this file
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
-          scripts/build-standalone-exe functests
-        env:
-          GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
-          GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
-          TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }}
-
-  build_packages:
-    # This job ensures the build-packages script is tested on each build, not only at release time
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      # Warning: changes on this step should be reflected in workflows/tag.yml
-      - name: Install packaging tools
-        shell: bash
-        run: |
-          curl -L https://github.com/goreleaser/nfpm/releases/download/v2.15.0/nfpm_amd64.deb -o nfpm_amd64.deb
-          sudo dpkg -i nfpm_amd64.deb
-
-          pip install shiv==1.0.1 build
-
-      # Append the abbreviated git sha1 to the version number to avoid confusing
-      # these packages with those produced at release time
-      - name: Fake version number
-        shell: bash
-        run: |
-          version=$(git describe --tags | sed -e 's/^v//' -e 's/-[0-9]*-g/+/')
-          echo "Set version number to '$version'"
-          sed -i "s/__version__ = .*/__version__ = \"$version\"/" ggshield/__init__.py
-
-      - name: Create packages
-        shell: bash
-        run: scripts/build-packages/build-packages
-
-      # Make packages downloadable from the workflow page
-      - name: Upload packages
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages
-          path: |
-            dist
-            packages
+  build_os_packages:
+    uses: ./.github/workflows/build_release_assets.yml
+    secrets: inherit
 
   test_github_secret_scan_action:
     name: Test GitHub action for `secret scan`

.github/workflows/tag.yml

@@ -9,6 +9,8 @@ jobs:
   build_release_assets:
     uses: ./.github/workflows/build_release_assets.yml
     secrets: inherit
+    with:
+      release_mode: true
 
   push_to_pypi:
     needs: build_release_assets
@@ -21,10 +23,11 @@ jobs:
         with:
           python-version: '3.x'
 
-      - name: Download packages
+      - name: Download wheel and sdist
         uses: actions/download-artifact@v4
         with:
-          name: packages
+          name: dist
+          path: dist
 
       - name: Publish distribution 📦 to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
@@ -54,16 +57,11 @@ jobs:
         run: |
           echo "tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
 
-      - name: Download packages
-        uses: actions/download-artifact@v4
-        with:
-          name: packages
-
-      - name: Download standalone binaries
+      - name: Download OS packages
         uses: actions/download-artifact@v4
         with:
-          pattern: standalone-binaries-*
-          path: standalone-binaries
+          pattern: os-packages-*
+          path: packages
           merge-multiple: true
 
       - name: Create release
@@ -78,10 +76,10 @@ jobs:
         run: |
           gh release upload \
             ${{ steps.tags.outputs.tag }} \
-            packages/ggshield-*.pyz \
+            packages/ggshield-*.pkg \
             packages/ggshield_*.deb \
             packages/ggshield-*.rpm \
-            standalone-binaries/ggshield-*.pkg
+            packages/ggshield-*.gz
 
   push_to_docker_hub:
     name: Push Docker image to Docker Hub
@@ -155,7 +153,9 @@ jobs:
       - name: Download packages
         uses: actions/download-artifact@v4
         with:
-          name: packages
+          pattern: os-packages-*
+          path: packages
+          merge-multiple: true
 
       - name: Install Cloudsmith CLI
         run: pip install cloudsmith-cli

changelog.d/20240417_180529_ggshield-test_standalone_linux_packages.md

@@ -0,0 +1,3 @@
+### Changed
+
+- Linux .deb and .rpm packages now use the binaries produced by pyinstaller. They no longer depend on Python.

doc/dev/standalone-executables.md → doc/dev/os-packages.md

@@ -1,16 +1,34 @@
-# Building standalone executables
+# Building OS packages
 
 ## Introduction
 
 `ggshield` is written in Python, and this sometimes makes deployment complicated.
 
 To solve those deployment issues, we provide standalone `ggshield` executables, that do not require a Python interpreter. This documentation explains how these executables are produced.
 
-The process of generating the executable is handled by the `scripts/build-standalone-exe` script. This script runs a series of "steps". It has a default list of steps, but you can tell it to run only specific steps using `scripts/build-standalone-exe step1 step2...`.
-
-All functions in the script starting with `step_` can be used as a step. This means you can get a list of all available steps with: `grep -o '^step_[a-z_]*' scripts/build-standalone-exe`.
-
-## Generating the executable
+The process of generating the packages is handled by the `scripts/build-os-packages/build-os-packages` script. This script runs a series of "steps". It has a default list of steps, but you can tell it to run only specific steps using `scripts/build-os-packages/build-os-packages step1 step2...`.
+
+All functions in the script starting with `step_` can be used as a step. This means you can get a list of all available steps with: `grep -o '^step_[a-z_]*' scripts/build-os-packages/build-os-packages`.
+
+Here is a high-level overview of the main steps (square boxes are steps):
+
+```mermaid
+flowchart TD
+    src[/source code/] --> build --> pyinstaller_dir[/"pyinstaller output
+    (dist/ggshield)"/]
+    pyinstaller_dir --> copy_files --> archive_dir[/"dir ready to be archived
+    (packages/ggshield-$version-$target)"/]
+    archive_dir --> test["test (run functional tests on archive dir)"]
+    test --> signing{Called with --sign?} -->|yes| sign
+    signing -->|no| create_archive
+    sign --> create_archive --> pkg[/"pkg 🍏"/]
+    create_archive --> zip[/"zip 🪟"/]
+    create_archive --> tar.gz[/"tar.gz 🐧"/]
+    create_archive --> deb[/"deb 🐧"/]
+    create_archive --> rpm[/"rpm 🐧"/]
+```
+
+## Generating the standalone executable
 
 We use [PyInstaller](https://pyinstaller.org) to generate `ggshield` standalone executable.