From 03830db154be2d50c6b365e239116f4923cffdd4 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Wed, 17 Apr 2024 11:07:42 +0200 Subject: [PATCH 1/8] ci: build wheel and sdist straight from CI Deprecate build-packages: - the .deb and .rpm part is going to move to build-standalone-exe - the .whl and .sdist part is now done by the CI itself (no need for a wrapper script: the command is simple enough) --- .github/workflows/build_release_assets.yml | 17 ++---- .github/workflows/tag.yml | 60 +++++++++++----------- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build_release_assets.yml b/.github/workflows/build_release_assets.yml index 8d2e021503..53ea2529ea 100644 --- a/.github/workflows/build_release_assets.yml +++ b/.github/workflows/build_release_assets.yml @@ -8,33 +8,26 @@ on: workflow_dispatch: jobs: - build_packages: + build_wheel_sdist: runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Install packaging tools - shell: bash run: | - curl -L https://github.com/goreleaser/nfpm/releases/download/v2.15.0/nfpm_amd64.deb -o nfpm_amd64.deb - sudo dpkg -i nfpm_amd64.deb - - pip install shiv==1.0.1 build + pip install build - name: Create packages - shell: bash - run: scripts/build-packages/build-packages + run: | + python -m build - name: Upload packages uses: actions/upload-artifact@v4 with: - name: packages + name: dist path: | dist - packages build_standalone_binaries: name: Build signed, standalone binaries diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 627de9d71a..3e96de9163 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -21,10 +21,11 @@ jobs: with: python-version: '3.x' - - name: Download packages + - name: Download wheel and sdist uses: actions/download-artifact@v4 with: - name: packages + name: dist + path: dist - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -54,11 +55,6 @@ jobs: run: | echo "tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT - - name: Download packages - uses: actions/download-artifact@v4 - with: - name: packages - - name: Download standalone binaries uses: actions/download-artifact@v4 with: @@ -78,11 +74,12 @@ jobs: run: | gh release upload \ ${{ steps.tags.outputs.tag }} \ - packages/ggshield-*.pyz \ - packages/ggshield_*.deb \ - packages/ggshield-*.rpm \ standalone-binaries/ggshield-*.pkg + # TODO: going to be brought back in next commits + # packages/ggshield_*.deb \ + # packages/ggshield-*.rpm \ + push_to_docker_hub: name: Push Docker image to Docker Hub runs-on: ubuntu-22.04 @@ -144,24 +141,25 @@ jobs: git push - push_to_cloudsmith: - needs: build_release_assets - runs-on: ubuntu-22.04 - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download packages - uses: actions/download-artifact@v4 - with: - name: packages - - - name: Install Cloudsmith CLI - run: pip install cloudsmith-cli - - - name: Push to Cloudsmith - run: | - scripts/push-to-cloudsmith - env: - CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} +# TODO: going to be brought back in next commits +# push_to_cloudsmith: +# needs: build_release_assets +# runs-on: ubuntu-22.04 +# if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# +# - name: Download packages +# uses: actions/download-artifact@v4 +# with: +# name: packages +# +# - name: Install Cloudsmith CLI +# run: pip install cloudsmith-cli +# +# - name: Push to Cloudsmith +# run: | +# scripts/push-to-cloudsmith +# env: +# CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} From cef8216dd8845e5d46c683ef1fccd3e963c459d2 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Thu, 18 Apr 2024 16:15:46 +0200 Subject: [PATCH 2/8] ci: rename build-standalone-exe to build-os-packages Rename build-standalone-exe to build-os-packages because the script is going to take care of building the .deb and .rpm, replacing build-packages. Other changes in the script: - Its output directory is now `packages` - It has its own directory inside `scripts`, to hold the files it is going to need (only the README template for now) --- .github/workflows/build_release_assets.yml | 14 ++--- .github/workflows/tag.yml | 8 +-- ...andalone-executables.md => os-packages.md} | 8 +-- .../README.md.tmpl} | 0 .../build-os-packages} | 52 +++++++++---------- 5 files changed, 41 insertions(+), 41 deletions(-) rename doc/dev/{standalone-executables.md => os-packages.md} (87%) rename scripts/{standalone-exe/README.md => build-os-packages/README.md.tmpl} (100%) rename scripts/{build-standalone-exe => build-os-packages/build-os-packages} (80%) diff --git a/.github/workflows/build_release_assets.yml b/.github/workflows/build_release_assets.yml index 53ea2529ea..5b08eaa88c 100644 --- a/.github/workflows/build_release_assets.yml +++ b/.github/workflows/build_release_assets.yml @@ -29,7 +29,7 @@ jobs: path: | dist - build_standalone_binaries: + build_os_packages: name: Build signed, standalone binaries runs-on: ${{ matrix.os }} strategy: @@ -135,7 +135,7 @@ jobs: - name: Build shell: bash run: | - scripts/build-standalone-exe --sign + scripts/build-os-packages/build-os-packages --sign - name: Override base Docker image used for functional tests on Windows if: matrix.os == 'windows-2022' @@ -150,7 +150,7 @@ jobs: # See note about steps requiring the GITGUARDIAN_API at the top of this file if: ${{ !github.event.pull_request.head.repo.fork }} run: | - scripts/build-standalone-exe functests + scripts/build-os-packages/build-os-packages functests env: GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }} @@ -159,8 +159,8 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: standalone-binaries-${{ matrix.os }} + name: os-packages-${{ matrix.os }} path: | - dist/ggshield-*.gz - dist/ggshield-*.pkg - dist/ggshield-*.zip + packages/ggshield-*.gz + packages/ggshield-*.pkg + packages/ggshield-*.zip diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 3e96de9163..154d4c5dbc 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -55,11 +55,11 @@ jobs: run: | echo "tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT - - name: Download standalone binaries + - name: Download OS packages uses: actions/download-artifact@v4 with: - pattern: standalone-binaries-* - path: standalone-binaries + pattern: os-packages-* + path: packages merge-multiple: true - name: Create release @@ -74,7 +74,7 @@ jobs: run: | gh release upload \ ${{ steps.tags.outputs.tag }} \ - standalone-binaries/ggshield-*.pkg + packages/ggshield-*.pkg # TODO: going to be brought back in next commits # packages/ggshield_*.deb \ diff --git a/doc/dev/standalone-executables.md b/doc/dev/os-packages.md similarity index 87% rename from doc/dev/standalone-executables.md rename to doc/dev/os-packages.md index fa023003e0..6d46d3b8f1 100644 --- a/doc/dev/standalone-executables.md +++ b/doc/dev/os-packages.md @@ -1,4 +1,4 @@ -# Building standalone executables +# Building OS packages ## Introduction @@ -6,11 +6,11 @@ To solve those deployment issues, we provide standalone `ggshield` executables, that do not require a Python interpreter. This documentation explains how these executables are produced. -The process of generating the executable is handled by the `scripts/build-standalone-exe` script. This script runs a series of "steps". It has a default list of steps, but you can tell it to run only specific steps using `scripts/build-standalone-exe step1 step2...`. +The process of generating the packages is handled by the `scripts/build-os-packages/build-os-packages` script. This script runs a series of "steps". It has a default list of steps, but you can tell it to run only specific steps using `scripts/build-os-packages/build-os-packages step1 step2...`. -All functions in the script starting with `step_` can be used as a step. This means you can get a list of all available steps with: `grep -o '^step_[a-z_]*' scripts/build-standalone-exe`. +All functions in the script starting with `step_` can be used as a step. This means you can get a list of all available steps with: `grep -o '^step_[a-z_]*' scripts/build-os-packages/build-os-packages`. -## Generating the executable +## Generating the standalone executable We use [PyInstaller](https://pyinstaller.org) to generate `ggshield` standalone executable. diff --git a/scripts/standalone-exe/README.md b/scripts/build-os-packages/README.md.tmpl similarity index 100% rename from scripts/standalone-exe/README.md rename to scripts/build-os-packages/README.md.tmpl diff --git a/scripts/build-standalone-exe b/scripts/build-os-packages/build-os-packages similarity index 80% rename from scripts/build-standalone-exe rename to scripts/build-os-packages/build-os-packages index db011c72c6..cd2b929bff 100755 --- a/scripts/build-standalone-exe +++ b/scripts/build-os-packages/build-os-packages @@ -2,12 +2,13 @@ set -euo pipefail PROGNAME=$(basename "$0") -ROOT_DIR=$(cd "$(dirname "$0")/.." ; pwd) -RESOURCES_DIR="$ROOT_DIR/scripts/standalone-exe" +SCRIPT_DIR=$(cd "$(dirname "$0")" ; pwd) +ROOT_DIR=$(cd "$SCRIPT_DIR/../.." ; pwd) DEFAULT_STEPS="req build copy_files test sign create_archive" -DIST_DIR=$PWD/dist +PYINSTALLER_OUTPUT_DIR=$ROOT_DIR/dist/ggshield +PACKAGES_DIR=$ROOT_DIR/packages REQUIREMENTS="pyinstaller" @@ -49,7 +50,7 @@ usage() { cat << EOF Usage: $PROGNAME [OPTION ...] [STEPS] -Build a standalone executable for ggshield. +Build OS specific packages for ggshield. Default steps are: $DEFAULT_STEPS @@ -57,7 +58,7 @@ Options: -h, --help Display this usage message and exit. --sign Sign the binary, on supported OSes. -For more details, see doc/dev/standalone-executables.md. +For more details, see doc/dev/os-packages.md. EOF exit 1 @@ -137,42 +138,42 @@ step_req() { step_build() { rm -rf build/ggshield - rm -rf "$DIST_DIR/ggshield" + rm -rf "$PYINSTALLER_OUTPUT_DIR" pyinstaller ggshield/__main__.py --name ggshield --noupx } step_copy_files() { - local pyinstaller_output_dir=$DIST_DIR/ggshield - if ! [ -d "$pyinstaller_output_dir" ] ; then - die "$pyinstaller_output_dir does not exist" + if ! [ -d "$PYINSTALLER_OUTPUT_DIR" ] ; then + die "$PYINSTALLER_OUTPUT_DIR does not exist" fi - local pyinstaller_ggshield=$pyinstaller_output_dir/ggshield$EXE_EXT + local pyinstaller_ggshield=$PYINSTALLER_OUTPUT_DIR/ggshield$EXE_EXT if ! [ -f "$pyinstaller_ggshield" ] ; then die "Can't find '$pyinstaller_ggshield', maybe 'build' step did not run?" fi + mkdir -p "$PACKAGES_DIR" case "$HUMAN_OS" in Linux|Windows) - local output_dir="$DIST_DIR/$ARCHIVE_DIR_NAME" + local output_dir="$PACKAGES_DIR/$ARCHIVE_DIR_NAME" info "Copying files to $output_dir" rm -rf "$output_dir" - cp -R "$pyinstaller_output_dir" "$output_dir" + cp -R "$PYINSTALLER_OUTPUT_DIR" "$output_dir" info "Generating README.md" sed \ -e "s/@HUMAN_OS@/$HUMAN_OS/" \ -e "s/@HUMAN_ARCH@/$HUMAN_ARCH/" \ - "$RESOURCES_DIR/README.md" \ + "$SCRIPT_DIR/README.md.tmpl" \ > "$output_dir/README.md" ;; macOS) - local output_dir="$DIST_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX" - local bin_dir="$DIST_DIR/$ARCHIVE_DIR_NAME/usr/local/bin" + local output_dir="$PACKAGES_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX" + local bin_dir="$PACKAGES_DIR/$ARCHIVE_DIR_NAME/usr/local/bin" info "Copying files to $output_dir" rm -rf "$output_dir" "$bin_dir" mkdir -p "$(dirname $output_dir)" - cp -R "$pyinstaller_output_dir" "$output_dir" + cp -R "$PYINSTALLER_OUTPUT_DIR" "$output_dir" info "Creating launcher symlink" mkdir -p "$bin_dir" @@ -197,7 +198,7 @@ sign_file() { } list_files_to_sign() { - local archive_dir="$DIST_DIR/$ARCHIVE_DIR_NAME" + local archive_dir="$PACKAGES_DIR/$ARCHIVE_DIR_NAME" echo "$archive_dir/$INSTALL_PREFIX/ggshield" find "$archive_dir" -name '*.so' -o -name '*.dylib' } @@ -219,30 +220,29 @@ step_sign() { step_test() { for args in --help --version ; do info "test: running $args" - "$DIST_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX/ggshield${EXE_EXT}" $args + "$PACKAGES_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX/ggshield${EXE_EXT}" $args info "test: running $args: OK" done } step_functests() { - PATH=$DIST_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX:$PATH pytest tests/functional + PATH=$PACKAGES_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX:$PATH pytest tests/functional } step_create_archive() { local archive_path - cd "$DIST_DIR" + cd "$PACKAGES_DIR" case "$HUMAN_OS" in Linux) - archive_path="$DIST_DIR/$ARCHIVE_DIR_NAME.tar.gz" + archive_path="$PACKAGES_DIR/$ARCHIVE_DIR_NAME.tar.gz" tar -czf "$archive_path" "$ARCHIVE_DIR_NAME" ;; macOS) - archive_path="$DIST_DIR/$ARCHIVE_DIR_NAME.pkg" - + archive_path="$PACKAGES_DIR/$ARCHIVE_DIR_NAME.pkg" pkgbuild \ --identifier com.gitguardian.ggshield \ --version "$VERSION" \ - --root "$DIST_DIR/$ARCHIVE_DIR_NAME" \ + --root "$PACKAGES_DIR/$ARCHIVE_DIR_NAME" \ "$archive_path" if [ "$DO_SIGN" -eq 1 ] ; then @@ -250,7 +250,7 @@ step_create_archive() { fi ;; Windows) - archive_path="$DIST_DIR/$ARCHIVE_DIR_NAME.zip" + archive_path="$PACKAGES_DIR/$ARCHIVE_DIR_NAME.zip" 7z a "$archive_path" "$ARCHIVE_DIR_NAME" ;; esac @@ -266,7 +266,7 @@ step_notarize() { rcodesign notary-submit \ --api-key-file "$MACOS_API_KEY_FILE" \ --staple \ - "$DIST_DIR/$ARCHIVE_DIR_NAME.pkg" + "$PACKAGES_DIR/$ARCHIVE_DIR_NAME.pkg" } steps="" From f44818159d3691a16756c18ca918efcbf07f8e70 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Wed, 17 Apr 2024 14:47:59 +0200 Subject: [PATCH 3/8] ci: make build-os-packages build .deb and .rpm on Linux Move code to build .deb and .rpm to build-os-packages. Remove build-packages script. --- .github/workflows/build_release_assets.yml | 24 +++-- .github/workflows/ci.yml | 88 +++++++------------ .github/workflows/tag.yml | 54 ++++++------ scripts/build-os-packages/build-os-packages | 38 +++++++- .../nfpm.yaml} | 19 ++-- scripts/build-packages/build-packages | 80 ----------------- scripts/build-packages/ggshield-wrapper | 39 -------- 7 files changed, 124 insertions(+), 218 deletions(-) rename scripts/{build-packages/nfpm.yaml.tmpl => build-os-packages/nfpm.yaml} (76%) delete mode 100755 scripts/build-packages/build-packages delete mode 100755 scripts/build-packages/ggshield-wrapper diff --git a/.github/workflows/build_release_assets.yml b/.github/workflows/build_release_assets.yml index 5b08eaa88c..a78091e742 100644 --- a/.github/workflows/build_release_assets.yml +++ b/.github/workflows/build_release_assets.yml @@ -30,7 +30,7 @@ jobs: dist build_os_packages: - name: Build signed, standalone binaries + name: Build packages runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -96,18 +96,30 @@ jobs: # Make it available cp rcodesign /usr/local/bin - - name: Install normal dependencies + - name: Install dependencies + shell: bash run: | python -m pip install --upgrade pip - python -m pip install --upgrade pipenv==2023.10.3 + python -m pip install --upgrade \ + pipenv==2023.10.3 \ + pyinstaller==6.5.0 pipenv install --system --dev env: # Disable lock otherwise Windows-only dependencies like colorama are not installed PIPENV_SKIP_LOCK: 1 - - name: Install standalone-specific dependencies + - name: Install Linux specific dependencies + if: matrix.os == 'ubuntu-22.04' run: | - python -m pip install pyinstaller==6.5.0 + NFPM_VERSION=2.36.1 + NFPM_CHECKSUM=05c17a1e09c470807b149fdd7bcd8f600eea044f459fc3ce81aa230103c0baf5 + + scripts/download \ + https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_amd64.deb \ + nfpm.deb \ + $NFPM_CHECKSUM + + sudo dpkg -i nfpm.deb - name: Prepare macOS secrets if: startsWith(matrix.os, 'macos-') @@ -164,3 +176,5 @@ jobs: packages/ggshield-*.gz packages/ggshield-*.pkg packages/ggshield-*.zip + packages/ggshield-*.rpm + packages/ggshield_*.deb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a924a58bc2..4c48639fa1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,17 +153,17 @@ jobs: GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }} TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }} - build-standalone: - name: Standalone exe + build_os_packages: + name: Build packages runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: - ubuntu-22.04 - - macos-13 - windows-2022 - - macos-14 + - macos-13 # x86-64 + - macos-14 # arm steps: - uses: actions/checkout@v4 with: @@ -175,32 +175,37 @@ jobs: with: python-version: '3.10' - - name: Install normal dependencies + - name: Install dependencies + shell: bash run: | python -m pip install --upgrade pip - python -m pip install --upgrade pipenv==2023.10.3 + python -m pip install --upgrade \ + pipenv==2023.10.3 \ + pyinstaller==6.5.0 pipenv install --system --dev env: # Disable lock otherwise Windows-only dependencies like colorama are not installed PIPENV_SKIP_LOCK: 1 - - name: Install standalone-specific dependencies + - name: Install Linux specific dependencies + if: matrix.os == 'ubuntu-22.04' run: | - python -m pip install pyinstaller==6.5.0 + NFPM_VERSION=2.36.1 + NFPM_CHECKSUM=05c17a1e09c470807b149fdd7bcd8f600eea044f459fc3ce81aa230103c0baf5 + + scripts/download \ + https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_amd64.deb \ + nfpm.deb \ + $NFPM_CHECKSUM + + sudo dpkg -i nfpm.deb - name: Build shell: bash run: | - scripts/build-standalone-exe - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: ggshield-${{ matrix.os }} - path: | - dist/ggshield-*.gz - dist/ggshield-*.pkg - dist/ggshield-*.zip + # Use --git-version to produce a package with a version number + # different from a release + scripts/build-os-packages/build-os-packages --git-version - name: Override base Docker image used for functional tests on Windows if: matrix.os == 'windows-2022' @@ -215,51 +220,22 @@ jobs: # See note about steps requiring the GITGUARDIAN_API at the top of this file if: ${{ !github.event.pull_request.head.repo.fork }} run: | - scripts/build-standalone-exe functests + scripts/build-os-packages/build-os-packages functests env: GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }} TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }} - build_packages: - # This job ensures the build-packages script is tested on each build, not only at release time - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Warning: changes on this step should be reflected in workflows/tag.yml - - name: Install packaging tools - shell: bash - run: | - curl -L https://github.com/goreleaser/nfpm/releases/download/v2.15.0/nfpm_amd64.deb -o nfpm_amd64.deb - sudo dpkg -i nfpm_amd64.deb - - pip install shiv==1.0.1 build - - # Append the abbreviated git sha1 to the version number to avoid confusing - # these packages with those produced at release time - - name: Fake version number - shell: bash - run: | - version=$(git describe --tags | sed -e 's/^v//' -e 's/-[0-9]*-g/+/') - echo "Set version number to '$version'" - sed -i "s/__version__ = .*/__version__ = \"$version\"/" ggshield/__init__.py - - - name: Create packages - shell: bash - run: scripts/build-packages/build-packages - - # Make packages downloadable from the workflow page - - name: Upload packages - uses: actions/upload-artifact@v3 + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: - name: packages + name: os-packages-${{ matrix.os }} path: | - dist - packages + packages/ggshield-*.gz + packages/ggshield-*.pkg + packages/ggshield-*.zip + packages/ggshield-*.rpm + packages/ggshield_*.deb test_github_secret_scan_action: name: Test GitHub action for `secret scan` diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 154d4c5dbc..c7e566e47f 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -74,11 +74,10 @@ jobs: run: | gh release upload \ ${{ steps.tags.outputs.tag }} \ - packages/ggshield-*.pkg - - # TODO: going to be brought back in next commits - # packages/ggshield_*.deb \ - # packages/ggshield-*.rpm \ + packages/ggshield-*.pkg \ + packages/ggshield_*.deb \ + packages/ggshield-*.rpm \ + packages/ggshield-*.gz push_to_docker_hub: name: Push Docker image to Docker Hub @@ -141,25 +140,26 @@ jobs: git push -# TODO: going to be brought back in next commits -# push_to_cloudsmith: -# needs: build_release_assets -# runs-on: ubuntu-22.04 -# if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# -# - name: Download packages -# uses: actions/download-artifact@v4 -# with: -# name: packages -# -# - name: Install Cloudsmith CLI -# run: pip install cloudsmith-cli -# -# - name: Push to Cloudsmith -# run: | -# scripts/push-to-cloudsmith -# env: -# CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} + push_to_cloudsmith: + needs: build_release_assets + runs-on: ubuntu-22.04 + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download packages + uses: actions/download-artifact@v4 + with: + pattern: os-packages-* + path: packages + merge-multiple: true + + - name: Install Cloudsmith CLI + run: pip install cloudsmith-cli + + - name: Push to Cloudsmith + run: | + scripts/push-to-cloudsmith + env: + CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} diff --git a/scripts/build-os-packages/build-os-packages b/scripts/build-os-packages/build-os-packages index cd2b929bff..1a697c7837 100755 --- a/scripts/build-os-packages/build-os-packages +++ b/scripts/build-os-packages/build-os-packages @@ -15,6 +15,11 @@ REQUIREMENTS="pyinstaller" # Whether we want a signed binary or not DO_SIGN=0 +# Where to find the version: +# - 0: read it from ggshield/__init__.py +# - 1: generate it using `git describe` +USE_GIT_VERSION=0 + MACOS_P12_FILE=${MACOS_P12_FILE:-} MACOS_P12_PASSWORD_FILE=${MACOS_P12_PASSWORD_FILE:-} @@ -57,6 +62,7 @@ Default steps are: $DEFAULT_STEPS Options: -h, --help Display this usage message and exit. --sign Sign the binary, on supported OSes. + --git-version Append "+COMMIT_SHA" to the version number. For more details, see doc/dev/os-packages.md. EOF @@ -66,6 +72,12 @@ EOF read_version() { VERSION=$(grep -o "[0-9]*\.[0-9]*\.[0-9]*" "$ROOT_DIR/ggshield/__init__.py") + if [ "$USE_GIT_VERSION" -eq 1 ] ; then + local commit_sha + commit_sha=$(git rev-parse --short HEAD) + VERSION="$VERSION+$commit_sha" + fi + info "VERSION=$VERSION" } init_system_vars() { @@ -95,6 +107,7 @@ init_system_vars() { EXE_EXT="" TARGET="$arch-unknown-linux-gnu" HUMAN_OS=Linux + REQUIREMENTS="$REQUIREMENTS nfpm" ;; Darwin) EXE_EXT="" @@ -229,21 +242,38 @@ step_functests() { PATH=$PACKAGES_DIR/$ARCHIVE_DIR_NAME/$INSTALL_PREFIX:$PATH pytest tests/functional } +create_linux_packages() { + for format in rpm deb ; do + info "Building $format" + + PYINSTALLER_OUTPUT_DIR=$PYINSTALLER_OUTPUT_DIR \ + VERSION=$VERSION \ + nfpm package \ + --packager $format \ + --config "$SCRIPT_DIR/nfpm.yaml" \ + --target "$PACKAGES_DIR" + done +} + step_create_archive() { local archive_path - cd "$PACKAGES_DIR" case "$HUMAN_OS" in Linux) archive_path="$PACKAGES_DIR/$ARCHIVE_DIR_NAME.tar.gz" + pushd "$PACKAGES_DIR" tar -czf "$archive_path" "$ARCHIVE_DIR_NAME" + popd + create_linux_packages ;; macOS) archive_path="$PACKAGES_DIR/$ARCHIVE_DIR_NAME.pkg" + pushd "$PACKAGES_DIR" pkgbuild \ --identifier com.gitguardian.ggshield \ --version "$VERSION" \ --root "$PACKAGES_DIR/$ARCHIVE_DIR_NAME" \ "$archive_path" + popd if [ "$DO_SIGN" -eq 1 ] ; then sign_file "$archive_path" @@ -251,10 +281,11 @@ step_create_archive() { ;; Windows) archive_path="$PACKAGES_DIR/$ARCHIVE_DIR_NAME.zip" + pushd "$PACKAGES_DIR" 7z a "$archive_path" "$ARCHIVE_DIR_NAME" + popd ;; esac - cd - info "Archive created in $archive_path" } @@ -278,6 +309,9 @@ while [ $# -gt 0 ] ; do --sign) DO_SIGN=1 ;; + --git-version) + USE_GIT_VERSION=1 + ;; -*) usage "Unknown option '$1'" ;; diff --git a/scripts/build-packages/nfpm.yaml.tmpl b/scripts/build-os-packages/nfpm.yaml similarity index 76% rename from scripts/build-packages/nfpm.yaml.tmpl rename to scripts/build-os-packages/nfpm.yaml index 0a20b4783c..6fa02b150d 100644 --- a/scripts/build-packages/nfpm.yaml.tmpl +++ b/scripts/build-os-packages/nfpm.yaml @@ -8,23 +8,24 @@ description: 'CLI application that runs in your local environment or in a CI env arch: amd64 platform: linux -version: @VERSION@ +version: ${VERSION} version_schema: semver release: 1 depends: - git - - python3 + contents: - - src: @GGSHIELD_WRAPPER@ + - src: /usr/libexec/ggshield/ggshield dst: /usr/bin/ggshield - file_info: - mode: 0755 - - src: @GGSHIELD_PYZ@ - dst: /usr/lib/ggshield/ggshield.pyz - file_info: - mode: 0755 + type: symlink + + - src: ${PYINSTALLER_OUTPUT_DIR} + dst: /usr/libexec/ggshield + expand: true + - src: README.md dst: /usr/share/doc/ggshield/README.md + - src: LICENSE dst: /usr/share/doc/ggshield/LICENSE diff --git a/scripts/build-packages/build-packages b/scripts/build-packages/build-packages deleted file mode 100755 index 00d11c0f25..0000000000 --- a/scripts/build-packages/build-packages +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/bash -set -euo pipefail - -# This script creates ggshield packages in various formats. -# -# It expects [nfpm][] and [shiv][] to be installed. -# -# Packages are created in the `packages` directory. -# -# [nfpm]: https://github.com/goreleaser/nfpm/ -# [shiv]: https://github.com/linkedin/shiv - -cd $(dirname $0) -NFPM_YAML_TMPL=$PWD/nfpm.yaml.tmpl -GGSHIELD_WRAPPER=$PWD/ggshield-wrapper - -# Move to the work-tree root -cd $(git rev-parse --show-toplevel) - -# Where `python -m build ...` places its packages -DIST_DIR=$PWD/dist - -# Where npfm places its packages. We do not use $DIST_DIR to ensure the "upload -# to pypi" build step does not try to upload them. -PKG_DIR=$PWD/packages - -VERSION=$(python -c 'import ggshield; print(ggshield.__version__)') - -GGSHIELD_WHL=$DIST_DIR/ggshield-$VERSION-py3-none-any.whl -GGSHIELD_SDIST=$DIST_DIR/ggshield-$VERSION.tar.gz -GGSHIELD_PYZ=$PKG_DIR/ggshield-$VERSION.pyz - -log_progress() { - local cstart="\e[35m" - local cend="\e[0m" - echo -e "${cstart}$*${cend}" -} - -build_whl_sdist() { - log_progress "Building wheel and sdist" - python -m build -} - -build_pyz() { - log_progress "Building $GGSHIELD_PYZ" - shiv -c ggshield --reproducible --compile-pyc --compressed -o "$GGSHIELD_PYZ" "$GGSHIELD_WHL" -} - -run_nfpm() { - # nfpm supports environment variables in its configuration file, but only - # in certain fields. It does not support them in `contents/src` [1], so we - # need to do a search-and-replace for this field. To be consistent, we do - # this for the `version` field as well, even if this one supports - # environment variables. - # [1]: https://github.com/goreleaser/nfpm/issues/449 - local nfpm_yaml=$(mktemp --tmpdir nfpm-XXXXX.yaml) - sed \ - -e "s,@VERSION@,$VERSION," \ - -e "s,@GGSHIELD_WRAPPER@,$GGSHIELD_WRAPPER," \ - -e "s,@GGSHIELD_PYZ@,$GGSHIELD_PYZ," \ - "$NFPM_YAML_TMPL" > "$nfpm_yaml" - - for format in rpm deb ; do - log_progress "Building $format" - nfpm package --packager $format --config "$nfpm_yaml" --target "$PKG_DIR" - done - rm "$nfpm_yaml" -} - -mkdir -p $PKG_DIR - -if [ ! -f "$GGSHIELD_WHL" ] || [ ! -f "$GGSHIELD_SDIST" ] ; then - build_whl_sdist -fi - -if [ ! -f "$GGSHIELD_PYZ" ] ; then - build_pyz -fi - -run_nfpm diff --git a/scripts/build-packages/ggshield-wrapper b/scripts/build-packages/ggshield-wrapper deleted file mode 100755 index 77028ada95..0000000000 --- a/scripts/build-packages/ggshield-wrapper +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# This wrapper looks for a recent enough Python interpreter and exits with a -# user-friendly message if it can't find one. -# This is useful when the `python3` interpreter is too old, but the system -# has a more recent `python3.x` interpreter installed. - -# The minimum Python version we need -MAJOR=3 -MINOR=8 - -GGSHIELD_PYZ=/usr/lib/ggshield/ggshield.pyz - -POSSIBLE_PYTHONS=\ -"python311 python3.11 -python310 python3.10 -python39 python3.9 -python38 python3.8 -python3 -python -" - -is_python_usable() { - local python_cmd=$1 - if ! command -v "$python_cmd" > /dev/null ; then - return 1 - fi - $python_cmd -c "import sys; sys.exit(0 if sys.version_info >= ($MAJOR, $MINOR) else 1)" -} - -for python_cmd in $POSSIBLE_PYTHONS ; do - if is_python_usable "$python_cmd" ; then - exec "$python_cmd" $GGSHIELD_PYZ "$@" - fi -done - -echo "Error: could not find a usable Python interpreter. ggshield needs at least Python $MAJOR.$MINOR." >&2 -exit 1 From 097bfbef64c4119a2a6956c1cee73f4d07fcf112 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Thu, 18 Apr 2024 11:38:45 +0200 Subject: [PATCH 4/8] fix: strip binaries on Linux and macOS --- scripts/build-os-packages/build-os-packages | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/build-os-packages/build-os-packages b/scripts/build-os-packages/build-os-packages index 1a697c7837..3005a65275 100755 --- a/scripts/build-os-packages/build-os-packages +++ b/scripts/build-os-packages/build-os-packages @@ -153,7 +153,14 @@ step_build() { rm -rf build/ggshield rm -rf "$PYINSTALLER_OUTPUT_DIR" - pyinstaller ggshield/__main__.py --name ggshield --noupx + local extra_args="" + if [ "$HUMAN_OS" != Windows ] ; then + # Only strip on Linux and macOS: pyinstaller docs says it's not + # recommended on Windows. + extra_args="--strip" + fi + + pyinstaller ggshield/__main__.py --name ggshield --noupx $extra_args } step_copy_files() { From d7bef319534f884b19e1dd38ce00f0c9fda682fa Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Thu, 18 Apr 2024 12:11:48 +0200 Subject: [PATCH 5/8] fix: fix issues reported by running lintian on the deb packages - define section - provide a short first line for description, and 80-column-wrapped content for the extended-description field - make sure files are not group-writable - fix absolute symbolic link - add dependency on libc6 --- scripts/build-os-packages/build-os-packages | 6 ++++++ scripts/build-os-packages/nfpm.yaml | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/build-os-packages/build-os-packages b/scripts/build-os-packages/build-os-packages index 3005a65275..ea5d511215 100755 --- a/scripts/build-os-packages/build-os-packages +++ b/scripts/build-os-packages/build-os-packages @@ -161,6 +161,12 @@ step_build() { fi pyinstaller ggshield/__main__.py --name ggshield --noupx $extra_args + + if [ "$HUMAN_OS" != Windows ] ; then + # Libraries do not need to be executable + find "$PYINSTALLER_OUTPUT_DIR" \( -name "*.so.*" -o -name "*.so" -o -name "*.dylib" \) \ + -exec chmod -x '{}' ';' + fi } step_copy_files() { diff --git a/scripts/build-os-packages/nfpm.yaml b/scripts/build-os-packages/nfpm.yaml index 6fa02b150d..762b18a54b 100644 --- a/scripts/build-os-packages/nfpm.yaml +++ b/scripts/build-os-packages/nfpm.yaml @@ -1,10 +1,14 @@ name: ggshield -section: default +section: utils vendor: GitGuardian maintainer: GitGuardian license: MIT homepage: https://github.com/GitGuardian/ggshield -description: 'CLI application that runs in your local environment or in a CI environment to help you detect leaked secrets, as well as other potential security vulnerabilities or policy breaks.' +description: |- + Detect leaked secrets and other potential security vulnerabilities + ggshield runs in your local environment or in a CI environment to help you + detect leaked secrets, as well as other potential security vulnerabilities or + policy breaks. arch: amd64 platform: linux @@ -15,8 +19,11 @@ release: 1 depends: - git +# Make sure files are not group-writable. lintian does not like that. +umask: 0o022 + contents: - - src: /usr/libexec/ggshield/ggshield + - src: ../libexec/ggshield/ggshield dst: /usr/bin/ggshield type: symlink @@ -29,3 +36,9 @@ contents: - src: LICENSE dst: /usr/share/doc/ggshield/LICENSE + +overrides: + deb: + depends: + - libc6 + - git From 648ca0a7fbdb4b48f526b6e9efd8662507e9f676 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Thu, 18 Apr 2024 16:11:58 +0200 Subject: [PATCH 6/8] ci: factorize CI Code Make ci.yml use build_release_assets.yml too. ci.yml was basically embedding a copy of build_release_assets.yml. This commit removes the copy and makes ci.yml call build_release_assets.yml. The only change in build_release_assets.yml is the addition of a "release_mode" input option, so that ci.yml produces unsigned packages with commit-sha in their version numbers, whereas tag.yml produces signed packages without commit-sha in version numbers. --- .github/workflows/build_release_assets.yml | 23 ++++-- .github/workflows/ci.yml | 84 +--------------------- .github/workflows/tag.yml | 2 + 3 files changed, 23 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build_release_assets.yml b/.github/workflows/build_release_assets.yml index a78091e742..d21526ed4e 100644 --- a/.github/workflows/build_release_assets.yml +++ b/.github/workflows/build_release_assets.yml @@ -1,11 +1,21 @@ name: Build release assets -# This workflow is used by the tag workflow to build all release assets. It -# can also be triggered manually. +# This workflow is used by the ci and tag workflows to build all release +# assets. It can also be triggered manually. on: workflow_call: + inputs: + release_mode: + description: 'Release mode (signed binaries, no commit sha in version number)' + type: boolean + default: false workflow_dispatch: + inputs: + release_mode: + description: 'Release mode (signed binaries, no commit sha in version number)' + type: boolean + default: false jobs: build_wheel_sdist: @@ -122,7 +132,7 @@ jobs: sudo dpkg -i nfpm.deb - name: Prepare macOS secrets - if: startsWith(matrix.os, 'macos-') + if: startsWith(matrix.os, 'macos-') && inputs.release_mode run: | set -euo pipefail SECRETS_DIR=$TMPDIR/secrets @@ -147,7 +157,12 @@ jobs: - name: Build shell: bash run: | - scripts/build-os-packages/build-os-packages --sign + if [ "${{ inputs.release_mode }}" = "true" ] ; then + args="--sign" + else + args="--git-version" + fi + scripts/build-os-packages/build-os-packages $args - name: Override base Docker image used for functional tests on Windows if: matrix.os == 'windows-2022' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c48639fa1..eb8c66b293 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,88 +154,8 @@ jobs: TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }} build_os_packages: - name: Build packages - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-22.04 - - windows-2022 - - macos-13 # x86-64 - - macos-14 # arm - steps: - - uses: actions/checkout@v4 - with: - # Get enough commits to run `ggshield secret scan commit-range` on ourselves - fetch-depth: 10 - - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade \ - pipenv==2023.10.3 \ - pyinstaller==6.5.0 - pipenv install --system --dev - env: - # Disable lock otherwise Windows-only dependencies like colorama are not installed - PIPENV_SKIP_LOCK: 1 - - - name: Install Linux specific dependencies - if: matrix.os == 'ubuntu-22.04' - run: | - NFPM_VERSION=2.36.1 - NFPM_CHECKSUM=05c17a1e09c470807b149fdd7bcd8f600eea044f459fc3ce81aa230103c0baf5 - - scripts/download \ - https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_amd64.deb \ - nfpm.deb \ - $NFPM_CHECKSUM - - sudo dpkg -i nfpm.deb - - - name: Build - shell: bash - run: | - # Use --git-version to produce a package with a version number - # different from a release - scripts/build-os-packages/build-os-packages --git-version - - - name: Override base Docker image used for functional tests on Windows - if: matrix.os == 'windows-2022' - # This is required because GitHub Windows runner is not configured to - # run Linux-based Docker images - shell: bash - run: | - echo "GGTEST_DOCKER_IMAGE=mcr.microsoft.com/windows/nanoserver:ltsc2022" >> $GITHUB_ENV - - - name: Functional tests - shell: bash - # See note about steps requiring the GITGUARDIAN_API at the top of this file - if: ${{ !github.event.pull_request.head.repo.fork }} - run: | - scripts/build-os-packages/build-os-packages functests - env: - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} - GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }} - TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }} - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: os-packages-${{ matrix.os }} - path: | - packages/ggshield-*.gz - packages/ggshield-*.pkg - packages/ggshield-*.zip - packages/ggshield-*.rpm - packages/ggshield_*.deb + uses: ./.github/workflows/build_release_assets.yml + secrets: inherit test_github_secret_scan_action: name: Test GitHub action for `secret scan` diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index c7e566e47f..ecf0b89f64 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -9,6 +9,8 @@ jobs: build_release_assets: uses: ./.github/workflows/build_release_assets.yml secrets: inherit + with: + release_mode: true push_to_pypi: needs: build_release_assets From a6143ae6c9d930c3457e437b9e8e78a394850fd0 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Thu, 18 Apr 2024 15:34:04 +0200 Subject: [PATCH 7/8] docs: add flowchart of build-os-packages steps --- doc/dev/os-packages.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/dev/os-packages.md b/doc/dev/os-packages.md index 6d46d3b8f1..476f181db9 100644 --- a/doc/dev/os-packages.md +++ b/doc/dev/os-packages.md @@ -10,6 +10,24 @@ The process of generating the packages is handled by the `scripts/build-os-packa All functions in the script starting with `step_` can be used as a step. This means you can get a list of all available steps with: `grep -o '^step_[a-z_]*' scripts/build-os-packages/build-os-packages`. +Here is a high-level overview of the main steps (square boxes are steps): + +```mermaid +flowchart TD + src[/source code/] --> build --> pyinstaller_dir[/"pyinstaller output + (dist/ggshield)"/] + pyinstaller_dir --> copy_files --> archive_dir[/"dir ready to be archived + (packages/ggshield-$version-$target)"/] + archive_dir --> test["test (run functional tests on archive dir)"] + test --> signing{Called with --sign?} -->|yes| sign + signing -->|no| create_archive + sign --> create_archive --> pkg[/"pkg 🍏"/] + create_archive --> zip[/"zip 🪟"/] + create_archive --> tar.gz[/"tar.gz 🐧"/] + create_archive --> deb[/"deb 🐧"/] + create_archive --> rpm[/"rpm 🐧"/] +``` + ## Generating the standalone executable We use [PyInstaller](https://pyinstaller.org) to generate `ggshield` standalone executable. From 5e785a54102e5c1014b32920e88a68314fefbd41 Mon Sep 17 00:00:00 2001 From: ggshield-test Date: Wed, 17 Apr 2024 18:06:28 +0200 Subject: [PATCH 8/8] docs: add changelog entry --- .../20240417_180529_ggshield-test_standalone_linux_packages.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/20240417_180529_ggshield-test_standalone_linux_packages.md diff --git a/changelog.d/20240417_180529_ggshield-test_standalone_linux_packages.md b/changelog.d/20240417_180529_ggshield-test_standalone_linux_packages.md new file mode 100644 index 0000000000..dba2c6373c --- /dev/null +++ b/changelog.d/20240417_180529_ggshield-test_standalone_linux_packages.md @@ -0,0 +1,3 @@ +### Changed + +- Linux .deb and .rpm packages now use the binaries produced by pyinstaller. They no longer depend on Python.