feat(standalone): add a script to build a standalone binary

.github/workflows/ci.yml

@@ -151,6 +151,57 @@ jobs:
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
           TEST_KNOWN_SECRET: ${{ secrets.TEST_KNOWN_SECRET }}
 
+  build-standalone:
+    name: Build standalone executable
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-2022]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Get enough commits to run `ggshield secret scan commit-range` on ourselves
+          fetch-depth: 10
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install normal dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade pipenv==2023.10.3
+          pipenv install --system --dev
+
+      - name: Install standalone-specific dependencies
+        run: |
+          python -m pip install --upgrade pyoxidizer
+
+      - name: Install macOS dependencies
+        if: matrix.os == 'macos-latest'
+        run: |
+          brew install upx
+
+      - name: Install Windows dependencies
+        if: matrix.os == 'windows-2022'
+        run: |
+          choco install upx
+
+      - name: Build
+        shell: bash
+        run: |
+          scripts/build-standalone-exe
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts
+          path: |
+            dist/ggshield-standalone-*.gz
+            dist/ggshield-standalone-*.zip
+
   build_packages:
     # This job ensures the build-packages script is tested on each build, not only at release time
     runs-on: ubuntu-latest

pyoxidizer.bzl

@@ -0,0 +1,122 @@
+# This file defines how PyOxidizer application building and packaging is
+# performed. See PyOxidizer's documentation at
+# https://gregoryszorc.com/docs/pyoxidizer/stable/pyoxidizer.html for details
+# of this configuration file format.
+
+# Configuration files consist of functions which define build "targets."
+# This function creates a Python executable and installs it in a destination
+# directory.
+def make_exe():
+    # Obtain the default PythonDistribution for our build target. We link
+    # this distribution into our produced executable and extract the Python
+    # standard library from it.
+    dist = default_python_distribution()
+
+    # This function creates a `PythonPackagingPolicy` instance, which
+    # influences how executables are built and how resources are added to
+    # the executable. You can customize the default behavior by assigning
+    # to attributes and calling functions.
+    policy = dist.make_python_packaging_policy()
+    policy.extension_module_filter = "all"
+    policy.include_distribution_sources = True
+    policy.include_distribution_resources = True
+    policy.include_test = False
+    policy.resources_location_fallback = "filesystem-relative:prefix"
+
+    #
+    # The configuration of the embedded Python interpreter can be modified
+    # by setting attributes on the instance. Some of these are
+    # documented below.
+    python_config = dist.make_python_interpreter_config()
+    python_config.run_command = "from ggshield.__main__ import main; main()"
+
+    # Produce a PythonExecutable from a Python distribution, embedded
+    # resources, and other options. The returned object represents the
+    # standalone executable that will be built.
+    exe = dist.to_python_executable(
+        name="ggshield",
+
+        # If no argument passed, the default `PythonPackagingPolicy` for the
+        # distribution is used.
+        packaging_policy=policy,
+
+        # If no argument passed, the default `PythonInterpreterConfig` is used.
+        config=python_config,
+    )
+
+    exe.add_python_resources(exe.pip_install([CWD]))
+
+    return exe
+
+def make_embedded_resources(exe):
+    return exe.to_embedded_resources()
+
+def make_install(exe):
+    # Create an object that represents our installed application file layout.
+    files = FileManifest()
+
+    # Add the generated executable to our install layout in the root directory.
+    files.add_python_resource(".", exe)
+
+    return files
+
+def make_msi(exe):
+    # See the full docs for more. But this will convert your Python executable
+    # into a `WiXMSIBuilder` Starlark type, which will be converted to a Windows
+    # .msi installer when it is built.
+    return exe.to_wix_msi_builder(
+        # Simple identifier of your app.
+        "myapp",
+        # The name of your application.
+        "My Application",
+        # The version of your application.
+        "1.0",
+        # The author/manufacturer of your application.
+        "Alice Jones"
+    )
+
+
+# Dynamically enable automatic code signing.
+def register_code_signers():
+    # You will need to run with `pyoxidizer build --var ENABLE_CODE_SIGNING 1` for
+    # this if block to be evaluated.
+    if not VARS.get("ENABLE_CODE_SIGNING"):
+        return
+
+    # Use a code signing certificate in a .pfx/.p12 file, prompting the
+    # user for its path and password to open.
+    # pfx_path = prompt_input("path to code signing certificate file")
+    # pfx_password = prompt_password(
+    #     "password for code signing certificate file",
+    #     confirm = True
+    # )
+    # signer = code_signer_from_pfx_file(pfx_path, pfx_password)
+
+    # Use a code signing certificate in the Windows certificate store, specified
+    # by its SHA-1 thumbprint. (This allows you to use YubiKeys and other
+    # hardware tokens if they speak to the Windows certificate APIs.)
+    # sha1_thumbprint = prompt_input(
+    #     "SHA-1 thumbprint of code signing certificate in Windows store"
+    # )
+    # signer = code_signer_from_windows_store_sha1_thumbprint(sha1_thumbprint)
+
+    # Choose a code signing certificate automatically from the Windows
+    # certificate store.
+    # signer = code_signer_from_windows_store_auto()
+
+    # Activate your signer so it gets called automatically.
+    # signer.activate()
+
+
+# Call our function to set up automatic code signers.
+register_code_signers()
+
+# Tell PyOxidizer about the build targets defined above.
+register_target("exe", make_exe)
+register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True)
+register_target("install", make_install, depends=["exe"], default=True)
+register_target("msi_installer", make_msi, depends=["exe"])
+
+# Resolve whatever targets the invoker of this configuration file is requesting
+# be resolved.
+resolve_targets()

scripts/build-standalone-exe

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PROGNAME=$(basename "$0")
+ROOT_DIR=$(cd "$(dirname "$0")/.." ; pwd)
+
+DEFAULT_STEPS="req build pack"
+
+DIST_DIR=$PWD/dist
+UPX_SPEED_OPTION=--best
+
+REQUIREMENTS="upx pyoxidizer strip"
+
+err() {
+    echo "$@" >&2
+}
+
+die() {
+    err "$PROGNAME: $*"
+    exit 1
+}
+
+usage() {
+    if [ "$*" != "" ] ; then
+        err "Error: $*"
+        err
+    fi
+
+    cat << EOF
+Usage: $PROGNAME [OPTION ...] [steps]
+Build a standalone executable for ggshield.
+
+Options:
+  -h, --help      Display this usage message and exit.
+  --fast          Use fast UPX compression. Useful for testing.
+EOF
+
+    exit 1
+}
+
+progress() {
+    echo "$PROGNAME: step $1"
+}
+
+read_version() {
+    VERSION=$(grep -o "[0-9]*\.[0-9]*\.[0-9]*" "$ROOT_DIR/ggshield/__init__.py")
+}
+
+init_system_vars() {
+    ARCH=$(uname -m)
+
+    local out
+    out=$(uname)
+
+    case "$out" in
+    Linux)
+        LIB_EXT=".so"
+        EXE_EXT=""
+        TARGET="$ARCH-unknown-linux-gnu"
+        ;;
+    Darwin)
+        LIB_EXT=".dylib"
+        EXE_EXT=""
+        TARGET="$ARCH-apple-darwin"
+        ;;
+    MINGW*|MSYS*)
+        LIB_EXT=".dll"
+        EXE_EXT=".exe"
+        TARGET="$ARCH-pc-windows-msvc"
+        ;;
+    *)
+        die "Unknown OS. uname printed '$out'"
+        ;;
+    esac
+}
+
+step_req() {
+    local fail=0
+    echo "Checking requirements"
+    for exe in $REQUIREMENTS ; do
+        err -n "$exe: "
+        if command -v "$exe" > /dev/null ; then
+            err OK
+        else
+            err FAIL
+            fail=1
+        fi
+    done
+    if [ $fail -ne 0 ] ; then
+        die "Not all requirements are installed"
+    fi
+}
+
+step_build() {
+    pyoxidizer run --release
+}
+
+step_pack() {
+    local oxidize_output_dir=$PWD/build/$TARGET/release/install
+    if ! [ -d "$oxidize_output_dir" ] ; then
+        ls "$PWD/build"
+        die "$oxidize_output_dir does not exist"
+    fi
+    local oxidized_ggshield=$oxidize_output_dir/ggshield$EXE_EXT
+    if ! [ -f "$oxidized_ggshield" ] ; then
+        die "Can't find '$oxidized_ggshield', maybe 'build' step did not run?"
+    fi
+
+    local archive_dir_name=ggshield-standalone-$VERSION-$TARGET
+    local output_dir="$DIST_DIR/$archive_dir_name"
+
+    # Copy our files to $output_dir
+    rm -rf "$output_dir"
+    mkdir -p "$output_dir"
+    cp -R "$oxidize_output_dir/prefix" "$oxidized_ggshield" "$output_dir"
+
+    # Strip all libs and the executable
+    find "$output_dir" -name $LIB_EXT -exec strip '{}' ';'
+    strip "$output_dir/ggshield$EXE_EXT"
+
+    # Compress the executable
+    upx $UPX_SPEED_OPTION "$output_dir/ggshield$EXE_EXT"
+
+    # Create archive
+    local archive_path="$DIST_DIR/$archive_dir_name.tar.gz"
+    tar -C "$DIST_DIR" -czf "$archive_path" "$archive_dir_name"
+    err "Archive created in $archive_path"
+}
+
+steps=""
+while [ $# -gt 0 ] ; do
+    case "$1" in
+    -h|--help)
+        usage
+        ;;
+    --fast)
+        UPX_SPEED_OPTION=-1
+        ;;
+    -*)
+        usage "Unknown option '$1'"
+        ;;
+    *)
+        steps="$steps $1"
+        ;;
+    esac
+    shift
+done
+
+if [ -z "$steps" ] ; then
+    steps=$DEFAULT_STEPS
+fi
+
+read_version
+init_system_vars
+for step in $steps ; do
+    progress "$step"
+    "step_$step"
+done