diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 21f4370..5d1b194 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,19 +5,58 @@ on: - v1.x - v2.x - v3.x + - v4.x jobs: - build: - name: Publish latest + stage1: + name: Stage 1 runs-on: ubuntu-latest steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and publish images - run: | - ./scripts/build -p . + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s toolchain + stage2: + name: Stage 2 + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s base + stage3: + name: Stage 3 + runs-on: ubuntu-latest + needs: stage2 + strategy: + matrix: + target: [golang, python, qt, rust, dotnet6] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s ${{ matrix.target }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1c61eee..d62326c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -5,15 +5,58 @@ on: - v1.x - v2.x - v3.x + - v4.x jobs: - build: - name: Publish latest + stage1: + name: Stage 1 runs-on: ubuntu-latest - permissions: - contents: read steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Build images - run: | - ./scripts/build . + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -g -s toolchain + stage2: + name: Stage 2 + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -g -s base + stage3: + name: Stage 3 + runs-on: ubuntu-latest + needs: stage2 + strategy: + matrix: + target: [golang, python, qt, rust, dotnet6] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -g -s ${{ matrix.target }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3eb7ab1..c893cac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,19 +4,68 @@ on: tags: - '*' jobs: - release: - name: Publish a release + stage1: + name: Stage 1 runs-on: ubuntu-latest steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and publish images + - name: Set up version run: | - version="$(echo "${{ github.ref }}" | cut -d / -f 3)" - ./scripts/build -p . "$version" + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "version=$VERSION" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s toolchain -v ${{ env.version }} + stage2: + name: Stage 2 + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "version=$VERSION" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s base -v ${{ env.version }} + stage3: + name: Stage 3 + runs-on: ubuntu-latest + needs: stage2 + strategy: + matrix: + target: [golang, python, qt, rust, dotnet6] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "version=$VERSION" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s ${{ matrix.target }} -v ${{ env.version }} diff --git a/base/Dockerfile b/base/Dockerfile index 48fcd71..7f5ebf0 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -174,7 +174,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && cd /root \ && mkdir openssl \ && cd openssl \ - && curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -o openssl.tar.gz \ + && curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -Lo openssl.tar.gz \ && echo "ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46 openssl.tar.gz" > sha256sums \ && sha256sum -c sha256sums \ && tar --strip-components=1 -xf openssl.tar.gz \ diff --git a/scripts/build b/scripts/build index c43fe74..5c01751 100755 --- a/scripts/build +++ b/scripts/build @@ -1,95 +1,93 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -set -e -source "${BASH_SOURCE%/*}"/lib +import argparse as ap +import subprocess as sp +import typing as tp +import os -usage="$0 [OPTION]... IMAGESDIR [VERSION] - -Build the Docker images under the IMAGESDIR directory. +parser = ap.ArgumentParser( + description=""" +Build the Docker images under the IMAGESDIR directory (default: .). Label each built image with the given VERSION number (default: latest). The build takes advantage of any reusable layer from previously built images. - -Options: - - -h Show this help message. - -p Publish each built image to the GitHub Container Registry - (you need to be logged in to ghcr.io with the appropriate - permissions for this to work)." - -helpflag= -publishflag= - -while getopts hp name; do - case $name in - h) helpflag=1 ;; - p) publishflag=1 ;; - *) error "Invalid option. Use the -h flag for more information." ;; - esac -done - -shift $((OPTIND - 1)) - -if [[ -n $helpflag ]]; then - echo "$usage" - exit -fi - -if [[ $# -eq 0 ]]; then - error "Missing IMAGESDIR arguments. Use the -h flag for more information." -fi - -if [[ $# -gt 2 ]]; then - error "Extraneous arguments. Use the -h flag for more information." -fi - -imagesdir="$1" -version="$2" - -# Enable BuildKit for better cache behavior -# See -export DOCKER_BUILDKIT=1 - -docker-build() { - from="$(image-name "$1" "$version")" - target="$(image-name "$2" "$version")" - status "Building image '$target'" - docker image build \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - --build-arg FROM="$from" \ - --cache-from "$(image-name "$2")" \ - --tag "$target" . - - if [[ -n $publishflag ]]; then - docker image push "$target" - fi +""" +) + +builds = ["toolchain", "base", "golang", "python", "qt", "rust", "dotnet6"] +requires = { + "toolchain": None, + "base": "toolchain", + "golang": "base", + "python": "base", + "qt": "base", + "rust": "base", + "dotnet6": "toolchain", } -pushd "$imagesdir"/toolchain -docker-build toolchain toolchain -popd - -pushd "$imagesdir"/base -docker-build toolchain base -popd - -pushd "$imagesdir"/golang -docker-build base golang -popd - -pushd "$imagesdir"/python -docker-build base python -popd - -pushd "$imagesdir"/qt -docker-build base qt -popd - -pushd "$imagesdir"/rust -docker-build base rust -popd - -pushd "$imagesdir"/dotnet6 -docker-build toolchain dotnet6 -popd - -status "Done" +parser.add_argument( + "-p", help="Publish built images to GitHub Container Registry", action="store_true" +) +parser.add_argument("-g", help="Use GitHub Actions for caching", action="store_true") +parser.add_argument("-d", help="Set image directory", default=".", metavar="IMAGESDIR") +parser.add_argument( + "-s", help="Just build single image given as argument", choices=builds +) +parser.add_argument( + "-v", help="Set version number", default="latest", metavar="VERSION" +) + + +def run_cmd(line: list[str]) -> None: + p = sp.Popen( + line, + stdout=sp.PIPE, + stderr=sp.STDOUT, + encoding="utf-8", + errors="replace", + ) + while True: + realtime_output = p.stdout.readline() + if realtime_output == "" and p.poll() is not None: + break + if realtime_output: + print(realtime_output.strip(), flush=True) + if p.returncode != 0: + exit(p.returncode) + + +def build( + image: str, version: str, publish: bool, github: bool, _from: tp.Optional[str] = None +) -> None: + cmd: list[str] = [ + "docker", + "buildx", + "build", + "--build-arg", + "BUILDKIT_INLINE_CACHE=1", + ] + if _from: + cmd.extend(["--build-arg", f"FROM=ghcr.io/toltec-dev/{_from}:{version}"]) + if github: + cmd.extend(["--cache-from", f"type=gha,scope={image}"]) + cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"]) + if publish: + cmd.extend(["--push"]) + cmd.extend(["--tag", f"ghcr.io/toltec-dev/{image}:{version}", "."]) + run_cmd(cmd) + + +args = parser.parse_args() +curdir = os.getcwd() +os.chdir(args.d) +workdir = os.getcwd() +if args.s: + os.chdir(args.s) + build(args.s, args.v, args.p, args.g, requires[args.s]) + os.chdir(workdir) +else: + for i in builds: + print(f"Building {i}...") + os.chdir(i) + build(i, args.v, args.p, args.g, requires[i]) + os.chdir(workdir) +os.chdir(curdir) diff --git a/scripts/lib b/scripts/lib deleted file mode 100644 index b42ba69..0000000 --- a/scripts/lib +++ /dev/null @@ -1,30 +0,0 @@ -# vim: set ft=sh: - -# Print an error message to error output then exit -# -# Arguments: -# -# $1 - Error message to print -error() { - echo "Error: $1" >&2 - exit 1 -} - -# Print a status message -# -# Arguments: -# -# $1 - Current status message -status() { - echo "=> $1" >&2 -} - -# Get the full tag of a Docker image -# -# Arguments: -# -# $1 - Name of the image -# $2 - Version of the image -image-name() { - echo "ghcr.io/toltec-dev/$1:${2:-latest}" -}