From 167b4d85150f98bbd357937700a20b3c57c5535e Mon Sep 17 00:00:00 2001 From: jschaul Date: Fri, 4 Nov 2022 00:33:25 +0100 Subject: [PATCH] remove buildah references --- Makefile | 20 -------- changelog.d/5-internal/buildah-drop-support | 1 + docs/src/developer/developer/dependencies.md | 6 --- docs/src/developer/developer/how-to.md | 13 ++--- hack/bin/buildah-clean.sh | 9 ---- hack/bin/buildah-compile.sh | 30 ------------ hack/bin/buildah-inside.sh | 16 ------ hack/bin/buildah-make-images-nginz.sh | 37 -------------- hack/bin/buildah-make-images.sh | 51 -------------------- hack/bin/buildah-purge-untagged.sh | 6 --- 10 files changed, 6 insertions(+), 183 deletions(-) create mode 100644 changelog.d/5-internal/buildah-drop-support delete mode 100755 hack/bin/buildah-clean.sh delete mode 100755 hack/bin/buildah-compile.sh delete mode 100755 hack/bin/buildah-inside.sh delete mode 100755 hack/bin/buildah-make-images-nginz.sh delete mode 100755 hack/bin/buildah-make-images.sh delete mode 100755 hack/bin/buildah-purge-untagged.sh diff --git a/Makefile b/Makefile index a607d23bfe..21f41da7db 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,7 @@ CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster fake-aws # this list could be generated from the folder names under ./charts/ like so: # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts) CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external elasticsearch-ephemeral minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn inbucket -BUILDAH_PUSH ?= 0 KIND_CLUSTER_NAME := wire-server -BUILDAH_KIND_LOAD ?= 1 package ?= all EXE_SCHEMA := ./dist/$(package)-schema @@ -409,24 +407,6 @@ upload-charts: charts-release echo-release-charts: @echo ${CHARTS_RELEASE} -.PHONY: buildah-docker -buildah-docker: buildah-docker-nginz - ./hack/bin/buildah-compile.sh all - BUILDAH_PUSH=${BUILDAH_PUSH} KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD} ./hack/bin/buildah-make-images.sh - -.PHONY: buildah-docker-nginz -buildah-docker-nginz: - BUILDAH_PUSH=${BUILDAH_PUSH} KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD} ./hack/bin/buildah-make-images-nginz.sh - -.PHONY: buildah-docker-% -buildah-docker-%: - ./hack/bin/buildah-compile.sh $(*) - BUILDAH_PUSH=${BUILDAH_PUSH} EXECUTABLES=$(*) KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD} ./hack/bin/buildah-make-images.sh - -.PHONY: buildah-clean -buildah-clean: - ./hack/bin/buildah-clean.sh - .PHONY: kind-cluster kind-cluster: kind create cluster --name $(KIND_CLUSTER_NAME) diff --git a/changelog.d/5-internal/buildah-drop-support b/changelog.d/5-internal/buildah-drop-support new file mode 100644 index 0000000000..2985ad2882 --- /dev/null +++ b/changelog.d/5-internal/buildah-drop-support @@ -0,0 +1 @@ +Remove support for compiling local docker images with buildah. Nix is used to build docker images these days diff --git a/docs/src/developer/developer/dependencies.md b/docs/src/developer/developer/dependencies.md index 15896f6d75..68db64e56d 100644 --- a/docs/src/developer/developer/dependencies.md +++ b/docs/src/developer/developer/dependencies.md @@ -254,12 +254,6 @@ Requirements: In both cases, you need to adjust the various integration configuration files and names so that this can work. -## Buildah (optional) - -[Buildah](https://buildah.io/) is used for local docker image creation during development. See [buildah installation](https://github.com/containers/buildah/blob/master/install.md) - -See `make buildah-docker` for an entry point here. - ## Helm chart development, integration tests in kubernetes You need `kubectl`, `helm`, `helmfile`, and a valid kubernetes context. Refer to https://docs.wire.com for details. diff --git a/docs/src/developer/developer/how-to.md b/docs/src/developer/developer/how-to.md index 3c98cfa869..0ed606399b 100644 --- a/docs/src/developer/developer/how-to.md +++ b/docs/src/developer/developer/how-to.md @@ -108,10 +108,10 @@ FUTUREWORK: this process is in development (update this section after it's confi ##### (i) Build images -1. Ensure `buildah` is available on your system. -2. Compile the image using `make buildah-docker`. This will try to upload the - images into a `kind` cluster. If you'd prefer uploading images to quay.io, - you can run it with `make buildah-docker BUILDAH_PUSH=1 BUILDAH_KIND_LOAD=0` +(FUTUREWORK: implement a convenient shortcut to build images without actually uploading them also) +``` +make upload-images-dev +``` ##### (ii) Run tests in kind @@ -120,7 +120,6 @@ FUTUREWORK: this process is in development (update this section after it's confi 2. Run tests using `make kind-integration-test`. 3. Run end2end integration tests: `make kind-integration-e2e`. -NOTE: debug this process further as some images (e.g. nginz) are missing from the default buildah steps. * Implement re-tagging development tags as your user tag? #### 2.4 Deploy your local code to a kubernetes cluster @@ -138,9 +137,7 @@ make kube-integration-setup Then build and push the `brig` image by running ``` -export DOCKER_TAG_LOCAL_BUILD=$USER -hack/bin/buildah-compile.sh all -DOCKER_TAG=$DOCKER_TAG_LOCAL_BUILD EXECUTABLES=brig BUILDAH_PUSH=1 ./hack/bin/buildah-make-images.sh +#FUTUREWORK ``` To update the release with brig's local image run diff --git a/hack/bin/buildah-clean.sh b/hack/bin/buildah-clean.sh deleted file mode 100755 index d0f49026e3..0000000000 --- a/hack/bin/buildah-clean.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -e -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -TOP_LEVEL="$(cd "$DIR/../.." && pwd)" - -rm -rf "$TOP_LEVEL"/buildah -# buildah rm wire-server-dev -buildah rm output diff --git a/hack/bin/buildah-compile.sh b/hack/bin/buildah-compile.sh deleted file mode 100755 index b49871aa36..0000000000 --- a/hack/bin/buildah-compile.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# This compiles wire-server inside an ubuntu-based container based on quay.io/wire/ubuntu20-builder. -# the tool 'buildah' is used to mount some folders in, and to -# keep the caches of /.root/.cabal and dist-newstyle (renamed to avoid conflicts) for the next compilation - -# After compilation, ./buildah-make-images.sh can be used -# to bake individual executables into their respective docker images used by kubernetes. - -set -ex - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -TOP_LEVEL="$(cd "$DIR/../.." && pwd)" - -# Note: keep the following names and paths in sync with the other buildah-* scripts. -mkdir -p "$TOP_LEVEL"/buildah/dot-cabal -mkdir -p "$TOP_LEVEL"/buildah/dist-newstyle -mkdir -p "$TOP_LEVEL"/buildah/dist - -CONTAINER_NAME=wire-server-dev - -# check for the existence of; or create a working container -buildah containers | awk '{print $5}' | grep "$CONTAINER_NAME" \ - || buildah from --name "$CONTAINER_NAME" -v "${TOP_LEVEL}":/src --pull quay.io/wire/ubuntu20-builder:develop - -# copy /root/.cabal out of the container -ls "$TOP_LEVEL"/buildah/dot-cabal/store 2> /dev/null \ - || buildah run "$CONTAINER_NAME" -- cp -a /root/.cabal/. /src/buildah/dot-cabal - -buildah run "$CONTAINER_NAME" -- /src/hack/bin/buildah-inside.sh "$@" diff --git a/hack/bin/buildah-inside.sh b/hack/bin/buildah-inside.sh deleted file mode 100755 index a750e21df8..0000000000 --- a/hack/bin/buildah-inside.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# This script is meant to be run from inside a buildah container. See buildah-compile.sh for details. - -set -e -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP_LEVEL="$(cd "$DIR/../.." && pwd)" - -cd "$TOP_LEVEL" - -cabal build \ - --prefix=./buildah/dot-cabal \ - --builddir=./buildah/dist-newstyle \ - "$@" - -DIST="$TOP_LEVEL"/buildah/dist PLAN_FILE="$TOP_LEVEL"/buildah/dist-newstyle/cache/plan.json ./hack/bin/cabal-install-artefacts.sh "$@" diff --git a/hack/bin/buildah-make-images-nginz.sh b/hack/bin/buildah-make-images-nginz.sh deleted file mode 100755 index dba85bf94f..0000000000 --- a/hack/bin/buildah-make-images-nginz.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Pulls nginz and nginz_disco images from quay.io into buildah store, and loads -# them into the kind cluster - -set -ex - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP_LEVEL="$(cd "$DIR/../.." && pwd)" - -DOCKER_DOWNLOAD_TAG=latest -DOCKER_TAG=${DOCKER_TAG:-$USER} -EXECUTABLES=${EXECUTABLES:-"nginz nginz_disco"} - -for EX in $EXECUTABLES; do - CONTAINER_NAME=$EX - buildah containers | awk '{print $5}' | grep "$CONTAINER_NAME" || - buildah from --name "$CONTAINER_NAME" -v "${TOP_LEVEL}":/src --pull "quay.io/wire/$CONTAINER_NAME:$DOCKER_DOWNLOAD_TAG" - buildah tag "quay.io/wire/$CONTAINER_NAME:$DOCKER_DOWNLOAD_TAG" "quay.io/wire/$CONTAINER_NAME:$DOCKER_TAG" - if [[ "$BUILDAH_KIND_LOAD" -eq "1" ]]; then - archiveDir=$(mktemp -d) - imgPath="$archiveDir/${EX}_${DOCKER_TAG}.tar" - imgName="quay.io/wire/$EX:$DOCKER_TAG" - buildah push "$imgName" "docker-archive:$imgPath:$imgName" - kind load image-archive --name "$KIND_CLUSTER_NAME" "$imgPath" - rm -rf "$archiveDir" - fi -done - -if [[ "$BUILDAH_PUSH" -eq "1" ]]; then - for EX in $EXECUTABLES; do - buildah push "quay.io/wire/$EX:$DOCKER_TAG" - done -fi - -# general cleanup -"$DIR/buildah-purge-untagged.sh" diff --git a/hack/bin/buildah-make-images.sh b/hack/bin/buildah-make-images.sh deleted file mode 100755 index cc06f2c102..0000000000 --- a/hack/bin/buildah-make-images.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP_LEVEL="$(cd "$DIR/../.." && pwd)" - -EXECUTABLES=${EXECUTABLES:-"cannon brig cargohold galley gundeck federator brig-index brig-schema galley-schema galley-migrate-data gundeck-schema proxy spar spar-schema spar-migrate-data brig-integration galley-integration spar-integration gundeck-integration cargohold-integration federator-integration"} -CONTAINER_NAME="output" -DOCKER_TAG=${DOCKER_TAG:-$USER} - -buildah containers | awk '{print $5}' | grep "$CONTAINER_NAME" || - buildah from --name "$CONTAINER_NAME" -v "${TOP_LEVEL}":/src --pull quay.io/wire/ubuntu20-deps:develop - -# Only brig needs these templates, but for simplicity we add them to all resulting images (optimization FUTUREWORK) -buildah run "$CONTAINER_NAME" -- sh -c 'mkdir -p /usr/share/wire/ && cp -r "/src/services/brig/deb/opt/brig/templates/." "/usr/share/wire/templates"' - -for EX in $EXECUTABLES; do - # Copy the main executable into the PATH on the container - buildah run "$CONTAINER_NAME" -- cp "/src/buildah/dist/$EX" "/usr/bin/$EX" - - # Start that executable by default when launching the docker image - buildah config --entrypoint "[ \"/usr/bin/dumb-init\", \"--\", \"/usr/bin/$EX\" ]" "$CONTAINER_NAME" - buildah config --cmd null "$CONTAINER_NAME" - - # Bake an image - buildah commit "$CONTAINER_NAME" quay.io/wire/"$EX":"$DOCKER_TAG" - - # remove executable from the image in preparation for the next iteration - buildah run "$CONTAINER_NAME" -- rm "/usr/bin/$EX" -done - -if [[ "$BUILDAH_PUSH" -eq "1" ]]; then - for EX in $EXECUTABLES; do - buildah push "quay.io/wire/$EX:$DOCKER_TAG" - done -fi - -if [[ "$BUILDAH_KIND_LOAD" -eq "1" ]]; then - archiveDir=$(mktemp -d) - for EX in $EXECUTABLES; do - imgPath="$archiveDir/${EX}_${DOCKER_TAG}.tar" - imgName="quay.io/wire/$EX:$DOCKER_TAG" - buildah push "$imgName" "docker-archive:$imgPath:$imgName" - kind load image-archive --name "$KIND_CLUSTER_NAME" "$imgPath" - done - rm -rf "$archiveDir" -fi - -# general cleanup -"$DIR/buildah-purge-untagged.sh" diff --git a/hack/bin/buildah-purge-untagged.sh b/hack/bin/buildah-purge-untagged.sh deleted file mode 100755 index da74d36854..0000000000 --- a/hack/bin/buildah-purge-untagged.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Remove untagged images (if there are any) in the buildah store -if buildah images | grep ""; then - buildah images | grep "" | awk '{print $3}' | xargs -n 1 buildah rmi -fi