Add new build scripts (#769)

* Update yq
* Add new scripts for building images
* Remove combo from build-chunk.sh
* Update docs for build-chunk

Co-authored-by: Kyle Brennan <[email protected]>

Author: Furisto

.gitpod.Dockerfile

@@ -8,10 +8,10 @@ ENV BUILDKIT_FILENAME=buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz
 USER root
 
 # Install dazzle, buildkit and pre-commit
-RUN cd /usr \
-    && curl -sSL https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/${BUILDKIT_FILENAME} | tar -xvz \
-    && curl -sSL https://github.com/gitpod-io/dazzle/releases/download/v0.1.8/dazzle_0.1.8_Linux_x86_64.tar.gz | tar -xvz dazzle \
+RUN curl -sSL https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/${BUILDKIT_FILENAME} | tar -xvz -C /usr \
+    && curl -sSL https://github.com/gitpod-io/dazzle/releases/download/v0.1.8/dazzle_0.1.8_Linux_x86_64.tar.gz | tar -xvz -C /usr/local/bin \
     && curl -sSL https://github.com/mvdan/sh/releases/download/v3.4.2/shfmt_v3.4.2_linux_amd64 -o /usr/bin/shfmt \
     && chmod +x /usr/bin/shfmt \
     && install-packages shellcheck \
-    && sudo pip3 install pre-commit
+    && sudo pip3 install pre-commit \
+    && curl -sSL https://github.com/mikefarah/yq/releases/download/v4.22.1/yq_linux_amd64 -o /usr/bin/yq && chmod +x /usr/bin/yq

.gitpod.yml

@@ -20,8 +20,9 @@ tasks:
     command: |
       gp await-port 5000
       REPO=localhost:5000/dazzle
-      echo "To build specific chunks and combinations 'time ./dazzle-up.sh  -c chunk1 -c chunk2 -n combo'"
-      echo "To build all the chunks and combinations 'time ./dazzle-up.sh'"
+      echo "To build specific chunks and combine them 'time ./build-chunk.sh  -c chunk1 -c chunk2:variant1.2.3 -n combo'"
+      echo "To build all the chunks and combinations 'time ./build-all.sh'"
+      echo "To build a specific combination 'time ./build-combo.sh combo'"
       echo "To list image chunks 'dazzle project image-name $REPO'"
       echo "To list hashes for image chunks 'dazzle project hash $REPO'"
       echo "To print the combined image maniest 'dazzle project manifest $REPO'"

CONTRIBUTING.md

@@ -31,7 +31,7 @@ Here is a list of dependencies and tools:
 
 ## Locally
 
-We ship a shell script [dazzle-up.sh](dazzle-up.sh) that can be used to build the images locally. See the following sub sections for usage.
+We ship a shell script [build-all.sh](build-all.sh) that can be used to build the images locally. See the following sub sections for usage.
 
 This script will first build the chunks and run tests followed by creation of container images. It uses `dazzle` to perform these tasks.
 
@@ -45,20 +45,32 @@ where `combo` is the name of the combination defined in [dazzle.yaml](dazzle.yam
 
 ### Build Specific Chunks
 
-Often, you would want to test only the chunks that you modify. You can do that by using the `-c` flag.
+Often, you would want to test only the chunks that you modify. You can do that with build-chunk.sh using the `-c` flag.
 
 ```console
-./dazzle-up.sh -c lang-c -c dep-cacert-update -n mychangecombo
+./build-chunk.sh -c lang-c -c dep-cacert-update -c lang-go:1.17.5
 ```
 
-Above command will build only chunks `lang-c` and `dep-cacert-update` and combine the created chunks (all variants, if any exists) as a combination with name `mychangecombo`.
+Above command will build only chunks `lang-c` and `dep-cacert-update`.
+
+The next step, is to test your changes with [./build-combo](#build-specific-combination).
+
+### Build Specific Combination
+
+Sometimes you only want to build one specific combination e.g. the `postgresql` or the `go` image. You can do that with
+
+```console
+./build-combo.sh <comboName> e.g. ./build-combo.sh postgresql
+```
+
+This will build all chunks that are referenced by the `go` combination and then combine them to create the `go` image.
 
 ### Build All Chunks
 
 Execute the following command to build using the default config `dazzle.yaml` shipped in this repo:
 
 ```bash
-./dazzle-up.sh
+./build-all.sh
 ```
 
 > **NOTE:** Building images locally consumes a lot of resources and is often slow.

build-all.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euo pipefail
+
+REPO=localhost:5000/dazzle
+# First, build chunks without hashes
+dazzle build $REPO -v --chunked-without-hash
+# Second, build again, but with hashes
+dazzle build $REPO -v
+# Third, create combinations of chunks
+dazzle combine $REPO --all -v

dazzle-up.sh renamed to build-chunk.sh

@@ -1,39 +1,25 @@
 #!/bin/bash
 set -euo pipefail
-trap ctrl_c INT
+trap ctrl_c EXIT
 
-readonly YELLOW=$(tput setaf 3)
-readonly NC=$(tput sgr0)
+# shellcheck source=/dev/null
+source build-common.sh
 
-readonly BACKUP_FILE=".dazzle.yaml.orig"
 readonly TEMP_FILE=".dazzle.yaml.temp"
-readonly ORIGINAL_FILE="dazzle.yaml"
-readonly AVAILABLE_CHUNKS=$(ls chunks/)
 readonly REPO="localhost:5000/dazzle"
 
 function usage() {
 	cat <<EOF
-Usage: ./dazzle-up.sh [OPTION]...
-Example: ./dazzle-up.sh  -c lang-c -c dep-cacert-update -n mychangecombo
+Usage: ./build-chunk.sh [OPTION]...
+Example: ./build-chunk.sh -c lang-c -c dep-cacert-update -c lang-go:1.17.5
 Options for build:
   -h, --help      Display this help
   -c, --chunk     Chunk to build, You can build multiple chunks: -c chunk1 -c chunk2. If no chunks are supplied then build using existing config
-  -n, --name      Combination name, by default a combination name 'default' is created. This flag only works when -c flag is specified
 EOF
 }
 
-function restore_original() {
-	echo "${YELLOW}Restoring backup file ${BACKUP_FILE} to original file ${ORIGINAL_FILE}${NC}"
-	cp ${BACKUP_FILE} ${ORIGINAL_FILE}
-}
-
-function ctrl_c() {
-	echo "${YELLOW}** Trapped CTRL-C${NC}"
-	restore_original
-}
-
-function build_and_combine() {
-	dazzle build ${REPO} -v --chunked-without-hash && dazzle build ${REPO} -v && dazzle combine ${REPO} --all -v
+function build_chunks() {
+	dazzle build ${REPO} -v --chunked-without-hash && dazzle build ${REPO} -v
 }
 
 function extract_variants() {
@@ -54,10 +40,9 @@ function extract_variants() {
 ARGS=$(getopt -o h,:c:n: --long help:,chunk:,name: -- "$@")
 eval set -- "${ARGS}"
 
-[ ! -f ${BACKUP_FILE} ] && echo "${YELLOW}Creating a backup of ${ORIGINAL_FILE} as it does not exist yet...${NC}" && cp ${ORIGINAL_FILE} ${BACKUP_FILE}
+save_original
 
 CHUNKS=""
-COMBINATION="default"
 
 while true; do
 	case "$1" in
@@ -75,17 +60,13 @@ while true; do
 		fi
 		shift 2
 		;;
-	-n | --name)
-		COMBINATION="$2"
-		shift 2
-		;;
 	--)
 		shift
 		break
 		;;
 	*)
 		echo Error: unknown flag "$1"
-		echo "${YELLOW}Run 'dazzle-up.sh --help' for usage.${NC}"
+		echo "${YELLOW}Run 'build-chunk.sh --help' for usage.${NC}"
 		exit 1
 		;;
 	esac
@@ -96,9 +77,9 @@ if [[ -z "${CHUNKS[*]}" ]]; then
 	echo "${YELLOW}No chunks specified, will build using the existing ${ORIGINAL_FILE}${NC}"
 else
 	# else build for supplied arguments
-	[ -f ${ORIGINAL_FILE} ] && echo "${YELLOW}Deleting ${ORIGINAL_FILE}, will produce a new config with supplied arguments${NC}" && rm ${ORIGINAL_FILE}
+	[ -f "${ORIGINAL_FILE}" ] && echo "${YELLOW}Deleting ${ORIGINAL_FILE}, will produce a new config with supplied arguments${NC}" && rm "${ORIGINAL_FILE}"
 
-	for CN in ${AVAILABLE_CHUNKS}; do
+	for CN in $(get_available_chunks); do
 		if [[ ! ${CHUNKS[*]} =~ ${CN} ]]; then
 			dazzle project ignore "${CN}"
 		else
@@ -109,12 +90,10 @@ else
 		fi
 	done
 
-	dazzle project add-combination "${COMBINATION}" "${CHUNKS_TO_COMBINE[@]}"
+	# dazzle project add-combination "${COMBINATION}" "${CHUNKS_TO_COMBINE[@]}"
 fi
 
-build_and_combine
+build_chunks
 
 echo "${YELLOW}Saving dazzle config used to generate build in ${TEMP_FILE}${NC}"
-cp ${ORIGINAL_FILE} ${TEMP_FILE}
-
-restore_original
+cp "${ORIGINAL_FILE}" ${TEMP_FILE}

build-combo.sh

@@ -0,0 +1,69 @@
+#! /bin/bash
+set -eo pipefail
+trap ctrl_c EXIT
+
+# shellcheck source=/dev/null
+source build-common.sh
+
+function build_combination() {
+	combination=$1
+
+	local exists
+	exists="$(yq e '.combiner.combinations[] | select (.name=="'"$combination"'")' dazzle.yaml)"
+	if [[ -z "$exists" ]]; then
+		echo "combination is not defined"
+		exit 1
+	fi
+
+	refs=$(get_refs "$combination")
+	required_chunks=$(get_chunks "$refs" | sort | uniq)
+	available_chunks=$(get_available_chunks)
+
+	for ac in $available_chunks; do
+		if [[ ! "${required_chunks[*]}" =~ ${ac} ]]; then
+			dazzle project ignore "$ac"
+		fi
+	done
+}
+
+function get_refs() {
+	local ref=$1
+	echo "$ref"
+
+	refs="$(yq e '.combiner.combinations[] | select (.name=="'"$ref"'") | .ref[]' dazzle.yaml)"
+	if [[ -z "$refs" ]]; then
+		return
+	fi
+
+	for ref in $refs; do
+		get_refs "$ref"
+	done
+}
+
+function get_chunks() {
+	# shellcheck disable=SC2068
+	for ref in $@; do
+		chunks=$(yq e '.combiner.combinations[] | select (.name=="'"$ref"'") | .chunks[]' dazzle.yaml)
+		echo "$chunks"
+	done
+}
+
+REPO=localhost:5000/dazzle
+
+save_original
+
+if [ -n "${1}" ]; then
+	build_combination "$1"
+fi
+
+# First, build chunks without hashes
+dazzle build $REPO -v --chunked-without-hash
+# Second, build again, but with hashes
+dazzle build $REPO -v
+
+# Third, create combinations of chunks
+if [[ -n "${1}" ]]; then
+	dazzle combine $REPO --combination "$1" -v
+else
+	dazzle combine $REPO --all -v
+fi

build-common.sh

@@ -0,0 +1,42 @@
+#! /bin/bash
+
+YELLOW=$(tput setaf 3)
+readonly YELLOW
+NC=$(tput sgr0)
+readonly NC
+
+readonly BACKUP_FILE=".dazzle.yaml.orig"
+readonly ORIGINAL_FILE="dazzle.yaml"
+
+function save_original() {
+	if [ ! -f ${BACKUP_FILE} ]; then
+		echo "${YELLOW}Creating a backup of ${ORIGINAL_FILE} as it does not exist yet...${NC}" && cp ${ORIGINAL_FILE} ${BACKUP_FILE}
+	fi
+}
+
+function restore_original() {
+	echo "${YELLOW}Restoring backup file ${BACKUP_FILE} to original file ${ORIGINAL_FILE}${NC}"
+	cp ${BACKUP_FILE} ${ORIGINAL_FILE}
+}
+
+function ctrl_c() {
+	echo "${YELLOW}** Trapped CTRL-C${NC}"
+	restore_original
+}
+
+function get_available_chunks() {
+	local chunk_defs
+	chunk_defs=$(ls chunks)
+	for chunk in $chunk_defs; do
+		local chunkYaml="chunks/${chunk}/chunk.yaml"
+		if [[ -f "$chunkYaml" ]]; then
+			variants=$(yq e '.variants[].name' "$chunkYaml")
+			for variant in $variants; do
+				echo "$chunk:$variant"
+			done
+		else
+			echo "$chunk"
+		fi
+
+	done
+}