Skip to content

Commit

Permalink
Add new scripts for building images
Browse files Browse the repository at this point in the history
  • Loading branch information
Furisto committed Mar 16, 2022
1 parent 6ca514b commit 405a4a8
Show file tree
Hide file tree
Showing 7 changed files with 150 additions and 29 deletions.
2 changes: 1 addition & 1 deletion .gitpod.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ ENV BUILDKIT_FILENAME=buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz

# Install custom tools, runtime, etc.
RUN sudo su -c "cd /usr; curl -L https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/${BUILDKIT_FILENAME} | tar xvz" && \
sudo su -c "cd /usr/bin; curl -o yq -L https://github.com/mikefarah/yq/releases/download/v4.22.1/yq_linux_amd64 && chmod +x yq"
sudo su -c "cd /usr/bin; curl -o yq -L https://github.com/mikefarah/yq/releases/download/v4.22.1/yq_linux_amd64 && chmod +x yq"

5 changes: 3 additions & 2 deletions .gitpod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ tasks:
command: |
gp await-port 5000
REPO=localhost:5000/dazzle
echo "To build specific chunks and combinations 'time ./dazzle-up.sh -c chunk1 -c chunk2 -n combo'"
echo "To build all the chunks and combinations 'time ./dazzle-up.sh'"
echo "To build specific chunks and combine them 'time ./build-chunk.sh -c chunk1 -c chunk2 -n combo'"
echo "To build all the chunks and combinations 'time ./build-all.sh'"
echo "To build a specific combination 'time ./build-combo.sh combo'"
echo "To list image chunks 'dazzle project image-name $REPO'"
echo "To list hashes for image chunks 'dazzle project hash $REPO'"
echo "To print the combined image maniest 'dazzle project manifest $REPO'"
Expand Down
18 changes: 14 additions & 4 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Here is a list of dependencies and tools:

## Locally

We ship a shell script [dazzle-up.sh](dazzle-up.sh) that can be used to build the images locally. See the following sub sections for usage.
We ship a shell script [build-all.sh](build-all.sh) that can be used to build the images locally. See the following sub sections for usage.

This script will first build the chunks and run tests followed by creation of container images. It uses `dazzle` to perform these tasks.

Expand All @@ -45,20 +45,30 @@ where `combo` is the name of the combination defined in [dazzle.yaml](dazzle.yam

### Build Specific Chunks

Often, you would want to test only the chunks that you modify. You can do that by using the `-c` flag.
Often, you would want to test only the chunks that you modify. You can do that with build-chunk.sh using the `-c` flag and specifying the name for your combination with `-n`.

```console
./dazzle-up.sh -c lang-c -c dep-cacert-update -n mychangecombo
./build-chunk.sh -c lang-c -c dep-cacert-update -n mychangecombo
```

Above command will build only chunks `lang-c` and `dep-cacert-update` and combine the created chunks (all variants, if any exists) as a combination with name `mychangecombo`.

### Build Specific Combination

Sometimes you only want to build one specific combination e.g. the `postgresql` or the `go` image. You can do that with

```console
./build-combo.sh <comboName> e.g. ./build-combo.sh postgresql
```

This will build all chunks that are referenced by the `go` combination and then combine them to create the `go` image.

### Build All Chunks

Execute the following command to build using the default config `dazzle.yaml` shipped in this repo:

```bash
./dazzle-up.sh
./build-all.sh
```

> **NOTE:** Building images locally consumes a lot of resources and is often slow.
Expand Down
10 changes: 10 additions & 0 deletions build-all.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash
set -euo pipefail

REPO=localhost:5000/dazzle
# First, build chunks without hashes
dazzle build $REPO -v --chunked-without-hash
# Second, build again, but with hashes
dazzle build $REPO -v
# Third, create combinations of chunks
dazzle combine $REPO --all -v
31 changes: 9 additions & 22 deletions dazzle-up.sh → build-chunk.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,36 +2,23 @@
set -euo pipefail
trap ctrl_c INT

readonly YELLOW=$(tput setaf 3)
readonly NC=$(tput sgr0)
# shellcheck source=/dev/null
source build-common.sh

readonly BACKUP_FILE=".dazzle.yaml.orig"
readonly TEMP_FILE=".dazzle.yaml.temp"
readonly ORIGINAL_FILE="dazzle.yaml"
readonly AVAILABLE_CHUNKS=$(ls chunks/)
readonly REPO="localhost:5000/dazzle"

function usage() {
cat <<EOF
Usage: ./dazzle-up.sh [OPTION]...
Example: ./dazzle-up.sh -c lang-c -c dep-cacert-update -n mychangecombo
Usage: ./build-chunk.sh [OPTION]...
Example: ./build-chunk.sh -c lang-c -c dep-cacert-update -n mychangecombo
Options for build:
-h, --help Display this help
-c, --chunk Chunk to build, You can build multiple chunks: -c chunk1 -c chunk2. If no chunks are supplied then build using existing config
-n, --name Combination name, by default a combination name 'default' is created. This flag only works when -c flag is specified
EOF
}

function restore_original() {
echo "${YELLOW}Restoring backup file ${BACKUP_FILE} to original file ${ORIGINAL_FILE}${NC}"
cp ${BACKUP_FILE} ${ORIGINAL_FILE}
}

function ctrl_c() {
echo "${YELLOW}** Trapped CTRL-C${NC}"
restore_original
}

function build_and_combine() {
dazzle build ${REPO} -v --chunked-without-hash && dazzle build ${REPO} -v && dazzle combine ${REPO} --all -v
}
Expand All @@ -54,7 +41,7 @@ function extract_variants() {
ARGS=$(getopt -o h,:c:n: --long help:,chunk:,name: -- "$@")
eval set -- "${ARGS}"

[ ! -f ${BACKUP_FILE} ] && echo "${YELLOW}Creating a backup of ${ORIGINAL_FILE} as it does not exist yet...${NC}" && cp ${ORIGINAL_FILE} ${BACKUP_FILE}
save_original

CHUNKS=""
COMBINATION="default"
Expand Down Expand Up @@ -85,7 +72,7 @@ while true; do
;;
*)
echo Error: unknown flag "$1"
echo "${YELLOW}Run 'dazzle-up.sh --help' for usage.${NC}"
echo "${YELLOW}Run 'build-chunk.sh --help' for usage.${NC}"
exit 1
;;
esac
Expand All @@ -96,9 +83,9 @@ if [[ -z "${CHUNKS[*]}" ]]; then
echo "${YELLOW}No chunks specified, will build using the existing ${ORIGINAL_FILE}${NC}"
else
# else build for supplied arguments
[ -f ${ORIGINAL_FILE} ] && echo "${YELLOW}Deleting ${ORIGINAL_FILE}, will produce a new config with supplied arguments${NC}" && rm ${ORIGINAL_FILE}
[ -f "${ORIGINAL_FILE}" ] && echo "${YELLOW}Deleting ${ORIGINAL_FILE}, will produce a new config with supplied arguments${NC}" && rm "${ORIGINAL_FILE}"

for CN in ${AVAILABLE_CHUNKS}; do
for CN in $(get_available_chunks); do
if [[ ! ${CHUNKS[*]} =~ ${CN} ]]; then
dazzle project ignore "${CN}"
else
Expand All @@ -115,6 +102,6 @@ fi
build_and_combine

echo "${YELLOW}Saving dazzle config used to generate build in ${TEMP_FILE}${NC}"
cp ${ORIGINAL_FILE} ${TEMP_FILE}
cp "${ORIGINAL_FILE}" ${TEMP_FILE}

restore_original
71 changes: 71 additions & 0 deletions build-combo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#! /bin/bash
set -eo pipefail
trap ctrl_c INT

# shellcheck source=/dev/null
source build-common.sh

function build_combination() {
combination=$1

local exists
exists="$(yq e '.combiner.combinations[] | select (.name=="'"$combination"'")' dazzle.yaml)"
if [[ -z "$exists" ]]; then
echo "combination is not defined"
exit 1
fi

refs=$(get_refs "$combination")
required_chunks=$(get_chunks "$refs" | sort | uniq)
available_chunks=$(get_available_chunks)

for ac in $available_chunks; do
if [[ ! "${required_chunks[*]}" =~ ${ac} ]]; then
dazzle project ignore "$ac"
fi
done
}

function get_refs() {
local ref=$1
echo "$ref"

refs="$(yq e '.combiner.combinations[] | select (.name=="'"$ref"'") | .ref[]' dazzle.yaml)"
if [[ -z "$refs" ]]; then
return
fi

for ref in $refs; do
get_refs "$ref"
done
}

function get_chunks() {
# shellcheck disable=SC2068
for ref in $@; do
chunks=$(yq e '.combiner.combinations[] | select (.name=="'"$ref"'") | .chunks[]' dazzle.yaml)
echo "$chunks"
done
}

REPO=localhost:5000/dazzle

save_original

if [ -n "${1}" ]; then
build_combination "$1"
fi

# First, build chunks without hashes
dazzle build $REPO -v --chunked-without-hash
# Second, build again, but with hashes
dazzle build $REPO -v

# Third, create combinations of chunks
if [[ -n "${1}" ]]; then
dazzle combine $REPO --combination "$1" -v
else
dazzle combine $REPO --all -v
fi

restore_original
42 changes: 42 additions & 0 deletions build-common.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#! /bin/bash

YELLOW=$(tput setaf 3)
readonly YELLOW
NC=$(tput sgr0)
readonly NC

readonly BACKUP_FILE=".dazzle.yaml.orig"
readonly ORIGINAL_FILE="dazzle.yaml"

function save_original {
if [ ! -f ${BACKUP_FILE} ]; then
echo "${YELLOW}Creating a backup of ${ORIGINAL_FILE} as it does not exist yet...${NC}" && cp ${ORIGINAL_FILE} ${BACKUP_FILE}
fi
}

function restore_original() {
echo "${YELLOW}Restoring backup file ${BACKUP_FILE} to original file ${ORIGINAL_FILE}${NC}"
cp ${BACKUP_FILE} ${ORIGINAL_FILE}
}

function ctrl_c() {
echo "${YELLOW}** Trapped CTRL-C${NC}"
restore_original
}

function get_available_chunks() {
local chunk_defs
chunk_defs=$(ls chunks)
for chunk in $chunk_defs;do
local chunkYaml="chunks/${chunk}/chunk.yaml"
if [[ -f "$chunkYaml" ]]; then
variants=$(yq e '.variants[].name' "$chunkYaml" )
for variant in $variants; do
echo "$chunk:$variant"
done
else
echo "$chunk"
fi

done
}

0 comments on commit 405a4a8

Please sign in to comment.