Consolidate build scripts (elementary#286)

* make sure all scripts exit on failure.

* move all terraform script tooling into the build script. Add a new option to allow config to choose whether or not to include the appcenter ppa.

* clean up comments related to old terraform script

* remove artifacts dir step, direct output to builds/amd64.

* move comments to better locations for logging

* allow for uploading other arch types in the future

* remove builds dir before moving over new builds

* Remove the builds directory before all build runs instead of before each.

Author: kgrubb

.github/workflows/daily-5.1.yml

@@ -22,5 +22,4 @@ jobs:
 
     - name: Build and upload daily .iso
       run: |
-        mkdir /artifacts
         ./workflows.sh etc/terraform-daily-5.1-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}"

.github/workflows/daily-6.0.yml

@@ -22,5 +22,4 @@ jobs:
 
     - name: Build and upload daily .iso
       run: |
-        mkdir /artifacts
         ./workflows.sh etc/terraform-daily-6.0-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}"

.github/workflows/distinst-weekly.yml

@@ -19,5 +19,4 @@ jobs:
 
     - name: Build and upload distinst .iso
       run: |
-        mkdir /artifacts
         ./workflows.sh etc/terraform-daily-distinst-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}"

.github/workflows/stable.yml

@@ -20,5 +20,4 @@ jobs:
 
     - name: Build and upload stable .iso
       run: |
-        mkdir /artifacts
         ./workflows.sh etc/terraform-stable-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}"

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
 tmp
 artifacts
+builds
 *~

README.md

@@ -36,17 +36,14 @@ The following example uses Docker and assumes you have Docker correctly installe
  3) Run the build:
 
     ```
-    mkdir artifacts
-    docker run --privileged -i \
-        -v /proc:/proc \
-        -v ${PWD}/artifacts:/artifacts \
+    docker run --privileged -i -v /proc:/proc \
         -v ${PWD}:/working_dir \
         -w /working_dir \
         debian:latest \
         /bin/bash -s etc/terraform.conf < build.sh
     ```
 
- 4) When done, your image will be in the `artifacts` folder.
+ 4) When done, your image will be in the `builds` folder.
 
 ## Further Information
 

build.sh

@@ -1,8 +1,21 @@
 #!/bin/bash
 
-CONFIG_FILE="$1"
+set -e
 
-source "$CONFIG_FILE"
+# check for root permissions
+if [[ "$(id -u)" != 0 ]]; then
+  echo "E: Requires root permissions" > /dev/stderr
+  exit 1
+fi
+
+# get config
+if [ -n "$1" ]; then
+  CONFIG_FILE="$1"
+else
+  CONFIG_FILE="etc/terraform.conf"
+fi
+BASE_DIR="$PWD"
+source "$BASE_DIR"/"$CONFIG_FILE"
 
 echo -e "
 #----------------------#
@@ -21,11 +34,69 @@ patch -d /usr/lib/live/build/ < live-build-fix-syslinux.patch
 # https://salsa.debian.org/installer-team/debootstrap/blob/master/debian/changelog
 ln -sfn /usr/share/debootstrap/scripts/gutsy /usr/share/debootstrap/scripts/focal
 
-echo -e "
-#----------------------#
-# RUN TERRAFORM SCRIPT #
-#----------------------#
+build () {
+  BUILD_ARCH="$1"
+
+  mkdir -p "$BASE_DIR/tmp/$BUILD_ARCH"
+  cd "$BASE_DIR/tmp/$BUILD_ARCH" || exit
+
+  # remove old configs and copy over new
+  rm -rf config auto
+  cp -r "$BASE_DIR"/etc/* .
+  # Make sure conffile specified as arg has correct name
+  cp -f "$BASE_DIR"/"$CONFIG_FILE" terraform.conf
+
+  # Symlink chosen package lists to where live-build will find them
+  ln -s "package-lists.$PACKAGE_LISTS_SUFFIX" "config/package-lists"
+  # symlink appcenter archive
+  if [ "$INCLUDE_APPCENTER" = "yes" ]; then
+    ln -s "appcenter/appcenter.list.binary" "archives/appcenter.list.binary"
+    ln -s "appcenter/appcenter.key.binary" "archives/appcenter.key.binary"
+  fi
+
+  echo -e "
+#------------------#
+# LIVE-BUILD CLEAN #
+#------------------#
 "
+  lb clean
+
+  echo -e "
+#-------------------#
+# LIVE-BUILD CONFIG #
+#-------------------#
+"
+  lb config
+
+  echo -e "
+#------------------#
+# LIVE-BUILD BUILD #
+#------------------#
+"
+  lb build
+
+  echo -e "
+#---------------------------#
+# MOVE OUTPUT TO BUILDS DIR #
+#---------------------------#
+"
+
+  YYYYMMDD="$(date +%Y%m%d)"
+  OUTPUT_DIR="$BASE_DIR/builds/$BUILD_ARCH"
+  mkdir -p "$OUTPUT_DIR"
+  FNAME="elementaryos-$VERSION-$CHANNEL.$YYYYMMDD$OUTPUT_SUFFIX"
+  mv "$BASE_DIR/tmp/$BUILD_ARCH/live-image-$BUILD_ARCH.hybrid.iso" "$OUTPUT_DIR/${FNAME}.iso"
+
+  md5sum "$OUTPUT_DIR/${FNAME}.iso" > "$OUTPUT_DIR/${FNAME}.md5.txt"
+  sha256sum "$OUTPUT_DIR/${FNAME}.iso" > "$OUTPUT_DIR/${FNAME}.sha256.txt"
+}
+
+# remove old builds before creating new ones
+rm -rf "$BASE_DIR"/builds
 
-./terraform.sh --config-path "$CONFIG_FILE"
-cp builds/amd64/* /artifacts/
+if [[ "$ARCH" == "all" ]]; then
+    build amd64
+    build i386
+else
+    build "$ARCH"
+fi

etc/config/archives/appcenter.key.binary etc/config/appcenter/appcenter.key.binary

@@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/"
 # use HWE kernel and packages?
 HWE="yes"
 
+# use appcenter ppa
+INCLUDE_APPCENTER="yes"
+
 # suffix for generated .iso files
 OUTPUT_SUFFIX=""
 

etc/config/archives/appcenter.list.binary etc/config/appcenter/appcenter.list.binary

@@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/"
 # use HWE kernel and packages?
 HWE=""
 
+# use appcenter ppa
+INCLUDE_APPCENTER=""
+
 # suffix for generated .iso files
 OUTPUT_SUFFIX="-distinst"
 

etc/terraform-daily-5.1-azure.conf

@@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/"
 # use HWE kernel and packages?
 HWE="yes"
 
+# use appcenter ppa
+INCLUDE_APPCENTER="yes"
+
 # suffix for generated .iso files
 OUTPUT_SUFFIX="-distinst"
 

etc/terraform-daily-6.0-azure.conf

@@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/"
 # use HWE kernel and packages?
 HWE="yes"
 
+# use appcenter ppa
+INCLUDE_APPCENTER="yes"
+
 # suffix for generated .iso files
 OUTPUT_SUFFIX=""
 

etc/terraform-daily-distinst-azure.conf

@@ -25,6 +25,9 @@ MIRROR_URL="http://archive.ubuntu.com/ubuntu/"
 # use HWE kernel and packages?
 HWE="yes"
 
+# use appcenter ppa
+INCLUDE_APPCENTER="yes"
+
 # suffix for generated .iso files
 OUTPUT_SUFFIX=""
 

etc/terraform-stable-azure.conf

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 CONFIG_FILE="$1"
 KEY="$2"
 SECRET="$3"
@@ -28,14 +30,17 @@ python3 get-pip.py
 pip install boto3
 
 # get the paths & filenames of the files to upload
-ISOPATH="$(find /artifacts -name "*.iso")"
-ISO="$CHANNEL/$(basename "$ISOPATH")"
-ISOTAG="$(basename "$ISOPATH" .iso)"
-SHAPATH="$(find /artifacts -name "*.sha256.txt")"
-SHASUM="$CHANNEL/$ISOTAG.sha256.txt"
-MD5PATH="$(find /artifacts -name "*.md5.txt")"
-MD5="$CHANNEL/$ISOTAG.md5.txt"
-
-python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$ISOPATH" "$ISO"
-python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$SHAPATH" "$SHASUM"
-python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$MD5PATH" "$MD5"
+ISOPATHS="$(find builds -name "*.iso")"
+while IFS= read -r ISOPATH; do
+  SHAPATH="${ISOPATH%.*}.sha256.txt"
+  MD5PATH="${ISOPATH%.*}.md5.txt"
+  ISO="$CHANNEL/$(basename "$ISOPATH")"
+  SHASUM="$CHANNEL/$(basename "$SHAPATH")"
+  MD5="$CHANNEL/$(basename "$MD5PATH")"
+  echo "uploading $ISO..."
+  python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$ISOPATH" "$ISO" || exit 1
+  echo "uploading $SHASUM..."
+  python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$SHAPATH" "$SHASUM" || exit 1
+  echo "uploading $MD5..."
+  python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$MD5PATH" "$MD5" || exit 1
+done <<< "$ISOPATHS"

etc/terraform.conf

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 CONFIG_FILE="$1"
 KEY="$2"
 SECRET="$3"