Merge branch 'master' into lomorage

Dockerfile

@@ -1,12 +1,14 @@
-FROM debian:buster
+ARG BASE_IMAGE=debian:buster
+FROM ${BASE_IMAGE}
 
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get -y update && \
-    apt-get -y install \
+    apt-get -y install --no-install-recommends \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
-        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod\
+        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
+        binfmt-support ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

README.md

@@ -1,20 +1,20 @@
 # pi-gen
 
-_Tool used to create the raspberrypi.org Raspbian images_
+Tool used to create Raspberry Pi OS images. (Previously known as Raspbian).
 
 
 ## Dependencies
 
-pi-gen runs on Debian based operating systems. Currently it is only supported on
+pi-gen runs on Debian-based operating systems. Currently it is only supported on
 either Debian Buster or Ubuntu Xenial and is known to have issues building on
 earlier releases of these systems. On other Linux distributions it may be possible
 to use the Docker build described below.
 
-To install the required dependencies for pi-gen you should run:
+To install the required dependencies for `pi-gen` you should run:
 
 ```bash
 apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
-dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl
+dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc
 ```
 
 The file `depends` contains a list of tools needed.  The format of this
@@ -36,6 +36,11 @@ The following environment variables are supported:
    but you should use something else for a customized version.  Export files
    in stages may add suffixes to `IMG_NAME`.
 
+ * `RELEASE` (Default: buster)
+
+   The release version to build images against. Valid values are jessie, stretch
+   buster, bullseye, and testing.
+
  * `APT_PROXY` (Default: unset)
 
    If you require the use of an apt proxy, set it here.  This proxy setting
@@ -62,8 +67,8 @@ The following environment variables are supported:
    be built and cached.  Note, `WORK_DIR` stores a complete copy of the target
    system for each build stage, amounting to tens of gigabytes in the case of
    Raspbian.
-   
-   **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build. Make sure this is a proper Linux filesystem.
+
+   **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build: make sure this is a proper Linux filesystem.
 
  * `DEPLOY_DIR`  (Default: `"$BASE_DIR/deploy"`)
 
@@ -78,6 +83,37 @@ The following environment variables are supported:
    Setting to '1' enables the QEMU mode - creating an image that can be mounted via QEMU for an emulated
    environment. These images include "-qemu" in the image file name.
 
+ * `LOCALE_DEFAULT` (Default: "en_GB.UTF-8" )
+
+   Default system locale.
+
+ * `TARGET_HOSTNAME` (Default: "raspberrypi" )
+
+   Setting the hostname to the specified value.
+
+ * `KEYBOARD_KEYMAP` (Default: "gb" )
+
+   Default keyboard keymap.
+
+   To get the current value from a running system, run `debconf-show
+   keyboard-configuration` and look at the
+   `keyboard-configuration/xkb-keymap` value.
+
+ * `KEYBOARD_LAYOUT` (Default: "English (UK)" )
+
+   Default keyboard layout.
+
+   To get the current value from a running system, run `debconf-show
+   keyboard-configuration` and look at the
+   `keyboard-configuration/variant` value.
+
+ * `TIMEZONE_DEFAULT` (Default: "Europe/London" )
+
+   Default keyboard layout.
+
+   To get the current value from a running system, look in
+   `/etc/timezone`.
+
  * `FIRST_USER_NAME` (Default: "pi" )
 
    Username for the first user
@@ -88,11 +124,23 @@ The following environment variables are supported:
 
  * `WPA_ESSID`, `WPA_PASSWORD` and `WPA_COUNTRY` (Default: unset)
 
-   If these are set, they are use to configure `wpa_supplicant.conf`, so that the raspberry pi can automatically connect to a wifi network on first boot.
+   If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wireless network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wireless network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters.
 
  * `ENABLE_SSH` (Default: `0`)
 
-   Setting to `1` will enable ssh server for remote log in. Note that if you are using a common password such as the defaults there is a high risk of attackers taking over you RaspberryPi.
+   Setting to `1` will enable ssh server for remote log in. Note that if you are using a common password such as the defaults there is a high risk of attackers taking over you Raspberry Pi.
+
+  * `PUBKEY_SSH_FIRST_USER` (Default: unset)
+
+   Setting this to a value will make that value the contents of the FIRST_USER_NAME's ~/.ssh/authorized_keys.  Obviously the value should
+   therefore be a valid authorized_keys file.  Note that this does not
+   automatically enable SSH.
+
+  * `PUBKEY_ONLY_SSH` (Default: `0`)
+
+   * Setting to `1` will disable password authentication for SSH and enable
+   public key authentication.  Note that if SSH is not enabled this will take
+   effect when SSH becomes enabled.
 
  * `STAGE_LIST` (Default: `stage*`)
 
@@ -228,7 +276,7 @@ maintenance and allows for more easy customization.
 
  - **Stage 2** - lite system.  This stage produces the Raspbian-Lite image.  It
    installs some optimized memory functions, sets timezone and charmap
-   defaults, installs fake-hwclock and ntp, wifi and bluetooth support,
+   defaults, installs fake-hwclock and ntp, wireless LAN and bluetooth support,
    dphys-swapfile, and other basics for managing the hardware.  It also
    creates necessary groups and gives the pi user access to sudo and the
    standard console hardware permission groups.
@@ -246,22 +294,21 @@ maintenance and allows for more easy customization.
    enhancements, etc.  This is a base desktop system, with some development
    tools installed.
 
- - **Stage 4** - Raspbian system meant to fit on a 4GB card.  More development
-   tools, an email client, learning tools like Scratch, specialized packages
-   like sonic-pi, system documentation, office productivity, etc.  This is the
-   stage that installs all of the things that make Raspbian friendly to new
-   users.
+ - **Stage 4** - Normal Raspbian image. System meant to fit on a 4GB card. This is the
+   stage that installs most things that make Raspbian friendly to new
+   users like system documentation.
 
- - **Stage 5** - The official Raspbian Desktop image. Right now only adds
-   Mathematica.
+ - **Stage 5** - The Raspbian Full image. More development
+   tools, an email client, learning tools like Scratch, specialized packages
+   like sonic-pi, office productivity, etc.  
 
 ### Stage specification
 
 If you wish to build up to a specified stage (such as building up to stage 2
 for a lite system), place an empty file named `SKIP` in each of the `./stage`
 directories you wish not to include.
 
-Then add an empty file named `SKIP_IMAGES` to `./stage4` (if building up to stage 2) or
+Then add an empty file named `SKIP_IMAGES` to `./stage4` and `./stage5` (if building up to stage 2) or
 to `./stage2` (if building a minimal system).
 
 ```bash

build-docker.sh

@@ -6,40 +6,42 @@ BUILD_OPTS="$*"
 DOCKER="docker"
 
 if ! ${DOCKER} ps >/dev/null 2>&1; then
-	DOCKER="sudo docker"
+DOCKER="sudo docker"
 fi
 if ! ${DOCKER} ps >/dev/null; then
-	echo "error connecting to docker:"
-	${DOCKER} ps
-	exit 1
+echo "error connecting to docker:"
+${DOCKER} ps
+exit 1
 fi
 
 CONFIG_FILE=""
 if [ -f "${DIR}/config" ]; then
-	CONFIG_FILE="${DIR}/config"
+CONFIG_FILE="${DIR}/config"
 fi
 
 while getopts "c:" flag
 do
-	case "${flag}" in
-		c)
-			CONFIG_FILE="${OPTARG}"
-			;;
-		*)
-			;;
-	esac
+case "${flag}" in
+	c)
+		CONFIG_FILE="${OPTARG}"
+		;;
+	*)
+		;;
+esac
 done
 
 # Ensure that the configuration file is an absolute path
-CONFIG_FILE=$(realpath -s "$CONFIG_FILE")
+if test -x /usr/bin/realpath; then
+	CONFIG_FILE=$(realpath -s "$CONFIG_FILE" || realpath "$CONFIG_FILE")
+fi
 
 # Ensure that the confguration file is present
 if test -z "${CONFIG_FILE}"; then
 	echo "Configuration file need to be present in '${DIR}/config' or path passed as parameter"
 	exit 1
 else
 	# shellcheck disable=SC1090
-	source "${CONFIG_FILE}"
+	source ${CONFIG_FILE}
 fi
 
 CONTAINER_NAME=${CONTAINER_NAME:-pigen_work}
@@ -71,7 +73,17 @@ fi
 # Modify original build-options to allow config file to be mounted in the docker container
 BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')"
 
-${DOCKER} build -t pi-gen "${DIR}"
+# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead
+case "$(uname -m)" in
+  x86_64|aarch64)
+    BASE_IMAGE=i386/debian:buster
+    ;;
+  *)
+    BASE_IMAGE=debian:buster
+    ;;
+esac
+${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
+
 if [ "${CONTAINER_EXISTS}" != "" ]; then
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
 	time ${DOCKER} run --rm --privileged \

build.sh

@@ -1,5 +1,4 @@
 #!/bin/bash -e
-set -x
 # shellcheck disable=SC2119
 run_sub_stage()
 {
@@ -21,7 +20,7 @@ EOF
 			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages-nr")"
 			if [ -n "$PACKAGES" ]; then
 				on_chroot << EOF
-apt-get install --no-install-recommends -y $PACKAGES
+apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
 EOF
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
@@ -31,7 +30,7 @@ EOF
 			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages")"
 			if [ -n "$PACKAGES" ]; then
 				on_chroot << EOF
-apt-get install -y $PACKAGES
+apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES
 EOF
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages"
@@ -123,6 +122,8 @@ if [ "$(id -u)" != "0" ]; then
 	exit 1
 fi
 
+BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export BASE_DIR
 
 if [ -f config ]; then
 	# shellcheck disable=SC1091
@@ -155,23 +156,33 @@ export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}"
 export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}"
 export ZIP_FILENAME="${ZIP_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}"
 
-BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 export SCRIPT_DIR="${BASE_DIR}/scripts"
 export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_DATE}-${IMG_NAME}"}"
 export DEPLOY_DIR=${DEPLOY_DIR:-"${BASE_DIR}/deploy"}
 export DEPLOY_ZIP="${DEPLOY_ZIP:-1}"
 export LOG_FILE="${WORK_DIR}/build.log"
 
+export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi}
+
 export FIRST_USER_NAME=${FIRST_USER_NAME:-pi}
 export FIRST_USER_PASS=${FIRST_USER_PASS:-raspberry}
+export RELEASE=${RELEASE:-buster}
 export WPA_ESSID
 export WPA_PASSWORD
 export WPA_COUNTRY
 export ENABLE_SSH="${ENABLE_SSH:-0}"
+export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}"
+
+export LOCALE_DEFAULT="${LOCALE_DEFAULT:-en_GB.UTF-8}"
+
+export KEYBOARD_KEYMAP="${KEYBOARD_KEYMAP:-gb}"
+export KEYBOARD_LAYOUT="${KEYBOARD_LAYOUT:-English (UK)}"
+
+export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}"
 
 export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
 
-export BASE_DIR
+export PUBKEY_SSH_FIRST_USER
 
 export CLEAN
 export IMG_NAME
@@ -213,6 +224,16 @@ if [[ -n "${APT_PROXY}" ]] && ! curl --silent "${APT_PROXY}" >/dev/null ; then
 	exit 1
 fi
 
+if [[ -n "${WPA_PASSWORD}" && ${#WPA_PASSWORD} -lt 8 || ${#WPA_PASSWORD} -gt 63  ]] ; then
+	echo "WPA_PASSWORD" must be between 8 and 63 characters
+	exit 1
+fi
+
+if [[ "${PUBKEY_ONLY_SSH}" = "1" && -z "${PUBKEY_SSH_FIRST_USER}" ]]; then
+	echo "Must set 'PUBKEY_SSH_FIRST_USER' to a valid SSH public key if using PUBKEY_ONLY_SSH"
+	exit 1
+fi
+
 mkdir -p "${WORK_DIR}"
 log "Begin ${BASE_DIR}"
 
@@ -240,7 +261,7 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	fi
 done
 
-if [ -x postrun.sh ]; then
+if [ -x ${BASE_DIR}/postrun.sh ]; then
 	log "Begin postrun.sh"
 	cd "${BASE_DIR}"
 	./postrun.sh

config

@@ -1,3 +1,10 @@
 IMG_NAME="lomorage"
-STAGE_LIST="stage0 stage1 stage2 stage-share stage-lomo"
+TIMEZONE_DEFAULT="America/Los_Angeles"
+LOCALE_DEFAULT="en_US.UTF-8"
+KEYBOARD_KEYMAP="us"
+KEYBOARD_LAYOUT="English (US)"
+TARGET_HOSTNAME="raspberrypi-lomorage"
+FIRST_USER_NAME="pi"
+FIRST_USER_PASS="raspberry"
 ENABLE_SSH=1
+STAGE_LIST="stage0 stage1 stage2 stage-share stage-lomo"

depends

@@ -16,3 +16,4 @@ xxd
 file
 git
 lsmod:kmod
+bc

export-image/01-set-sources/01-run.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -e
 
+rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
+find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete
 on_chroot << EOF
 apt-get update
 apt-get -y dist-upgrade

export-image/04-finalise/01-run.sh

@@ -16,7 +16,6 @@ if [ -d "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" ]; then
 	chmod 700 "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config"
 fi
 
-rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
 rm -f "${ROOTFS_DIR}/usr/bin/qemu-arm-static"
 
 if [ "${USE_QEMU}" != "1" ]; then