Add Raspberry Pi build script (elementary#343)

davidmhewitt · meisenzahl · web-flow · commit 44e76494eac4 · 2020-12-03T08:47:04.000-07:00
* Add Raspberry Pi build script * Optimize build script for Raspberry Pi 4 (elementary#407) * Update README.md * Add oneshot service to resize rootfs on first boot * Fix path * Get firmware directly from RPi foundation Means we can keep the correct kernel version from focal * Run the same hooks as the amd64 image * Fix paths * Fix paths some more * Make sure we have git dependency * Add some multithreading to xz * Rework image size calculation * Fix firmware package install * Make sure we have certificates to verify https from github * Remove old firmware copy * Remove systemd growfs stuff * Fix resize script * Try to make plymouth show * Use SVG as OEM logo * Add Pi 400 support to flash-kernel * Upload images to bucket * Add a workflow file Co-authored-by: Marius Meisenzahl <meisenzahl@users.noreply.github.com> Co-authored-by: Marius Meisenzahl <mariusmeisenzahl@gmail.com>
diff --git a/.github/workflows/daily-6.0-rpi.yml b/.github/workflows/daily-6.0-rpi.yml
@@ -0,0 +1,22 @@
+name: daily-6.0-rpi
+
+on:
+  push:
+    branches:
+      - master
+      - github-update
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Clone build scripts
+      uses: actions/checkout@v1
+
+    - name: Build and upload image
+      run: |
+        sudo ./build-rpi.sh "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}"
diff --git a/README.md b/README.md
@@ -45,6 +45,17 @@ The following example uses Docker and assumes you have Docker correctly installe
 
  4) When done, your image will be in the `builds` folder.
 
+## Raspberry Pi 4
+
+```
+docker run --privileged -i -v /proc:/proc \
+    -v ${PWD}:/working_dir \
+    -w /working_dir \
+    debian:unstable \
+    ./build-rpi.sh
+```
+
+
 ## Further Information
 
 More information about the concepts behind `live-build` and the technical decisions made to arrive at this set of tools to build an .iso can be found [on the wiki](https://github.com/elementary/os/wiki/Building-iso-Images).
diff --git a/build-rpi.sh b/build-rpi.sh
@@ -0,0 +1,261 @@
+#!/bin/bash
+
+set -e
+
+# Install dependencies in host system
+apt-get update
+apt-get install -y --no-install-recommends ubuntu-keyring ca-certificates debootstrap git qemu-user-static qemu-utils qemu-system-arm binfmt-support parted kpartx rsync dosfstools xz-utils
+
+# Make sure cross-running ARM ELF executables is enabled
+update-binfmts --enable
+
+rootdir=`pwd`
+basedir=`pwd`/artifacts/elementary-rpi
+
+# Free space on rootfs in MiB
+free_space="500"
+
+export packages="elementary-minimal elementary-desktop elementary-standard"
+export architecture="arm64"
+export codename="focal"
+export channel="daily"
+
+version=6.0
+YYYYMMDD="$(date +%Y%m%d)"
+imagename=elementaryos-$version-$channel-rpi-$YYYYMMDD
+
+mkdir -p ${basedir}
+cd ${basedir}
+
+# Bootstrap an ubuntu minimal system
+debootstrap --foreign --arch $architecture $codename elementary-$architecture http://ports.ubuntu.com/ubuntu-ports
+
+# Add the QEMU emulator for running ARM executables
+cp /usr/bin/qemu-arm-static elementary-$architecture/usr/bin/
+
+# Run the second stage of the bootstrap in QEMU
+LANG=C chroot elementary-$architecture /debootstrap/debootstrap --second-stage
+
+# Copy Raspberry Pi specific files
+cp -r ${rootdir}/rpi/rootfs/writable/* elementary-${architecture}/
+
+# Add the rest of the ubuntu repos
+cat << EOF > elementary-$architecture/etc/apt/sources.list
+deb http://ports.ubuntu.com/ubuntu-ports $codename main restricted universe multiverse
+deb http://ports.ubuntu.com/ubuntu-ports $codename-updates main restricted universe multiverse
+EOF
+
+# Copy in the elementary PPAs/keys/apt config
+for f in ${rootdir}/etc/config/archives/*.list; do cp -- "$f" "elementary-$architecture/etc/apt/sources.list.d/$(basename -- $f)"; done
+for f in ${rootdir}/etc/config/archives/*.key; do cp -- "$f" "elementary-$architecture/etc/apt/trusted.gpg.d/$(basename -- $f).asc"; done
+for f in ${rootdir}/etc/config/archives/*.pref; do cp -- "$f" "elementary-$architecture/etc/apt/preferences.d/$(basename -- $f)"; done
+
+# Set codename/channel in added repos
+sed -i "s/@CHANNEL/$channel/" elementary-$architecture/etc/apt/sources.list.d/*.list*
+sed -i "s/@BASECODENAME/$codename/" elementary-$architecture/etc/apt/sources.list.d/*.list*
+
+# Set codename in added preferences
+sed -i "s/@BASECODENAME/$codename/" elementary-$architecture/etc/apt/preferences.d/*.pref*
+
+echo "elementary" > elementary-$architecture/etc/hostname
+
+cat << EOF > elementary-${architecture}/etc/hosts
+127.0.0.1       elementary    localhost
+::1             localhost ip6-localhost ip6-loopback
+fe00::0         ip6-localnet
+ff00::0         ip6-mcastprefix
+ff02::1         ip6-allnodes
+ff02::2         ip6-allrouters
+EOF
+
+# Configure mount points
+cat << EOF > elementary-${architecture}/etc/fstab
+# <file system> <mount point>   <type>  <options>       <dump>  <pass>
+proc /proc proc nodev,noexec,nosuid 0  0
+LABEL=writable    /     ext4    defaults    0 0
+LABEL=system-boot       /boot/firmware  vfat    defaults        0       1
+EOF
+
+export LC_ALL=C
+export DEBIAN_FRONTEND=noninteractive
+# Config to stop flash-kernel trying to detect the hardware in chroot
+export FK_MACHINE=none
+
+mount -t proc proc elementary-$architecture/proc
+mount -o bind /dev/ elementary-$architecture/dev/
+mount -o bind /dev/pts elementary-$architecture/dev/pts
+
+# Make a third stage that installs all of the metapackages
+cat << EOF > elementary-$architecture/third-stage
+#!/bin/bash
+apt-get update
+apt-get --yes upgrade
+apt-get --yes install $packages
+
+rm -f /third-stage
+EOF
+
+chmod +x elementary-$architecture/third-stage
+LANG=C chroot elementary-$architecture /third-stage
+
+
+# Install Raspberry Pi specific packages
+cat << EOF > elementary-$architecture/hardware
+#!/bin/bash
+
+# Make a dummy folder for the boot partition so packages install properly,
+# we'll recreate it on the actual partition later
+mkdir -p /boot/firmware
+
+apt-get --yes install linux-image-raspi linux-firmware-raspi2
+
+rm -rf /boot/firmware
+
+rm -f hardware
+EOF
+
+chmod +x elementary-$architecture/hardware
+LANG=C chroot elementary-$architecture /hardware
+
+# Copy in any file overrides
+cp -r ${rootdir}/etc/config/includes.chroot/* elementary-$architecture/
+
+mkdir elementary-$architecture/hooks
+cp ${rootdir}/etc/config/hooks/live/*.chroot elementary-$architecture/hooks
+
+for f in elementary-$architecture/hooks/*
+do
+    base=`basename ${f}`
+    LANG=C chroot elementary-$architecture "/hooks/${base}"
+done
+
+rm -r "elementary-$architecture/hooks"
+
+# Add a oneshot service to grow the rootfs on first boot
+install -m 755 -o root -g root ${rootdir}/rpi/files/resizerootfs "elementary-$architecture/usr/sbin/resizerootfs"
+install -m 644 -o root -g root ${rootdir}/pinebookpro/files/resizerootfs.service "elementary-$architecture/etc/systemd/system"
+mkdir -p "elementary-$architecture/etc/systemd/system/systemd-remount-fs.service.requires/"
+ln -s /etc/systemd/system/resizerootfs.service "elementary-$architecture/etc/systemd/system/systemd-remount-fs.service.requires/resizerootfs.service"
+
+
+# Support for kernel updates on the Pi 400
+cat << EOF >> elementary-$architecture/etc/flash-kernel/db
+
+Machine: Raspberry Pi 400 Rev 1.0
+Method: pi
+Kernel-Flavors: raspi raspi2
+DTB-Id: bcm2711-rpi-4-b.dtb
+U-Boot-Script-Name: bootscr.rpi
+Required-Packages: u-boot-tools
+EOF
+
+# Calculate the space to create the image.
+root_size=$(du -s -B1K elementary-$architecture | cut -f1)
+raw_size=$(($((${free_space}*1024))+${root_size}))
+
+# Create the disk and partition it
+echo "Creating image file"
+
+# Sometimes fallocate fails if the filesystem or location doesn't support it, fallback to slower dd in this case
+if ! fallocate -l $(echo ${raw_size}Ki | numfmt --from=iec-i --to=si --format=%.1f) ${basedir}/${imagename}.img
+then
+    dd if=/dev/zero of=${basedir}/${imagename}.img bs=1024 count=${raw_size}
+fi
+
+parted ${imagename}.img --script -- mklabel msdos
+parted ${imagename}.img --script -- mkpart primary fat32 0 256
+parted ${imagename}.img --script -- mkpart primary ext4 256 -1
+
+# Set the partition variables
+loopdevice=`losetup -f --show ${basedir}/${imagename}.img`
+device=`kpartx -va $loopdevice| sed -E 's/.*(loop[0-9])p.*/\1/g' | head -1`
+device="/dev/mapper/${device}"
+bootp=${device}p1
+rootp=${device}p2
+
+# Create file systems
+mkfs.vfat -n system-boot $bootp
+mkfs.ext4 -L writable $rootp
+
+# Create the dirs for the partitions and mount them
+mkdir -p ${basedir}/bootp ${basedir}/root
+mount -t vfat $bootp ${basedir}/bootp
+mount $rootp ${basedir}/root
+
+mkdir -p elementary-$architecture/boot/firmware
+mount -o bind ${basedir}/bootp/ elementary-$architecture/boot/firmware
+
+# Copy Raspberry Pi specific files
+cp -r ${rootdir}/rpi/rootfs/system-boot/* elementary-${architecture}/boot/firmware/
+
+# Copy kernels and firemware to boot partition
+cat << EOF > elementary-$architecture/hardware
+#!/bin/bash
+
+cp /boot/vmlinuz /boot/firmware/vmlinuz
+cp /boot/initrd.img /boot/firmware/initrd.img
+
+# Copy device-tree blobs to fat32 partition
+cp -r /lib/firmware/*-raspi/device-tree/broadcom/* /boot/firmware/
+cp -r /lib/firmware/*-raspi/device-tree/overlays /boot/firmware/
+
+rm -f hardware
+EOF
+
+chmod +x elementary-$architecture/hardware
+LANG=C chroot elementary-$architecture /hardware
+
+# Grab some updated firmware from the Raspberry Pi foundation
+git clone -b '1.20201022' --single-branch --depth 1 https://github.com/raspberrypi/firmware raspi-firmware
+cp raspi-firmware/boot/*.elf ${basedir}/bootp/
+cp raspi-firmware/boot/*.dat ${basedir}/bootp/
+cp raspi-firmware/boot/bootcode.bin ${basedir}/bootp/
+
+umount elementary-$architecture/dev/pts
+umount elementary-$architecture/dev/
+umount elementary-$architecture/proc
+umount elementary-$architecture/boot/firmware
+
+echo "Rsyncing rootfs into image file"
+rsync -HPavz -q ${basedir}/elementary-$architecture/ ${basedir}/root/
+
+# Unmount partitions
+umount $bootp
+umount $rootp
+kpartx -dv $loopdevice
+losetup -d $loopdevice
+
+echo "Compressing ${imagename}.img"
+xz -T0 -z ${basedir}/${imagename}.img
+
+cd "${basedir}"
+
+md5sum ${imagename}.img.xz > ${imagename}.md5.txt
+sha256sum ${imagename}.img.xz > ${imagename}.sha256.txt
+
+cd "${rootdir}"
+
+KEY="$1"
+SECRET="$2"
+ENDPOINT="$3"
+BUCKET="$4"
+IMGPATH="${basedir}"/${imagename}.img.xz
+IMGNAME=${channel}-rpi/$(basename "$IMGPATH")
+
+apt-get install -y curl python3 python3-distutils
+
+curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+python3 get-pip.py
+pip install boto3
+
+python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$IMGPATH" "$IMGNAME" || exit 1
+
+CHECKSUMPATH="${basedir}"/${imagename}.md5.txt
+CHECKSUMNAME=${channel}-rpi/$(basename "$CHECKSUMPATH")
+
+python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$CHECKSUMPATH" "$CHECKSUMNAME" || exit 1
+
+CHECKSUMPATH="${basedir}"/${imagename}.sha256.txt
+CHECKSUMNAME=${channel}-rpi/$(basename "$CHECKSUMPATH")
+
+python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$CHECKSUMPATH" "$CHECKSUMNAME" || exit 1
diff --git a/rpi/files/resizerootfs b/rpi/files/resizerootfs
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+rootpart="$(findmnt -n -o SOURCE /)"
+rootdev="/dev/$(lsblk -no pkname "$rootpart")"
+
+flock $rootdev sfdisk -f $rootdev -N 2 <<EOF
+,+
+EOF
+
+sleep 5
+
+udevadm settle
+
+sleep 5
+
+flock $rootdev partprobe $rootdev
+
+mount -o remount,rw $rootpart
+
+resize2fs $rootpart
+
+exit 0
diff --git a/rpi/rootfs/system-boot/cmdline.txt b/rpi/rootfs/system-boot/cmdline.txt
@@ -0,0 +1 @@
+net.ifnames=0 dwc_otg.lpm_enable=0 console=ttyAMA0,115200 console=tty1 root=LABEL=writable rootfstype=ext4 elevator=deadline rootwait fixrtc quiet splash plymouth.ignore-serial-consoles
diff --git a/rpi/rootfs/system-boot/config.txt b/rpi/rootfs/system-boot/config.txt
@@ -0,0 +1,32 @@
+[pi4]
+max_framebuffers=2
+
+[all]
+arm_64bit=1
+kernel=vmlinuz
+cmdline=cmdline.txt
+initramfs initrd.img followkernel
+
+# Enable the audio output, I2C and SPI interfaces on the GPIO header
+dtparam=audio=on
+dtparam=i2c_arm=on
+dtparam=spi=on
+
+# Enable the FKMS ("Fake" KMS) graphics overlay, enable the camera firmware
+# and allocate 128Mb to the GPU memory
+dtoverlay=vc4-fkms-v3d
+gpu_mem=128
+start_x=1
+
+# Comment out the following line if the edges of the desktop appear outside
+# the edges of your display
+disable_overscan=1
+
+# If you have issues with audio, you may try uncommenting the following line
+# which forces the HDMI output into HDMI mode instead of DVI (which doesn't
+# support audio output)
+#hdmi_drive=2
+
+# If you have a CM4, uncomment the following line to enable the USB2 outputs
+# on the IO board (assuming your CM4 is plugged into such a board)
+#dtoverlay=dwc2,dr_mode=host
diff --git a/rpi/rootfs/writable/etc/oem.conf b/rpi/rootfs/writable/etc/oem.conf
@@ -0,0 +1,5 @@
+[OEM]
+Manufacturer=Raspberry Pi Foundation
+Product=Raspberry Pi
+Logo=/etc/oem/logo.svg
+URL=https://www.raspberrypi.org/
diff --git a/rpi/rootfs/writable/etc/oem/logo.svg b/rpi/rootfs/writable/etc/oem/logo.svg

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+net.ifnames=0 dwc_otg.lpm_enable=0 console=ttyAMA0,115200 console=tty1 root=LABEL=writable rootfstype=ext4 elevator=deadline rootwait fixrtc quiet splash plymouth.ignore-serial-consoles`