Upgrade docker build images

build/README.md

@@ -11,3 +11,7 @@ This makes it simple to build any release without exotic hardware or software.
 
 An image can be built and tagged using `build_builder`,
 and run on a source tarball using `run_builder`.
+
+Tip: Are you developing an image that relies on QEmu usermode emulation?
+     It's easy to accidentally depend on binfmt\_misc on the host OS.
+     Do a `echo 0 | sudo tee /proc/sys/fs/binfmt_misc/status` before testing.

build/darwin.aarch64/build

@@ -4,7 +4,6 @@ set -xe
   tar xzv --strip-components=1
   chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
-  cabal update
   ( IFS=';'; cabal build $CABALOPTS )
   find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
   ls -l "$TARGETNAME"

build/darwin.x86_64/build

@@ -4,7 +4,6 @@ set -xe
   tar xzv --strip-components=1
   chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
-  cabal update
   ( IFS=';'; cabal build $CABALOPTS )
   find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
   ls -l "$TARGETNAME"

build/linux.aarch64/Dockerfile

@@ -28,7 +28,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-in
 
 # Due to an apparent cabal bug, we specify our options directly to cabal
 # It won't reuse caches if ghc-options are specified in ~/.cabal/config
-ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
+ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg;-c;hashable -arch-native"
 
 # Prebuild the dependencies
 RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck

build/linux.aarch64/build

@@ -4,7 +4,6 @@ set -xe
   tar xzv --strip-components=1
   chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
-  cabal update
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )
   find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
   ls -l "$TARGETNAME"

build/linux.armv6hf/Dockerfile

@@ -1,60 +1,42 @@
-# I've again spent days trying to get a working armv6hf compiler going.
-# God only knows how many recompilations of GCC, GHC, libraries, and
-# ShellCheck itself, has gone into it.
-#
-# I tried Debian's toolchain.  I tried my custom one built according to
-# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but
-# nothing has yielded an armv6hf binary that does not immediately
-# segfault on qemu-arm-static or the RPi itself.
-#
-# I then tried the same but with armv7hf. Same story.
-#
-# Emulating the entire userspace with balenalib again?  Very strange build
-# failures where programs would fail to execute with > ~100 arguments.
-#
-# Finally, creating our own appears to work when using a custom QEmu
-# patched to follow execve calls.
-#
-# PS: $100 bounty for getting a RPi1 compatible static build going
-#     with cross-compilation, similar to what the aarch64 build does.
-#
-
-FROM ubuntu:20.04
+# This Docker file uses a custom QEmu fork with patches to follow execve
+# to build all of ShellCheck emulated.
+
+FROM ubuntu:24.04
 
 ENV TARGETNAME linux.armv6hf
 
 # Build QEmu with execve follow support
 USER root
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update
-RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev
-WORKDIR /build
-RUN git clone --depth 1 https://github.com/koalaman/qemu
-RUN cd qemu && ./configure --static && cd build && ninja qemu-arm
-RUN cp qemu/build/qemu-arm /build/qemu-arm-static
+RUN apt-get install -y --no-install-recommends build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev python3-setuptools ca-certificates debootstrap
+WORKDIR /qemu
+RUN git clone --depth 1 https://github.com/koalaman/qemu .
+RUN ./configure --static --disable-werror && cd build && ninja qemu-arm
 ENV QEMU_EXECVE 1
 
+# Convenience utility
+COPY scutil /bin/scutil
+COPY scutil /chroot/bin/scutil
+RUN chmod +x /bin/scutil /chroot/bin/scutil
+
 # Set up an armv6 userspace
 WORKDIR /
-RUN apt-get install -y debootstrap qemu-user-static
-# We expect this to fail if the host doesn't have binfmt qemu support
-RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ]
-RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static
-RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun
-# If the debootstrap process didn't finish, continue it
-RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage
+RUN debootstrap --arch armhf --variant=minbase --foreign bookworm /chroot http://mirrordirector.raspbian.org/raspbian
+RUN cp /qemu/build/qemu-arm /chroot/bin/qemu
+RUN scutil emu /debootstrap/debootstrap --second-stage
 
 # Install deps in the chroot
-RUN pirun apt-get update
-RUN pirun apt-get install -y ghc cabal-install
+RUN scutil emu apt-get update
+RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install
+RUN scutil emu cabal update
 
 # Finally we can build the current dependencies. This takes hours.
 ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections"
-RUN pirun cabal update
-RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck
-RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl
+# Generated with `cabal freeze --constraint 'hashable -arch-native'`
+COPY cabal.project.freeze /chroot/etc
+RUN IFS=";" && scutil install_from_freeze /chroot/etc/cabal.project.freeze emu cabal install $CABALOPTS
 
 # Copy the build script
-WORKDIR /pi/scratch
-COPY build /pi/usr/bin
-ENTRYPOINT ["/bin/pirun", "/usr/bin/build"]
+COPY build /chroot/bin
+ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"]

build/linux.armv6hf/build

@@ -1,8 +1,9 @@
 #!/bin/sh
 set -xe
-cd /scratch
+mkdir /scratch && cd /scratch
 {
   tar xzv --strip-components=1
+  cp /etc/cabal.project.freeze .
   chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   # This script does not cabal update because compiling anything new is slow

build/linux.armv6hf/cabal.project.freeze

@@ -0,0 +1,93 @@
+active-repositories: hackage.haskell.org:merge
+constraints: any.Diff ==0.5,
+             any.OneTuple ==0.4.2,
+             any.QuickCheck ==2.14.3,
+             QuickCheck -old-random +templatehaskell,
+             any.StateVar ==1.2.2,
+             any.aeson ==2.2.3.0,
+             aeson +ordered-keymap,
+             any.array ==0.5.4.0,
+             any.assoc ==1.1.1,
+             assoc -tagged,
+             any.base ==4.15.1.0,
+             any.base-orphans ==0.9.2,
+             any.bifunctors ==5.6.2,
+             bifunctors +tagged,
+             any.binary ==0.8.8.0,
+             any.bytestring ==0.10.12.1,
+             any.character-ps ==0.1,
+             any.comonad ==5.0.8,
+             comonad +containers +distributive +indexed-traversable,
+             any.containers ==0.6.4.1,
+             any.contravariant ==1.5.5,
+             contravariant +semigroups +statevar +tagged,
+             any.data-array-byte ==0.1.0.1,
+             any.data-fix ==0.3.3,
+             any.deepseq ==1.4.5.0,
+             any.directory ==1.3.6.2,
+             any.distributive ==0.6.2.1,
+             distributive +semigroups +tagged,
+             any.dlist ==1.0,
+             dlist -werror,
+             any.exceptions ==0.10.4,
+             any.fgl ==5.8.2.0,
+             fgl +containers042,
+             any.filepath ==1.4.2.1,
+             any.foldable1-classes-compat ==0.1,
+             foldable1-classes-compat +tagged,
+             any.generically ==0.1.1,
+             any.ghc-bignum ==1.1,
+             any.ghc-boot-th ==9.0.2,
+             any.ghc-prim ==0.7.0,
+             any.hashable ==1.4.6.0,
+             hashable -arch-native +integer-gmp -random-initial-seed,
+             any.indexed-traversable ==0.1.4,
+             any.indexed-traversable-instances ==0.1.2,
+             any.integer-conversion ==0.1.1,
+             any.integer-logarithms ==1.0.3.1,
+             integer-logarithms -check-bounds +integer-gmp,
+             any.mtl ==2.2.2,
+             any.network-uri ==2.6.4.2,
+             any.parsec ==3.1.14.0,
+             any.pretty ==1.1.3.6,
+             any.primitive ==0.9.0.0,
+             any.process ==1.6.13.2,
+             any.random ==1.2.1.2,
+             any.regex-base ==0.94.0.2,
+             any.regex-tdfa ==1.3.2.2,
+             regex-tdfa +doctest -force-o2,
+             any.rts ==1.0.2,
+             any.scientific ==0.3.8.0,
+             scientific -integer-simple,
+             any.semialign ==1.3.1,
+             semialign +semigroupoids,
+             any.semigroupoids ==6.0.1,
+             semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers,
+             any.splitmix ==0.1.0.5,
+             splitmix -optimised-mixer,
+             any.stm ==2.5.0.0,
+             any.strict ==0.5,
+             any.tagged ==0.8.8,
+             tagged +deepseq +transformers,
+             any.template-haskell ==2.17.0.0,
+             any.text ==1.2.5.0,
+             any.text-iso8601 ==0.1.1,
+             any.text-short ==0.1.6,
+             text-short -asserts,
+             any.th-abstraction ==0.7.0.0,
+             any.th-compat ==0.1.5,
+             any.these ==1.2.1,
+             any.time ==1.9.3,
+             any.time-compat ==1.9.7,
+             any.transformers ==0.5.6.2,
+             any.transformers-compat ==0.7.2,
+             transformers-compat -five +five-three -four +generic-deriving +mtl -three -two,
+             any.unix ==2.7.2.2,
+             any.unordered-containers ==0.2.20,
+             unordered-containers -debug,
+             any.uuid-types ==1.0.6,
+             any.vector ==0.13.1.0,
+             vector +boundschecks -internalchecks -unsafechecks -wall,
+             any.vector-stream ==0.1.0.1,
+             any.witherable ==0.5
+index-state: hackage.haskell.org 2024-06-18T02:21:19Z

build/linux.armv6hf/scutil

@@ -0,0 +1,48 @@
+#!/bin/dash
+# Various ShellCheck build utility functions
+
+# Generally set a ulimit to avoid QEmu using too much memory
+ulimit -v "$((10*1024*1024))"
+# If we happen to invoke or run under QEmu, make sure to follow execve.
+# This requires a patched QEmu.
+export QEMU_EXECVE=1
+
+# Retry a command until it succeeds
+# Usage: scutil retry 3 mycmd
+retry() {
+  n="$1"
+  ret=1
+  shift
+  while [ "$n" -gt 0 ]
+  do
+    "$@"
+    ret=$?
+    [ "$ret" = 0 ] && break
+    n=$((n-1))
+  done
+  return "$ret"
+}
+
+# Install all dependencies from a freeze file
+# Usage: scutil install_from_freeze /path/cabal.project.freeze cabal install
+install_from_freeze() {
+  linefeed=$(printf '\nx')
+  linefeed=${linefeed%x}
+  flags=$(
+    sed 's/constraints:/&\n /' "$1" |
+    grep -vw -e rts -e base |
+    sed -n -e 's/^  *\([^,]*\).*/\1/p' |
+    sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/--constraint\n&/; :e')
+  shift
+  # shellcheck disable=SC2086
+  ( IFS=$linefeed; set -x; "$@" $flags )
+}
+
+# Run a command under emulation.
+# This assumes the correct emulator is named 'qemu' and the chroot is /chroot
+# Usage: scutil emu echo "Hello World"
+emu() {
+  chroot /chroot /bin/qemu /usr/bin/env "$@"
+}
+
+"$@"

build/linux.riscv64/Dockerfile

@@ -10,46 +10,37 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update -y
 
 # Install qemu
-RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools
+RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools debootstrap
 WORKDIR /qemu
 RUN git clone --depth 1 https://github.com/koalaman/qemu .
-#RUN git clone https://github.com/balena-io/qemu .
-# Release 7.0.0
-#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86
 RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror
 RUN cd build && ninja qemu-riscv64
 ENV QEMU_EXECVE 1
 
+# Convenience utility
+COPY scutil /bin/scutil
+# We have to copy to /usr/bin because debootstrap will try to symlink /bin and fail if it exists
+COPY scutil /chroot/usr/bin/scutil
+RUN chmod +x /bin/scutil /chroot/usr/bin/scutil
 
 # Set up a riscv64 userspace
-RUN apt-get install -y --no-install-recommends debootstrap
-RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports
-RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static
-
-# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB
-# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate
-# less and reduce RAM usage.
-RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"'
-RUN chmod +x /bin/rv
-RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage
+WORKDIR /
+RUN debootstrap --arch=riscv64 --variant=minbase --components=main,universe --foreign noble /chroot http://ports.ubuntu.com/ubuntu-ports
+RUN cp /qemu/build/qemu-riscv64 /chroot/bin/qemu
+RUN scutil emu /debootstrap/debootstrap --second-stage
 
 # Install deps in the chroot
-RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe'
-RUN rv apt-get update -y
-RUN rv apt-get install -y --no-install-recommends ghc cabal-install
+RUN scutil emu apt-get update
+RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install
+RUN scutil emu cabal update
 
-RUN rv cabal update
 # Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it.
-COPY cabal.project.freeze /rvfs/etc
-# Awful hack to install everything from the freeze file
-# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version,
-# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint.
-RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^  *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags
-# Finally we can build the current dependencies. This takes hours.
-# There's apparently a random segfault during assembly, so retry a few times.
-RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret
+COPY cabal.project.freeze /chroot/etc
+
+# Build all dependencies from the freeze file. The emulator segfaults at random,
+# so retry a few times.
+RUN scutil install_from_freeze /chroot/etc/cabal.project.freeze retry 5 emu cabal install --keep-going
 
 # Copy the build script
-WORKDIR /rvfs/scratch
-COPY build /rvfs/usr/bin/build
-ENTRYPOINT ["/bin/rv", "/usr/bin/build"]
+COPY build /chroot/bin/build
+ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"]

build/linux.riscv64/build

@@ -2,14 +2,16 @@
 set -xe
 IFS=';'
 {
+  mkdir -p /tmp/scratch
+  cd /tmp/scratch
   tar xzv --strip-components=1
   chmod +x striptests && ./striptests
   # Use a freeze file to ensure we use the same dependencies we cached during
   # the docker image build. We don't want to spend time compiling anything new.
   cp /etc/cabal.project.freeze .
   mkdir "$TARGETNAME"
   # Retry in case of random segfault
-  cabal build --enable-executable-static || cabal build --enable-executable-static
+  scutil retry 3 cabal build --enable-executable-static
   find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
   ls -l "$TARGETNAME"
   "$TARGET-strip" -s "$TARGETNAME/shellcheck"

build/windows.x86_64/build

@@ -8,7 +8,6 @@ set -xe
   tar xzv --strip-components=1
   chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
-  cabal update
   ( IFS=';'; cabal build $CABALOPTS )
   find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \;
   ls -l "$TARGETNAME"