ci: ask the AppImage what it bundles, and whether it starts

.github/dependabot.yml

@@ -193,6 +193,15 @@ updates:
   # carries no version for Dependabot to compare), so a bump here is usually the
   # tag moving to a new major -- exactly the case the group excludes. In
   # practice this ecosystem will open single pull requests rather than a batch.
+  #
+  # This covers `uses:` and nothing else. `runs-on:` is not a dependency --
+  # there is no manifest and no version to resolve -- so nothing here or
+  # anywhere else proposes `ubuntu-24.04` -> `ubuntu-26.04`. That move is
+  # manual, and until someone makes it the build sits on an image that is
+  # quietly ageing: test.yml and test_build.yml were on ubuntu-22.04 from the
+  # commits that created them until 2026-08-11, two Ubuntus behind what
+  # build.yml releases from. `releaseWorkflow.test.ts` now fails if the three
+  # disagree, which makes the move all-or-nothing rather than proposing it.
   - package-ecosystem: github-actions
     directory: /
     schedule:

.github/workflows/build.yml

@@ -310,6 +310,20 @@ jobs:
           fi
           ls -la "$APPIMAGE" "$APPIMAGE.sig"
 
+      # The last thing before the artifact becomes downloadable, and the only
+      # check in this pipeline that starts Markpad rather than inspecting it.
+      # It runs on another distribution's userspace, because the runner cannot
+      # disagree with libraries that were copied out of the runner.
+      #
+      # Placed after the re-sign so it starts the exact bytes latest.json will
+      # point at, and before the upload so a failure leaves the draft release
+      # without a Linux artifact. generate-update-feed needs `build` to have
+      # succeeded, so it does not run either -- nothing is published, and
+      # nothing was ever installable.
+      - name: The AppImage starts on a distribution it was not built on
+        if: matrix.os == 'linux'
+        run: bash scripts/smoke-appimage.sh "$APPIMAGE"
+
       - name: Upload Linux Artifacts
         if: matrix.os == 'linux'
         shell: bash

.github/workflows/test_build.yml

@@ -192,6 +192,48 @@ jobs:
           NODE_OPTIONS: "--max_old_space_size=4096"
         run: npm run tauri build -- --no-sign ${{ matrix.args }}
 
+      # The AppImage this job just built is the un-stripped one, and for this
+      # question that is the artifact to ask, not a worse one: it shows what
+      # linuxdeploy put in before anything was taken back out. What decides that
+      # is linuxdeploy-plugin-gtk, fetched from the tip of someone else's branch
+      # on every build -- so the answer can change with no commit here.
+      #
+      # Only possible since this job moved to the release's Ubuntu. Before that
+      # it bundled jammy's libraries and the comparison meant nothing.
+      - name: The AppImage bundles nothing that has to come from the host
+        if: matrix.os-name == 'linux'
+        run: |
+          set -euo pipefail
+          APPIMAGE=$(find src-tauri/target/release/bundle/appimage -maxdepth 1 -name '*.AppImage' | head -1)
+          if [ -z "$APPIMAGE" ]; then
+            echo "::error::the Linux build produced no AppImage to check" >&2
+            exit 1
+          fi
+          bash scripts/check-appimage-libraries.sh "$APPIMAGE"
+          echo "APPIMAGE=$(realpath "$APPIMAGE")" >> "$GITHUB_ENV"
+
+      # The release strips the AppImage and then starts it. Doing both here as
+      # well is what makes those two steps testable at all: strip-appimage.sh
+      # has never run outside a release -- it failed in two of the three v2.7.2
+      # release attempts -- and smoke-appimage.sh would otherwise ship into the
+      # release path having never run anywhere.
+      #
+      # The order matters and is the release's order. Un-stripped, this AppImage
+      # still carries the libraries #498 was about, so smoking it before the
+      # strip would be reproducing the defect rather than testing for it.
+      #
+      # Not free: the strip repacks, and the smoke pulls a container image and a
+      # graphics stack. Roughly three minutes on a twelve-minute job. If that
+      # stops being worth it, path-filter these two steps rather than deleting
+      # them -- the release depends on both scripts either way.
+      - name: Strip the AppImage, as the release does
+        if: matrix.os-name == 'linux'
+        run: bash scripts/strip-appimage.sh "$APPIMAGE"
+
+      - name: The AppImage starts on a distribution it was not built on
+        if: matrix.os-name == 'linux'
+        run: bash scripts/smoke-appimage.sh "$APPIMAGE"
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v7
         with:

scripts/check-appimage-libraries.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# Ask a freshly built AppImage which host-coupled libraries linuxdeploy put in
+# it, and require the answer to be exactly the list strip-appimage.sh removes.
+#
+# strip-appimage.sh already proves its six are gone afterwards. That is "we
+# removed what we meant to remove"; it cannot say whether six is still the right
+# number. #463 and #498 were both a library that had to come from the host being
+# bundled anyway, and the second was found by a user on Arch weeks after release.
+#
+# What decides the answer is not in this repository. `tauri build` fetches
+# linuxdeploy-plugin-gtk from the tip of someone else's branch on every run --
+#
+#   Downloading https://raw.githubusercontent.com/tauri-apps/linuxdeploy-plugin-gtk/master/linuxdeploy-plugin-gtk.sh
+#
+# -- so the set of libraries copied into the AppDir can change with no commit
+# here, no pull request, and no notification. That is the same shape as a runner
+# image moving under `macos-latest`, one repository further away.
+#
+# Run against the AppImage a pull request builds, which is the un-stripped one:
+# for this question that is the right artifact, not a worse one. It shows what
+# linuxdeploy actually did before anything was taken back out.
+#
+# The authority on "must come from the host" is the AppImage project's own
+# excludelist, fetched rather than vendored. It growing is the signal we want,
+# and a copy here would be a second place to keep current. A failed fetch is a
+# failed check, never a pass.
+#
+# Usage: scripts/check-appimage-libraries.sh <path-to.AppImage>
+
+set -euo pipefail
+
+APPIMAGE="${1:-}"
+if [ -z "$APPIMAGE" ] || [ ! -f "$APPIMAGE" ]; then
+	echo "usage: $0 <path-to.AppImage>" >&2
+	exit 2
+fi
+APPIMAGE=$(realpath "$APPIMAGE")
+HERE=$(cd "$(dirname "$0")" && pwd)
+
+EXCLUDELIST_URL=${EXCLUDELIST_URL:-https://raw.githubusercontent.com/AppImageCommunity/pkg2appimage/master/excludelist}
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+# The strip list has one home, and this is not it -- read it out of the script
+# that acts on it rather than keeping a second copy in step.
+mapfile -t stripped < <(
+	sed -n '/^EXCLUDED=(/,/^)/p' "$HERE/strip-appimage.sh" |
+		grep -oE '\blib[A-Za-z0-9_.+-]*\.so\.[0-9]+' | sort -u
+)
+if [ "${#stripped[@]}" -eq 0 ]; then
+	echo "::error::found no EXCLUDED entries in strip-appimage.sh; this check has nothing to compare against" >&2
+	exit 1
+fi
+
+curl -fsSL --max-time 60 "$EXCLUDELIST_URL" -o "$work/excludelist"
+grep -oE '^[A-Za-z0-9_.+-]+\.so[.0-9]*' "$work/excludelist" | sort -u >"$work/must-come-from-host"
+if [ ! -s "$work/must-come-from-host" ]; then
+	echo "::error::$EXCLUDELIST_URL answered, but with no library names in it" >&2
+	exit 1
+fi
+
+( cd "$work" && "$APPIMAGE" --appimage-extract >/dev/null )
+find "$work/squashfs-root" -name '*.so*' -printf '%f\n' | sort -u >"$work/bundled"
+
+comm -12 "$work/bundled" "$work/must-come-from-host" >"$work/host-coupled"
+printf '%s\n' "${stripped[@]}" | sort -u >"$work/expected"
+
+echo "Bundled libraries the excludelist says must come from the host:"
+sed 's/^/  /' "$work/host-coupled"
+
+appeared=$(comm -23 "$work/host-coupled" "$work/expected")
+vanished=$(comm -13 "$work/host-coupled" "$work/expected")
+
+status=0
+if [ -n "$appeared" ]; then
+	echo "::error::linuxdeploy bundled a host-coupled library strip-appimage.sh does not remove:" >&2
+	printf '  %s\n' $appeared >&2
+	echo "::error::this is the shape of #463 and #498. Add it to EXCLUDED in scripts/strip-appimage.sh, or explain why it is safe to bundle." >&2
+	status=1
+fi
+# Reported and not failed, because the list is deliberately wider than what any
+# one build bundles. #499 named libwayland-client.so.0 as necessary and
+# sufficient and added five more that "belong to the host graphics stack for the
+# same reason and are on the same excludelist" -- prophylaxis, and `rm -f` costs
+# nothing when they are absent.
+#
+# It is worth printing because the two builds disagree. On the release's package
+# list, `rm -v` reported all six removed in v2.7.2 and v2.7.3; on the pull
+# request's shorter list only libwayland-client.so.0 is there. Which of the two
+# differences does it -- the packages, or the linuxdeploy build.yml
+# pre-downloads -- is not isolated. After the package lists are the same, the
+# next release's `removed` lines answer it.
+if [ -n "$vanished" ]; then
+	echo "::notice::strip-appimage.sh also removes these, and this build did not bundle them:" >&2
+	printf '  %s\n' $vanished >&2
+fi
+
+[ "$status" -eq 0 ] && echo "The ${#stripped[@]} libraries strip-appimage.sh removes are exactly the ones bundled."
+exit "$status"

scripts/releaseWorkflow.test.ts

@@ -10,6 +10,8 @@ const testBuildWorkflow = readSource('.github/workflows/test_build.yml');
 const releasing = readSource('RELEASING.md');
 const snapcraft = readSource('snapcraft.yaml');
 const stripAppImage = readSource('scripts/strip-appimage.sh');
+const checkAppImageLibraries = readSource('scripts/check-appimage-libraries.sh');
+const smokeAppImage = readSource('scripts/smoke-appimage.sh');
 const readme = readSource('README.md');
 const cargoToml = readSource('src-tauri/Cargo.toml');
 const packageJson = JSON.parse(readSource('package.json')) as {
@@ -308,6 +310,67 @@ test('the AppImage strip is a script, and its excludelist has one home', () => {
 	assert.doesNotMatch(workflow, /libwayland/);
 });
 
+test('the excludelist is checked against what is actually bundled', () => {
+	// The strip proving its six are gone says nothing about whether six is still
+	// the right number, and what decides that is not in this repository:
+	// `tauri build` fetches linuxdeploy-plugin-gtk from the tip of someone else's
+	// branch on every run, so the libraries copied into the AppDir can change
+	// with no commit here.
+	//
+	// On the pull request build, deliberately. That AppImage is the un-stripped
+	// one, which for this question is the right artifact rather than a worse one
+	// — and it only became comparable when this job moved to the release's
+	// Ubuntu, since before that it bundled jammy's libraries.
+	assert.match(testBuildWorkflow, /bash scripts\/check-appimage-libraries\.sh/);
+	assert.doesNotMatch(workflow, /check-appimage-libraries\.sh/);
+	// It reads the strip list rather than restating it, so the two cannot drift.
+	// Comments are exempt, as in the Ubuntu assertion above: naming a library
+	// while explaining what was measured is not a second copy of the list.
+	assert.match(checkAppImageLibraries, /strip-appimage\.sh/);
+	const restated = checkAppImageLibraries
+		.split('\n')
+		.filter((line) => !/^\s*#/.test(line))
+		.filter((line) => /\blib[A-Za-z0-9_.+-]*\.so\.[0-9]/.test(line));
+	assert.deepEqual(restated, [], 'the library list has a second copy in check-appimage-libraries.sh');
+});
+
+test('the AppImage is started before it can be downloaded', () => {
+	// The only check that runs Markpad rather than reading it, and the only one
+	// that could have caught #463 or #498 before a user did. It has to be on the
+	// release build: the file it starts is the repacked, re-signed one, which is
+	// the one latest.json points at and the one no pull request produces.
+	//
+	// Order is the whole safety argument. After the re-sign, so it starts the
+	// bytes that ship; before the upload, so a failure leaves the draft without
+	// a Linux artifact and generate-update-feed — which requires `build` to have
+	// succeeded — never runs.
+	const linuxSteps = sliceBetween(workflow, 'Re-sign the repacked AppImage', 'MacOS Build');
+	const smoke = linuxSteps.indexOf('scripts/smoke-appimage.sh');
+	const upload = linuxSteps.indexOf('Upload Linux Artifacts');
+	assert.ok(smoke > 0, 'the release build never starts the AppImage it is about to publish');
+	assert.ok(upload > 0, 'the Linux upload step moved; this assertion no longer means anything');
+	assert.ok(smoke < upload, 'the AppImage is uploaded before anything checks that it starts');
+
+	// Liveness alone would have passed both defects: #499 records that the GTK
+	// shell survives while WebKit's WebProcess aborts behind a blank window. The
+	// abort line is the actual signal.
+	assert.match(smokeAppImage, /EGL_BAD_PARAMETER/);
+	assert.match(smokeAppImage, /docker run/);
+
+	// And it runs on pull requests too, after a strip, because otherwise both
+	// scripts would reach the release path having never run anywhere: the strip
+	// failed in two of the three v2.7.2 release attempts, and a smoke test that
+	// only a release can exercise is the problem it exists to solve.
+	const prLinux = sliceFrom(testBuildWorkflow, 'The AppImage bundles nothing');
+	const prStrip = prLinux.indexOf('scripts/strip-appimage.sh');
+	const prSmoke = prLinux.indexOf('scripts/smoke-appimage.sh');
+	assert.ok(prStrip > 0 && prSmoke > 0, 'a pull request never runs the strip or the smoke test');
+	assert.ok(
+		prStrip < prSmoke,
+		'the pull request smokes the AppImage before stripping it, which reproduces #498 rather than testing for it',
+	);
+});
+
 test('the strip proves itself against the artifact that ships', () => {
 	// A CI smoke test cannot catch this defect: the runner's Mesa is the one the
 	// libraries were copied from, so nothing mismatches there and the AppImage

scripts/smoke-appimage.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+#
+# Start the AppImage that is about to be released, on a distribution that is not
+# the one it was built on, and fail if it does not come up.
+#
+# strip-appimage.sh says a CI smoke test cannot catch this defect, because the
+# runner's Mesa is the one the libraries were copied from. That is true of the
+# runner and only of the runner. A container carries another distribution's
+# userspace on the same kernel, which is enough: #463 and #498 were both a
+# bundled library shadowing a newer one on the host, and a host with a newer one
+# is exactly what this rents for ninety seconds.
+#
+# Two things make it testable at all, and both are recorded in those issues:
+# the abort happens without a GPU -- LIBGL_ALWAYS_SOFTWARE, WEBKIT_DISABLE_DMABUF_RENDERER
+# and WEBKIT_DISABLE_COMPOSITING_MODE were all tried and none of them helped, so
+# software rendering under Xvfb reproduces it -- and it prints the same line
+# every time.
+#
+# That line is why "did the process survive" is not the check. #499: "the window
+# comes up blank while the GTK shell survives". The main process stays up; it is
+# WebKit's WebProcess that aborts. A liveness check alone would have passed
+# through both defects this exists to catch.
+#
+# Limits, so nobody reads more into a green run than it says. The container
+# shares the host kernel. One distribution is not every distribution -- it is
+# the "newer than the builder" case, which is the case that broke. And it reads
+# specific strings, so a reworded abort would pass; the process-death check is
+# the backstop for that.
+#
+# Usage: scripts/smoke-appimage.sh <path-to.AppImage>
+#   IMAGE          container image to run it on   (default: archlinux:latest)
+#   SECONDS_TO_LIVE  how long it must stay up     (default: 25)
+
+set -euo pipefail
+
+APPIMAGE="${1:-}"
+if [ -z "$APPIMAGE" ] || [ ! -f "$APPIMAGE" ]; then
+	echo "usage: $0 <path-to.AppImage>" >&2
+	exit 2
+fi
+APPIMAGE=$(realpath "$APPIMAGE")
+
+IMAGE=${IMAGE:-archlinux:latest}
+SECONDS_TO_LIVE=${SECONDS_TO_LIVE:-25}
+
+echo "Starting $(basename "$APPIMAGE") on $IMAGE for ${SECONDS_TO_LIVE}s"
+
+# `-i` with a heredoc keeps the inner script out of a quoting maze.
+# APPIMAGE_EXTRACT_AND_RUN because the container has no FUSE to mount with.
+docker run --rm -i \
+	-v "$APPIMAGE:/tmp/Markpad.AppImage:ro" \
+	-e SECONDS_TO_LIVE="$SECONDS_TO_LIVE" \
+	-e APPIMAGE_EXTRACT_AND_RUN=1 \
+	"$IMAGE" bash -s <<'INNER'
+set -euo pipefail
+
+# A bare container is not a desktop. These are the host's side of the bargain:
+# an AppImage deliberately does not carry them -- they are on the same
+# excludelist as the six strip-appimage.sh removes, for the same reason -- so a
+# host without them fails to start the app for a reason that says nothing about
+# the build.
+#
+# `gtk3` rather than a list of individual libraries, because naming them one at
+# a time is a round of CI each: run 31494803899 wanted libfontconfig.so.1, and
+# with that added run 31497012965 wanted libfribidi.so.0. The set is not
+# arbitrary -- it is GTK's own dependency closure, which is what a machine that
+# can run a GTK app has. Installing GTK asks for all of it at once.
+pacman -Sy --noconfirm --needed xorg-server-xvfb mesa gtk3 ttf-dejavu >/dev/null 2>&1
+
+cd /tmp
+cp Markpad.AppImage app.AppImage
+chmod +x app.AppImage
+
+echo "host graphics stack:"
+pacman -Q mesa | sed 's/^/  /'
+
+set +e
+timeout --signal=TERM "$SECONDS_TO_LIVE" xvfb-run -a ./app.AppImage >/tmp/output 2>&1
+rc=$?
+set -e
+
+echo "--- output ---"
+cat /tmp/output
+echo "--- exit code: $rc ---"
+
+# `timeout` returns 124 when it had to stop the process, which is the pass: the
+# app was still running when its time was up.
+if [ "$rc" != "124" ]; then
+	if grep -q 'error while loading shared libraries' /tmp/output; then
+		echo "::error::the container is missing a library the AppImage expects the host to provide. Add it to the pacman line in scripts/smoke-appimage.sh; this is the test's environment, not a defect in the build." >&2
+	else
+		echo "::error::Markpad exited on its own after less than ${SECONDS_TO_LIVE}s (exit $rc). It should still have been running." >&2
+	fi
+	exit 1
+fi
+
+# The failure that does not kill the process. Both #463 and #498 printed this.
+if grep -qE 'EGL_BAD_PARAMETER|Could not create default EGL display|Aborting\.\.\.' /tmp/output; then
+	echo "::error::Markpad started but WebKit aborted -- the blank-window failure of #463 and #498, on a host newer than the builder." >&2
+	exit 1
+fi
+
+echo "Markpad was still running after ${SECONDS_TO_LIVE}s with no abort in its output."
+INNER

scripts/strip-appimage.sh

@@ -76,10 +76,18 @@ ARCH=x86_64 "$tools/appimagetool" --appimage-extract-and-run \
 	"$work/squashfs-root" "$APPIMAGE"
 
 # Assert the property on the artifact that ships, not on the AppDir it was
-# built from. A CI smoke test cannot catch this defect at all — the runner's
-# Mesa is the one the libraries came from, so nothing mismatches there — which
-# leaves "these libraries are absent" as the only thing that can actually be
-# checked before a user on a newer distro finds out.
+# built from.
+#
+# This used to say a CI smoke test cannot catch the defect at all, because the
+# runner's Mesa is the one the libraries came from. That is true of the runner
+# and only of the runner: scripts/smoke-appimage.sh now starts the repacked
+# AppImage on another distribution's userspace in a container, which is a host
+# that can disagree. This check is still worth keeping — it is the one that says
+# the strip did what it meant to, and it costs nothing.
+#
+# The other half is scripts/check-appimage-libraries.sh, on every pull request:
+# this list being removed successfully says nothing about whether it is still
+# the right list.
 ( cd "$check" && "$APPIMAGE" --appimage-extract >/dev/null )
 failed=0
 for lib in "${EXCLUDED[@]}"; do