Fix deterministic CI race conditions

.github/workflows/pr_validation.yml

@@ -62,19 +62,6 @@ jobs:
       - name: "dotnet build"
         run: dotnet build -c Release
 
-      # Pre-warm the PowerShell host probe so cold-start (Defender scan, first-run
-      # init) doesn't push the in-test 5s probe timeout on loaded Windows runners.
-      # Warm both pwsh.exe and the powershell.exe fallback — the resolver probes
-      # the fallback when pwsh is missing or its probe fails. Use -Command "exit 0"
-      # (no interpolated variables) so the outer pwsh shell can't mangle the args.
-      - name: "Warm PowerShell host probe"
-        if: runner.os == 'Windows'
-        shell: pwsh
-        run: |
-          pwsh -NoLogo -NoProfile -NonInteractive -Command "exit 0"
-          & "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" -NoLogo -NoProfile -NonInteractive -Command "exit 0"
-          exit 0
-
       # .NET Framework tests can't run reliably on Linux, so we only do .NET 8
 
       - name: "dotnet test"
@@ -208,7 +195,7 @@ jobs:
 
       - name: "Install shell integration test dependencies"
         if: runner.os == 'Linux'
-        run: sudo apt-get update && sudo apt-get install --yes fish zsh
+        run: bash scripts/smoke/install-shell-test-dependencies.sh
 
       - name: "install.sh smoke test"
         if: runner.os != 'Windows'

.github/workflows/smoke.yml

@@ -199,9 +199,6 @@ jobs:
           name: netclaw-native-binaries-linux-x64-${{ github.run_id }}-${{ github.run_attempt }}
           path: ./publish
 
-      - name: Install ImageMagick
-        run: sudo apt-get install -y --no-install-recommends imagemagick
-
       - name: Mark binaries executable
         run: |
           chmod +x publish/cli/netclaw publish/daemon/netclawd publish/mcp-server/Netclaw.SmokeMcpServer

scripts/smoke/count-png-differences.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Count pixels that differ between two PNG files."""
+
+from pathlib import Path
+import struct
+import subprocess
+import sys
+
+
+def decode_rgba(path: Path) -> tuple[tuple[int, int], bytes]:
+    header = path.read_bytes()[:24]
+    if len(header) != 24 or header[:8] != b"\x89PNG\r\n\x1a\n":
+        raise ValueError(f"{path} is not a PNG file.")
+
+    dimensions = struct.unpack(">II", header[16:24])
+    result = subprocess.run(
+        [
+            "ffmpeg",
+            "-v",
+            "error",
+            "-i",
+            str(path),
+            "-f",
+            "rawvideo",
+            "-pix_fmt",
+            "rgba",
+            "-frames:v",
+            "1",
+            "-",
+        ],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    return dimensions, result.stdout
+
+
+def main() -> int:
+    if len(sys.argv) != 3:
+        print("Usage: count-png-differences.py <baseline.png> <actual.png>", file=sys.stderr)
+        return 2
+
+    try:
+        baseline_dimensions, baseline = decode_rgba(Path(sys.argv[1]))
+        actual_dimensions, actual = decode_rgba(Path(sys.argv[2]))
+    except (OSError, subprocess.CalledProcessError, ValueError) as error:
+        print(error, file=sys.stderr)
+        return 1
+
+    if baseline_dimensions != actual_dimensions or len(baseline) != len(actual):
+        print("PNG dimensions do not match.", file=sys.stderr)
+        return 1
+
+    differences = sum(
+        baseline[index : index + 4] != actual[index : index + 4]
+        for index in range(0, len(baseline), 4)
+    )
+    print(differences)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/smoke/install-shell-test-dependencies.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Install the real fish and zsh processes that the installer smoke test uses.
+# The fixed files avoid apt index and mirror resolution during the CI gate.
+
+set -euo pipefail
+
+FISH_VERSION="${FISH_VERSION:-4.8.1}"
+FISH_SHA256="${FISH_SHA256:-39cab35242ab77bfdbce73b473000c3b045aaf2fe0951b042199bb7fdba3df78}"
+ZSH_VERSION="${ZSH_VERSION:-5.9-6ubuntu2}"
+ZSH_SHA256="${ZSH_SHA256:-bd5cc8dd3a01a6db38c0a815d75202c356a9c7f378674ba7bed9bc86dcba8af0}"
+SHELL_TEST_BIN_DIR="${SHELL_TEST_BIN_DIR:-/usr/local/bin}"
+
+if [[ "$(uname -s)/$(uname -m)" != "Linux/x86_64" ]]; then
+  echo "ERROR: The pinned shell test files support Linux x86_64 only." >&2
+  exit 1
+fi
+
+for dependency in curl dpkg-deb sha256sum tar; do
+  if ! command -v "$dependency" >/dev/null 2>&1; then
+    echo "ERROR: '${dependency}' is required to install the shell test files." >&2
+    exit 1
+  fi
+done
+
+install_file() {
+  local source="$1"
+  local destination="$2"
+
+  if [[ -w "$SHELL_TEST_BIN_DIR" ]]; then
+    install -m 0755 "$source" "$destination"
+  else
+    sudo install -m 0755 "$source" "$destination"
+  fi
+}
+
+if [[ ! -d "$SHELL_TEST_BIN_DIR" ]]; then
+  if [[ -w "$(dirname "$SHELL_TEST_BIN_DIR")" ]]; then
+    install -d "$SHELL_TEST_BIN_DIR"
+  else
+    sudo install -d "$SHELL_TEST_BIN_DIR"
+  fi
+fi
+
+temporary_dir="$(mktemp -d)"
+trap 'rm -rf "$temporary_dir"' EXIT
+
+if command -v fish >/dev/null 2>&1; then
+  fish_path="$(command -v fish)"
+else
+  fish_archive="$temporary_dir/fish.tar.xz"
+  fish_url="https://github.com/fish-shell/fish-shell/releases/download/${FISH_VERSION}/fish-${FISH_VERSION}-linux-x86_64.tar.xz"
+
+  echo "Downloading fish ${FISH_VERSION} from its fixed upstream file."
+  curl -fsSL "$fish_url" -o "$fish_archive"
+  echo "${FISH_SHA256}  ${fish_archive}" | sha256sum -c -
+  tar -xJf "$fish_archive" -C "$temporary_dir"
+
+  fish_path="$SHELL_TEST_BIN_DIR/fish"
+  install_file "$temporary_dir/fish" "$fish_path"
+fi
+
+if command -v zsh >/dev/null 2>&1; then
+  zsh_path="$(command -v zsh)"
+else
+  zsh_package="$temporary_dir/zsh.deb"
+  zsh_root="$temporary_dir/zsh-root"
+  zsh_url="https://archive.ubuntu.com/ubuntu/pool/main/z/zsh/zsh_${ZSH_VERSION}_amd64.deb"
+
+  echo "Downloading zsh ${ZSH_VERSION} from its fixed Ubuntu archive file."
+  curl -fsSL "$zsh_url" -o "$zsh_package"
+  echo "${ZSH_SHA256}  ${zsh_package}" | sha256sum -c -
+  mkdir -p "$zsh_root"
+  dpkg-deb -x "$zsh_package" "$zsh_root"
+
+  zsh_path="$SHELL_TEST_BIN_DIR/zsh"
+  install_file "$zsh_root/bin/zsh" "$zsh_path"
+fi
+
+"$fish_path" --version
+"$zsh_path" --version

scripts/smoke/install-smoke.sh

@@ -521,17 +521,18 @@ fi
 # Zsh: resolve a non-exported ZDOTDIR from .zshenv, then execute the selected
 # startup file under zsh so a Bash-compatible false positive cannot pass.
 if command -v zsh >/dev/null 2>&1; then
+  ZSH_EXECUTABLE="$(command -v zsh)"
   ZSH_HOME="$WORK/shell-zsh"
   ZDOT_DIR="$ZSH_HOME/custom-zdotdir"
   ZSH_INSTALL="$ZSH_HOME/netclaw install's/bin"
   mkdir -p "$ZDOT_DIR"
   printf "ZDOTDIR='%s'\n" "$ZDOT_DIR" > "$ZSH_HOME/.zshenv"
   printf '# existing zsh config\n' > "$ZDOT_DIR/.zshrc"
-  if (unset ZDOTDIR; run_unix_installer "$(command -v zsh)" "$ZSH_HOME" "$ZSH_INSTALL" >/dev/null) \
-      && (unset ZDOTDIR; run_unix_installer "$(command -v zsh)" "$ZSH_HOME" "$ZSH_INSTALL" >/dev/null); then
+  if (unset ZDOTDIR; run_unix_installer "$ZSH_EXECUTABLE" "$ZSH_HOME" "$ZSH_INSTALL" >/dev/null) \
+      && (unset ZDOTDIR; run_unix_installer "$ZSH_EXECUTABLE" "$ZSH_HOME" "$ZSH_INSTALL" >/dev/null); then
     ZSH_INSTALL_PHYSICAL=$(cd "$ZSH_INSTALL" && pwd -P)
     zsh_path=$(PATH="/usr/bin:/bin" ZDOTDIR="$ZDOT_DIR" \
-      zsh -f -c 'source "$ZDOTDIR/.zshrc"; print -rn -- "$PATH"')
+      "$ZSH_EXECUTABLE" -f -c 'source "$ZDOTDIR/.zshrc"; print -rn -- "$PATH"')
     assert_path_once "zsh" "$zsh_path" "$ZSH_INSTALL_PHYSICAL"
     if [ ! -e "$ZSH_HOME/.zshrc" ]; then
       pass "zsh: non-exported ZDOTDIR is authoritative"
@@ -547,15 +548,16 @@ fi
 
 # Fish owns a native conf.d file. Execute that file with fish, not Bash.
 if command -v fish >/dev/null 2>&1; then
+  FISH_EXECUTABLE="$(command -v fish)"
   FISH_HOME="$WORK/shell-fish"
   FISH_INSTALL="$FISH_HOME/netclaw install's/bin"
   FISH_RC="$FISH_HOME/.config/fish/conf.d/netclaw.fish"
   if XDG_CONFIG_HOME="$FISH_HOME/.config" \
-      run_unix_installer "$(command -v fish)" "$FISH_HOME" "$FISH_INSTALL" >/dev/null \
+      run_unix_installer "$FISH_EXECUTABLE" "$FISH_HOME" "$FISH_INSTALL" >/dev/null \
       && XDG_CONFIG_HOME="$FISH_HOME/.config" \
-      run_unix_installer "$(command -v fish)" "$FISH_HOME" "$FISH_INSTALL" >/dev/null; then
+      run_unix_installer "$FISH_EXECUTABLE" "$FISH_HOME" "$FISH_INSTALL" >/dev/null; then
     FISH_INSTALL_PHYSICAL=$(cd "$FISH_INSTALL" && pwd -P)
-    fish_path=$(PATH="/usr/bin:/bin" fish --no-config -c \
+    fish_path=$(PATH="/usr/bin:/bin" "$FISH_EXECUTABLE" --no-config -c \
       "source '$FISH_RC'; string join : -- \$PATH")
     assert_path_once "fish" "$fish_path" "$FISH_INSTALL_PHYSICAL"
   else

scripts/smoke/install-vhs.sh

@@ -2,9 +2,9 @@
 # Ensure VHS (charmbracelet/vhs) is installed for the interactive tape harness.
 #
 # Installation strategy:
-#   - If `vhs` is already on PATH, do nothing.
-#   - On Linux x86_64: install vhs from the upstream release with SHA256 verification,
-#     and ensure ttyd / ffmpeg are present (apt-get if available).
+#   - If the pinned vhs and its runtime tools exist, do nothing.
+#   - On Linux x86_64: install vhs and ttyd from pinned upstream releases.
+#     Install the imageio-ffmpeg static binary. Verify all files with SHA256.
 #   - On macOS: install vhs / ttyd / ffmpeg via Homebrew.
 #   - Other platforms: print install hints and fail.
 #
@@ -21,15 +21,22 @@ VHS_VERSION="${VHS_VERSION:-0.11.0}"
 # comment above. The pin matters for screenshot regression: a VHS bump can
 # change the bundled font/renderer and silently drift every baseline PNG.
 VHS_LINUX_X64_SHA256="${VHS_LINUX_X64_SHA256:-99cb634587eaae0473c1ea377db80c3a048c27f99fe0a7febb1a1e8cb7ee5009}"
+TTYD_VERSION="${TTYD_VERSION:-1.7.7}"
+# SHA256 of ttyd.x86_64 from the upstream SHA256SUMS file.
+TTYD_LINUX_X64_SHA256="${TTYD_LINUX_X64_SHA256:-8a217c968aba172e0dbf3f34447218dc015bc4d5e59bf51db2f2cd12b7be4f55}"
+# SHA256 of the imageio-ffmpeg 0.6.0 manylinux2014 x86_64 wheel from PyPI.
+FFMPEG_WHEEL_SHA256="${FFMPEG_WHEEL_SHA256:-c7e46fcec401dd990405049d2e2f475e2b397779df2519b544b8aab515195282}"
 # 0.11.0 is the minimum that supports `Wait+Screen /pattern/`.
 # Earlier versions (e.g. 0.8.0) parse `Wait` as an unknown command.
 
 have() { command -v "$1" >/dev/null 2>&1; }
 
 if have vhs; then
-  installed="$(vhs --version 2>/dev/null | awk '{print $NF}' | sed 's/^v//')"
-  echo "vhs ${installed:-unknown} already installed at $(command -v vhs)"
-  exit 0
+  installed="$(vhs --version 2>/dev/null | sed -n 's/.*version v\([^ ]*\).*/\1/p')"
+  if [[ "$installed" == "$VHS_VERSION" ]] && have ttyd && have ffmpeg && have python3; then
+    echo "vhs ${installed} and its runtime tools are already installed."
+    exit 0
+  fi
 fi
 
 uname_s="$(uname -s)"
@@ -76,30 +83,47 @@ EOF
 esac
 
 # Linux x86_64 path.
+for dep in curl python3; do
+  if ! have "$dep"; then
+    echo "ERROR: '$dep' is required to install and run vhs." >&2
+    exit 1
+  fi
+done
 
-if have apt-get; then
-  echo "Installing vhs runtime deps (ttyd, ffmpeg) via apt-get..."
-  # Force non-interactive mode so apt never tries to open whiptail dialogs
-  # (e.g., the kernel-upgrade prompt) when running on a CI runner or under
-  # an SSH/agent session.
-  export DEBIAN_FRONTEND=noninteractive
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+if ! have ffmpeg; then
+  ffmpeg_wheel="$tmp/imageio-ffmpeg.whl"
+  ffmpeg_url="https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl"
+  echo "Downloading the pinned imageio-ffmpeg binary..."
+  curl -fsSL "$ffmpeg_url" -o "$ffmpeg_wheel"
+  echo "${FFMPEG_WHEEL_SHA256}  ${ffmpeg_wheel}" | sha256sum -c -
+  python3 -m zipfile -e "$ffmpeg_wheel" "$tmp/imageio-ffmpeg"
+
+  ffmpeg_binary="$tmp/imageio-ffmpeg/imageio_ffmpeg/binaries/ffmpeg-linux-x86_64-v7.0.2"
+  ffmpeg_dest="${FFMPEG_INSTALL_PATH:-/usr/local/bin/ffmpeg}"
   if [[ "${EUID:-$(id -u)}" -eq 0 ]]; then
-    apt-get update
-    apt-get install -y --no-install-recommends ttyd ffmpeg ca-certificates curl
+    install -m 0755 "$ffmpeg_binary" "$ffmpeg_dest"
   else
-    sudo -E apt-get update
-    sudo -E apt-get install -y --no-install-recommends ttyd ffmpeg ca-certificates curl
+    sudo install -m 0755 "$ffmpeg_binary" "$ffmpeg_dest"
   fi
-else
-  for dep in ttyd ffmpeg curl; do
-    if ! have "$dep"; then
-      echo "WARNING: '$dep' is not on PATH and apt-get is unavailable. vhs will likely fail." >&2
-    fi
-  done
 fi
 
-tmp="$(mktemp -d)"
-trap 'rm -rf "$tmp"' EXIT
+if ! have ttyd; then
+  ttyd_binary="$tmp/ttyd"
+  ttyd_url="https://github.com/tsl0922/ttyd/releases/download/${TTYD_VERSION}/ttyd.x86_64"
+  echo "Downloading ttyd ${TTYD_VERSION} from ${ttyd_url}..."
+  curl -fsSL "$ttyd_url" -o "$ttyd_binary"
+  echo "${TTYD_LINUX_X64_SHA256}  ${ttyd_binary}" | sha256sum -c -
+
+  ttyd_dest="${TTYD_INSTALL_PATH:-/usr/local/bin/ttyd}"
+  if [[ "${EUID:-$(id -u)}" -eq 0 ]]; then
+    install -m 0755 "$ttyd_binary" "$ttyd_dest"
+  else
+    sudo install -m 0755 "$ttyd_binary" "$ttyd_dest"
+  fi
+fi
 
 archive="$tmp/vhs.tar.gz"
 url="https://github.com/charmbracelet/vhs/releases/download/v${VHS_VERSION}/vhs_${VHS_VERSION}_Linux_x86_64.tar.gz"

scripts/smoke/lib/common.sh

@@ -9,13 +9,11 @@
 #
 # Environment knobs:
 #   START_TIMEOUT_SECONDS  daemon start/health timeout (default: 180)
-#   STOP_TIMEOUT_SECONDS   daemon stop timeout         (default: 90)
 #   STEP_TIMEOUT_SECONDS   per-command timeout         (default: 120)
 #   DAEMON_BASE_URL        health endpoint base        (default loopback:56199)
 #   DAEMON_PORT            daemon listen port          (default: port from DAEMON_BASE_URL or 56199)
 
 START_TIMEOUT_SECONDS="${START_TIMEOUT_SECONDS:-180}"
-STOP_TIMEOUT_SECONDS="${STOP_TIMEOUT_SECONDS:-90}"
 STEP_TIMEOUT_SECONDS="${STEP_TIMEOUT_SECONDS:-120}"
 DAEMON_BASE_URL="${DAEMON_BASE_URL:-http://127.0.0.1:56199}"
 DAEMON_PORT="${DAEMON_PORT:-${DAEMON_BASE_URL##*:}}"
@@ -217,13 +215,18 @@ wait_for_health() {
   return 1
 }
 
-# stop_daemon — best-effort daemon stop. Never fails the caller.
+# stop_daemon — stop smoke-owned daemon processes. Never fail the caller.
 stop_daemon() {
-  : "${NETCLAW_SMOKE_CLI:?NETCLAW_SMOKE_CLI must be set}"
-  run_timed "$STOP_TIMEOUT_SECONDS" "$NETCLAW_SMOKE_CLI" daemon stop >/dev/null 2>&1 || true
-  # `daemon stop` only signals the PID in this NETCLAW_HOME's PID file; make
-  # sure the listening socket is actually released before the next daemon
-  # tries to bind it.
+  local holders
+  holders="$(lsof -ti "tcp:${DAEMON_PORT}" -sTCP:LISTEN 2>/dev/null || true)"
+  local pid
+  for pid in $holders; do
+    if pid_is_smoke_daemon "$pid"; then
+      log "stopping smoke daemon (pid=${pid})."
+      kill "$pid" 2>/dev/null || true
+    fi
+  done
+
   ensure_daemon_port_free || true
 }