fix(smoke): make the screenshot-regression suite deterministic (pixel compare + blank/partial retry + Termina seam fixes)

.github/workflows/smoke.yml

@@ -192,6 +192,9 @@ jobs:
           name: netclaw-native-binaries-linux-x64-${{ github.run_id }}-${{ github.run_attempt }}
           path: ./publish
 
+      - name: Install ImageMagick
+        run: sudo apt-get install -y --no-install-recommends imagemagick
+
       - name: Mark binaries executable
         run: |
           chmod +x publish/cli/netclaw publish/daemon/netclawd

Directory.Packages.props

@@ -63,7 +63,7 @@
     <PackageVersion Include="Cronos" Version="0.13.0" />
     <PackageVersion Include="Netclaw.SkillClient" Version="0.3.1" />
     <PackageVersion Include="ShellSyntaxTree" Version="0.1.5" />
-    <PackageVersion Include="Termina" Version="0.14.0-beta.1" />
+    <PackageVersion Include="Termina" Version="0.14.0-beta3" />
   </ItemGroup>
   <!-- Serialization -->
   <ItemGroup>

scripts/smoke/run-smoke.sh

@@ -87,6 +87,41 @@ SHOT_FRAMES=(
   config-search-saved
 )
 
+# Which frames each capture tape emits. Used by the blank-frame retry
+# (run_shot_tape_with_retry): after a tape runs, only its own captures are
+# inspected for the transient blank described below. Keep in sync with the
+# `Screenshot` directives in tests/smoke/tapes/screenshots/<tape>.tape.
+#
+# A function with a case is used instead of `declare -A` because macOS ships
+# bash 3.2, which has no associative arrays — `declare -A` there parses the
+# `[help]=...` entries as indexed-array assignments and aborts under set -u
+# (`help: unbound variable`), breaking the non-screenshot smoke modes too.
+shot_tape_frames() {
+  case "$1" in
+    help)             echo "help" ;;
+    wizard-screens)   echo "wizard-provider-picker wizard-security-posture" ;;
+    provider-manager) echo "provider-manager-empty" ;;
+    mcp-permissions)  echo "mcp-permissions-server-list mcp-permissions-tool-grid" ;;
+    config-search)    echo "config-search-selection config-search-brave-entry config-search-saved" ;;
+    *)                echo "" ;;
+  esac
+}
+
+# Max attempts per tape when a transient blank frame is detected. A TUI screen
+# can render momentarily blank because Termina emits a full-screen clear
+# () + repaint as one write on a startup resize event, and VHS can
+# sample the PNG between the clear and the repaint half of that same write.
+# The write is atomic from the app's side (real users never see it); only VHS's
+# mid-write PTY sampling does. Re-running the tape re-captures a settled frame.
+SHOT_BLANK_RETRIES="${SHOT_BLANK_RETRIES:-3}"
+
+# Pixel tolerance (ImageMagick AE) for a frame to count as matching its
+# baseline. Shared by compare_shot_frame (the pass/fail gate) and the retry
+# trigger (a capture above this differs enough to re-run). ~2 character cells —
+# clears a single shell-cursor cell (~493 px) while still failing on real
+# content changes (thousands of px). See compare_shot_frame for the rationale.
+SHOT_AE_TOLERANCE="${SHOT_AE_TOLERANCE:-1000}"
+
 usage() {
   sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'
   exit 2
@@ -356,6 +391,80 @@ run_shot_tape() {
   fi
 }
 
+# frame_is_blank <png> — true if the capture is a near-uniform frame, i.e. the
+# transient Termina full-refresh blank (see SHOT_BLANK_RETRIES). Baseline-
+# independent: it counts unique colors with ImageMagick `identify %k`. A blank
+# frame is the solid theme background (~1 color); any populated TUI screen has
+# hundreds. The threshold (16) sits far below the sparsest real frame and far
+# above a blank, so it never misclassifies a real screen as blank.
+frame_is_blank() {
+  local png="$1"
+  command -v identify >/dev/null 2>&1 || return 1   # can't tell → treat as not blank
+  [[ -f "$png" ]] || return 1                        # missing capture is handled elsewhere
+  local colors
+  colors=$(identify -format '%k' "$png" 2>/dev/null || echo "")
+  [[ "$colors" =~ ^[0-9]+$ ]] || return 1
+  (( colors < 16 ))
+}
+
+# frame_needs_retry <frame> — true if the capture looks like a transient Termina
+# full-refresh artifact that re-running the tape can clear. Two shapes:
+#   * fully blank (frame_is_blank) — VHS sampled the [2J-cleared frame.
+#   * partial/garbled — VHS sampled mid-repaint, so only the top rows landed
+#     (e.g. the MCP tool grid captured before its lower rows painted). Such a
+#     frame has plenty of colors (so frame_is_blank misses it) but differs from
+#     baseline by far more than the cursor tolerance.
+# A genuine regression also trips the second branch, but it reproduces every
+# attempt and so still fails at compare time — only the latency differs.
+frame_needs_retry() {
+  local frame="$1"
+  local capture="/tmp/shot-${frame}.png"
+  local baseline="${SHOT_BASELINE_DIR}/${frame}.approved.png"
+  [[ -f "$capture" ]] || return 1
+  frame_is_blank "$capture" && return 0
+  command -v compare >/dev/null 2>&1 || return 1
+  [[ -f "$baseline" ]] || return 1
+  local ae ae_int
+  ae=$(compare -metric AE "$baseline" "$capture" /dev/null 2>&1 || true)
+  ae_int="${ae%%.*}"; ae_int="${ae_int// /}"
+  [[ "$ae_int" =~ ^[0-9]+$ ]] || return 1
+  (( ae_int > SHOT_AE_TOLERANCE ))
+}
+
+# run_shot_tape_with_retry <tape> — run a capture tape, then inspect the frames
+# it emits. If any is a transient blank (SHOT_BLANK_RETRIES), re-run the whole
+# tape so the next attempt captures a settled frame. Bounded so a genuinely
+# broken tape still fails at compare time instead of looping forever.
+run_shot_tape_with_retry() {
+  local tape="$1"
+  local frames
+  frames="$(shot_tape_frames "$tape")"
+  local attempt=1
+  while :; do
+    run_shot_tape "$tape"
+    [[ -z "$frames" ]] && return          # no frame map → accept the single run
+
+    local bad="" f
+    for f in $frames; do
+      if frame_needs_retry "$f"; then
+        bad="$f"
+        break
+      fi
+    done
+    [[ -z "$bad" ]] && return              # all frames settled → done
+
+    if (( attempt >= SHOT_BLANK_RETRIES )); then
+      echo "  WARN: ${tape} produced a transient frame (${bad}) on all ${attempt} attempts;" >&2
+      echo "        leaving it for compare_shot_frame to fail on." >&2
+      return
+    fi
+    echo "  RETRY: ${tape} attempt ${attempt} produced a transient frame (${bad}) —" >&2
+    echo "         re-running tape (blank or partial Termina full-refresh capture)." >&2
+    attempt=$((attempt + 1))
+    for f in $frames; do rm -f "/tmp/shot-${f}.png"; done   # clear stale captures
+  done
+}
+
 # compare_shot_frame <frame> — compare /tmp/shot-<frame>.png against the
 # committed baseline. Records a failure (and writes review PNGs) on a
 # missing capture, missing baseline, or pixel mismatch.
@@ -381,9 +490,32 @@ compare_shot_frame() {
     return
   fi
 
-  if cmp -s "$baseline" "$capture"; then
-    echo "  PASS: ${frame} — pixel-identical to baseline."
-    return
+  # Use ImageMagick pixel comparison rather than cmp -s (byte-for-byte).
+  # Two sources of false failures are tolerated:
+  #   1. VHS PNG zlib encoder jitter — same pixels, different byte streams
+  #      across process invocations (AE = 0, always passes).
+  #   2. Terminal cursor block — Set CursorBlink false freezes the cursor but
+  #      not its on/off state; the shell-prompt cursor cell can appear or not
+  #      between runs. The block is one character cell (measured AE≈493 at this
+  #      geometry). AE_CURSOR_TOLERANCE is set to ~2 cells so a single cursor
+  #      cell passes with margin, while real regressions still fail — a changed
+  #      word/line differs by thousands of px, a blank screen by ~68,000.
+  # Fall back to cmp -s only if ImageMagick is absent. The tolerance
+  # (SHOT_AE_TOLERANCE) is shared with the retry trigger (frame_needs_retry).
+  if command -v compare >/dev/null 2>&1; then
+    local ae
+    ae=$(compare -metric AE "$baseline" "$capture" /dev/null 2>&1 || true)
+    local ae_int="${ae%%.*}"
+    ae_int="${ae_int// /}"
+    if [[ "${ae_int:-0}" -le "$SHOT_AE_TOLERANCE" ]]; then
+      echo "  PASS: ${frame} — pixel-close to baseline (AE=${ae_int:-0})."
+      return
+    fi
+  else
+    if cmp -s "$baseline" "$capture"; then
+      echo "  PASS: ${frame} — pixel-identical to baseline."
+      return
+    fi
   fi
 
   # Mismatch — keep the actual, and a visual diff if ImageMagick is around.
@@ -407,7 +539,7 @@ if [[ "$shots_mode" -eq 1 ]]; then
   # Fresh /tmp so a stale capture from an earlier run cannot be compared.
   rm -f /tmp/shot-*.png
   for tape in "${SHOT_TAPES[@]}"; do
-    run_shot_tape "$tape"
+    run_shot_tape_with_retry "$tape"
   done
 
   echo

tests/smoke/tapes/screenshots/config-search.tape

@@ -22,20 +22,30 @@ Down 5
 Enter
 
 # ─── Frame 1: Provider selection ───────────────────────────────────────────
+# SelectBackendForEditing is synchronous. Two anchors are required: the title
+# confirms the screen loaded, and the DuckDuckGo description confirms the
+# cursor is on row 1 (DuckDuckGo). Without the second anchor, a Down key
+# leaked from the "Down 5" Settings navigation can move the cursor to Brave
+# before the screenshot fires, capturing the wrong row.
 Wait+Screen@10s /Choose the backend Netclaw uses for web search/
-Sleep 1s
+Wait+Screen@5s /DuckDuckGo works without setup/
 Screenshot "/tmp/shot-config-search-selection.png"
-Sleep 1s
 
 # ─── Frame 2: Brave entry state ────────────────────────────────────────────
+# SelectBackendForEditing("brave") is synchronous — Wait+Screen is sufficient.
 Down
 Enter
 Wait+Screen@10s /Brave Search requires an API key/
-Sleep 1s
 Screenshot "/tmp/shot-config-search-brave-entry.png"
-Sleep 1s
 
 # ─── Frame 3: Saved state ──────────────────────────────────────────────────
+# SaveWithoutProbeOverride cascades multiple ReactiveProperty notifications
+# and a ReloadPersistedDraft disk read before the frame settles. A single
+# Wait+Screen on the success text is not enough — a second anchor on the
+# key-binding bar text guarantees both CurrentScreen==Saved and
+# ActiveDialog==None have committed to a stable rendered frame.
+# "[Enter] Settings Areas" is emitted by BuildKeyBindings only in that state
+# (SearchConfigEditorPage.cs line 212).
 Escape
 Down
 Enter
@@ -46,9 +56,8 @@ Wait+Screen@10s /Search Validation Warning/
 Down 2
 Enter
 Wait+Screen@10s /validated and saved/
-Sleep 1s
+Wait+Screen@5s /\[Enter\] Settings Areas/
 Screenshot "/tmp/shot-config-search-saved.png"
-Sleep 1s
 
 Ctrl+Q
 Wait+Screen@10s /TAPE\$/

tests/smoke/tapes/screenshots/help.tape

@@ -13,13 +13,13 @@ Output "/tmp/tape-shot-help.gif"
 
 Type "netclaw --help"
 Enter
-# Anchor on the usage header, then settle. VHS Screenshot can otherwise
-# catch a frame mid-render; the Sleep after keeps the next keystroke from
-# leaking into the capture.
+# Anchor on the usage header, then sleep to let the full terminal output
+# finish rendering. `netclaw --help` writes multiple lines and VHS may not
+# have flushed all of them into the terminal buffer by the time Wait+Screen
+# matches the first visible line.
 Wait+Screen@10s /Usage:/
 Sleep 1s
 Screenshot "/tmp/shot-help.png"
-Sleep 1s
 
 Type "exit"
 Enter

tests/smoke/tapes/screenshots/mcp-permissions.tape

@@ -59,21 +59,29 @@ Enter
 # ─── Frame 1: ServerList ─────────────────────────────────────────────
 # smoke-math should appear as "Connected, 3 tools".
 Wait+Screen@15s /smoke-math/
-Sleep 1s
+Wait+Screen@5s /3 tools/
 Screenshot "/tmp/shot-mcp-permissions-server-list.png"
-Sleep 1s
 
 # ─── Navigate into the ToolGrid ──────────────────────────────────────
+# Sleep 1s here is an intentional settle guard: the daemon can push a
+# server-state update (tool re-index, reconnect) immediately after the
+# server-list frame stabilises. Navigating into the tool grid during such
+# an update causes Enter to hit the TUI mid-transition, producing a blank
+# screen rather than the tool grid. The sleep lets any in-flight update
+# complete before Enter is delivered.
+Sleep 1s
 Enter
 
 # ─── Frame 2: ToolGrid ───────────────────────────────────────────────
 # All header rows (Server, Audience, Server enabled, Server default) plus
 # all tool rows (add, echo, record-tasks) must be visible simultaneously.
 # If the #1424 regression reappears, tool rows will overwrite the header.
-Wait+Screen@10s /record-tasks/
-Sleep 1s
+# Timeout is 15s to match the server-list anchor; /Server default:/ (with
+# colon) matches the rendered "Server default: [Auto]" line, which is
+# unique to the tool-grid view and absent from the server-list scrollback.
+Wait+Screen@15s /record-tasks/
+Wait+Screen@10s /Server default:/
 Screenshot "/tmp/shot-mcp-permissions-tool-grid.png"
-Sleep 1s
 
 # ─── Exit TUI ────────────────────────────────────────────────────────
 Ctrl+Q

tests/smoke/tapes/screenshots/provider-manager.tape

@@ -16,8 +16,8 @@
 #
 # Navigation + anchors mirror tapes/provider-add.tape; see that file and
 # src/Netclaw.Cli/Tui/ProviderManagerPage.cs for the synchronization
-# rationale. The Screenshot is bracketed by Sleep 1s to settle the frame
-# and keep the next keystroke from leaking into the capture.
+# rationale. Wait+Screen anchors on the list content, then Sleep 1s lets
+# the full TUI frame flush before the screenshot is captured.
 #
 # Prepended preamble: tapes/screenshot-preamble.tape (determinism-pinned).
 
@@ -31,10 +31,14 @@ Enter
 # With no providers configured the list shows every known type as a
 # "(not configured)" row plus the "+ Add new provider..." sentinel.
 # Anchor on "Ollama" (a guaranteed type row) so we know the list rendered.
+# Sleep lets VHS flush the full frame; the SECOND Wait+Screen is an anti-blank
+# re-assert immediately before capture — if a Termina startup full-refresh
+# blanked the screen during the Sleep, this blocks until the list repaints.
+# (Defence in depth: run-smoke.sh also re-runs the tape if a blank slips through.)
 Wait+Screen@10s /Ollama/
 Sleep 1s
+Wait+Screen@5s /Ollama/
 Screenshot "/tmp/shot-provider-manager-empty.png"
-Sleep 1s
 
 # Frame captured; quit the TUI back to the shell.
 Ctrl+Q

tests/smoke/tapes/screenshots/wizard-screens.tape

@@ -17,8 +17,9 @@
 #
 # Navigation + anchors mirror tapes/init-wizard.tape; see that file and
 # src/Netclaw.Cli/Tui/Wizard/Steps/*StepView.cs for the synchronization
-# rationale. Each Screenshot is bracketed by Sleep 1s — VHS can otherwise
-# capture an unsettled frame, or let the next keystroke leak into the shot.
+# rationale. Wizard step transitions are synchronous — Wait+Screen is the
+# only gate needed. The Sleep 1s after Ctrl+Q below is intentional: it is
+# the alt-screen restore guard (CSI ?1049l), not a screenshot timing guard.
 #
 # Prepended preamble: tapes/screenshot-preamble.tape (determinism-pinned).
 
@@ -30,9 +31,10 @@ Enter
 
 # ─── Frame 1: Provider picker ────────────────────────────────────────
 Wait+Screen@10s /Choose your LLM provider:/
-Sleep 1s
+# The heading can appear before the full provider list is painted on slower
+# CI runners. Anchor on the last row so the screenshot captures the settled list.
+Wait+Screen@5s /7\. Venice\.ai/
 Screenshot "/tmp/shot-wizard-provider-picker.png"
-Sleep 1s
 
 # Provider list ordering is alphabetical by TypeKey:
 #   anthropic, github-copilot, ollama, openai, openai-compatible, openrouter
@@ -78,9 +80,7 @@ Enter
 
 # ─── Frame 2: Security posture ───────────────────────────────────────
 Wait+Screen@10s /Who will interact with this Netclaw instance/
-Sleep 1s
 Screenshot "/tmp/shot-wizard-security-posture.png"
-Sleep 1s
 
 # Both frames captured; abandon the wizard. Ctrl+Q is the TUI quit
 # shortcut (Ctrl+C requires a double-press under raw input mode).