diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 32093920e..f52a78d43 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -192,6 +192,9 @@ jobs: name: netclaw-native-binaries-linux-x64-${{ github.run_id }}-${{ github.run_attempt }} path: ./publish + - name: Install ImageMagick + run: sudo apt-get install -y --no-install-recommends imagemagick + - name: Mark binaries executable run: | chmod +x publish/cli/netclaw publish/daemon/netclawd diff --git a/Directory.Packages.props b/Directory.Packages.props index f44e458cb..32c33cc9d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -63,7 +63,7 @@ - + diff --git a/scripts/smoke/run-smoke.sh b/scripts/smoke/run-smoke.sh index 45e2cf1b0..bcb941215 100755 --- a/scripts/smoke/run-smoke.sh +++ b/scripts/smoke/run-smoke.sh @@ -87,6 +87,41 @@ SHOT_FRAMES=( config-search-saved ) +# Which frames each capture tape emits. Used by the blank-frame retry +# (run_shot_tape_with_retry): after a tape runs, only its own captures are +# inspected for the transient blank described below. Keep in sync with the +# `Screenshot` directives in tests/smoke/tapes/screenshots/.tape. +# +# A function with a case is used instead of `declare -A` because macOS ships +# bash 3.2, which has no associative arrays — `declare -A` there parses the +# `[help]=...` entries as indexed-array assignments and aborts under set -u +# (`help: unbound variable`), breaking the non-screenshot smoke modes too. +shot_tape_frames() { + case "$1" in + help) echo "help" ;; + wizard-screens) echo "wizard-provider-picker wizard-security-posture" ;; + provider-manager) echo "provider-manager-empty" ;; + mcp-permissions) echo "mcp-permissions-server-list mcp-permissions-tool-grid" ;; + config-search) echo "config-search-selection config-search-brave-entry config-search-saved" ;; + *) echo "" ;; + esac +} + +# Max attempts per tape when a transient blank frame is detected. A TUI screen +# can render momentarily blank because Termina emits a full-screen clear +# () + repaint as one write on a startup resize event, and VHS can +# sample the PNG between the clear and the repaint half of that same write. +# The write is atomic from the app's side (real users never see it); only VHS's +# mid-write PTY sampling does. Re-running the tape re-captures a settled frame. +SHOT_BLANK_RETRIES="${SHOT_BLANK_RETRIES:-3}" + +# Pixel tolerance (ImageMagick AE) for a frame to count as matching its +# baseline. Shared by compare_shot_frame (the pass/fail gate) and the retry +# trigger (a capture above this differs enough to re-run). ~2 character cells — +# clears a single shell-cursor cell (~493 px) while still failing on real +# content changes (thousands of px). See compare_shot_frame for the rationale. +SHOT_AE_TOLERANCE="${SHOT_AE_TOLERANCE:-1000}" + usage() { sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' exit 2 @@ -356,6 +391,80 @@ run_shot_tape() { fi } +# frame_is_blank — true if the capture is a near-uniform frame, i.e. the +# transient Termina full-refresh blank (see SHOT_BLANK_RETRIES). Baseline- +# independent: it counts unique colors with ImageMagick `identify %k`. A blank +# frame is the solid theme background (~1 color); any populated TUI screen has +# hundreds. The threshold (16) sits far below the sparsest real frame and far +# above a blank, so it never misclassifies a real screen as blank. +frame_is_blank() { + local png="$1" + command -v identify >/dev/null 2>&1 || return 1 # can't tell → treat as not blank + [[ -f "$png" ]] || return 1 # missing capture is handled elsewhere + local colors + colors=$(identify -format '%k' "$png" 2>/dev/null || echo "") + [[ "$colors" =~ ^[0-9]+$ ]] || return 1 + (( colors < 16 )) +} + +# frame_needs_retry — true if the capture looks like a transient Termina +# full-refresh artifact that re-running the tape can clear. Two shapes: +# * fully blank (frame_is_blank) — VHS sampled the [2J-cleared frame. +# * partial/garbled — VHS sampled mid-repaint, so only the top rows landed +# (e.g. the MCP tool grid captured before its lower rows painted). Such a +# frame has plenty of colors (so frame_is_blank misses it) but differs from +# baseline by far more than the cursor tolerance. +# A genuine regression also trips the second branch, but it reproduces every +# attempt and so still fails at compare time — only the latency differs. +frame_needs_retry() { + local frame="$1" + local capture="/tmp/shot-${frame}.png" + local baseline="${SHOT_BASELINE_DIR}/${frame}.approved.png" + [[ -f "$capture" ]] || return 1 + frame_is_blank "$capture" && return 0 + command -v compare >/dev/null 2>&1 || return 1 + [[ -f "$baseline" ]] || return 1 + local ae ae_int + ae=$(compare -metric AE "$baseline" "$capture" /dev/null 2>&1 || true) + ae_int="${ae%%.*}"; ae_int="${ae_int// /}" + [[ "$ae_int" =~ ^[0-9]+$ ]] || return 1 + (( ae_int > SHOT_AE_TOLERANCE )) +} + +# run_shot_tape_with_retry — run a capture tape, then inspect the frames +# it emits. If any is a transient blank (SHOT_BLANK_RETRIES), re-run the whole +# tape so the next attempt captures a settled frame. Bounded so a genuinely +# broken tape still fails at compare time instead of looping forever. +run_shot_tape_with_retry() { + local tape="$1" + local frames + frames="$(shot_tape_frames "$tape")" + local attempt=1 + while :; do + run_shot_tape "$tape" + [[ -z "$frames" ]] && return # no frame map → accept the single run + + local bad="" f + for f in $frames; do + if frame_needs_retry "$f"; then + bad="$f" + break + fi + done + [[ -z "$bad" ]] && return # all frames settled → done + + if (( attempt >= SHOT_BLANK_RETRIES )); then + echo " WARN: ${tape} produced a transient frame (${bad}) on all ${attempt} attempts;" >&2 + echo " leaving it for compare_shot_frame to fail on." >&2 + return + fi + echo " RETRY: ${tape} attempt ${attempt} produced a transient frame (${bad}) —" >&2 + echo " re-running tape (blank or partial Termina full-refresh capture)." >&2 + attempt=$((attempt + 1)) + for f in $frames; do rm -f "/tmp/shot-${f}.png"; done # clear stale captures + done +} + # compare_shot_frame — compare /tmp/shot-.png against the # committed baseline. Records a failure (and writes review PNGs) on a # missing capture, missing baseline, or pixel mismatch. @@ -381,9 +490,32 @@ compare_shot_frame() { return fi - if cmp -s "$baseline" "$capture"; then - echo " PASS: ${frame} — pixel-identical to baseline." - return + # Use ImageMagick pixel comparison rather than cmp -s (byte-for-byte). + # Two sources of false failures are tolerated: + # 1. VHS PNG zlib encoder jitter — same pixels, different byte streams + # across process invocations (AE = 0, always passes). + # 2. Terminal cursor block — Set CursorBlink false freezes the cursor but + # not its on/off state; the shell-prompt cursor cell can appear or not + # between runs. The block is one character cell (measured AE≈493 at this + # geometry). AE_CURSOR_TOLERANCE is set to ~2 cells so a single cursor + # cell passes with margin, while real regressions still fail — a changed + # word/line differs by thousands of px, a blank screen by ~68,000. + # Fall back to cmp -s only if ImageMagick is absent. The tolerance + # (SHOT_AE_TOLERANCE) is shared with the retry trigger (frame_needs_retry). + if command -v compare >/dev/null 2>&1; then + local ae + ae=$(compare -metric AE "$baseline" "$capture" /dev/null 2>&1 || true) + local ae_int="${ae%%.*}" + ae_int="${ae_int// /}" + if [[ "${ae_int:-0}" -le "$SHOT_AE_TOLERANCE" ]]; then + echo " PASS: ${frame} — pixel-close to baseline (AE=${ae_int:-0})." + return + fi + else + if cmp -s "$baseline" "$capture"; then + echo " PASS: ${frame} — pixel-identical to baseline." + return + fi fi # Mismatch — keep the actual, and a visual diff if ImageMagick is around. @@ -407,7 +539,7 @@ if [[ "$shots_mode" -eq 1 ]]; then # Fresh /tmp so a stale capture from an earlier run cannot be compared. rm -f /tmp/shot-*.png for tape in "${SHOT_TAPES[@]}"; do - run_shot_tape "$tape" + run_shot_tape_with_retry "$tape" done echo diff --git a/tests/smoke/screenshots/config-search-brave-entry.approved.png b/tests/smoke/screenshots/config-search-brave-entry.approved.png index 9b2262995..0b8f97fd1 100644 Binary files a/tests/smoke/screenshots/config-search-brave-entry.approved.png and b/tests/smoke/screenshots/config-search-brave-entry.approved.png differ diff --git a/tests/smoke/screenshots/help.approved.png b/tests/smoke/screenshots/help.approved.png index bef9e7ff5..b9b636fe3 100644 Binary files a/tests/smoke/screenshots/help.approved.png and b/tests/smoke/screenshots/help.approved.png differ diff --git a/tests/smoke/tapes/screenshots/config-search.tape b/tests/smoke/tapes/screenshots/config-search.tape index 45049ad2e..0e6c716e1 100644 --- a/tests/smoke/tapes/screenshots/config-search.tape +++ b/tests/smoke/tapes/screenshots/config-search.tape @@ -22,20 +22,30 @@ Down 5 Enter # ─── Frame 1: Provider selection ─────────────────────────────────────────── +# SelectBackendForEditing is synchronous. Two anchors are required: the title +# confirms the screen loaded, and the DuckDuckGo description confirms the +# cursor is on row 1 (DuckDuckGo). Without the second anchor, a Down key +# leaked from the "Down 5" Settings navigation can move the cursor to Brave +# before the screenshot fires, capturing the wrong row. Wait+Screen@10s /Choose the backend Netclaw uses for web search/ -Sleep 1s +Wait+Screen@5s /DuckDuckGo works without setup/ Screenshot "/tmp/shot-config-search-selection.png" -Sleep 1s # ─── Frame 2: Brave entry state ──────────────────────────────────────────── +# SelectBackendForEditing("brave") is synchronous — Wait+Screen is sufficient. Down Enter Wait+Screen@10s /Brave Search requires an API key/ -Sleep 1s Screenshot "/tmp/shot-config-search-brave-entry.png" -Sleep 1s # ─── Frame 3: Saved state ────────────────────────────────────────────────── +# SaveWithoutProbeOverride cascades multiple ReactiveProperty notifications +# and a ReloadPersistedDraft disk read before the frame settles. A single +# Wait+Screen on the success text is not enough — a second anchor on the +# key-binding bar text guarantees both CurrentScreen==Saved and +# ActiveDialog==None have committed to a stable rendered frame. +# "[Enter] Settings Areas" is emitted by BuildKeyBindings only in that state +# (SearchConfigEditorPage.cs line 212). Escape Down Enter @@ -46,9 +56,8 @@ Wait+Screen@10s /Search Validation Warning/ Down 2 Enter Wait+Screen@10s /validated and saved/ -Sleep 1s +Wait+Screen@5s /\[Enter\] Settings Areas/ Screenshot "/tmp/shot-config-search-saved.png" -Sleep 1s Ctrl+Q Wait+Screen@10s /TAPE\$/ diff --git a/tests/smoke/tapes/screenshots/help.tape b/tests/smoke/tapes/screenshots/help.tape index 85cd92618..5f0dab4ca 100644 --- a/tests/smoke/tapes/screenshots/help.tape +++ b/tests/smoke/tapes/screenshots/help.tape @@ -13,13 +13,13 @@ Output "/tmp/tape-shot-help.gif" Type "netclaw --help" Enter -# Anchor on the usage header, then settle. VHS Screenshot can otherwise -# catch a frame mid-render; the Sleep after keeps the next keystroke from -# leaking into the capture. +# Anchor on the usage header, then sleep to let the full terminal output +# finish rendering. `netclaw --help` writes multiple lines and VHS may not +# have flushed all of them into the terminal buffer by the time Wait+Screen +# matches the first visible line. Wait+Screen@10s /Usage:/ Sleep 1s Screenshot "/tmp/shot-help.png" -Sleep 1s Type "exit" Enter diff --git a/tests/smoke/tapes/screenshots/mcp-permissions.tape b/tests/smoke/tapes/screenshots/mcp-permissions.tape index eb0dd80c2..967f2f983 100644 --- a/tests/smoke/tapes/screenshots/mcp-permissions.tape +++ b/tests/smoke/tapes/screenshots/mcp-permissions.tape @@ -59,21 +59,29 @@ Enter # ─── Frame 1: ServerList ───────────────────────────────────────────── # smoke-math should appear as "Connected, 3 tools". Wait+Screen@15s /smoke-math/ -Sleep 1s +Wait+Screen@5s /3 tools/ Screenshot "/tmp/shot-mcp-permissions-server-list.png" -Sleep 1s # ─── Navigate into the ToolGrid ────────────────────────────────────── +# Sleep 1s here is an intentional settle guard: the daemon can push a +# server-state update (tool re-index, reconnect) immediately after the +# server-list frame stabilises. Navigating into the tool grid during such +# an update causes Enter to hit the TUI mid-transition, producing a blank +# screen rather than the tool grid. The sleep lets any in-flight update +# complete before Enter is delivered. +Sleep 1s Enter # ─── Frame 2: ToolGrid ─────────────────────────────────────────────── # All header rows (Server, Audience, Server enabled, Server default) plus # all tool rows (add, echo, record-tasks) must be visible simultaneously. # If the #1424 regression reappears, tool rows will overwrite the header. -Wait+Screen@10s /record-tasks/ -Sleep 1s +# Timeout is 15s to match the server-list anchor; /Server default:/ (with +# colon) matches the rendered "Server default: [Auto]" line, which is +# unique to the tool-grid view and absent from the server-list scrollback. +Wait+Screen@15s /record-tasks/ +Wait+Screen@10s /Server default:/ Screenshot "/tmp/shot-mcp-permissions-tool-grid.png" -Sleep 1s # ─── Exit TUI ──────────────────────────────────────────────────────── Ctrl+Q diff --git a/tests/smoke/tapes/screenshots/provider-manager.tape b/tests/smoke/tapes/screenshots/provider-manager.tape index 58d4385b2..1901a7ae1 100644 --- a/tests/smoke/tapes/screenshots/provider-manager.tape +++ b/tests/smoke/tapes/screenshots/provider-manager.tape @@ -16,8 +16,8 @@ # # Navigation + anchors mirror tapes/provider-add.tape; see that file and # src/Netclaw.Cli/Tui/ProviderManagerPage.cs for the synchronization -# rationale. The Screenshot is bracketed by Sleep 1s to settle the frame -# and keep the next keystroke from leaking into the capture. +# rationale. Wait+Screen anchors on the list content, then Sleep 1s lets +# the full TUI frame flush before the screenshot is captured. # # Prepended preamble: tapes/screenshot-preamble.tape (determinism-pinned). @@ -31,10 +31,14 @@ Enter # With no providers configured the list shows every known type as a # "(not configured)" row plus the "+ Add new provider..." sentinel. # Anchor on "Ollama" (a guaranteed type row) so we know the list rendered. +# Sleep lets VHS flush the full frame; the SECOND Wait+Screen is an anti-blank +# re-assert immediately before capture — if a Termina startup full-refresh +# blanked the screen during the Sleep, this blocks until the list repaints. +# (Defence in depth: run-smoke.sh also re-runs the tape if a blank slips through.) Wait+Screen@10s /Ollama/ Sleep 1s +Wait+Screen@5s /Ollama/ Screenshot "/tmp/shot-provider-manager-empty.png" -Sleep 1s # Frame captured; quit the TUI back to the shell. Ctrl+Q diff --git a/tests/smoke/tapes/screenshots/wizard-screens.tape b/tests/smoke/tapes/screenshots/wizard-screens.tape index 854f0209d..b4789b749 100644 --- a/tests/smoke/tapes/screenshots/wizard-screens.tape +++ b/tests/smoke/tapes/screenshots/wizard-screens.tape @@ -17,8 +17,9 @@ # # Navigation + anchors mirror tapes/init-wizard.tape; see that file and # src/Netclaw.Cli/Tui/Wizard/Steps/*StepView.cs for the synchronization -# rationale. Each Screenshot is bracketed by Sleep 1s — VHS can otherwise -# capture an unsettled frame, or let the next keystroke leak into the shot. +# rationale. Wizard step transitions are synchronous — Wait+Screen is the +# only gate needed. The Sleep 1s after Ctrl+Q below is intentional: it is +# the alt-screen restore guard (CSI ?1049l), not a screenshot timing guard. # # Prepended preamble: tapes/screenshot-preamble.tape (determinism-pinned). @@ -30,9 +31,10 @@ Enter # ─── Frame 1: Provider picker ──────────────────────────────────────── Wait+Screen@10s /Choose your LLM provider:/ -Sleep 1s +# The heading can appear before the full provider list is painted on slower +# CI runners. Anchor on the last row so the screenshot captures the settled list. +Wait+Screen@5s /7\. Venice\.ai/ Screenshot "/tmp/shot-wizard-provider-picker.png" -Sleep 1s # Provider list ordering is alphabetical by TypeKey: # anthropic, github-copilot, ollama, openai, openai-compatible, openrouter @@ -78,9 +80,7 @@ Enter # ─── Frame 2: Security posture ─────────────────────────────────────── Wait+Screen@10s /Who will interact with this Netclaw instance/ -Sleep 1s Screenshot "/tmp/shot-wizard-security-posture.png" -Sleep 1s # Both frames captured; abandon the wizard. Ctrl+Q is the TUI quit # shortcut (Ctrl+C requires a double-press under raw input mode).