Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 57 additions & 9 deletions .github/workflows/qwen-triage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2073,6 +2073,17 @@ jobs:
(cd "${RUNNER_TEMP:?}" && npm install -g --registry=https://registry.npmjs.org '@qwen-code/qwen-code@latest')
qwen --version

# Chromium system dependencies (apt packages) for evidence
# screenshots. install-deps is unpinned so the apt list tracks
# current Playwright (a superset of the lockfile binary's needs);
# the version-sensitive BINARY is downloaded after checkout by the
# "Install evidence browser" step, using the checkout's own
# Playwright so it always matches the lockfile. Best-effort: a
# failure here must not fail the job.
if ! (cd "${RUNNER_TEMP:?}" && npx --yes playwright install-deps chromium); then
echo "::warning::Chromium system deps install failed; the verification agent will produce a text-only report."
fi

# This container mounts the persistent runner workspace, and a previous
# run EXECUTED PR code as the node user with write access to .git —
# hooks or config planted then would fire during the checkout below (as
Expand Down Expand Up @@ -2320,6 +2331,26 @@ jobs:
fi
echo "Install/build completed before verification." >> "$GITHUB_STEP_SUMMARY"

- name: 'Install evidence browser'
if: "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''"
run: |-
set -uo pipefail
# Download the browser binary matching the checkout's Playwright
# (node_modules/.bin/playwright resolves the exact lockfile
# version). System deps were installed as root in the tools step;
# this runs as node so the tree is agent-readable without chmod.
# Best-effort: failure degrades to a text-only report.
PW_PATH="${RUNNER_TEMP:?}/pw-browsers"
mkdir -p "$PW_PATH"
chown node:node "$PW_PATH"
if runuser -u node -- env PLAYWRIGHT_BROWSERS_PATH="$PW_PATH" \
npx playwright install chromium; then
printf '%s' "$PW_PATH" > "${RUNNER_TEMP:?}/verify-chromium-path"
echo "Chromium available for evidence screenshots." >> "$GITHUB_STEP_SUMMARY"
else
echo "::warning::Chromium browser download failed; the verification agent will produce a text-only report."
fi

- name: 'Run verification agent'
if: "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''"
id: 'run'
Expand Down Expand Up @@ -2754,6 +2785,19 @@ jobs:
"QWEN_CI_REAL_GH=${QWEN_CI_REAL_GH:-}"
"QWEN_CI_REAL_GIT=${QWEN_CI_REAL_GIT:-}"
)
# Evidence screenshots: the "Install evidence browser" step
# downloaded chromium into a shared path using the checkout's
# Playwright. Both variables are set only when that install
# actually succeeded, so their ABSENCE is the agent's signal to
# skip captures rather than burn budget on a download it cannot
# complete.
if CHROMIUM_PATH="$(cat "${RUNNER_TEMP:?}/verify-chromium-path" 2>/dev/null)" &&
[ -n "$CHROMIUM_PATH" ]; then
QWEN_ENV+=(
"PLAYWRIGHT_BROWSERS_PATH=$CHROMIUM_PATH"
"QWEN_VERIFY_CHROMIUM=1"
)
fi
if [ -n "${OPENAI_MODEL:-}" ]; then
QWEN_ENV+=("OPENAI_MODEL=$OPENAI_MODEL")
fi
Expand Down Expand Up @@ -3012,8 +3056,8 @@ jobs:
printf '</code></pre>\n\n</details>\n\n'
}

# Host the agent's evidence images (if any) on the pr-assets branch
# — the same convention hand-run verification rounds use — and build
# Host the agent's evidence images (if any) on a per-PR branch
# (pr-assets/pr<N>-verify, matching hand-run convention) and build
# a markdown section referencing them. Image bytes come from a run
# that executed PR code: inert but untrusted, so filenames pass a
# strict allowlist, count/size are capped (8 files, <2 MB each; the
Expand All @@ -3024,6 +3068,7 @@ jobs:
collect_and_host_evidence() {
local imgs=() f base safe seen=' ' hosted=0 total=0 skipped=0
local dest_dir="verify/pr${PR_NUMBER}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
local assets_branch="pr-assets/pr${PR_NUMBER}-verify"
Comment thread
wenshao marked this conversation as resolved.
Resolved
local clone_dir="${RUNNER_TEMP:-/tmp}/pr-assets"
local remote="${VERIFY_ASSETS_REMOTE:-https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git}"
# `|| true` on every find: the artifact download is
Expand All @@ -3043,9 +3088,12 @@ jobs:
return 0
fi
rm -rf "$clone_dir"
if ! git clone -q --depth 1 --branch pr-assets "$remote" "$clone_dir" 2>/dev/null; then
echo "::warning::pr-assets branch unavailable; posting a text-only report." >&2
return 0
if ! git clone -q --depth 1 --branch "$assets_branch" "$remote" "$clone_dir" 2>/dev/null; then
rm -rf "$clone_dir"
mkdir -p "$clone_dir"
git -C "$clone_dir" init -q
git -C "$clone_dir" checkout -q --orphan "$assets_branch"
git -C "$clone_dir" remote add origin "$remote"
fi
# Rebase in the racing-push retry needs a committer identity, so
# set it once on the clone instead of per-command -c flags.
Expand Down Expand Up @@ -3081,19 +3129,19 @@ jobs:
git add "$dest_dir" &&
git commit -q -m "verify evidence for PR #${PR_NUMBER} (run ${GITHUB_RUN_ID})" &&
{
git push -q origin HEAD:pr-assets 2>/dev/null ||
git push -q origin "HEAD:$assets_branch" 2>/dev/null ||
{
# One retry after a racing push from another assets job.
git pull -q --rebase origin pr-assets 2>/dev/null &&
git push -q origin HEAD:pr-assets 2>/dev/null
git pull -q --rebase origin "$assets_branch" 2>/dev/null &&
git push -q origin "HEAD:$assets_branch" 2>/dev/null
}
}
); then
echo "::warning::Failed to push evidence images; posting a text-only report." >&2
EVIDENCE_SECTION=''
return 0
fi
local raw_base="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/pr-assets/${dest_dir}"
local raw_base="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${assets_branch}/${dest_dir}"
EVIDENCE_SECTION=$'### Evidence images\n\n'
for f in "$clone_dir/$dest_dir"/*.png; do
[ -f "$f" ] || continue
Expand Down
10 changes: 8 additions & 2 deletions .qwen/skills/terminal-capture/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,14 @@ Ensure the following dependencies are installed before running:

```bash
npm install # Install project dependencies.
npx playwright install chromium # Install Playwright browser
npx playwright install chromium # Install Playwright browser (skip in CI: see note below)
```

> **CI / verify context:** when `QWEN_VERIFY_CHROMIUM=1` is set, the browser
> is already installed and `PLAYWRIGHT_BROWSERS_PATH` points at it. Do **not**
> run `playwright install` — it downloads ~170 MB and fails on system deps
> the agent user cannot install.

## Architecture

```
Expand Down Expand Up @@ -227,7 +232,8 @@ This tool is commonly used for visual verification during PR reviews.

- Playwright error `browser not found`
Cause: browser not installed.
Solution: `npx playwright install chromium`.
Solution: `npx playwright install chromium` (local dev only — in CI verify
runs, this means the pre-install step failed; report it, do not install).
- Blank screenshot
Cause: process starts slowly or build failed.
Solution: check build success and the spawn command.
Expand Down
38 changes: 27 additions & 11 deletions .qwen/skills/verify-pr/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -325,17 +325,33 @@ workflow globs). It must contain:
- `assertions.json` — `{"pass": <int>, "fail": <int>, "total": <int>}`,
counting **only scripted assertions that actually executed**.
- Harness scripts and raw logs (per-cell stdout/stderr, build logs).
- Optionally `evidence/*.png` — rendered image evidence. The publish job
hosts these on the `pr-assets` branch and appends them below the report,
capped at **8 images, 2 MB each**; anything beyond stays in the run
artifacts only. Use them when text cannot carry the oracle: TUI rendering
(`terminal-capture` skill: node-pty → xterm → Playwright PNG;
`npx playwright install chromium` on demand) or a one-image harness
summary. Name each file as a kebab-case caption that binds image to claim
(`01-bundle-ab-base-vs-head.png`, `02-repaint-after-sigcont.png`) — the
filename becomes the published caption — and reference it from report.md
prose by that name. Before/after pairs beat single "after" shots; a
screenshot that does not name what to look at proves nothing.
- `evidence/*.png` — image evidence. **Produce these whenever you ran a
harness**, not only for TUI work. A table in the report is your _claim_
about what happened; a capture of the run is a _witness_ that the numbers
came from a real execution, and it is the part a reviewer cannot get any
other way. The highest-value shots, in order: the A/B cells side by side,
the mutation matrix as it printed, and the raw harness output behind a
headline number. One capture of the terminal showing `2999 → 0` is worth
more than the sentence asserting it.

**Chromium is pre-installed for you** when `QWEN_VERIFY_CHROMIUM=1` is set;
`PLAYWRIGHT_BROWSERS_PATH` already points at it. Do **not** run
`playwright install` — you run as `node` with a fresh `HOME` and no apt
rights, so it downloads ~170 MB and then fails on system deps. If
`QWEN_VERIFY_CHROMIUM` is unset the capability is unavailable in this run:
ship the text-only report and note it under _Not covered_ in one line, do
not spend budget working around it.

Route: `terminal-capture` skill (node-pty → xterm.js → Playwright PNG).
The publish job hosts what you produce on a per-PR branch
(`pr-assets/pr<N>-verify`) and appends it below the report, capped at
**8 images, 2 MB each**; anything
beyond stays in the run artifacts. Name each file as a kebab-case caption
that binds image to claim (`01-bundle-ab-base-vs-head.png`,
`02-repaint-after-sigcont.png`) — the filename becomes the published
caption — and reference it from report.md prose by that name. Before/after
pairs beat single "after" shots; a screenshot that does not name what to
look at proves nothing.

`verdict.txt` meanings: `merge-ready` = every executed assertion passed and no
new blocking finding; `findings` = evidence produced concrete problems worth a
Expand Down
86 changes: 81 additions & 5 deletions scripts/tests/qwen-triage-workflow.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -1004,7 +1004,7 @@ describe('qwen-triage verify hardening', () => {
// qwen under `env -i`, which needs no per-variable stripping.
const prepare = verifyJob.slice(
verifyJob.indexOf('Install and build PR app'),
verifyJob.indexOf('Run verification agent'),
verifyJob.indexOf("name: 'Install evidence browser'"),
);
const commands = prepare.match(/runuser -u node -- env[\s\S]*?\n/g) ?? [];
expect(commands.length).toBe(2);
Expand Down Expand Up @@ -1353,7 +1353,7 @@ describe('qwen-triage verify hardening round 2', () => {
// A bare remote with a pr-assets branch, plus a gh stub.
sh(`git init -q --bare "${dir}/assets.git"`);
sh(
`mkdir -p "${dir}/seed" && cd "${dir}/seed" && git init -q && git checkout -q -b pr-assets && echo s > s.txt && git add . && git -c user.name=t -c user.email=t@t commit -qm s && git push -q "${dir}/assets.git" pr-assets`,
`mkdir -p "${dir}/seed" && cd "${dir}/seed" && git init -q && git checkout -q -b pr-assets/pr7999-verify && echo s > s.txt && git add . && git -c user.name=t -c user.email=t@t commit -qm s && git push -q "${dir}/assets.git" pr-assets/pr7999-verify`,
);
writeFileSync(
join(dir, 'gh'),
Expand Down Expand Up @@ -1415,7 +1415,7 @@ describe('qwen-triage verify hardening round 2', () => {
});
expect(res.status).toBe(0);
const hosted = sh(
`git -C "${dir}/assets.git" ls-tree -r --name-only pr-assets | grep verify/ || true`,
`git -C "${dir}/assets.git" ls-tree -r --name-only pr-assets/pr7999-verify | grep verify/ || true`,
)
.stdout.trim()
.split('\n')
Expand All @@ -1431,8 +1431,8 @@ describe('qwen-triage verify hardening round 2', () => {
expect(comment).not.toContain('02-fake');
expect(comment).toContain('did not pass the hosting checks');

// No reachable pr-assets branch -> text-only, never an aborted report.
sh(`git init -q --bare "${dir}/empty.git"`);
// Unreachable remote -> text-only, never an aborted report.
sh(`rm -rf "${dir}/empty.git" && mkdir -p "${dir}/empty.git"`);
Comment thread
wenshao marked this conversation as resolved.
Resolved
const out2 = join(dir, 'comment2.md');
const res2 = sh(script, {
cwd: work,
Expand Down Expand Up @@ -2140,6 +2140,82 @@ describe('qwen-triage verify round-3 hardening', () => {
expect(group).toContain("vars.MAINTAINER_ECS_RUNNER_DISABLED != 'true'");
});

// Evidence images: the agent cannot install chromium (runs as `node`,
// `env -i`, fresh HOME, no apt). The tools step installs system deps
// as root; a post-checkout step downloads the browser binary using the
// checkout's own Playwright so the version always matches the lockfile.
it('pre-installs chromium and hands it to the agent', () => {
// System deps only — no browser binary, no version-sensitive download.
const tools = stepIn('verify', 'Install verify runner tools');
expect(tools).toContain('install-deps chromium');
expect(tools).not.toContain('install --with-deps');
expect(tools).toContain('::warning::Chromium system deps install failed');
// No hardcoded Playwright pin here either: the apt list must track
// current Playwright so it covers the lockfile-matched binary below.
expect(tools).not.toMatch(/playwright@[\d.]/);

// Browser binary: downloaded after npm ci so npx resolves the
// checkout's lockfile version — never a hardcoded pin (M5).
const browser = stepIn('verify', 'Install evidence browser');
expect(browser).toContain('npx playwright install chromium');
expect(browser).not.toMatch(/playwright@[\d.]/);
expect(browser).toContain('PLAYWRIGHT_BROWSERS_PATH');
// Marker is written ONLY inside the success branch (M6): the if
// condition must precede the marker write, and the else branch must
// carry the warning instead.
const ifIdx = browser.indexOf('if runuser');
const markerIdx = browser.indexOf('verify-chromium-path');
const elseIdx = browser.indexOf(
'::warning::Chromium browser download failed',
);
expect(ifIdx).toBeGreaterThan(-1);
expect(markerIdx).toBeGreaterThan(ifIdx);
expect(elseIdx).toBeGreaterThan(markerIdx);
// Best-effort: a failed download must not fail a verification.
expect(browser).not.toContain('exit 1');

// The agent is told ONLY when the install actually succeeded, so the
// variable's absence is a real signal rather than a stale promise.
// The guard must be CONDITIONAL (M1b): QWEN_VERIFY_CHROMIUM=1 must
// appear inside the if-block that reads the marker, not unconditionally.
const runStep = stepIn('verify', 'Run verification agent');
expect(runStep).toContain('verify-chromium-path');
expect(runStep).toContain('QWEN_VERIFY_CHROMIUM=1');
Comment thread
wenshao marked this conversation as resolved.
Resolved
const guard = runStep.indexOf('verify-chromium-path');
const marker = runStep.indexOf('QWEN_VERIFY_CHROMIUM=1');
expect(guard).toBeLessThan(marker);
// The if-fi block must wrap the QWEN_ENV+= assignment.
const ifBlock = runStep.slice(guard - 20, marker + 30);
expect(ifBlock).toContain('if');
expect(ifBlock).toContain('then');

// The tmux lane is untouched: it has no evidence-image path.
expect(stepIn('tmux-testing', 'Run tmux real-user testing')).not.toContain(
'QWEN_VERIFY_CHROMIUM',
);

// And the skill must stop calling captures optional, must name the
// gate, and must forbid the install the agent cannot complete.
const flat = verifySkill.replace(/\s+/g, ' ');
expect(flat).toContain('Produce these whenever you ran a harness');
expect(flat).toContain('QWEN_VERIFY_CHROMIUM=1');
expect(flat).toContain('Do **not** run `playwright install`');
expect(flat).not.toContain('Optionally `evidence/*.png`');
});

// The publish job must host images on a per-PR branch that can coexist
// with the existing pr-assets/* namespace — a bare `pr-assets` leaf
// cannot be created while `pr-assets/…` children exist.
it('hosts evidence on a per-PR branch, not a bare pr-assets leaf', () => {
const publish = stepIn(
'publish-verify',
'Post verification report comment',
);
expect(publish).toContain('pr-assets/pr${PR_NUMBER}-verify');
expect(publish).toContain('checkout -q --orphan');
expect(publish).not.toMatch(/--branch pr-assets["\s]/);
});

// Cleanups must never descend through a PR-writable parent, and an
// outward-resolving hooks entry must be removed rather than reported.
it('survives symlink escapes in the workspace cleanup', () => {
Expand Down
Loading