diff --git a/.github/scripts/web-shell-visuals-publish.mjs b/.github/scripts/web-shell-visuals-publish.mjs new file mode 100644 index 00000000000..730ed23e873 --- /dev/null +++ b/.github/scripts/web-shell-visuals-publish.mjs @@ -0,0 +1,282 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Staging + comment generation for the web-shell visuals publish workflow. + * + * Extracted from the inline workflow so the image validation and comment + * construction โ€” the parts that consume UNTRUSTED PR output and were + * previously untested โ€” have unit coverage. (A shell sanitizer bug once + * appended `_` to every filename and silently produced an empty preview; the + * pure functions here are covered by web-shell-visuals-publish.test.mjs.) + * + * The pure helpers (`sanitizeName`, `classifyMagic`, `selectImages`, + * `buildComment`) are exported and tested. The file also runs as a CLI for the + * workflow: + * node web-shell-visuals-publish.mjs stage + * node web-shell-visuals-publish.mjs comment + */ + +import { + closeSync, + copyFileSync, + mkdirSync, + openSync, + readdirSync, + readSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { basename, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +// Bounds on UNTRUSTED artifact content: cap files EXAMINED (so a flood of junk +// can't burn the budget before valid files), files ACCEPTED, and per-file size. +export const MAX_CANDIDATES = 200; +export const MAX_SCREENSHOTS = 20; +export const MAX_GIFS = 6; +export const MAX_BYTES = 3 * 1024 * 1024; + +const PNG_MAGIC = '89504e470d0a1a0a'; +const GIF_MAGICS = new Set(['474946383961', '474946383761']); // GIF89a / GIF87a + +const FLOW_LABELS = { + 'model-switch': 'Open the slash menu and switch model', + 'prompt-stream': 'Submit a prompt and watch the reply stream in', +}; + +/** + * Sanitize to the hosted-filename charset WITHOUT corrupting the extension. + * (The shell version captured `basename` through a pipe, turning its trailing + * newline into `_` and breaking the `.png`/`.gif` filter โ€” this cannot.) + */ +export function sanitizeName(name) { + return String(name).replace(/[^A-Za-z0-9._-]/g, '_'); +} + +/** Classify by first-bytes magic hex โ†’ 'png' | 'gif' | null. */ +export function classifyMagic(ext, magicHex) { + const hex = String(magicHex).toLowerCase(); + if (ext === 'png') return hex.slice(0, 16) === PNG_MAGIC ? 'png' : null; + if (ext === 'gif') return GIF_MAGICS.has(hex.slice(0, 12)) ? 'gif' : null; + return null; +} + +/** + * Pure selection over candidates `[{ name, ext, size, magic }]` (in order): + * apply the examined/accepted/size caps and magic validation. Returns + * `{ accepted: [{ name, safeName, kind }], warnings: string[] }`. + */ +export function selectImages(candidates, opts = {}) { + const maxCandidates = opts.maxCandidates ?? MAX_CANDIDATES; + const maxBytes = opts.maxBytes ?? MAX_BYTES; + // Per-kind caps so a large screenshot set can't starve the flow GIFs: a + // shared total cap over PNG-first candidates would let >=N screenshots + // silently drop every GIF from the preview. + const maxPerKind = { + png: opts.maxScreenshots ?? MAX_SCREENSHOTS, + gif: opts.maxGifs ?? MAX_GIFS, + }; + const kindCount = { png: 0, gif: 0 }; + const accepted = []; + const warnings = []; + let examined = 0; + for (const c of candidates) { + examined += 1; + if (examined > maxCandidates) { + warnings.push(`examined ${maxCandidates} candidate files; stopping`); + break; + } + if (c.size > maxBytes) { + warnings.push(`${c.name} exceeds ${maxBytes} bytes; skipping`); + continue; + } + const kind = classifyMagic(c.ext, c.magic); + if (!kind) { + warnings.push(`${c.name} is not a valid ${c.ext}; skipping`); + continue; + } + if (kindCount[kind] >= maxPerKind[kind]) { + warnings.push( + `reached the ${kind} cap (${maxPerKind[kind]}); skipping ${c.name}`, + ); + continue; + } + kindCount[kind] += 1; + accepted.push({ + name: c.name, + safeName: sanitizeName(basename(c.name)), + kind, + }); + } + return { accepted, warnings }; +} + +/** Self-defending HTML escaping for interpolated values. */ +export const esc = (s) => + String(s) + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(//g, '>'); + +export const pretty = (s) => + s.replace(/[-_]+/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()); + +/** + * Pure comment builder. `files` is the list of staged filenames (png + gif). + * `ctx` is `{ rawBase, shortSha, runUrl }`. Returns the markdown body. + */ +export function buildComment(files, ctx = {}) { + const rawBase = ctx.rawBase ?? ''; + const shortSha = ctx.shortSha ?? ''; + const runUrl = ctx.runUrl ?? ''; + const url = (name) => `${rawBase}/${encodeURIComponent(name)}`; + + const shots = files.filter((f) => /\.png$/i.test(f)); + const views = new Map(); + for (const f of shots) { + const m = f.match(/^(.*)-(light|dark)\.png$/i); + if (!m) continue; + const [, view, theme] = m; + const entry = views.get(view) || {}; + entry[theme.toLowerCase()] = f; + views.set(view, entry); + } + const gifs = files.filter((f) => /\.gif$/i.test(f)).sort(); + + const out = []; + out.push(''); + out.push('### ๐Ÿ–ผ๏ธ web-shell visual preview'); + out.push( + `Auto-rendered from this PR head \`${esc(shortSha)}\` against a mock daemon (no real backend). Refreshes on every push.`, + ); + out.push(''); + + if (views.size > 0) { + out.push('#### Screenshots ยท light / dark'); + out.push(''); + out.push(''); + out.push(''); + for (const [view, pair] of [...views.entries()].sort()) { + const light = pair.light + ? `${esc(view)} light` + : 'โ€”'; + const dark = pair.dark + ? `${esc(view)} dark` + : 'โ€”'; + out.push( + ``, + ); + } + out.push('
viewlightdark
${esc(pretty(view))}${light}${dark}
'); + out.push(''); + } + + if (gifs.length > 0) { + out.push('#### Flows'); + out.push(''); + for (const g of gifs) { + const key = g.replace(/\.gif$/i, ''); + // Own-property only: `FLOW_LABELS[key]` would otherwise inherit + // Object.prototype members, so a `toString.gif` would render the function + // source as the label. + const label = Object.hasOwn(FLOW_LABELS, key) + ? FLOW_LABELS[key] + : pretty(key); + out.push(`**${esc(label)}**`); + out.push(''); + out.push(`${esc(key)} flow`); + out.push(''); + } + } + + if (runUrl) { + out.push( + `Full-resolution recordings (.webm) are attached to the workflow run.`, + ); + } + out.push(''); + out.push('โ€” _Qwen Code ยท web-shell visuals_'); + return out.join('\n') + '\n'; +} + +// --- I/O layer (exercised by the CLI; not part of the unit-tested surface) --- + +function readMagicHex(path, n = 8) { + const fd = openSync(path, 'r'); + try { + const buf = Buffer.alloc(n); + const read = readSync(fd, buf, 0, n, 0); + return buf.subarray(0, read).toString('hex'); + } finally { + closeSync(fd); + } +} + +function gatherCandidates(dir, ext) { + let names; + try { + names = readdirSync(dir); + } catch { + return []; + } + return names + .filter((n) => n.toLowerCase().endsWith(`.${ext}`)) + .sort() + .map((n) => { + const path = join(dir, n); + let size = Infinity; + let magic = ''; + try { + size = statSync(path).size; + magic = readMagicHex(path); + } catch { + // Unreadable entry: leave size=Infinity/magic='' so it is skipped. + } + return { name: n, ext, size, magic, path }; + }); +} + +function stageCli(screenshotsDir, gifsDir, stageDir) { + const candidates = [ + ...gatherCandidates(screenshotsDir, 'png'), + ...gatherCandidates(gifsDir, 'gif'), + ]; + const { accepted, warnings } = selectImages(candidates); + for (const w of warnings) process.stderr.write(`::warning::${w}\n`); + mkdirSync(stageDir, { recursive: true }); + const byName = new Map(candidates.map((c) => [c.name, c.path])); + for (const a of accepted) { + copyFileSync(byName.get(a.name), join(stageDir, a.safeName)); + } + // stdout = accepted count (the workflow reads it to decide whether to post). + process.stdout.write(`${accepted.length}\n`); +} + +function commentCli(stageDir, rawBase, shortSha, runUrl, bodyFile) { + let files = []; + try { + files = readdirSync(stageDir); + } catch { + // Missing stage dir โ†’ empty preview body. + } + const body = buildComment(files, { rawBase, shortSha, runUrl }); + writeFileSync(bodyFile, body); + process.stderr.write(`Comment body: ${body.split('\n').length} lines.\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + const [cmd, ...rest] = process.argv.slice(2); + if (cmd === 'stage') { + stageCli(rest[0], rest[1], rest[2]); + } else if (cmd === 'comment') { + commentCli(rest[0], rest[1], rest[2], rest[3], rest[4]); + } else { + process.stderr.write(`unknown command: ${cmd ?? '(none)'}\n`); + process.exit(2); + } +} diff --git a/.github/scripts/web-shell-visuals-publish.test.mjs b/.github/scripts/web-shell-visuals-publish.test.mjs new file mode 100644 index 00000000000..1eb50b9e99f --- /dev/null +++ b/.github/scripts/web-shell-visuals-publish.test.mjs @@ -0,0 +1,152 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + buildComment, + classifyMagic, + MAX_BYTES, + MAX_CANDIDATES, + MAX_GIFS, + MAX_SCREENSHOTS, + sanitizeName, + selectImages, +} from './web-shell-visuals-publish.mjs'; + +const PNG = '89504e470d0a1a0a'; +const GIF89 = '474946383961'; +const GIF87 = '474946383761'; + +test('sanitizeName preserves the extension (regression: a trailing char broke the .png filter)', () => { + assert.equal( + sanitizeName('session-transcript-light.png'), + 'session-transcript-light.png', + ); + assert.match(sanitizeName('model-dialog-dark.png'), /\.png$/); + assert.match(sanitizeName('model-switch.gif'), /\.gif$/); + // Disallowed characters become `_`, but the extension is untouched. + assert.equal(sanitizeName('weird name!.png'), 'weird_name_.png'); + assert.equal(sanitizeName('trailing.png\n'), 'trailing.png_'); +}); + +test('classifyMagic accepts real PNG/GIF magic and rejects mismatches', () => { + assert.equal(classifyMagic('png', PNG), 'png'); + assert.equal(classifyMagic('gif', GIF89), 'gif'); + assert.equal(classifyMagic('gif', GIF87), 'gif'); + assert.equal(classifyMagic('png', GIF89), null); // GIF bytes in a .png + assert.equal(classifyMagic('gif', PNG), null); // PNG bytes in a .gif + assert.equal(classifyMagic('png', 'deadbeefdeadbeef'), null); + assert.equal(classifyMagic('svg', PNG), null); // unknown extension +}); + +test('selectImages accepts valid images and keeps safe, extension-correct names', () => { + const { accepted, warnings } = selectImages([ + { name: 'a-light.png', ext: 'png', size: 100, magic: PNG }, + { name: 'a-dark.png', ext: 'png', size: 100, magic: PNG }, + { name: 'model-switch.gif', ext: 'gif', size: 100, magic: GIF89 }, + ]); + assert.equal(accepted.length, 3); + assert.deepEqual( + accepted.map((a) => a.safeName), + ['a-light.png', 'a-dark.png', 'model-switch.gif'], + ); + assert.deepEqual(warnings, []); +}); + +test('selectImages skips oversized and magic-invalid files', () => { + let r = selectImages([ + { name: 'big.png', ext: 'png', size: MAX_BYTES + 1, magic: PNG }, + ]); + assert.equal(r.accepted.length, 0); + assert.ok(r.warnings.some((w) => w.includes('exceeds'))); + + r = selectImages([{ name: 'fake.png', ext: 'png', size: 10, magic: GIF89 }]); + assert.equal(r.accepted.length, 0); + assert.ok(r.warnings.some((w) => w.includes('not a valid'))); +}); + +test('selectImages caps screenshots per-kind WITHOUT starving gifs', () => { + const many = [ + ...Array.from({ length: MAX_SCREENSHOTS + 5 }, (_, i) => ({ + name: `s${i}-light.png`, + ext: 'png', + size: 10, + magic: PNG, + })), + { name: 'model-switch.gif', ext: 'gif', size: 10, magic: GIF89 }, + ]; + const { accepted } = selectImages(many); + const png = accepted.filter((a) => a.kind === 'png').length; + const gif = accepted.filter((a) => a.kind === 'gif').length; + assert.equal(png, MAX_SCREENSHOTS); // screenshots capped + assert.equal(gif, 1); // the gif survives the screenshot flood (not starved) +}); + +test('selectImages caps gifs per-kind', () => { + const gifs = Array.from({ length: MAX_GIFS + 3 }, (_, i) => ({ + name: `flow${i}.gif`, + ext: 'gif', + size: 10, + magic: GIF89, + })); + assert.equal(selectImages(gifs).accepted.length, MAX_GIFS); +}); + +test('selectImages bounds EXAMINED candidates so a junk flood cannot run forever', () => { + const flood = Array.from({ length: MAX_CANDIDATES + 50 }, (_, i) => ({ + name: `x${i}.png`, + ext: 'png', + size: 10, + magic: '00000000', // all invalid + })); + const { accepted, warnings } = selectImages(flood); + assert.equal(accepted.length, 0); + assert.ok(warnings.some((w) => w.includes('candidate files'))); +}); + +test('buildComment pairs light/dark, lists gifs, labels flows, escapes, links the run', () => { + const body = buildComment( + [ + 'session-transcript-light.png', + 'session-transcript-dark.png', + 'model-switch.gif', + ], + { + rawBase: 'https://raw.example/imgs', + shortSha: 'abc1234', + runUrl: 'https://run.example/1', + }, + ); + assert.match(body, //); + assert.match(body, /session-transcript-light\.png/); + assert.match(body, /session-transcript-dark\.png/); + assert.match(body, /model-switch\.gif/); + assert.match(body, /Open the slash menu and switch model/); // flow label + assert.match(body, /abc1234/); + assert.match(body, /https:\/\/run\.example\/1/); + // Exactly one screenshot row: the single view with light+dark paired. + const rows = body.split('\n').filter((l) => l.startsWith(' { + const body = buildComment(['toString.gif', 'constructor.gif'], { + rawBase: 'r', + }); + assert.doesNotMatch(body, /native code/); + assert.match(body, /\*\*ToString\*\*/); // falls back to the prettified filename +}); + +test('buildComment is empty-safe and marks a missing pair with an em dash', () => { + const empty = buildComment([], {}); + assert.match(empty, /web-shell visual preview/); + assert.doesNotMatch(empty, //); // no screenshots section + + const onlyLight = buildComment(['home-light.png'], { rawBase: 'r' }); + assert.match(onlyLight, /
โ€”<\/td>/); // the missing dark cell +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d6ce0c0cc9..8960ed493ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,7 +199,7 @@ jobs: node scripts/lint.js --setup node scripts/lint.js --actionlint node scripts/lint.js --yamllint - node --test .github/scripts/pr-safety-precheck.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/resolve-sandbox-image.test.mjs + node --test .github/scripts/pr-safety-precheck.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs # Self-hosted can't reach nodejs.org reliably; reuse the machine's Node. - name: 'Set up Node.js 22.x (hosted)' diff --git a/.github/workflows/web-shell-visuals-cleanup.yml b/.github/workflows/web-shell-visuals-cleanup.yml new file mode 100644 index 00000000000..ffee20ef376 --- /dev/null +++ b/.github/workflows/web-shell-visuals-cleanup.yml @@ -0,0 +1,34 @@ +name: 'Web-shell Visuals Cleanup' + +# When a PR closes, delete its per-PR visuals asset branch so the `pr-assets/*` +# refs (one per PR that ever produced a preview) don't accumulate without bound +# in the base repository. Runs in the base context (pull_request_target) but +# never checks out or runs PR code โ€” it only deletes one ref by name. +on: + pull_request_target: + types: + - 'closed' + +permissions: + contents: 'read' + +jobs: + delete-asset-branch: + if: "${{ github.repository == 'QwenLM/qwen-code' }}" + runs-on: 'ubuntu-latest' + timeout-minutes: 5 + steps: + - name: 'Delete the PR asset branch' + env: + # Deleting a ref needs contents:write, which the CI_BOT_PAT carries. + GH_TOKEN: '${{ secrets.CI_BOT_PAT }}' + PR_NUMBER: '${{ github.event.pull_request.number }}' + run: |- + set -euo pipefail + branch="pr-assets/web-shell-visuals-${PR_NUMBER}" + if gh api "repos/${GITHUB_REPOSITORY}/git/refs/heads/${branch}" >/dev/null 2>&1; then + gh api -X DELETE "repos/${GITHUB_REPOSITORY}/git/refs/heads/${branch}" + echo "Deleted ${branch}." + else + echo "No asset branch ${branch}; nothing to delete." + fi diff --git a/.github/workflows/web-shell-visuals-publish.yml b/.github/workflows/web-shell-visuals-publish.yml new file mode 100644 index 00000000000..fbf171cdbaf --- /dev/null +++ b/.github/workflows/web-shell-visuals-publish.yml @@ -0,0 +1,260 @@ +name: 'Web-shell Visuals Publish' + +# Privileged companion to `web-shell-visuals.yml`. It runs AFTER that workflow +# via workflow_run, so it executes in the base-repo context with a write token +# but NEVER checks out or runs PR code โ€” it only downloads the image artifact +# (opaque bytes), hosts it on the `pr-assets` branch, and posts an inline +# comment. This is the GitHub-recommended split for commenting on fork PRs with +# the results of untrusted-code execution. +on: + workflow_run: + workflows: + - 'Web-shell Visuals' + types: + - 'completed' + +# GITHUB_TOKEN only needs to read the triggering run's artifact; the branch push +# and PR comment are done with CI_BOT_PAT, which carries its own scope. +permissions: + actions: 'read' + +# Serialize publishes for the SAME PR (identified by its source repo + branch), +# since they force-push to that PR's own `pr-assets/web-shell-visuals-` +# branch; serializing means the force-push never has to reconcile a concurrent +# same-PR snapshot (a bounded retry below covers transient failures). Different +# PRs โ€” including forks that happen to share a branch name like `main` โ€” get +# distinct groups and publish in parallel. Never cancel an in-flight publish. +concurrency: + group: >- + web-shell-visuals-publish-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +defaults: + run: + shell: 'bash' + +jobs: + publish: + name: 'Publish web-shell visuals to the PR' + if: >- + github.repository == 'QwenLM/qwen-code' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + runs-on: 'ubuntu-latest' + timeout-minutes: 10 + steps: + # Trusted base-repo script (staging + comment builder). workflow_run + # checks out the default branch, never PR code. Sparse โ€” just the script. + - name: 'Checkout the publish script' + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + with: + sparse-checkout: '.github/scripts/web-shell-visuals-publish.mjs' + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: 'Download visuals artifact' + id: 'download' + continue-on-error: true + uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v5.0.0 + with: + name: 'web-shell-visuals' + run-id: '${{ github.event.workflow_run.id }}' + github-token: '${{ secrets.GITHUB_TOKEN }}' + path: 'visuals' + + - name: 'Publish visuals to the PR' + env: + GH_TOKEN: '${{ secrets.CI_BOT_PAT }}' + RUN_ID: '${{ github.event.workflow_run.id }}' + RUN_URL: '${{ github.event.workflow_run.html_url }}' + # Authenticated (NOT artifact-sourced) head SHA of the run that + # triggered this publish โ€” used to bind the artifact to its real PR. + RUN_HEAD_SHA: '${{ github.event.workflow_run.head_sha }}' + # Authenticated head repo + branch. A (repo, branch) pair maps to at + # most one open PR, so binding on it rejects a sibling PR that merely + # shares the same head commit SHA. + RUN_HEAD_REPO: '${{ github.event.workflow_run.head_repository.full_name }}' + RUN_HEAD_BRANCH: '${{ github.event.workflow_run.head_branch }}' + run: |- + set -euo pipefail + + ART='visuals' + if [ ! -d "${ART}" ]; then + echo "::notice::No visuals artifact was downloaded; nothing to publish." + exit 0 + fi + + # --- Validate + bind the untrusted PR number ---------------------- + # pr.txt is written by a job that ran PR code, so it is UNTRUSTED: a + # malicious PR could put a *victim* PR number here. Sanitize it, then + # BIND it to this run โ€” require the PR's current head SHA to equal the + # authenticated head SHA of the run that produced the artifact. A + # victim PR's head won't match, so the misdirection is rejected. + PR="$(tr -dc '0-9' < "${ART}/pr.txt" 2>/dev/null | head -c 12 || true)" + if [ -z "${PR}" ]; then + echo "::warning::Artifact has no valid PR number; aborting." + exit 0 + fi + + # Authenticated bot identity โ€” the dedup lookup filters on this so a + # participant who posts the marker can't hijack or redirect the update. + BOT_LOGIN="$(gh api user --jq '.login' 2>/dev/null || true)" + if [ -z "${BOT_LOGIN}" ]; then + echo "::warning::Could not resolve the bot identity; skipping to avoid mis-targeting a comment." + exit 0 + fi + + # Re-runnable gate: PR must be open AND its current head must equal this + # run's authenticated head SHA. This binds the untrusted artifact PR + # number to the real run, and is re-checked right before the comment + # write to close the validate->write TOCTOU window (a push/close during + # asset upload must not let a stale run post). + # Returns 0 = valid, 1 = genuinely invalid (closed / head mismatch), + # 2 = TRANSIENT API failure (empty after retries) โ€” distinguished so a + # blip fails loudly (re-triggerable) instead of silently skipping. + validate_pr() { + local j st hd rr rf attempt + j='' + for attempt in 1 2 3; do + j="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR}" 2>/dev/null || true)" + [ -n "${j}" ] && break + sleep 2 + done + [ -n "${j}" ] || { echo "::warning::Could not fetch PR #${PR} after retries."; return 2; } + st="$(jq -r '.state // empty' <<< "${j}")" + hd="$(jq -r '.head.sha // empty' <<< "${j}")" + rr="$(jq -r '.head.repo.full_name // empty' <<< "${j}")" + rf="$(jq -r '.head.ref // empty' <<< "${j}")" + [ "${st}" = "open" ] || { echo "::notice::PR #${PR} not open (state=${st:-unknown})."; return 1; } + { [ -n "${RUN_HEAD_SHA}" ] && [ "${hd}" = "${RUN_HEAD_SHA}" ]; } || { + echo "::warning::PR #${PR} head (${hd:-none}) != run head (${RUN_HEAD_SHA:-none}); refusing." + return 1 + } + # Also bind repo+branch: two open PRs can share a head SHA (same + # commit on different branches) but not the same (repo, branch). + { [ -n "${RUN_HEAD_REPO}" ] && [ "${rr}" = "${RUN_HEAD_REPO}" ] && [ "${rf}" = "${RUN_HEAD_BRANCH}" ]; } || { + echo "::warning::PR #${PR} head ${rr:-none}#${rf:-none} != run ${RUN_HEAD_REPO:-none}#${RUN_HEAD_BRANCH:-none}; refusing." + return 1 + } + return 0 + } + # Gate: proceed when valid; a genuine invalid state SKIPS quietly, but a + # transient failure EXITS 1 so the publish surfaces as re-triggerable. + gate() { + local rc + if validate_pr; then rc=0; else rc=$?; fi + [ "${rc}" -eq 0 ] && return 0 + if [ "${rc}" -eq 2 ]; then + echo "::error::Transient API failure validating PR #${PR}; failing so the publish can be re-triggered." + exit 1 + fi + echo "::notice::PR #${PR} no longer valid for publishing; skipping." + exit 0 + } + gate + SHORT_SHA="${RUN_HEAD_SHA:0:7}" + + # --- Stage + validate the images ---------------------------------- + # Delegated to the unit-tested script: magic-byte validation, filename + # sanitization, and the examined/accepted/size caps over untrusted + # artifact content live in web-shell-visuals-publish.mjs (covered by + # web-shell-visuals-publish.test.mjs). It prints the accepted count. + STAGE="${RUNNER_TEMP}/visuals-stage" + rm -rf "${STAGE}" + count="$(node .github/scripts/web-shell-visuals-publish.mjs stage "${ART}/screenshots" "${ART}/gifs" "${STAGE}")" + if [ -z "${count}" ] || [ "${count}" = "0" ]; then + echo "::notice::No valid images in the artifact; nothing to publish." + exit 0 + fi + echo "Staged ${count} image(s) for PR #${PR}." + + # Re-validate right before the force-push too: a close/new-head during + # download+staging must not force-push a stale snapshot (which would + # orphan the commit the existing comment still points at). + gate + + # --- Host on the PR's own pr-assets branch (bounded) -------------- + # Replace the branch with a SINGLE orphan snapshot each run and + # force-push, so untrusted PR content can't accumulate unbounded + # history in the base repo. The comment always points at the new + # immutable commit SHA; the previous snapshot becomes unreachable and + # is GC'd. Per-PR branch under the existing pr-assets/ convention + # (a bare `pr-assets` branch would D/F-conflict with pr-assets/*), and + # the per-PR concurrency group serializes same-PR runs so the + # force-push never drops a concurrent snapshot. + BRANCH="pr-assets/web-shell-visuals-${PR}" + AUTH_URL="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + WORK="${RUNNER_TEMP}/pr-assets" + rm -rf "${WORK}" + mkdir -p "${WORK}/imgs" + cp "${STAGE}"/* "${WORK}/imgs/" + cd "${WORK}" + git init -q + git config user.name 'qwen-code-bot' + git config user.email 'qwen-code-bot@users.noreply.github.com' + git checkout -q --orphan snapshot + git add imgs + git commit -q -m "web-shell visuals: PR #${PR} (run ${RUN_ID})" + # Bounded retry for transient push failures (network / brief lock). + # Same-PR runs are serialized by concurrency, so this never needs to + # reconcile a concurrent snapshot โ€” a plain retry suffices. + pushed=0 + for attempt in 1 2 3; do + if git push -q --force "${AUTH_URL}" "HEAD:${BRANCH}"; then + pushed=1 + break + fi + echo "::notice::force-push attempt ${attempt} failed; retrying." + sleep 2 + done + if [ "${pushed}" -ne 1 ]; then + echo "::error::Failed to push web-shell visuals to ${BRANCH} after retries." + exit 1 + fi + ASSET_SHA="$(git rev-parse HEAD)" + cd "${GITHUB_WORKSPACE}" + echo "web-shell visuals hosted on ${BRANCH} at ${ASSET_SHA}." + + # --- Build the comment body --------------------------------------- + # Delegated to the same unit-tested script (light/dark pairing, flow + # labels, and HTML escaping live in web-shell-visuals-publish.mjs). + BODY_FILE="${RUNNER_TEMP}/visuals-comment.md" + node .github/scripts/web-shell-visuals-publish.mjs comment \ + "${STAGE}" \ + "https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${ASSET_SHA}/imgs" \ + "${SHORT_SHA}" "${RUN_URL}" "${BODY_FILE}" + + # --- Post or update the PR comment -------------------------------- + # Dedup only against OUR OWN prior comment (bot author + marker), and + # only after a SUCCESSFUL listing โ€” a failed/partial list must not read + # as "no existing comment" (that would POST a duplicate on every failure). + MARKER='' + EXISTING='' + listed=0 + for attempt in 1 2 3; do + if resp="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --method GET --paginate -F per_page=100 2>/dev/null)"; then + EXISTING="$(jq -sr --arg m "${MARKER}" --arg u "${BOT_LOGIN}" \ + '[.[][] | select((.user.login == $u) and (.body | contains($m)))] | last | .id // empty' <<< "${resp}")" + listed=1 + break + fi + echo "::notice::comment lookup attempt ${attempt} failed; retrying." + sleep 2 + done + if [ "${listed}" -ne 1 ]; then + echo "::error::Could not list PR comments after retries; not posting (avoids duplicates)." + exit 1 + fi + + # Final TOCTOU gate: re-validate (open + head SHA + repo/branch) AFTER + # the paginated/retried lookup, immediately before the write โ€” a + # close or new head during validation/lookup must not post a stale one. + gate + + if [ -n "${EXISTING}" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING}" -F body=@"${BODY_FILE}" >/dev/null + echo "Updated existing web-shell visuals comment on PR #${PR}." + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@"${BODY_FILE}" >/dev/null + echo "Posted web-shell visuals comment on PR #${PR}." + fi diff --git a/.github/workflows/web-shell-visuals.yml b/.github/workflows/web-shell-visuals.yml new file mode 100644 index 00000000000..951021c0724 --- /dev/null +++ b/.github/workflows/web-shell-visuals.yml @@ -0,0 +1,183 @@ +name: 'Web-shell Visuals' + +# Auto-capture web-shell screenshots (light + dark) and short flow recordings +# for PRs that touch the web-shell UI, then hand the images to the companion +# `web-shell-visuals-publish.yml` (workflow_run) which posts them inline on the +# PR. +# +# Security model: this workflow BUILDS AND RENDERS untrusted PR code, so it runs +# on the `pull_request` trigger (fork PRs get a read-only token and NO secrets), +# on an ephemeral hosted runner, with `contents: read` and no secrets of its +# own. It produces only image/video bytes as an artifact. The privileged step +# that needs a write token โ€” pushing the images and commenting on the PR โ€” lives +# in the separate workflow_run workflow that never checks out PR code. +on: + pull_request: + branches: + - 'main' + - 'release/**' + # Matches the /tmux flow's web-shell surface: only the client UI, so + # doc/config-only PRs don't trigger a full build + render. + paths: + - 'packages/web-shell/client/**' + - 'packages/web-shell/package.json' + - 'packages/web-shell/vite.config.ts' + - 'packages/web-shell/playwright.visuals.config.ts' + # The visuals dev server aliases these runtime sources (see the `resolve. + # alias` block in packages/web-shell/vite.config.ts), so UI-affecting + # changes there must also refresh the preview. + - 'packages/webui/src/**' + - 'packages/sdk-typescript/src/**' + # The capture pipeline itself, so a workflow-only change is exercised. + - '.github/workflows/web-shell-visuals.yml' + +permissions: + contents: 'read' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.pull_request.number }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +jobs: + capture: + name: 'Capture web-shell visuals (ubuntu-latest, Node 22.x)' + if: "${{ github.repository == 'QwenLM/qwen-code' }}" + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + steps: + - name: 'Checkout PR head' + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + with: + ref: '${{ github.event.pull_request.head.sha }}' + fetch-depth: 1 + persist-credentials: false + + - name: 'Set up Node.js 22.x' + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + with: + node-version: '22.x' + cache: 'npm' + cache-dependency-path: 'package-lock.json' + registry-url: 'https://registry.npmjs.org/' + + - name: 'Configure npm for rate limiting' + run: |- + npm config set fetch-retry-mintimeout 20000 + npm config set fetch-retry-maxtimeout 120000 + npm config set fetch-retries 5 + npm config set fetch-timeout 300000 + + - name: 'Install dependencies' + run: 'npm ci --prefer-offline --no-audit --progress=false' + + - name: 'Install Playwright Chromium' + run: 'npx playwright install --with-deps chromium' + + - name: 'Choose web-shell Playwright port' + run: |- + port="$(node -e "const net=require('node:net');const server=net.createServer();server.listen(0,'127.0.0.1',()=>{console.log(server.address().port);server.close();});")" + echo "PLAYWRIGHT_PORT=${port}" >> "${GITHUB_ENV}" + echo "Using web-shell Playwright port ${port}" + + - name: 'Capture screenshots and flow recordings' + env: + WEB_SHELL_VISUALS_OUTPUT_DIR: '${{ runner.temp }}/web-shell-visuals' + run: 'npm run test:e2e:visuals --workspace=packages/web-shell' + + - name: 'Convert flow recordings to inline GIFs' + env: + OUT_DIR: '${{ runner.temp }}/web-shell-visuals' + run: |- + set -euo pipefail + if ! command -v ffmpeg >/dev/null 2>&1; then + echo "::warning::ffmpeg not found on the runner; skipping GIF conversion (raw .webm is still uploaded)." + exit 0 + fi + mkdir -p "${OUT_DIR}/gifs" + shopt -s nullglob + converted=0 + for webm in "${OUT_DIR}"/video/*.webm; do + name="$(basename "${webm%.webm}")" + gif="${OUT_DIR}/gifs/${name}.gif" + # -ss 1 trims the ~1s blank while the page loads. Two-pass palette + # (generate + apply) keeps the GIF sharp at a fraction of naive size. + if err="$(ffmpeg -y -ss 1 -i "${webm}" \ + -vf "fps=12,scale=960:-1:flags=lanczos,split[s0][s1];[s0]palettegen=stats_mode=diff[p];[s1][p]paletteuse=dither=bayer:bayer_scale=3" \ + "${gif}" 2>&1)"; then + echo "converted ${name}.webm -> gifs/${name}.gif ($(du -h "${gif}" | cut -f1))" + converted=$((converted + 1)) + else + # Surface ffmpeg's own diagnostic (codec/container/filter error) + # instead of a bare "failed", so a future breakage is actionable. + detail="$(printf '%s' "${err}" | tr '\n' ' ' | tail -c 400)" + echo "::warning::ffmpeg failed to convert ${name}.webm; skipping its GIF. ${detail}" + rm -f "${gif}" + fi + done + echo "GIFs produced: ${converted}" + + - name: 'Record PR metadata for the publish workflow' + env: + OUT_DIR: '${{ runner.temp }}/web-shell-visuals' + PR_NUMBER: '${{ github.event.pull_request.number }}' + run: |- + set -euo pipefail + # Ensure both dirs exist so the counts below are robust even when an + # upstream step produced none (e.g. no ffmpeg -> no gifs/). (The finds + # sit inside `echo "$(...)"`, so a missing dir wouldn't actually trip + # set -e โ€” echo masks it โ€” but create them for clarity all the same.) + mkdir -p "${OUT_DIR}/screenshots" "${OUT_DIR}/gifs" + # Bound artifact contents BEFORE upload: this job ran untrusted PR + # code, so drop oversized files and cap the count per directory โ€” + # otherwise a hostile spec could bloat the published artifact (which + # the privileged publisher downloads) or the retained video artifact. + MAX_FILE_BYTES=$((6 * 1024 * 1024)) + MAX_FILES=40 + for d in screenshots gifs video; do + dir="${OUT_DIR}/${d}" + [ -d "${dir}" ] || continue + find "${dir}" -maxdepth 1 -type f -size "+${MAX_FILE_BYTES}c" \ + -printf '::warning::dropping oversized artifact file %p\n' -delete || true + find "${dir}" -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort \ + | tail -n "+$((MAX_FILES + 1))" \ + | while IFS= read -r extra; do + echo "::warning::dropping excess artifact file ${d}/${extra}" + rm -f "${dir}/${extra}" + done + done + # PR number for the workflow_run publish job (which validates it). + # The head SHA is intentionally NOT shipped in the artifact: the + # publish job binds to the authenticated github.event.workflow_run + # .head_sha, and an artifact-sourced SHA would be untrusted. + printf '%s\n' "${PR_NUMBER}" > "${OUT_DIR}/pr.txt" + echo "Screenshots: $(find "${OUT_DIR}/screenshots" -name '*.png' | wc -l | tr -d ' ')" + echo "GIFs: $(find "${OUT_DIR}/gifs" -name '*.gif' | wc -l | tr -d ' ')" + + # The privileged publish workflow downloads THIS artifact, so keep the raw + # videos out of it: an untrusted PR could drop a multi-GB file under + # video/ and exhaust the publisher's bandwidth/disk/timeout. Screenshots + # and GIFs (which the publisher hosts) are size-capped again on that side. + - name: 'Upload web-shell visuals artifact (published)' + uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1 + with: + name: 'web-shell-visuals' + path: |- + ${{ runner.temp }}/web-shell-visuals/screenshots + ${{ runner.temp }}/web-shell-visuals/gifs + ${{ runner.temp }}/web-shell-visuals/pr.txt + if-no-files-found: 'warn' + retention-days: 7 + + # Raw recordings live in a SEPARATE artifact the publish workflow never + # downloads โ€” they're only the "full-resolution recordings" link target. + - name: 'Upload raw flow recordings' + uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1 + with: + name: 'web-shell-visuals-video' + path: '${{ runner.temp }}/web-shell-visuals/video' + if-no-files-found: 'ignore' + retention-days: 7 diff --git a/.gitignore b/.gitignore index cb894b60070..e09c78687b6 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,8 @@ junit.xml packages/*/coverage/ packages/web-shell/client/e2e/playwright-report/ packages/web-shell/client/e2e/test-results/ +packages/web-shell/client/e2e/visuals/output/ +packages/web-shell/client/e2e/visuals/.playwright/ # PR body draft pr_body.md diff --git a/packages/web-shell/client/e2e/visuals/constants.ts b/packages/web-shell/client/e2e/visuals/constants.ts new file mode 100644 index 00000000000..3f97743fdc3 --- /dev/null +++ b/packages/web-shell/client/e2e/visuals/constants.ts @@ -0,0 +1,12 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Fixed capture viewport โ€” the single source of truth shared by + * playwright.visuals.config.ts and the visuals harness/specs, so the value + * cannot drift between the config and what actually renders. + */ +export const VISUAL_VIEWPORT = { width: 1280, height: 800 } as const; diff --git a/packages/web-shell/client/e2e/visuals/flows.spec.ts b/packages/web-shell/client/e2e/visuals/flows.spec.ts new file mode 100644 index 00000000000..6999e060be0 --- /dev/null +++ b/packages/web-shell/client/e2e/visuals/flows.spec.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import { expect, test } from '@playwright/test'; +import { + assistantTextEvent, + createWebShellDaemonScenario, + turnCompleteEvent, +} from '../utils/mockDaemon'; +import { + beat, + fillComposer, + gotoSession, + installScenario, + recordFlow, + resolveBaseURL, +} from './harness'; + +// Flows are recorded as video (later converted to an inline GIF). Each flow +// runs in its own browser context (recordFlow) rather than the shared page +// fixture. Dark theme only โ€” the animation, not the palette, is the point. +// (No serial mode: a flake in one flow must not skip/lose the other's recording.) + +test('flow: open the slash menu and switch model', async ({ + browser, +}, testInfo) => { + const url = resolveBaseURL(testInfo); + await recordFlow(browser, url, 'model-switch', async (page) => { + const scenario = createWebShellDaemonScenario(); + const daemon = await installScenario(page, scenario, url); + await gotoSession(page, scenario, daemon, 'dark'); + await beat(page); + + const editor = page.locator('[data-web-shell-composer-editor] .cm-content'); + await editor.click(); + await page.keyboard.type('/'); + await expect(page.locator('[data-web-shell-slash-menu]')).toBeVisible(); + await beat(page); + await page.keyboard.type('model'); + await beat(page); + await page.locator('[data-web-shell-composer-submit]').click(); + + await expect(page.locator('[data-web-shell-model-dialog]')).toBeVisible(); + await beat(page); + await page + .locator('[data-web-shell-model-option][data-model-id="qwen-test-alt"]') + .click(); + await expect(page.locator('[data-web-shell-model-dialog]')).toHaveCount(0); + // Confirm the switch actually reached the daemon (not just the dialog UI + // closing), so the "switch model" GIF reflects a real model change. + await expect.poll(() => daemon.modelRequests().length).toBe(1); + await beat(page, 900); + }); +}); + +test('flow: submit a prompt and watch the reply stream in', async ({ + browser, +}, testInfo) => { + const url = resolveBaseURL(testInfo); + await recordFlow(browser, url, 'prompt-stream', async (page) => { + const scenario = createWebShellDaemonScenario(); + const daemon = await installScenario(page, scenario, url); + await gotoSession(page, scenario, daemon, 'dark'); + await beat(page); + + await fillComposer(page, 'Summarize the web-shell architecture.'); + await beat(page); + await page.locator('[data-web-shell-composer-submit]').click(); + + await expect.poll(() => daemon.promptRequests().length).toBe(1); + await daemon.sse.split( + assistantTextEvent('Streaming a reply from the mock daemon', { id: 10 }), + ); + await beat(page); + // The mock returns promptId 'prompt-e2e' for a prompt with no + // _meta.promptId, so complete the live turn with that id (clears the + // streaming spinner before the recording ends). + await daemon.sendEvent(turnCompleteEvent('prompt-e2e', { id: 11 })); + + await expect(page.locator('[data-web-shell-message-list]')).toContainText( + 'Streaming a reply', + ); + await beat(page, 900); + }); +}); + +// Guards the error-handling path in recordFlow: a throwing `drive` must +// surface its own error (not a masked video-save / context-close error), even +// though the video is saved best-effort. +test('flow: a drive error propagates instead of being masked', async ({ + browser, +}, testInfo) => { + const url = resolveBaseURL(testInfo); + await expect( + recordFlow(browser, url, 'drive-error', async () => { + throw new Error('drive-boom'); + }), + ).rejects.toThrow('drive-boom'); +}); diff --git a/packages/web-shell/client/e2e/visuals/harness.ts b/packages/web-shell/client/e2e/visuals/harness.ts new file mode 100644 index 00000000000..21a5dcca635 --- /dev/null +++ b/packages/web-shell/client/e2e/visuals/harness.ts @@ -0,0 +1,218 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import { mkdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + expect, + type Browser, + type BrowserContext, + type Page, + type TestInfo, +} from '@playwright/test'; +import { + installMockDaemon, + replayCompleteEvent, + type MockDaemonController, + type WebShellDaemonScenario, +} from '../utils/mockDaemon'; +import { VISUAL_VIEWPORT } from './constants'; + +export type VisualTheme = 'dark' | 'light'; + +export { VISUAL_VIEWPORT }; + +/** localStorage key the web-shell reads for its persisted theme (see index.html). */ +const THEME_STORAGE_KEY = 'qwen-code-web-shell-theme'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * Root the capture pipeline collects. The CI job points + * WEB_SHELL_VISUALS_OUTPUT_DIR at a temp dir; locally it defaults next to the + * spec so `npm run test:e2e:visuals` drops artifacts under the package. + */ +export const VISUALS_OUTPUT_DIR = process.env['WEB_SHELL_VISUALS_OUTPUT_DIR'] + ? resolve(process.env['WEB_SHELL_VISUALS_OUTPUT_DIR']) + : join(HERE, 'output'); + +export const SCREENSHOTS_DIR = join(VISUALS_OUTPUT_DIR, 'screenshots'); +export const VIDEO_DIR = join(VISUALS_OUTPUT_DIR, 'video'); +/** Playwright writes raw per-context videos here before we save them by name. */ +const VIDEO_RAW_DIR = join(VISUALS_OUTPUT_DIR, 'video-raw'); + +/** + * Force a theme deterministically: seed localStorage before any app code runs, + * then navigate with `?theme=`. `getInitialTheme()` consumes the query param on + * load (main.tsx strips it afterwards), and the localStorage seed is the + * belt-and-suspenders fallback if the app ever re-reads. + */ +async function primeTheme(page: Page, theme: VisualTheme): Promise { + await page.addInitScript( + ([key, value]) => { + try { + window.localStorage.setItem(key, value); + } catch { + // Private-mode / storage-disabled: the ?theme= param still applies. + } + }, + [THEME_STORAGE_KEY, theme] as const, + ); +} + +export function resolveBaseURL(testInfo: TestInfo): string { + const value = testInfo.project.use.baseURL; + if (!value) + throw new Error('Expected a Playwright baseURL to be configured.'); + return value; +} + +export async function installScenario( + page: Page, + scenario: WebShellDaemonScenario, + baseURL: string, +): Promise { + return installMockDaemon(page, scenario, { baseURL }); +} + +/** + * Navigate to a session in the requested theme and wait for the replayed + * transcript to settle. Asserts the theme actually took effect so a + * mislabelled light/dark capture fails loudly instead of shipping silently. + */ +export async function gotoSession( + page: Page, + scenario: WebShellDaemonScenario, + daemon: MockDaemonController, + theme: VisualTheme, +): Promise { + await primeTheme(page, theme); + await page.goto( + `/session/${encodeURIComponent(scenario.sessionId)}?theme=${theme}`, + ); + await expect(page.locator('[data-web-shell-root]')).toBeVisible(); + await expect(page.locator('html')).toHaveClass(new RegExp(`theme-${theme}`)); + await completeReplay( + page, + daemon, + scenario.sessionId, + scenario.events.length, + ); +} + +export async function completeReplay( + page: Page, + daemon: MockDaemonController, + sessionId?: string, + replayedCount = 0, +): Promise { + const connection = await daemon.sse.waitForConnection(sessionId); + await daemon.sendEvent( + replayCompleteEvent({ sessionId: connection.sessionId, replayedCount }), + ); + await expect(page.getByText('Loading...')).toHaveCount(0); +} + +export async function fillComposer(page: Page, text: string): Promise { + const editor = page.locator('[data-web-shell-composer-editor] .cm-content'); + await editor.click(); + await page.keyboard.press( + process.platform === 'darwin' ? 'Meta+A' : 'Control+A', + ); + await page.keyboard.type(text); +} + +export async function submitLocalCommand( + page: Page, + text: string, +): Promise { + await fillComposer(page, text); + await page.locator('[data-web-shell-composer-submit]').click(); +} + +/** Capture the current viewport to `/screenshots/.png`. */ +export async function captureScreenshot( + page: Page, + name: string, +): Promise { + mkdirSync(SCREENSHOTS_DIR, { recursive: true }); + await page.screenshot({ + path: join(SCREENSHOTS_DIR, `${name}.png`), + animations: 'disabled', + }); +} + +/** + * Record a continuous flow to `/video/.webm`. A dedicated + * browser context owns the video lifecycle so the file can be saved under a + * stable name (the CI job converts it to an inline GIF). + */ +export async function recordFlow( + browser: Browser, + baseURL: string, + name: string, + drive: (page: Page) => Promise, +): Promise { + mkdirSync(VIDEO_DIR, { recursive: true }); + mkdirSync(VIDEO_RAW_DIR, { recursive: true }); + const context: BrowserContext = await browser.newContext({ + baseURL, + viewport: { ...VISUAL_VIEWPORT }, + recordVideo: { dir: VIDEO_RAW_DIR, size: { ...VISUAL_VIEWPORT } }, + }); + let page: Page | undefined; + // Track failure with an explicit boolean, not the truthiness of the caught + // value: `throw undefined` / `throw null` / `Promise.reject()` must still mark + // the flow failed (otherwise an aborted flow would look passed). + let driveFailed = false; + let driveError: unknown; + try { + page = await context.newPage(); + await drive(page); + } catch (error) { + driveFailed = true; + driveError = error; + } finally { + try { + await context.close(); + } catch (closeError) { + // If the drive already failed, keep that original error (the close error + // is secondary). But if the drive SUCCEEDED, a close failure is a real + // problem (it can also leave the video unfinalized) โ€” promote it so the + // flow fails instead of masking it. + if (!driveFailed) { + driveFailed = true; + driveError = closeError; + } + } + } + + const video = page?.video(); + if (driveFailed) { + // The flow errored โ€” discard the partial recording rather than publishing a + // meaningless "failed flow" video into the artifact. + await video?.delete().catch(() => {}); + throw driveError; + } + + // Drive succeeded, so the recording IS the deliverable: let a save failure or + // a missing recording FAIL the flow (a silent pass with no .webm makes the + // downstream GIF-conversion step fail confusingly). Deleting the raw copy is + // best-effort. + if (!video) { + throw new Error( + `No video recorded for flow "${name}" โ€” recording did not start.`, + ); + } + await video.saveAs(join(VIDEO_DIR, `${name}.webm`)); + await video.delete().catch(() => {}); +} + +/** A short, human-readable pause so a recorded flow is legible as a GIF. */ +export async function beat(page: Page, ms = 650): Promise { + await page.waitForTimeout(ms); +} diff --git a/packages/web-shell/client/e2e/visuals/screenshots.spec.ts b/packages/web-shell/client/e2e/visuals/screenshots.spec.ts new file mode 100644 index 00000000000..50a5fa6c265 --- /dev/null +++ b/packages/web-shell/client/e2e/visuals/screenshots.spec.ts @@ -0,0 +1,122 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import { expect, test } from '@playwright/test'; +import { + assistantTextEvent, + createWebShellDaemonScenario, + permissionRequestEvent, + turnCompleteEvent, + userTextEvent, +} from '../utils/mockDaemon'; +import { + captureScreenshot, + fillComposer, + gotoSession, + installScenario, + resolveBaseURL, + submitLocalCommand, + VISUAL_VIEWPORT, + type VisualTheme, +} from './harness'; + +const THEMES: readonly VisualTheme[] = ['dark', 'light']; + +test.use({ viewport: { ...VISUAL_VIEWPORT } }); + +for (const theme of THEMES) { + test.describe(`web-shell screenshots (${theme})`, () => { + test(`session transcript`, async ({ page }, testInfo) => { + const scenario = createWebShellDaemonScenario({ + events: [ + userTextEvent('Render the web-shell so I can review the layout.', { + id: 1, + }), + assistantTextEvent( + 'Here is a **streamed** reply with a code block:\n\n```ts\nexport const greeting = "hello from web-shell";\n```', + { id: 2 }, + ), + turnCompleteEvent('prompt-visual', { id: 3 }), + ], + }); + const daemon = await installScenario( + page, + scenario, + resolveBaseURL(testInfo), + ); + await gotoSession(page, scenario, daemon, theme); + + await expect(page.locator('[data-web-shell-message-list]')).toContainText( + 'Here is a', + ); + // Shiki swaps in `
` asynchronously; wait for it so the
+      // code block is captured highlighted (not the plain fallback) every run.
+      await expect(
+        page.locator('[data-web-shell-message-list] pre.shiki').first(),
+      ).toBeVisible();
+      await captureScreenshot(page, `session-transcript-${theme}`);
+    });
+
+    test(`slash menu`, async ({ page }, testInfo) => {
+      const scenario = createWebShellDaemonScenario();
+      const daemon = await installScenario(
+        page,
+        scenario,
+        resolveBaseURL(testInfo),
+      );
+      await gotoSession(page, scenario, daemon, theme);
+
+      await fillComposer(page, '/');
+      await expect(page.locator('[data-web-shell-slash-menu]')).toBeVisible();
+      await captureScreenshot(page, `slash-menu-${theme}`);
+    });
+
+    test(`model dialog`, async ({ page }, testInfo) => {
+      const scenario = createWebShellDaemonScenario();
+      const daemon = await installScenario(
+        page,
+        scenario,
+        resolveBaseURL(testInfo),
+      );
+      await gotoSession(page, scenario, daemon, theme);
+
+      await submitLocalCommand(page, '/model');
+      await expect(page.locator('[data-web-shell-model-dialog]')).toBeVisible();
+      await captureScreenshot(page, `model-dialog-${theme}`);
+    });
+
+    test(`theme dialog`, async ({ page }, testInfo) => {
+      const scenario = createWebShellDaemonScenario();
+      const daemon = await installScenario(
+        page,
+        scenario,
+        resolveBaseURL(testInfo),
+      );
+      await gotoSession(page, scenario, daemon, theme);
+
+      await submitLocalCommand(page, '/theme');
+      await expect(page.locator('[data-web-shell-theme-dialog]')).toBeVisible();
+      await captureScreenshot(page, `theme-dialog-${theme}`);
+    });
+
+    test(`permission panel`, async ({ page }, testInfo) => {
+      const scenario = createWebShellDaemonScenario({
+        events: [permissionRequestEvent('perm-visual', { id: 1 })],
+      });
+      const daemon = await installScenario(
+        page,
+        scenario,
+        resolveBaseURL(testInfo),
+      );
+      await gotoSession(page, scenario, daemon, theme);
+
+      await expect(
+        page.locator('[data-web-shell-permission-panel]'),
+      ).toBeVisible();
+      await captureScreenshot(page, `permission-panel-${theme}`);
+    });
+  });
+}
diff --git a/packages/web-shell/package.json b/packages/web-shell/package.json
index 6245f614322..e1c379aeff7 100644
--- a/packages/web-shell/package.json
+++ b/packages/web-shell/package.json
@@ -28,6 +28,7 @@
     "test:coverage": "vitest run --config vitest.config.ts --coverage",
     "test:e2e:smoke": "playwright test --config playwright.config.ts --grep @smoke",
     "test:e2e": "playwright test --config playwright.config.ts",
+    "test:e2e:visuals": "playwright test --config playwright.visuals.config.ts",
     "test:e2e:report": "playwright show-report client/e2e/playwright-report",
     "verify": "npm run lint && npm run format:check && npm run typecheck && npm run test:ci"
   },
diff --git a/packages/web-shell/playwright.config.ts b/packages/web-shell/playwright.config.ts
index d872e8b956b..6ea39dbe099 100644
--- a/packages/web-shell/playwright.config.ts
+++ b/packages/web-shell/playwright.config.ts
@@ -6,6 +6,9 @@ const baseURL =
 
 export default defineConfig({
   testDir: './client/e2e',
+  // The visuals suite (screenshot/video capture) runs under its own
+  // playwright.visuals.config.ts; keep it out of the smoke/e2e runs.
+  testIgnore: '**/visuals/**',
   outputDir: './client/e2e/test-results',
   timeout: 60_000,
   expect: {
diff --git a/packages/web-shell/playwright.visuals.config.ts b/packages/web-shell/playwright.visuals.config.ts
new file mode 100644
index 00000000000..15db8ed2477
--- /dev/null
+++ b/packages/web-shell/playwright.visuals.config.ts
@@ -0,0 +1,51 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { defineConfig, devices } from '@playwright/test';
+import { VISUAL_VIEWPORT } from './client/e2e/visuals/constants';
+
+// Separate port default from playwright.config.ts (5174) so a stray base-config
+// dev server does not collide when both are run locally back to back.
+const port = Number(process.env['PLAYWRIGHT_PORT'] ?? 5175);
+const baseURL =
+  process.env['PLAYWRIGHT_BASE_URL'] ?? `http://127.0.0.1:${port}`;
+
+// Single source of truth for the capture viewport (shared with the harness).
+const viewport = { ...VISUAL_VIEWPORT };
+
+export default defineConfig({
+  testDir: './client/e2e/visuals',
+  outputDir: './client/e2e/visuals/.playwright',
+  // Retry in CI so one transient flake doesn't sink the whole preview (the job
+  // is all-or-nothing). Output filenames are deterministic, so a retry just
+  // overwrites the same PNG/webm. No auto-screenshots/traces we don't collect.
+  retries: process.env['CI'] ? 2 : 0,
+  timeout: 60_000,
+  expect: { timeout: 10_000 },
+  forbidOnly: !!process.env['CI'],
+  reporter: [['line']],
+  webServer: {
+    command: `npm run dev -- --host 127.0.0.1 --port ${port}`,
+    url: baseURL,
+    reuseExistingServer: !process.env['CI'],
+    timeout: 120_000,
+  },
+  use: {
+    baseURL,
+    viewport,
+    trace: 'off',
+    // Screenshots are captured explicitly; flows record video via their own
+    // browser context (client/e2e/visuals/harness.ts) for stable filenames.
+    screenshot: 'off',
+    video: 'off',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'], viewport },
+    },
+  ],
+});