Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 46 additions & 13 deletions .github/workflows/qwen-triage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -272,11 +272,11 @@ jobs:
verdict: '${{ steps.run.outputs.verdict || steps.prepare.outputs.verdict || steps.pr.outputs.verdict }}'
failure_phase: '${{ steps.prepare.outputs.failure_phase }}'
steps:
- name: 'Install tmux runner tools'
- name: 'Install PR resolver tools'
run: |-
set -euo pipefail
apt-get update
apt-get install -y --no-install-recommends ca-certificates curl git gnupg jq tmux util-linux
apt-get install -y --no-install-recommends ca-certificates curl git gnupg jq

install -d -m 755 /etc/apt/keyrings
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
Expand All @@ -287,10 +287,7 @@ jobs:
apt-get update
apt-get install -y --no-install-recommends gh

npm install -g '@qwen-code/qwen-code@latest'
qwen --version
gh --version
tmux -V

- name: 'Resolve PR and check state'
id: 'pr'
Expand Down Expand Up @@ -365,6 +362,20 @@ jobs:
ref: 'refs/pull/${{ steps.pr.outputs.pr_number }}/merge'
fetch-depth: 1

# Heavy tmux/qwen/util-linux setup is deferred here (gated on
# decision == 'run') so non-TUI PRs skip it. Any step added between
# 'Resolve PR and check state' and this one must not need tmux, qwen, or
# util-linux/runuser; those are installed here, not in resolver tools.
- name: 'Install tmux runner tools'
Comment thread
yiliang114 marked this conversation as resolved.
Resolved
if: "steps.pr.outputs.decision == 'run'"
run: |-
set -euo pipefail
apt-get install -y --no-install-recommends tmux util-linux

npm install -g '@qwen-code/qwen-code@latest'
qwen --version
tmux -V

- name: 'Install and build PR app'
id: 'prepare'
if: "steps.pr.outputs.decision == 'run'"
Expand Down Expand Up @@ -663,6 +674,9 @@ jobs:
"QWEN_CI_REAL_GH=${QWEN_CI_REAL_GH:-}"
"QWEN_CI_REAL_GIT=${QWEN_CI_REAL_GIT:-}"
)
if [ -n "${OPENAI_MODEL:-}" ]; then
QWEN_ENV+=("OPENAI_MODEL=$OPENAI_MODEL")
fi

set +e
timeout --kill-after=10s 20m runuser -u node -- env -i "${QWEN_ENV[@]}" "${QWEN_CMD[@]}" \
Expand Down Expand Up @@ -772,15 +786,23 @@ jobs:
# backtick fence (breakable by a long enough ``` run) cannot guarantee.
# Order matters: escape & first so the < / > entities aren't re-escaped.
emit_block() {
local summary="$1" file="$2" max="$3" content
local summary="$1" file="$2" max="$3" content truncated=''
[ -n "$file" ] && [ -f "$file" ] || return 0
content="$(head -c "$max" "$file")"
if [ "$(wc -c < "$file")" -gt "$max" ]; then
content="${content}"$'\n\n...truncated -- full log in the run artifacts.'
truncated=$'\n\n...truncated -- full log in the run artifacts.'
fi
if ! content="$(
set -o pipefail
{
head_status=0
head -c "$max" "$file" || head_status=$?
printf '%s' "$truncated"
exit "$head_status"
} | tr -d '\000' | sed -e 's/&/\&amp;/g' -e 's/</\&lt;/g' -e 's/>/\&gt;/g'
)"; then
echo "::warning::emit_block failed while rendering $summary; see run artifacts." >&2
content='<em>Log could not be rendered; see run artifacts.</em>'
fi
content="${content//&/&amp;}"
content="${content//</&lt;}"
content="${content//>/&gt;}"
printf '<details>\n<summary>%s</summary>\n\n<pre><code>\n' "$summary"
printf '%s\n' "$content"
printf '</code></pre>\n\n</details>\n\n'
Expand Down Expand Up @@ -819,10 +841,21 @@ jobs:
else
REPORT="$(find tmux-results -name 'report.md' 2>/dev/null | head -1 || true)"
TRANSCRIPT="$(find tmux-results -name 'tmux-readable-full.log' 2>/dev/null | head -1 || true)"
VERDICT_LABEL="${VERDICT:-unknown}"
if [ "${VERDICT:-}" = "infra-error" ]; then
VERDICT_LABEL='infra-error (crash/OOM)'
DESCRIPTION='The tmux test did not complete because the qwen process failed or was killed. This is not a pass/fail result for the affected flow; check runner resources and PR code for crashes or memory leaks.'
elif [ "${VERDICT:-}" = "timeout" ]; then
DESCRIPTION='The tmux test did not complete before the time limit. This is not a pass/fail result for the affected flow; see the workflow run and artifacts for details.'
elif [ "${VERDICT:-}" = "pass" ] || [ "${VERDICT:-}" = "fail" ]; then
DESCRIPTION='Launched the changed app in a real tmux session and exercised the affected flow.'
else
DESCRIPTION='The tmux test produced an unrecognized verdict, so this is not a pass/fail result for the affected flow. See the workflow run and artifacts for details.'
fi
{
printf '%s\n\n' '<!-- qwen-triage:tmux -->'
printf '**tmux real-user testing: %s** - [workflow run](%s)\n\n' "$VERDICT" "$RUN_URL"
printf 'Launched the changed app in a real tmux session and exercised the affected flow.\n\n'
printf '**tmux real-user testing: %s** - [workflow run](%s)\n\n' "$VERDICT_LABEL" "$RUN_URL"
printf '%s\n\n' "$DESCRIPTION"
emit_block 'E2E test report' "$REPORT" 20000
emit_block 'Full tmux transcript' "$TRANSCRIPT" 30000
printf '%s\n' '— _Qwen Code · tmux real-user testing_'
Expand Down
86 changes: 86 additions & 0 deletions scripts/tests/qwen-triage-workflow.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/**
* @license
* Copyright 2025 Qwen Team
* SPDX-License-Identifier: Apache-2.0
*/

import { readFileSync } from 'node:fs';
import { describe, expect, it } from 'vitest';

const workflow = readFileSync('.github/workflows/qwen-triage.yml', 'utf8');

function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

function step(name) {
const escaped = escapeRegExp(name);
const match = workflow.match(
new RegExp(
Comment thread
yiliang114 marked this conversation as resolved.
Resolved
`\\n\\s+- name:\\s*(['"])${escaped}\\1[\\s\\S]*?(?=\\n\\s+- name:\\s*['"]|\\n\\s{2}[a-zA-Z0-9_-]+:|$)`,
),
);
return match?.[0] ?? '';
}

describe('qwen-triage tmux workflow', () => {
it('escapes embedded tmux artifacts without bash pattern replacement ampersands', () => {
const postStep = step('Post tmux result comment');

expect(postStep).not.toContain('content="${content//&/&amp;}"');
expect(postStep).not.toContain('content="${content//</&lt;}"');
expect(postStep).not.toContain('content="${content//>/&gt;}"');
expect(postStep).toContain("sed -e 's/&/\\&amp;/g'");
expect(postStep).toContain("-e 's/</\\&lt;/g'");
expect(postStep).toContain("-e 's/>/\\&gt;/g'");
expect(postStep).toContain("tr -d '\\000'");
expect(postStep).toContain('Log could not be rendered');
expect(postStep).toContain('if ! content="$(');
expect(postStep).toContain('set -o pipefail');
expect(postStep).toContain('head_status=0');
expect(postStep).toContain('::warning::emit_block failed');
});

it('passes the selected OpenAI model into the app under tmux test', () => {
const runStep = step('Run tmux real-user testing');

expect(runStep).toContain('if [ -n "${OPENAI_MODEL:-}" ]; then');
expect(runStep).toContain('"OPENAI_MODEL=$OPENAI_MODEL"');
});

it('reports timeout and infra-error without claiming the flow was exercised', () => {
const postStep = step('Post tmux result comment');

expect(postStep).toContain('if [ "${VERDICT:-}" = "infra-error" ]; then');
expect(postStep).toContain('elif [ "${VERDICT:-}" = "timeout" ]; then');
expect(postStep).toContain(
'elif [ "${VERDICT:-}" = "pass" ] || [ "${VERDICT:-}" = "fail" ]; then',
);
expect(postStep).toContain('VERDICT_LABEL="${VERDICT:-unknown}"');
expect(postStep).toContain("VERDICT_LABEL='infra-error (crash/OOM)'");
expect(postStep).toContain('The tmux test did not complete');
expect(postStep).toContain(
'The tmux test did not complete before the time limit',
);
expect(postStep).toContain('not a pass/fail result');
expect(postStep).toContain('crashes or memory leaks');
expect(postStep).toContain(
'Launched the changed app in a real tmux session and exercised the affected flow.',
);
expect(postStep).toContain('produced an unrecognized verdict');
});

it('installs the heavy tmux test harness only for runnable PRs', () => {
const installStep = step('Install tmux runner tools');
const resolverStep = step('Install PR resolver tools');

expect(resolverStep).toContain('apt-get install');
expect(installStep).toContain('if: "steps.pr.outputs.decision == \'run\'"');
expect(resolverStep).not.toContain('tmux');
expect(resolverStep).not.toContain('npm install');
expect(resolverStep).not.toContain('qwen --version');
expect(
workflow.indexOf("- name: 'Resolve PR and check state'"),
).toBeLessThan(workflow.indexOf("- name: 'Install tmux runner tools'"));
});
});
Loading