diff --git a/.github/workflows/.size-baseline b/.github/workflows/.size-baseline index fe5abc3454f..422c4dae449 100644 --- a/.github/workflows/.size-baseline +++ b/.github/workflows/.size-baseline @@ -25,7 +25,7 @@ 31677 desktop-release.yml 2038 docs-page-action.yml 10005 dsw-swe-verified-release.yml -16534 e2e.yml +21437 e2e.yml 11394 finalize-release.yml 16647 live-host-release.yml 5950 live-host.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 36e2bae74f9..5bcd7f0acb7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -71,6 +71,13 @@ jobs: node-version: - '22.x' steps: + # The sandbox:none retry gate below budgets against the whole 60-minute + # job, not just the test step — setup spent ~29 of those minutes on two + # of the runners that motivated the retry. Recording the start here is + # what lets the gate see that spend. + - name: 'Record job start epoch' + run: 'echo "E2E_JOB_START_EPOCH=$(date +%s)" >> "${GITHUB_ENV}"' + # Fail-fast capability preflight (#9556): the docker leg needs a live # daemon, and without this check a missing one surfaces only after # npm ci + build. Runs on hosted and pool runners alike — both are @@ -256,7 +263,36 @@ jobs: if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' else - npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' + run_shard() { + npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' + } + # One bounded retry: pool runners' sandbox:none shards die under + # shared-host pressure with every test green and no vitest FAIL + # line — runs 33293739505, 33302550436 and 33317457036 failed + # this leg on three different hosts while sibling shards of the + # same runs passed, and the shard passes on re-run. The same + # transient class the sandbox-image build retry above covers + # (#10355). A deterministic test failure fails both attempts and + # keeps the shard red. The retry is budget-gated on job-elapsed + # time: two of those three runs died with ~10-12 of the 60 job + # minutes left, less than one shard-time, where a retry can only + # be cancelled mid-flight by timeout-minutes. The docker leg + # keeps no retry: two ~30min attempts would outrun the job's + # timeout-minutes. + run_shard || { + elapsed=$(( $(date +%s) - ${E2E_JOB_START_EPOCH:-0} )) + # Budget gate: a retry needs a full shard-time inside the job's + # timeout-minutes. 3600s minus a 25-minute reserve — the worst + # measured shard is ~21min under shared-host pressure — leaves + # 2100s; past that, GitHub cancels the retry mid-flight, burns + # pool time, and the failure degrades to a timeout signature. + if (( elapsed > 2100 )); then + echo "::error::sandbox:none shard failed on ${RUNNER_NAME:-this runner} after ${elapsed}s of the 3600s job budget — not enough left for a retry" + exit 1 + fi + echo "::warning::sandbox:none shard failed on ${RUNNER_NAME:-this runner} after ${elapsed}s; retrying once (transient shared-host pressure class)" + run_shard + } fi # Commit-qualified CI images are pruned under the exclusive daemon lock diff --git a/scripts/tests/e2e-shard-retry.test.js b/scripts/tests/e2e-shard-retry.test.js new file mode 100644 index 00000000000..d40c3f37c4c --- /dev/null +++ b/scripts/tests/e2e-shard-retry.test.js @@ -0,0 +1,166 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { execFileSync } from 'node:child_process'; +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { parse } from 'yaml'; + +// Executes the e2e workflow's 'Run E2E tests' script under GitHub Actions' +// default Linux step shell — `bash -e {0}`: the step has no `shell:` override +// and e2e.yml no `defaults:` block (both absences pinned in +// e2e-workflow.test.js), so no pipefail — with npm stubbed and the +// clock pinned, so the retry's exit-code semantics and the budget gate's +// exact threshold are witnessed by bash rather than by shape assertions +// alone. A failure-swallowing mutation (a group-level `|| true`) or a missing +// budget gate turns these red. Bash-driven, so it is excluded from the +// Windows lanes in vitest.config.ts. +describe('e2e workflow sandbox:none shard retry execution', () => { + const yml = parse(readFileSync('.github/workflows/e2e.yml', 'utf8')); + const steps = yml.jobs['e2e-test-linux'].steps; + const runStep = steps.find((step) => step.name === 'Run E2E tests'); + const script = runStep.run + .replaceAll('${{ matrix.sandbox }}', 'sandbox:none') + .replaceAll('${{ matrix.shard }}', '1/3'); + + function runStepScript({ failCalls, elapsedSeconds }) { + const dir = mkdtempSync(join(tmpdir(), 'qwen-e2e-retry-')); + try { + const callCountFile = join(dir, 'npm-call-count'); + const npmStub = join(dir, 'npm'); + writeFileSync(callCountFile, '0'); + writeFileSync( + npmStub, + [ + '#!/usr/bin/env bash', + 'call=$(( $(cat "$NPM_CALL_COUNT_FILE") + 1 ))', + 'printf "%s" "$call" > "$NPM_CALL_COUNT_FILE"', + 'for bad in $NPM_FAIL_CALLS; do', + ' [ "$call" = "$bad" ] && exit 1', + 'done', + 'exit 0', + ].join('\n'), + ); + chmodSync(npmStub, 0o755); + // The clock is pinned so the budget-gate boundary cases are exact: + // with the real clock, elapsed only ever grows between this process + // and the script's `date +%s`, which would race the 2100s threshold. + const now = Math.floor(Date.now() / 1000); + const dateStub = join(dir, 'date'); + writeFileSync( + dateStub, + ['#!/usr/bin/env bash', `printf '%s' '${now}'`].join('\n'), + ); + chmodSync(dateStub, 0o755); + const scriptFile = join(dir, 'run-e2e-tests.sh'); + writeFileSync(scriptFile, script); + let exitCode = 0; + let output = ''; + try { + output = execFileSync('bash', ['-e', scriptFile], { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + NPM_CALL_COUNT_FILE: callCountFile, + NPM_FAIL_CALLS: failCalls, + E2E_JOB_START_EPOCH: String(now - elapsedSeconds), + }, + encoding: 'utf8', + }); + } catch (err) { + exitCode = err.status; + output = `${err.stdout ?? ''}${err.stderr ?? ''}`; + } + return { + exitCode, + output, + npmCalls: Number(readFileSync(callCountFile, 'utf8')), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + } + + it('runs a green shard straight through with no retry noise', () => { + // The green first-attempt path needs its own witness: without one, an + // unconditional pre-gate side effect (a spurious ::warning:: before + // `run_shard || {`) ships with every other witness green. + const { exitCode, npmCalls, output } = runStepScript({ + failCalls: '', + elapsedSeconds: 1200, + }); + expect(npmCalls).toBe(1); + expect(output).not.toContain('::warning::'); + expect(output).not.toContain('::error::'); + expect(exitCode).toBe(0); + }); + + it('retries a shard that dies once and passes on the second attempt', () => { + // The transient class the retry exists for: first attempt dead, re-run + // green (runs 33293739505, 33302550436, 33317457036). + const { exitCode, npmCalls, output } = runStepScript({ + failCalls: '1', + elapsedSeconds: 1200, + }); + expect(npmCalls).toBe(2); + expect(output).toContain('::warning::'); + expect(exitCode).toBe(0); + }); + + it('keeps the step red when the shard fails both attempts', () => { + // A deterministic failure must not be absorbed: appending `|| true` to + // the brace group turns this red. + const { exitCode, npmCalls } = runStepScript({ + failCalls: '1 2', + elapsedSeconds: 1200, + }); + expect(npmCalls).toBe(2); + expect(exitCode).not.toBe(0); + }); + + it('retries at exactly the 2100s budget-gate threshold', () => { + // The gate admits a retry at elapsed <= 2100. Threshold mutations in + // either direction must not ship silently between the 1200/3000 probes. + const { exitCode, npmCalls, output } = runStepScript({ + failCalls: '1', + elapsedSeconds: 2100, + }); + expect(npmCalls).toBe(2); + expect(output).toContain('::warning::'); + expect(exitCode).toBe(0); + }); + + it('refuses a retry one second past the 2100s budget-gate threshold', () => { + const { exitCode, npmCalls, output } = runStepScript({ + failCalls: '1', + elapsedSeconds: 2101, + }); + expect(npmCalls).toBe(1); + expect(output).toContain('::error::'); + expect(exitCode).not.toBe(0); + }); + + it('fails fast when the remaining job budget cannot fit a retried shard', () => { + // 3000s spent of the 3600s job budget leaves 10 minutes — the exact + // shape of runs 33293739505 and 33302550436. An unconditional retry + // would call npm again and be cancelled mid-flight by timeout-minutes. + const { exitCode, npmCalls, output } = runStepScript({ + failCalls: '1', + elapsedSeconds: 3000, + }); + expect(npmCalls).toBe(1); + expect(output).toContain('::error::'); + expect(exitCode).not.toBe(0); + }); +}); diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index 8b1dded3351..2b01e16036e 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -100,6 +100,125 @@ describe('e2e workflow', () => { }); }); + describe('sandbox:none shard retry', () => { + // Runs 33293739505, 33302550436 and 33317457036 each failed the + // sandbox:none leg at the 'Run E2E tests' step with zero vitest FAIL + // lines — an all-green shard exiting red under shared-host pressure, + // with sibling shards of the same runs green and the shard green on + // re-run. The bounded retry absorbs one such transient death; a + // deterministic test failure fails both attempts and keeps the job red. + const steps = yml.jobs['e2e-test-linux'].steps; + const runStep = steps.find((step) => step.name === 'Run E2E tests'); + const epochStep = steps.find( + (step) => step.name === 'Record job start epoch', + ); + + it('records the job start epoch before the expensive setup steps', () => { + // The retry gate budgets against the whole 60-minute job; an epoch + // recorded at the test step would hide ~30 minutes of setup spend. + expect(epochStep.run).toContain( + 'echo "E2E_JOB_START_EPOCH=$(date +%s)" >> "${GITHUB_ENV}"', + ); + expect(steps.indexOf(epochStep)).toBeLessThan( + steps.indexOf( + steps.find((step) => step.name === 'Install dependencies'), + ), + ); + // An `if:` here would skip the record on one leg, where the gate's + // ${E2E_JOB_START_EPOCH:-0} fallback then always takes the ::error:: + // branch and the retry never fires on the leg it exists for. + expect(epochStep.if).toBeUndefined(); + }); + + it('wraps the sandbox:none shard command in a retryable function', () => { + expect(runStep.run).toContain('run_shard() {'); + }); + + it('retries the full shard command, shard and excludes included', () => { + // Everything after `--` is forwarded to vitest by the npm script, so + // shard and exclude coverage lives only in this argument list. The + // excludes are shared verbatim with the docker leg above. + expect(runStep.run).toContain( + "npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'", + ); + }); + + it('retries the sandbox:none shard exactly once', () => { + expect(runStep.run).toContain('run_shard || {'); + // Definition + first attempt + one retry: the second attempt's exit + // status is the step's, and a third attempt would burn pool time for + // nothing. + expect(runStep.run.match(/run_shard/g)).toHaveLength(3); + // End-anchored scope: the retry is the group's last command and the + // group is the script's last statement. A retry moved outside the + // `|| { ... }` would run unconditionally, re-running green shards too. + expect(runStep.run).toMatch(/run_shard\s*\n\s*\}\s*\n\s*fi\s*$/); + }); + + it('gates the retry on the remaining job budget', () => { + // The retried run_shard is reachable only behind an elapsed-time check + // that exits the step when the job cannot fit another shard. Shape + // only — bash itself witnesses the execution semantics in + // e2e-shard-retry.test.js. + const group = runStep.run.slice(runStep.run.indexOf('run_shard || {')); + expect(group).toMatch(/elapsed[\s\S]*exit 1[\s\S]*run_shard\s*\n\s*\}/); + }); + + it('pins the job timeout the budget-gate arithmetic is built on', () => { + // The 2100s threshold is 3600s minus a 25-minute reserve; the 3600s + // comes from this timeout. Editing one without the other mis-budgets + // the retry in both directions with every other witness green. + expect(yml.jobs['e2e-test-linux']['timeout-minutes']).toBe(60); + }); + + it('keeps the run step red when the shard stays red', () => { + // continue-on-error sits above the script exit code that every other + // witness observes: with it, two failing attempts still report green. + // Pin both levels — a job-level key computes the job conclusion green + // whatever the run step exits. The sandbox-image build step's + // deliberate step-level key and isolated-nightly's deliberate job-level + // key stay untouched — this pins the run step and e2e-test-linux only. + expect(runStep['continue-on-error']).toBeUndefined(); + expect(yml.jobs['e2e-test-linux']['continue-on-error']).toBeUndefined(); + }); + + it('keeps the default step shell the execution harness assumes', () => { + // e2e-shard-retry.test.js executes this step's script under `bash -e`, + // GitHub's default Linux step shell only while the step carries no + // `shell:` override and neither the workflow nor the job a `defaults:` + // block. Any of those switches the lane's shell semantics — explicit + // `bash` expands to `bash --noprofile --norc -e -o pipefail {0}` — + // while the harness keeps executing the old shell, so every execution + // witness stays green for a contract the lane no longer runs. Absence + // only: this pins e2e.yml, not workflows that deliberately set a shell. + expect(runStep.shell).toBeUndefined(); + expect(yml.defaults).toBeUndefined(); + expect(yml.jobs['e2e-test-linux'].defaults).toBeUndefined(); + }); + + it('does not retry the docker leg', () => { + // Two ~30min docker attempts would outrun the job's timeout-minutes. + expect(runStep.run.match(/QWEN_SANDBOX=docker vitest run/g)).toHaveLength( + 1, + ); + // Structure, not just count: wrapping the docker command in a + // function and calling it twice keeps the literal count at one. The + // docker leg defines its own helper functions, so match by brace + // depth rather than any earlier definition: every `${...}` brace in + // the script is balanced, leaving the command at depth zero unless + // something wraps it. + const commandIndex = runStep.run.indexOf( + 'QWEN_SANDBOX=docker vitest run', + ); + let depth = 0; + for (const ch of runStep.run.slice(0, commandIndex)) { + if (ch === '{') depth += 1; + if (ch === '}') depth -= 1; + } + expect(depth).toBe(0); + }); + }); + it('routes Linux E2E scratch files away from /tmp', () => { const runStep = yml.jobs['e2e-test-linux'].steps.find( (step) => step.name === 'Run E2E tests', diff --git a/scripts/tests/vitest.config.ts b/scripts/tests/vitest.config.ts index cbec8a7cafe..ad7d02d675b 100644 --- a/scripts/tests/vitest.config.ts +++ b/scripts/tests/vitest.config.ts @@ -18,6 +18,7 @@ export default defineConfig({ process.platform === 'win32' ? [ ...configDefaults.exclude, + 'scripts/tests/e2e-shard-retry.test.js', 'scripts/tests/pr-self-report-label.test.js', // Bash-driven workflow suites cannot run on Windows; pure // YAML-parse workflow suites still do.