fix(ci): retry transient sandbox:none E2E shard failures once (#10566)

.github/workflows/e2e.yml

@@ -71,6 +71,13 @@ jobs:
         node-version:
           - '22.x'
     steps:
+      # The sandbox:none retry gate below budgets against the whole 60-minute
+      # job, not just the test step — setup spent ~29 of those minutes on two
+      # of the runners that motivated the retry. Recording the start here is
+      # what lets the gate see that spend.
+      - name: 'Record job start epoch'
+        run: 'echo "E2E_JOB_START_EPOCH=$(date +%s)" >> "${GITHUB_ENV}"'
+
       # Fail-fast capability preflight (#9556): the docker leg needs a live
       # daemon, and without this check a missing one surfaces only after
       # npm ci + build. Runs on hosted and pool runners alike — both are
@@ -235,7 +242,36 @@ jobs:
           if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then
             npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
           else
-            npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
+            run_shard() {
+              npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
+            }
+            # One bounded retry: pool runners' sandbox:none shards die under
+            # shared-host pressure with every test green and no vitest FAIL
+            # line — runs 33293739505, 33302550436 and 33317457036 failed
+            # this leg on three different hosts while sibling shards of the
+            # same runs passed, and the shard passes on re-run. The same
+            # transient class the sandbox-image build retry above covers
+            # (#10355). A deterministic test failure fails both attempts and
+            # keeps the shard red. The retry is budget-gated on job-elapsed
+            # time: two of those three runs died with ~10-12 of the 60 job
+            # minutes left, less than one shard-time, where a retry can only
+            # be cancelled mid-flight by timeout-minutes. The docker leg
+            # keeps no retry: two ~30min attempts would outrun the job's
+            # timeout-minutes.
+            run_shard || {
+              elapsed=$(( $(date +%s) - ${E2E_JOB_START_EPOCH:-0} ))
+              # Budget gate: a retry needs a full shard-time inside the job's
+              # timeout-minutes. 3600s minus a 25-minute reserve — the worst
+              # measured shard is ~21min under shared-host pressure — leaves
+              # 2100s; past that, GitHub cancels the retry mid-flight, burns
+              # pool time, and the failure degrades to a timeout signature.
+              if (( elapsed > 2100 )); then
+                echo "::error::sandbox:none shard failed on ${RUNNER_NAME:-this runner} after ${elapsed}s of the 3600s job budget — not enough left for a retry"
+                exit 1
+              fi
+              echo "::warning::sandbox:none shard failed on ${RUNNER_NAME:-this runner} after ${elapsed}s; retrying once (transient shared-host pressure class)"
+              run_shard
+            }
           fi
 
       # The sandbox build retags the same image name every run, so on a

scripts/tests/e2e-shard-retry.test.js

@@ -0,0 +1,166 @@
+/**
+ * @license
+ * Copyright 2026 Qwen Team
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { execFileSync } from 'node:child_process';
+import {
+  chmodSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { parse } from 'yaml';
+
+// Executes the e2e workflow's 'Run E2E tests' script under GitHub Actions'
+// default Linux step shell — `bash -e {0}`: the step has no `shell:` override
+// and e2e.yml no `defaults:` block (both absences pinned in
+// e2e-workflow.test.js), so no pipefail — with npm stubbed and the
+// clock pinned, so the retry's exit-code semantics and the budget gate's
+// exact threshold are witnessed by bash rather than by shape assertions
+// alone. A failure-swallowing mutation (a group-level `|| true`) or a missing
+// budget gate turns these red. Bash-driven, so it is excluded from the
+// Windows lanes in vitest.config.ts.
+describe('e2e workflow sandbox:none shard retry execution', () => {
+  const yml = parse(readFileSync('.github/workflows/e2e.yml', 'utf8'));
+  const steps = yml.jobs['e2e-test-linux'].steps;
+  const runStep = steps.find((step) => step.name === 'Run E2E tests');
+  const script = runStep.run
+    .replaceAll('${{ matrix.sandbox }}', 'sandbox:none')
+    .replaceAll('${{ matrix.shard }}', '1/3');
+
+  function runStepScript({ failCalls, elapsedSeconds }) {
+    const dir = mkdtempSync(join(tmpdir(), 'qwen-e2e-retry-'));
+    try {
+      const callCountFile = join(dir, 'npm-call-count');
+      const npmStub = join(dir, 'npm');
+      writeFileSync(callCountFile, '0');
+      writeFileSync(
+        npmStub,
+        [
+          '#!/usr/bin/env bash',
+          'call=$(( $(cat "$NPM_CALL_COUNT_FILE") + 1 ))',
+          'printf "%s" "$call" > "$NPM_CALL_COUNT_FILE"',
+          'for bad in $NPM_FAIL_CALLS; do',
+          '  [ "$call" = "$bad" ] && exit 1',
+          'done',
+          'exit 0',
+        ].join('\n'),
+      );
+      chmodSync(npmStub, 0o755);
+      // The clock is pinned so the budget-gate boundary cases are exact:
+      // with the real clock, elapsed only ever grows between this process
+      // and the script's `date +%s`, which would race the 2100s threshold.
+      const now = Math.floor(Date.now() / 1000);
+      const dateStub = join(dir, 'date');
+      writeFileSync(
+        dateStub,
+        ['#!/usr/bin/env bash', `printf '%s' '${now}'`].join('\n'),
+      );
+      chmodSync(dateStub, 0o755);
+      const scriptFile = join(dir, 'run-e2e-tests.sh');
+      writeFileSync(scriptFile, script);
+      let exitCode = 0;
+      let output = '';
+      try {
+        output = execFileSync('bash', ['-e', scriptFile], {
+          env: {
+            ...process.env,
+            PATH: `${dir}:${process.env.PATH}`,
+            NPM_CALL_COUNT_FILE: callCountFile,
+            NPM_FAIL_CALLS: failCalls,
+            E2E_JOB_START_EPOCH: String(now - elapsedSeconds),
+          },
+          encoding: 'utf8',
+        });
+      } catch (err) {
+        exitCode = err.status;
+        output = `${err.stdout ?? ''}${err.stderr ?? ''}`;
+      }
+      return {
+        exitCode,
+        output,
+        npmCalls: Number(readFileSync(callCountFile, 'utf8')),
+      };
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  }
+
+  it('runs a green shard straight through with no retry noise', () => {
+    // The green first-attempt path needs its own witness: without one, an
+    // unconditional pre-gate side effect (a spurious ::warning:: before
+    // `run_shard || {`) ships with every other witness green.
+    const { exitCode, npmCalls, output } = runStepScript({
+      failCalls: '',
+      elapsedSeconds: 1200,
+    });
+    expect(npmCalls).toBe(1);
+    expect(output).not.toContain('::warning::');
+    expect(output).not.toContain('::error::');
+    expect(exitCode).toBe(0);
+  });
+
+  it('retries a shard that dies once and passes on the second attempt', () => {
+    // The transient class the retry exists for: first attempt dead, re-run
+    // green (runs 33293739505, 33302550436, 33317457036).
+    const { exitCode, npmCalls, output } = runStepScript({
+      failCalls: '1',
+      elapsedSeconds: 1200,
+    });
+    expect(npmCalls).toBe(2);
+    expect(output).toContain('::warning::');
+    expect(exitCode).toBe(0);
+  });
+
+  it('keeps the step red when the shard fails both attempts', () => {
+    // A deterministic failure must not be absorbed: appending `|| true` to
+    // the brace group turns this red.
+    const { exitCode, npmCalls } = runStepScript({
+      failCalls: '1 2',
+      elapsedSeconds: 1200,
+    });
+    expect(npmCalls).toBe(2);
+    expect(exitCode).not.toBe(0);
+  });
+
+  it('retries at exactly the 2100s budget-gate threshold', () => {
+    // The gate admits a retry at elapsed <= 2100. Threshold mutations in
+    // either direction must not ship silently between the 1200/3000 probes.
+    const { exitCode, npmCalls, output } = runStepScript({
+      failCalls: '1',
+      elapsedSeconds: 2100,
+    });
+    expect(npmCalls).toBe(2);
+    expect(output).toContain('::warning::');
+    expect(exitCode).toBe(0);
+  });
+
+  it('refuses a retry one second past the 2100s budget-gate threshold', () => {
+    const { exitCode, npmCalls, output } = runStepScript({
+      failCalls: '1',
+      elapsedSeconds: 2101,
+    });
+    expect(npmCalls).toBe(1);
+    expect(output).toContain('::error::');
+    expect(exitCode).not.toBe(0);
+  });
+
+  it('fails fast when the remaining job budget cannot fit a retried shard', () => {
+    // 3000s spent of the 3600s job budget leaves 10 minutes — the exact
+    // shape of runs 33293739505 and 33302550436. An unconditional retry
+    // would call npm again and be cancelled mid-flight by timeout-minutes.
+    const { exitCode, npmCalls, output } = runStepScript({
+      failCalls: '1',
+      elapsedSeconds: 3000,
+    });
+    expect(npmCalls).toBe(1);
+    expect(output).toContain('::error::');
+    expect(exitCode).not.toBe(0);
+  });
+});

scripts/tests/e2e-workflow.test.js

@@ -87,6 +87,116 @@ describe('e2e workflow', () => {
     });
   });
 
+  describe('sandbox:none shard retry', () => {
+    // Runs 33293739505, 33302550436 and 33317457036 each failed the
+    // sandbox:none leg at the 'Run E2E tests' step with zero vitest FAIL
+    // lines — an all-green shard exiting red under shared-host pressure,
+    // with sibling shards of the same runs green and the shard green on
+    // re-run. The bounded retry absorbs one such transient death; a
+    // deterministic test failure fails both attempts and keeps the job red.
+    const steps = yml.jobs['e2e-test-linux'].steps;
+    const runStep = steps.find((step) => step.name === 'Run E2E tests');
+    const epochStep = steps.find(
+      (step) => step.name === 'Record job start epoch',
+    );
+
+    it('records the job start epoch before the expensive setup steps', () => {
+      // The retry gate budgets against the whole 60-minute job; an epoch
+      // recorded at the test step would hide ~30 minutes of setup spend.
+      expect(epochStep.run).toContain(
+        'echo "E2E_JOB_START_EPOCH=$(date +%s)" >> "${GITHUB_ENV}"',
+      );
+      expect(steps.indexOf(epochStep)).toBeLessThan(
+        steps.indexOf(
+          steps.find((step) => step.name === 'Install dependencies'),
+        ),
+      );
+      // An `if:` here would skip the record on one leg, where the gate's
+      // ${E2E_JOB_START_EPOCH:-0} fallback then always takes the ::error::
+      // branch and the retry never fires on the leg it exists for.
+      expect(epochStep.if).toBeUndefined();
+    });
+
+    it('wraps the sandbox:none shard command in a retryable function', () => {
+      expect(runStep.run).toContain('run_shard() {');
+    });
+
+    it('retries the full shard command, shard and excludes included', () => {
+      // Everything after `--` is forwarded to vitest by the npm script, so
+      // shard and exclude coverage lives only in this argument list. The
+      // excludes are shared verbatim with the docker leg above.
+      expect(runStep.run).toContain(
+        "npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'",
+      );
+    });
+
+    it('retries the sandbox:none shard exactly once', () => {
+      expect(runStep.run).toContain('run_shard || {');
+      // Definition + first attempt + one retry: the second attempt's exit
+      // status is the step's, and a third attempt would burn pool time for
+      // nothing.
+      expect(runStep.run.match(/run_shard/g)).toHaveLength(3);
+      // End-anchored scope: the retry is the group's last command and the
+      // group is the script's last statement. A retry moved outside the
+      // `|| { ... }` would run unconditionally, re-running green shards too.
+      expect(runStep.run).toMatch(/run_shard\s*\n\s*\}\s*\n\s*fi\s*$/);
+    });
+
+    it('gates the retry on the remaining job budget', () => {
+      // The retried run_shard is reachable only behind an elapsed-time check
+      // that exits the step when the job cannot fit another shard. Shape
+      // only — bash itself witnesses the execution semantics in
+      // e2e-shard-retry.test.js.
+      const group = runStep.run.slice(runStep.run.indexOf('run_shard || {'));
+      expect(group).toMatch(/elapsed[\s\S]*exit 1[\s\S]*run_shard\s*\n\s*\}/);
+    });
+
+    it('pins the job timeout the budget-gate arithmetic is built on', () => {
+      // The 2100s threshold is 3600s minus a 25-minute reserve; the 3600s
+      // comes from this timeout. Editing one without the other mis-budgets
+      // the retry in both directions with every other witness green.
+      expect(yml.jobs['e2e-test-linux']['timeout-minutes']).toBe(60);
+    });
+
+    it('keeps the run step red when the shard stays red', () => {
+      // continue-on-error sits above the script exit code that every other
+      // witness observes: with it, two failing attempts still report green.
+      // Pin both levels — a job-level key computes the job conclusion green
+      // whatever the run step exits. The sandbox-image build step's
+      // deliberate step-level key and isolated-nightly's deliberate job-level
+      // key stay untouched — this pins the run step and e2e-test-linux only.
+      expect(runStep['continue-on-error']).toBeUndefined();
+      expect(yml.jobs['e2e-test-linux']['continue-on-error']).toBeUndefined();
+    });
+
+    it('keeps the default step shell the execution harness assumes', () => {
+      // e2e-shard-retry.test.js executes this step's script under `bash -e`,
+      // GitHub's default Linux step shell only while the step carries no
+      // `shell:` override and the workflow no `defaults:` block. Either one
+      // switches the lane's shell semantics — explicit `bash` expands to
+      // `bash --noprofile --norc -e -o pipefail {0}` — while the harness
+      // keeps executing the old shell, so every execution witness stays
+      // green for a contract the lane no longer runs. Absence only: this
+      // pins e2e.yml, not workflows that deliberately set a shell.
+      expect(runStep.shell).toBeUndefined();
+      expect(yml.defaults).toBeUndefined();
+    });
+
+    it('does not retry the docker leg', () => {
+      // Two ~30min docker attempts would outrun the job's timeout-minutes.
+      expect(runStep.run.match(/QWEN_SANDBOX=docker vitest run/g)).toHaveLength(
+        1,
+      );
+      // Structure, not just count: wrapping the docker command in a
+      // function and calling it twice keeps the literal count at one. The
+      // match spans inner braces so a body with its own `}` (a preflight
+      // brace group folded into the wrapper) cannot hide the wrapping.
+      expect(runStep.run).not.toMatch(
+        /[A-Za-z_]+\(\)\s*\{.*?QWEN_SANDBOX=docker/s,
+      );
+    });
+  });
+
   it('routes Linux E2E scratch files away from /tmp', () => {
     const runStep = yml.jobs['e2e-test-linux'].steps.find(
       (step) => step.name === 'Run E2E tests',

scripts/tests/vitest.config.ts

@@ -18,6 +18,7 @@ export default defineConfig({
       process.platform === 'win32'
         ? [
             ...configDefaults.exclude,
+            'scripts/tests/e2e-shard-retry.test.js',
             'scripts/tests/pr-self-report-label.test.js',
             // Bash-driven workflow suites cannot run on Windows; pure
             // YAML-parse workflow suites still do.