diff --git a/.github/workflows/.size-baseline b/.github/workflows/.size-baseline index 15a35943dd6..ccbd67f7189 100644 --- a/.github/workflows/.size-baseline +++ b/.github/workflows/.size-baseline @@ -24,6 +24,7 @@ 31677 desktop-release.yml 2038 docs-page-action.yml 10005 dsw-swe-verified-release.yml +2061 e2e-flaky-rerun.yml 12340 e2e.yml 11394 finalize-release.yml 16647 live-host-release.yml diff --git a/.github/workflows/e2e-flaky-rerun.yml b/.github/workflows/e2e-flaky-rerun.yml new file mode 100644 index 00000000000..52ac00496b3 --- /dev/null +++ b/.github/workflows/e2e-flaky-rerun.yml @@ -0,0 +1,46 @@ +# Re-runs the failed lanes of a post-merge E2E run once before leaving main +# red. The recurring 'Main CI failed: E2E Tests' issues are overwhelmingly +# transient model-endpoint connectivity flakes, not regressions: run 32966197395 +# (2026-08-26) failed a Linux shard with `connect ETIMEDOUT` to the endpoint +# and passed the identical tree on attempt 2. One automatic re-run turns those +# green again; a failure that survives both attempts stays red and still +# raises the Main CI Failure issue. + +name: 'E2E Flaky Rerun' + +on: + workflow_run: + workflows: ['E2E Tests'] + types: ['completed'] + branches: ['main'] + +permissions: + contents: 'read' + +jobs: + rerun-failed-lanes: + name: 'Re-run failed E2E lanes (once)' + # First-attempt push runs only: the attempt-2 completion event carries + # run_attempt == 2, so the re-run cannot re-trigger itself, and a failure + # that survives both attempts reaches the issue bot untouched. Scheduled + # and dispatched runs stay manual: they exist to surface flakiness, not to + # gate main. + if: |- + ${{ github.repository == 'QwenLM/qwen-code' && github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.run_attempt == 1 }} + # Deliberately hosted, NOT the ECS pool — same reasoning as + # main-ci-failure-issue.yml#file_issue: a recovery job must not queue + # behind the infrastructure it is recovering from. + runs-on: 'ubuntu-latest' + timeout-minutes: 5 + steps: + # Checks out nothing and runs no repository code: the job holding the + # bot PAT only calls the Actions API. + - name: 'Re-run the failed jobs' + env: + # GitHub writes use CI_BOT_PAT; keep the generated GITHUB_TOKEN + # read-only. + GH_TOKEN: '${{ secrets.CI_BOT_PAT }}' + REPO: '${{ github.repository }}' + RUN_ID: '${{ github.event.workflow_run.id }}' + run: |- + gh run rerun --failed "${RUN_ID}" --repo "${REPO}" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c3bf7c069a7..aaeb05f4055 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -38,6 +38,11 @@ jobs: e2e-test-linux: name: 'E2E Test (Linux) - ${{ matrix.sandbox }} - shard ${{ matrix.shard }}' runs-on: 'ubuntu-latest' + # Healthy lanes finish in ~17min. Without a cap, a lane that loses + # connectivity to the model endpoint burns every test's full 300s timeout + # (plus retries) before reporting — the 2026-08-26 run took over 4 hours + # to fail. Cap the lane so a hung endpoint surfaces within the hour. + timeout-minutes: 60 # Skip on fork PRs: forks have no access to repository secrets # (OPENAI_*, DOCKERHUB_*), so the matrix would fail unconditionally # and show misleading red status. Same-repo PRs run normally. @@ -150,6 +155,8 @@ jobs: e2e-test-macos: name: 'E2E Test - macOS - shard ${{ matrix.shard }}' runs-on: 'macos-latest' + # Same cap rationale as e2e-test-linux. + timeout-minutes: 60 # Skip on fork PRs (no secrets) — see e2e-test-linux above. if: |- ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -206,6 +213,8 @@ jobs: isolated-nightly: name: '${{ matrix.label }} (nightly)' runs-on: 'ubuntu-latest' + # Same cap rationale as e2e-test-linux. + timeout-minutes: 60 if: |- ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} continue-on-error: true @@ -264,6 +273,8 @@ jobs: web-shell-browser-regression: name: 'web-shell Browser Regression' runs-on: 'ubuntu-latest' + # Same cap rationale as e2e-test-linux. + timeout-minutes: 60 if: |- ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} steps: diff --git a/scripts/tests/e2e-flaky-rerun-workflow.test.js b/scripts/tests/e2e-flaky-rerun-workflow.test.js new file mode 100644 index 00000000000..48d54b0cc1b --- /dev/null +++ b/scripts/tests/e2e-flaky-rerun-workflow.test.js @@ -0,0 +1,53 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { parse } from 'yaml'; + +describe('e2e flaky rerun workflow', () => { + const workflow = readFileSync( + '.github/workflows/e2e-flaky-rerun.yml', + 'utf8', + ); + const yml = parse(workflow); + const job = yml.jobs['rerun-failed-lanes']; + + it('watches completed E2E Tests runs on main only', () => { + expect(workflow).toContain('workflow_run:'); + expect(yml.on.workflow_run.workflows).toEqual(['E2E Tests']); + expect(yml.on.workflow_run.types).toEqual(['completed']); + expect(yml.on.workflow_run.branches).toEqual(['main']); + }); + + it('retries only first-attempt failed push runs, exactly once', () => { + // The attempt-2 completion event carries run_attempt == 2: without that + // guard the re-run could re-trigger itself forever, and with it a failure + // that survives both attempts reaches the issue bot untouched. Scheduled + // and dispatched runs exist to surface flakiness, not to gate main, so + // push is the only event that earns a retry. Pin the whole clause so a + // connective mutation fails here. + expect(job.if).toContain("github.repository == 'QwenLM/qwen-code'"); + expect(job.if).toContain( + "github.event.workflow_run.conclusion == 'failure'", + ); + expect(job.if).toContain("github.event.workflow_run.event == 'push'"); + expect(job.if).toContain("github.event.workflow_run.head_branch == 'main'"); + expect(job.if).toContain('github.event.workflow_run.run_attempt == 1'); + }); + + it('re-runs only the failed jobs with the bot PAT, not repository code', () => { + // The job holding the PAT must stay a pure Actions-API call: no checkout, + // no repository code. `--failed` keeps the green lanes' results so the + // re-run covers exactly the lanes that flaked. + expect(workflow).not.toContain('actions/checkout'); + expect(workflow).toContain("GH_TOKEN: '${{ secrets.CI_BOT_PAT }}'"); + expect(workflow).toContain( + 'gh run rerun --failed "${RUN_ID}" --repo "${REPO}"', + ); + expect(workflow).toContain("RUN_ID: '${{ github.event.workflow_run.id }}'"); + }); +}); diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index 4331ada05d5..33e761e2c7a 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -32,4 +32,20 @@ describe('e2e workflow', () => { expect(group).toContain('github.event_name'); expect(group).toContain('github.head_ref || github.ref_name'); }); + + it('caps every lane so a hung model endpoint cannot stall the run', () => { + // A lane that loses connectivity to the model endpoint burns every test's + // full 300s timeout before reporting — the 2026-08-26 run took over + // 4 hours to fail while healthy lanes finish in ~17 minutes. Dropping a + // cap would silently reintroduce that, so every lane's timeout is pinned. + const lanes = [ + 'e2e-test-linux', + 'e2e-test-macos', + 'isolated-nightly', + 'web-shell-browser-regression', + ]; + for (const lane of lanes) { + expect(yml.jobs[lane]['timeout-minutes'], lane).toBeGreaterThan(0); + } + }); });