From 64a703cade01e200b116288ac3b10208c2768684 Mon Sep 17 00:00:00 2001 From: Qwen-Coder Date: Mon, 31 Aug 2026 03:21:48 +0000 Subject: [PATCH] fix(ci): keep contended ECS E2E shards above the flat 60-minute ceiling (#10591) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The post-merge E2E run 33345905817 on cd5d5af2fb failed before any test result was reported: the per-commit issue body from main-ci-failure-issue.yml means no failed job log carried a vitest FAIL line, and the run's ~63-minute wall clock lines up with the flat timeout-minutes: 60 killing a shard still working on a contended shared ECS host. ci.yml's pool lanes already got extended, ECS-scoped ceilings for the same contention (#10552), and #10567 capped each E2E shard to one vitest fork on the pool; the Linux E2E job was the last shared-pool lane still on the pre-contention bound. Extend the ceiling to 90 on the ECS routing only — the same condition runs-on uses — so hosted fallbacks (forks and the kill-switch) keep the pre-contention 60 and a genuine hang there does not burn the extra 30 minutes. A lane-contract test evaluates the real timeout expression for both routings, the way ci-platform-lanes.test.js does, so a regression to an unconditional ceiling fails there instead of reading as a passing constant. --- .github/workflows/e2e.yml | 8 +++-- scripts/tests/e2e-workflow.test.js | 52 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1877d8bfedd..1f69f1ed8c0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -46,8 +46,12 @@ jobs: # Kill-switch: MAINTAINER_ECS_RUNNER_DISABLED. runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' # A wedged shard must free the pool runner well before GitHub's - # 360-minute default; 60 matches ci.yml's pool jobs. - timeout-minutes: 60 + # 360-minute default. The ceiling is extended on the ECS pool only, + # mirroring ci.yml's pool lanes: run 33345905817 (issue #10591) expired + # at the old flat 60 on a contended shared host before its log reported + # any test result. Hosted fallbacks keep the pre-contention ceiling so + # a genuine hang there does not burn the extra 30 minutes. + timeout-minutes: "${{ fromJSON((github.repository == 'QwenLM/qwen-code' && vars.MAINTAINER_ECS_RUNNER_DISABLED != 'true') && '90' || '60') }}" # Skip on fork PRs: forks have no access to repository secrets # (OPENAI_*, DOCKERHUB_*), so the matrix would fail unconditionally # and show misleading red status. Same-repo PRs run normally. diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index c8838dc4325..6d8b93fbfbe 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -94,4 +94,56 @@ describe('e2e workflow', () => { expect(runStep.run).toContain('mktemp -d /var/tmp/qwen-ci-XXXXXX'); expect(runStep.run).toContain('trap \'rm -rf "$TMPDIR"'); }); + + describe('job timeout lane contract', () => { + // Run 33345905817 (issue #10591) expired at the old flat 60-minute + // ceiling on a contended shared ECS host before its log reported any + // test result. The extended ceiling is + // scoped to the pool routing — the same condition `runs-on` uses: + // hosted fallbacks (forks and MAINTAINER_ECS_RUNNER_DISABLED=true) + // keep the pre-contention bound, so a genuine hang there does not + // burn the extra 30 minutes. Evaluate the real timeout expression for + // both routings instead of pinning a bare number — the same + // substitute-then-evaluate technique ci-platform-lanes.test.js uses — + // so a regression to an unconditional ceiling fails here instead of + // reading as a passing constant. + const timeoutMinutesOn = ({ repository, ecsDisabled }) => { + const expr = String(yml.jobs['e2e-test-linux']['timeout-minutes']) + .replace(/^\$\{\{\s*/, '') + .replace(/\s*\}\}$/, '') + .replace(/github\.repository/g, JSON.stringify(repository)) + .replace( + /vars\.MAINTAINER_ECS_RUNNER_DISABLED/g, + JSON.stringify(ecsDisabled), + ) + .replace(/fromJSON\(/g, '('); + if (/github\.|vars\.|needs\.|steps\.|fromJSON\(/.test(expr)) { + throw new Error( + `e2e-test-linux timeout carries a term this guard does not model: ${expr}`, + ); + } + return Number(new Function(`return (${expr});`)()); + }; + + it('keeps a contended ECS shard above the install/build/test budget', () => { + expect( + timeoutMinutesOn({ repository: 'QwenLM/qwen-code', ecsDisabled: '' }), + ).toBe(90); + }); + + it('keeps the hosted fallback on the pre-contention ceiling', () => { + expect( + timeoutMinutesOn({ + repository: 'some-fork/qwen-code', + ecsDisabled: '', + }), + ).toBe(60); + expect( + timeoutMinutesOn({ + repository: 'QwenLM/qwen-code', + ecsDisabled: 'true', + }), + ).toBe(60); + }); + }); });