From 659a61ab4d0706386571179d55f01ab54a22e312 Mon Sep 17 00:00:00 2001 From: yiliang114 <1204183885@qq.com> Date: Wed, 2 Sep 2026 12:19:33 +0900 Subject: [PATCH 1/4] fix(ci): keep release validation off contended ECS hosts Pin every release validation job to the hk4 host label, move the nightly and preview crons out of the pool's peak hours and apart from each other, give the web-shell Vitest project the same shared-ECS timeout ceiling as cli/core, and retry timing-only workspace test failures twice. Claude-Session: https://claude.ai/code/session_01AWWgJEqafyAT1Mc75T8N7h --- .github/actionlint.yaml | 1 + .github/workflows/release.yml | 41 ++++++++++++++--------- docs/developers/development/npm.md | 8 ++--- packages/web-shell/vitest.config.ts | 8 +++++ scripts/tests/release-workflow.test.js | 18 ++++++++-- scripts/tests/unit-vitest-configs.test.ts | 12 +++++-- 6 files changed, 64 insertions(+), 24 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 9e38b8a0d73..71a0c4d2539 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,7 @@ self-hosted-runner: labels: - 'ecs-qwen' + - 'ecs-qwen-hk4-host' - 'ecs-win' - 'ecs-update-hk-1' - 'ecs-update-hk-2' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e46f52cf5b6..3fb2f62c032 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,14 @@ name: 'Release' on: schedule: - # Runs every day at midnight UTC for the nightly release. - - cron: '0 0 * * *' - # Runs every Tuesday at 23:59 UTC for the preview release. - - cron: '59 23 * * 2' + # Both crons run in the shared ECS pool's quiet hours (UTC+8 early + # morning) and hours apart from each other and from the CI/E2E + # nightlies, so one release validation never shares a host with + # another. The strings are compared verbatim in prepare below. + # Runs every day at 21:00 UTC for the nightly release. + - cron: '0 21 * * *' + # Runs every Tuesday at 17:00 UTC for the preview release. + - cron: '0 17 * * 2' workflow_dispatch: inputs: version: @@ -39,12 +43,17 @@ on: default: false jobs: + # Validation runs on the ecs-qwen pool but is pinned to the hk4 host: + # every runner on a host shares its CPUs, and the sharded workspace tests + # failed on timing whenever they landed on a host that was also running + # a dozen PR CI / E2E jobs. Dropping the ecs-qwen label from the hk4 + # runners makes the host release-only without touching this file. prepare: name: 'Prepare Release Metadata' # Process cleanup belongs to the runner service/cgroup boundary. This # workflow only resets job-owned filesystem state on the shared pool. # MAINTAINER_ECS_RUNNER_DISABLED restores the hosted fallback. - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 if: |- ${{ github.repository == 'QwenLM/qwen-code' }} @@ -232,13 +241,13 @@ jobs: DRY_RUN_INPUT: '${{ github.event.inputs.dry_run }}' run: |- is_nightly="false" - if [[ "${CRON}" == "0 0 * * *" || "${CREATE_NIGHTLY_RELEASE}" == "true" ]]; then + if [[ "${CRON}" == "0 21 * * *" || "${CREATE_NIGHTLY_RELEASE}" == "true" ]]; then is_nightly="true" fi echo "is_nightly=${is_nightly}" >> "${GITHUB_OUTPUT}" is_preview="false" - if [[ "${CRON}" == "59 23 * * 2" || "${CREATE_PREVIEW_RELEASE}" == "true" ]]; then + if [[ "${CRON}" == "0 17 * * 2" || "${CREATE_PREVIEW_RELEASE}" == "true" ]]; then is_preview="true" fi echo "is_preview=${is_preview}" >> "${GITHUB_OUTPUT}" @@ -319,7 +328,7 @@ jobs: quality_static: name: 'Quality Checks (Static)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 needs: 'prepare' if: |- @@ -383,7 +392,7 @@ jobs: quality_build: name: 'Quality Checks (Build)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 45 needs: 'prepare' if: |- @@ -440,7 +449,7 @@ jobs: quality_typecheck: name: 'Quality Checks (Typecheck)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 needs: - 'prepare' @@ -475,7 +484,7 @@ jobs: workspace_tests: name: 'Workspace Tests (${{ matrix.shard }}/3)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 45 needs: - 'prepare' @@ -506,11 +515,13 @@ jobs: VITEST_MIN_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" VITEST_MAX_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}" VITEST_MIN_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" - run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests' + # --retry absorbs the timing-only failures a shared host still + # produces; a test that fails three times in a row is a real failure. + run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry=2' quality_scripts: name: 'Quality Checks (Scripts)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 needs: - 'prepare' @@ -578,7 +589,7 @@ jobs: integration_none: name: 'Integration Tests (No Sandbox)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 120 needs: 'prepare' if: |- @@ -613,7 +624,7 @@ jobs: integration_docker: name: 'Integration Tests (Docker)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 120 needs: 'prepare' if: |- diff --git a/docs/developers/development/npm.md b/docs/developers/development/npm.md index e0c4068b883..71c21d37cc4 100644 --- a/docs/developers/development/npm.md +++ b/docs/developers/development/npm.md @@ -41,16 +41,16 @@ Regular stable releases for production use. ### Preview Releases -Weekly preview releases every Tuesday at 23:59 UTC for early access to upcoming features. +Weekly preview releases every Tuesday at 17:00 UTC for early access to upcoming features. ### Nightly Releases -Daily nightly releases at midnight UTC for bleeding-edge development testing. +Daily nightly releases at 21:00 UTC for bleeding-edge development testing. ## Automated Release Schedule -- **Nightly**: Every day at midnight UTC -- **Preview**: Every Tuesday at 23:59 UTC +- **Nightly**: Every day at 21:00 UTC +- **Preview**: Every Tuesday at 17:00 UTC - **Stable**: Manual releases triggered by maintainers ### How to Use Different Release Types diff --git a/packages/web-shell/vitest.config.ts b/packages/web-shell/vitest.config.ts index ae7987245b5..d6aa7e2196c 100644 --- a/packages/web-shell/vitest.config.ts +++ b/packages/web-shell/vitest.config.ts @@ -14,6 +14,14 @@ export default defineConfig({ }, test: { setupFiles: ['./test/setup.ts'], + // Same shared-ECS ceilings as packages/cli: the jsdom component tests + // here were the bulk of the release failures at vitest's 5s default. + testTimeout: process.env['RUNNER_NAME']?.startsWith('ecs-qwen-') + ? 60_000 + : undefined, + hookTimeout: process.env['RUNNER_NAME']?.startsWith('ecs-qwen-') + ? 60_000 + : undefined, exclude: [...configDefaults.exclude, 'e2e/**'], reporters: ['default', ['junit', { suiteName: '@qwen-code/web-shell' }]], outputFile: { diff --git a/scripts/tests/release-workflow.test.js b/scripts/tests/release-workflow.test.js index afceafa063b..9fc465e0932 100644 --- a/scripts/tests/release-workflow.test.js +++ b/scripts/tests/release-workflow.test.js @@ -538,7 +538,7 @@ describe('release workflow', () => { (step) => step.name === 'Run Workspace Tests', ); expect(testStep.run).toBe( - 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests', + 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry=2', ); const workspacePackages = getTestCiWorkspaces(); @@ -1557,7 +1557,7 @@ describe('Live Host feed contract', () => { describe('release lane runner routing', () => { const ecsRunsOn = - '${{ (github.repository == \'QwenLM/qwen-code\' && vars.MAINTAINER_ECS_RUNNER_DISABLED != \'true\') && fromJSON(\'["self-hosted", "linux", "x64", "ecs-qwen"]\') || fromJSON(\'["ubuntu-latest"]\') }}'; + '${{ (github.repository == \'QwenLM/qwen-code\' && vars.MAINTAINER_ECS_RUNNER_DISABLED != \'true\') && fromJSON(\'["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]\') || fromJSON(\'["ubuntu-latest"]\') }}'; it('routes validation jobs to ECS with a hosted emergency fallback', () => { const validationJobs = [ @@ -1577,6 +1577,20 @@ describe('release lane runner routing', () => { } }); + it('classifies each schedule cron by the exact string it fires with', () => { + // prepare tells nightly from preview by comparing github.event.schedule + // against the cron text; a cron edited here without its comparison + // silently turns that schedule into a no-op run. + const crons = releaseYaml.on.schedule.map((entry) => entry.cron); + expect(crons).toHaveLength(2); + const vars = releaseYaml.jobs.prepare.steps.find( + (step) => step.id === 'vars', + ); + for (const cron of crons) { + expect(vars.run).toContain(`"\${CRON}" == "${cron}"`); + } + }); + it('passes the runner environment to integration test configuration', () => { for (const [name, expectedSteps] of [ ['integration_none', 2], diff --git a/scripts/tests/unit-vitest-configs.test.ts b/scripts/tests/unit-vitest-configs.test.ts index cf00c02e9e1..773598e8ef2 100644 --- a/scripts/tests/unit-vitest-configs.test.ts +++ b/scripts/tests/unit-vitest-configs.test.ts @@ -105,10 +105,11 @@ describe('autofix gate load clamps', () => { vi.resetModules(); // Re-imported under the stub: the configs read the env at import time, // and the static imports above already resolved the non-ECS branch. - const [core, cli, acpBridge] = await Promise.all([ + const [core, cli, acpBridge, webShell] = await Promise.all([ import('../../packages/core/vitest.config.js'), import('../../packages/cli/vitest.config.js'), import('../../packages/acp-bridge/vitest.config.js'), + import('../../packages/web-shell/vitest.config.js'), ]); vi.unstubAllEnvs(); @@ -135,8 +136,13 @@ describe('autofix gate load clamps', () => { ); // 60_000 / 60_000 / '25%' on the ECS branch of core and cli; - // acp-bridge sets the two timeouts but defines no maxWorkers. - for (const config of [core.default, cli.default, acpBridge.default]) { + // acp-bridge and web-shell set the two timeouts but define no maxWorkers. + for (const config of [ + core.default, + cli.default, + acpBridge.default, + webShell.default, + ]) { expect(String(config.test?.testTimeout)).toBe(clamps['testTimeout']); expect(String(config.test?.hookTimeout)).toBe(clamps['hookTimeout']); } From 965e5137be475a0979b1ac1a0afd82d3365c078f Mon Sep 17 00:00:00 2001 From: yiliang114 Date: Wed, 2 Sep 2026 11:53:28 +0800 Subject: [PATCH 2/4] fix(ci): harden release validation isolation Co-authored-by: Qwen-Coder --- .github/workflows/release.yml | 50 ++++++++++++++++---------- scripts/tests/package-scripts.test.js | 25 ++++++++----- scripts/tests/release-workflow.test.js | 15 ++++++-- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3fb2f62c032..30e744754ed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,10 @@ name: 'Release' on: schedule: - # Both crons run in the shared ECS pool's quiet hours (UTC+8 early - # morning) and hours apart from each other and from the CI/E2E - # nightlies, so one release validation never shares a host with - # another. The strings are compared verbatim in prepare below. + # Both crons run in the ECS pool's quiet hours (UTC+8 early morning), + # hours apart from each other and from the CI/E2E nightlies. Scheduled + # release validation is also serialized below. The strings are compared + # verbatim in prepare. # Runs every day at 21:00 UTC for the nightly release. - cron: '0 21 * * *' # Runs every Tuesday at 17:00 UTC for the preview release. @@ -42,18 +42,25 @@ on: type: 'boolean' default: false +concurrency: + # A delayed preview must not overlap the nightly validation on the same + # reserved host. Manual runs keep independent groups. + group: "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}" + cancel-in-progress: false + jobs: - # Validation runs on the ecs-qwen pool but is pinned to the hk4 host: + # Validation is pinned to the hk4 host through its dedicated label: # every runner on a host shares its CPUs, and the sharded workspace tests # failed on timing whenever they landed on a host that was also running - # a dozen PR CI / E2E jobs. Dropping the ecs-qwen label from the hk4 - # runners makes the host release-only without touching this file. + # a dozen PR CI / E2E jobs. The workflow deliberately does not require the + # shared ecs-qwen label, so infra can remove that label from hk4 and keep + # the host release-only without making these jobs unroutable. prepare: name: 'Prepare Release Metadata' # Process cleanup belongs to the runner service/cgroup boundary. This # workflow only resets job-owned filesystem state on the shared pool. # MAINTAINER_ECS_RUNNER_DISABLED restores the hosted fallback. - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 if: |- ${{ github.repository == 'QwenLM/qwen-code' }} @@ -328,7 +335,7 @@ jobs: quality_static: name: 'Quality Checks (Static)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 needs: 'prepare' if: |- @@ -392,7 +399,7 @@ jobs: quality_build: name: 'Quality Checks (Build)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 45 needs: 'prepare' if: |- @@ -449,7 +456,7 @@ jobs: quality_typecheck: name: 'Quality Checks (Typecheck)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 needs: - 'prepare' @@ -484,7 +491,7 @@ jobs: workspace_tests: name: 'Workspace Tests (${{ matrix.shard }}/3)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 45 needs: - 'prepare' @@ -515,13 +522,14 @@ jobs: VITEST_MIN_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" VITEST_MAX_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}" VITEST_MIN_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" - # --retry absorbs the timing-only failures a shared host still - # produces; a test that fails three times in a row is a real failure. - run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry=2' + # Nightly and preview releases may absorb a transient timing failure; + # stable/manual releases remain strict and never retry a failed test. + VITEST_RETRY: "${{ (needs.prepare.outputs.is_nightly == 'true' || needs.prepare.outputs.is_preview == 'true') && '2' || '0' }}" + run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry="${VITEST_RETRY}"' quality_scripts: name: 'Quality Checks (Scripts)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 needs: - 'prepare' @@ -542,6 +550,12 @@ jobs: - *unpack_release_build - name: 'Run Script Tests' + env: + # Match the tunable per-process bound used by the main CI gate. + VITEST_MAX_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}" + VITEST_MIN_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" + VITEST_MAX_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}" + VITEST_MIN_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" run: 'npm run test:scripts' quality: @@ -589,7 +603,7 @@ jobs: integration_none: name: 'Integration Tests (No Sandbox)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 120 needs: 'prepare' if: |- @@ -624,7 +638,7 @@ jobs: integration_docker: name: 'Integration Tests (Docker)' - runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 120 needs: 'prepare' if: |- diff --git a/scripts/tests/package-scripts.test.js b/scripts/tests/package-scripts.test.js index 4c6959d8799..509255ade10 100644 --- a/scripts/tests/package-scripts.test.js +++ b/scripts/tests/package-scripts.test.js @@ -448,6 +448,10 @@ describe('package scripts', () => { workspaceTestJob, 'Run Workspace Tests', ); + const scriptsTestStep = getWorkflowStep( + getWorkflowJob(workflow, 'quality_scripts'), + 'Run Script Tests', + ); expect(buildJob).toContain("name: 'Check Serve Fast Path Bundle'"); expect(buildJob).toContain('npm run check:serve-fast-path-bundle'); @@ -456,15 +460,18 @@ describe('package scripts', () => { ); expect(workspaceTestStep).toContain('npm run test:release:workspaces'); expect(workspaceTestStep).not.toContain('npm run test:ci'); - for (const name of ['VITEST_MAX_THREADS', 'VITEST_MAX_FORKS']) { - expect(workspaceTestStep).toContain( - `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"`, - ); - } - for (const name of ['VITEST_MIN_THREADS', 'VITEST_MIN_FORKS']) { - expect(workspaceTestStep).toContain( - `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"`, - ); + expect(scriptsTestStep).toContain('npm run test:scripts'); + for (const cappedStep of [workspaceTestStep, scriptsTestStep]) { + for (const name of ['VITEST_MAX_THREADS', 'VITEST_MAX_FORKS']) { + expect(cappedStep).toContain( + `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"`, + ); + } + for (const name of ['VITEST_MIN_THREADS', 'VITEST_MIN_FORKS']) { + expect(cappedStep).toContain( + `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"`, + ); + } } }); diff --git a/scripts/tests/release-workflow.test.js b/scripts/tests/release-workflow.test.js index 9fc465e0932..b1085dd6d71 100644 --- a/scripts/tests/release-workflow.test.js +++ b/scripts/tests/release-workflow.test.js @@ -538,7 +538,10 @@ describe('release workflow', () => { (step) => step.name === 'Run Workspace Tests', ); expect(testStep.run).toBe( - 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry=2', + 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry="${VITEST_RETRY}"', + ); + expect(testStep.env.VITEST_RETRY).toBe( + "${{ (needs.prepare.outputs.is_nightly == 'true' || needs.prepare.outputs.is_preview == 'true') && '2' || '0' }}", ); const workspacePackages = getTestCiWorkspaces(); @@ -1557,7 +1560,7 @@ describe('Live Host feed contract', () => { describe('release lane runner routing', () => { const ecsRunsOn = - '${{ (github.repository == \'QwenLM/qwen-code\' && vars.MAINTAINER_ECS_RUNNER_DISABLED != \'true\') && fromJSON(\'["self-hosted", "linux", "x64", "ecs-qwen", "ecs-qwen-hk4-host"]\') || fromJSON(\'["ubuntu-latest"]\') }}'; + '${{ (github.repository == \'QwenLM/qwen-code\' && vars.MAINTAINER_ECS_RUNNER_DISABLED != \'true\') && fromJSON(\'["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]\') || fromJSON(\'["ubuntu-latest"]\') }}'; it('routes validation jobs to ECS with a hosted emergency fallback', () => { const validationJobs = [ @@ -1591,6 +1594,14 @@ describe('release lane runner routing', () => { } }); + it('serializes scheduled release validation without coupling manual runs', () => { + expect(releaseYaml.concurrency).toEqual({ + group: + "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}", + 'cancel-in-progress': false, + }); + }); + it('passes the runner environment to integration test configuration', () => { for (const [name, expectedSteps] of [ ['integration_none', 2], From e1a3b24c292bb4575399fd636332bcf1330479a4 Mon Sep 17 00:00:00 2001 From: yiliang114 Date: Wed, 2 Sep 2026 11:55:32 +0800 Subject: [PATCH 3/4] fix(ci): serialize release validation runs Co-authored-by: Qwen-Coder --- .github/workflows/release.yml | 8 ++++---- scripts/tests/release-workflow.test.js | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 30e744754ed..9f4d4562d84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,9 +43,9 @@ on: default: false concurrency: - # A delayed preview must not overlap the nightly validation on the same - # reserved host. Manual runs keep independent groups. - group: "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}" + # Only one release validation may use the reserved host at a time, including + # delayed schedules and manually dispatched dry runs or releases. + group: 'release-validation' cancel-in-progress: false jobs: @@ -523,7 +523,7 @@ jobs: VITEST_MAX_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}" VITEST_MIN_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}" # Nightly and preview releases may absorb a transient timing failure; - # stable/manual releases remain strict and never retry a failed test. + # stable releases remain strict and never retry a failed test. VITEST_RETRY: "${{ (needs.prepare.outputs.is_nightly == 'true' || needs.prepare.outputs.is_preview == 'true') && '2' || '0' }}" run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry="${VITEST_RETRY}"' diff --git a/scripts/tests/release-workflow.test.js b/scripts/tests/release-workflow.test.js index b1085dd6d71..a69ce795afb 100644 --- a/scripts/tests/release-workflow.test.js +++ b/scripts/tests/release-workflow.test.js @@ -1594,10 +1594,9 @@ describe('release lane runner routing', () => { } }); - it('serializes scheduled release validation without coupling manual runs', () => { + it('serializes every release validation on the reserved host', () => { expect(releaseYaml.concurrency).toEqual({ - group: - "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}", + group: 'release-validation', 'cancel-in-progress': false, }); }); From f875f060ca1039a6554d1dad0962364481815f84 Mon Sep 17 00:00:00 2001 From: yiliang114 Date: Wed, 2 Sep 2026 11:56:44 +0800 Subject: [PATCH 4/4] fix(ci): preserve manual release dispatches Co-authored-by: Qwen-Coder --- .github/workflows/release.yml | 7 ++++--- scripts/tests/release-workflow.test.js | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f4d4562d84..643ce30d677 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,9 +43,10 @@ on: default: false concurrency: - # Only one release validation may use the reserved host at a time, including - # delayed schedules and manually dispatched dry runs or releases. - group: 'release-validation' + # A delayed preview must not overlap the nightly validation on the same + # reserved host. Manual runs keep unique groups so GitHub's non-FIFO + # concurrency queue can never supersede a pending real release with a dry run. + group: "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}" cancel-in-progress: false jobs: diff --git a/scripts/tests/release-workflow.test.js b/scripts/tests/release-workflow.test.js index a69ce795afb..b1085dd6d71 100644 --- a/scripts/tests/release-workflow.test.js +++ b/scripts/tests/release-workflow.test.js @@ -1594,9 +1594,10 @@ describe('release lane runner routing', () => { } }); - it('serializes every release validation on the reserved host', () => { + it('serializes scheduled release validation without coupling manual runs', () => { expect(releaseYaml.concurrency).toEqual({ - group: 'release-validation', + group: + "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}", 'cancel-in-progress': false, }); });