fix(ci): isolate and bound release validation workloads

.github/actionlint.yaml

@@ -1,6 +1,7 @@
 self-hosted-runner:
   labels:
     - 'ecs-qwen'
+    - 'ecs-qwen-hk4-host'
     - 'ecs-win'
     - 'ecs-update-hk-1'
     - 'ecs-update-hk-2'

.github/workflows/release.yml

@@ -2,10 +2,14 @@ name: 'Release'
 
 on:
   schedule:
-    # Runs every day at midnight UTC for the nightly release.
-    - cron: '0 0 * * *'
-    # Runs every Tuesday at 23:59 UTC for the preview release.
-    - cron: '59 23 * * 2'
+    # Both crons run in the ECS pool's quiet hours (UTC+8 early morning),
+    # hours apart from each other and from the CI/E2E nightlies. Scheduled
+    # release validation is also serialized below. The strings are compared
+    # verbatim in prepare.
+    # Runs every day at 21:00 UTC for the nightly release.
+    - cron: '0 21 * * *'
+    # Runs every Tuesday at 17:00 UTC for the preview release.
+    - cron: '0 17 * * 2'
   workflow_dispatch:
     inputs:
       version:
@@ -38,13 +42,26 @@ on:
         type: 'boolean'
         default: false
 
+concurrency:
+  # A delayed preview must not overlap the nightly validation on the same
+  # reserved host. Manual runs keep unique groups so GitHub's non-FIFO
+  # concurrency queue can never supersede a pending real release with a dry run.
+  group: "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}"
+  cancel-in-progress: false
+
 jobs:
+  # Validation is pinned to the hk4 host through its dedicated label:
+  # every runner on a host shares its CPUs, and the sharded workspace tests
+  # failed on timing whenever they landed on a host that was also running
+  # a dozen PR CI / E2E jobs. The workflow deliberately does not require the
+  # shared ecs-qwen label, so infra can remove that label from hk4 and keep
+  # the host release-only without making these jobs unroutable.
   prepare:
     name: 'Prepare Release Metadata'
     # Process cleanup belongs to the runner service/cgroup boundary. This
     # workflow only resets job-owned filesystem state on the shared pool.
     # MAINTAINER_ECS_RUNNER_DISABLED restores the hosted fallback.
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 30
     if: |-
       ${{ github.repository == 'QwenLM/qwen-code' }}
@@ -232,13 +249,13 @@ jobs:
           DRY_RUN_INPUT: '${{ github.event.inputs.dry_run }}'
         run: |-
           is_nightly="false"
-          if [[ "${CRON}" == "0 0 * * *" || "${CREATE_NIGHTLY_RELEASE}" == "true" ]]; then
+          if [[ "${CRON}" == "0 21 * * *" || "${CREATE_NIGHTLY_RELEASE}" == "true" ]]; then
             is_nightly="true"
           fi
           echo "is_nightly=${is_nightly}" >> "${GITHUB_OUTPUT}"
 
           is_preview="false"
-          if [[ "${CRON}" == "59 23 * * 2" || "${CREATE_PREVIEW_RELEASE}" == "true" ]]; then
+          if [[ "${CRON}" == "0 17 * * 2" || "${CREATE_PREVIEW_RELEASE}" == "true" ]]; then
             is_preview="true"
           fi
           echo "is_preview=${is_preview}" >> "${GITHUB_OUTPUT}"
@@ -319,7 +336,7 @@ jobs:
 
   quality_static:
     name: 'Quality Checks (Static)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 30
     needs: 'prepare'
     if: |-
@@ -383,7 +400,7 @@ jobs:
 
   quality_build:
     name: 'Quality Checks (Build)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 45
     needs: 'prepare'
     if: |-
@@ -440,7 +457,7 @@ jobs:
 
   quality_typecheck:
     name: 'Quality Checks (Typecheck)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 30
     needs:
       - 'prepare'
@@ -475,7 +492,7 @@ jobs:
 
   workspace_tests:
     name: 'Workspace Tests (${{ matrix.shard }}/3)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 45
     needs:
       - 'prepare'
@@ -506,11 +523,14 @@ jobs:
           VITEST_MIN_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"
           VITEST_MAX_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"
           VITEST_MIN_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"
-        run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests'
+          # Nightly and preview releases may absorb a transient timing failure;
+          # stable releases remain strict and never retry a failed test.
+          VITEST_RETRY: "${{ (needs.prepare.outputs.is_nightly == 'true' || needs.prepare.outputs.is_preview == 'true') && '2' || '0' }}"
+        run: 'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry="${VITEST_RETRY}"'
 
   quality_scripts:
     name: 'Quality Checks (Scripts)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 30
     needs:
       - 'prepare'
@@ -531,6 +551,12 @@ jobs:
       - *unpack_release_build
 
       - name: 'Run Script Tests'
+        env:
+          # Match the tunable per-process bound used by the main CI gate.
+          VITEST_MAX_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"
+          VITEST_MIN_THREADS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"
+          VITEST_MAX_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"
+          VITEST_MIN_FORKS: "${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"
         run: 'npm run test:scripts'
 
   quality:
@@ -578,7 +604,7 @@ jobs:
 
   integration_none:
     name: 'Integration Tests (No Sandbox)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 120
     needs: 'prepare'
     if: |-
@@ -613,7 +639,7 @@ jobs:
 
   integration_docker:
     name: 'Integration Tests (Docker)'
-    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
+    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]'') || fromJSON(''["ubuntu-latest"]'') }}'
     timeout-minutes: 120
     needs: 'prepare'
     if: |-

docs/developers/development/npm.md

@@ -41,16 +41,16 @@ Regular stable releases for production use.
 
 ### Preview Releases
 
-Weekly preview releases every Tuesday at 23:59 UTC for early access to upcoming features.
+Weekly preview releases every Tuesday at 17:00 UTC for early access to upcoming features.
 
 ### Nightly Releases
 
-Daily nightly releases at midnight UTC for bleeding-edge development testing.
+Daily nightly releases at 21:00 UTC for bleeding-edge development testing.
 
 ## Automated Release Schedule
 
-- **Nightly**: Every day at midnight UTC
-- **Preview**: Every Tuesday at 23:59 UTC
+- **Nightly**: Every day at 21:00 UTC
+- **Preview**: Every Tuesday at 17:00 UTC
 - **Stable**: Manual releases triggered by maintainers
 
 ### How to Use Different Release Types

packages/web-shell/vitest.config.ts

@@ -14,6 +14,14 @@ export default defineConfig({
   },
   test: {
     setupFiles: ['./test/setup.ts'],
+    // Same shared-ECS ceilings as packages/cli: the jsdom component tests
+    // here were the bulk of the release failures at vitest's 5s default.
+    testTimeout: process.env['RUNNER_NAME']?.startsWith('ecs-qwen-')
+      ? 60_000
+      : undefined,
+    hookTimeout: process.env['RUNNER_NAME']?.startsWith('ecs-qwen-')
+      ? 60_000
+      : undefined,
     exclude: [...configDefaults.exclude, 'e2e/**'],
     reporters: ['default', ['junit', { suiteName: '@qwen-code/web-shell' }]],
     outputFile: {

scripts/tests/package-scripts.test.js

@@ -448,6 +448,10 @@ describe('package scripts', () => {
       workspaceTestJob,
       'Run Workspace Tests',
     );
+    const scriptsTestStep = getWorkflowStep(
+      getWorkflowJob(workflow, 'quality_scripts'),
+      'Run Script Tests',
+    );
 
     expect(buildJob).toContain("name: 'Check Serve Fast Path Bundle'");
     expect(buildJob).toContain('npm run check:serve-fast-path-bundle');
@@ -456,15 +460,18 @@ describe('package scripts', () => {
     );
     expect(workspaceTestStep).toContain('npm run test:release:workspaces');
     expect(workspaceTestStep).not.toContain('npm run test:ci');
-    for (const name of ['VITEST_MAX_THREADS', 'VITEST_MAX_FORKS']) {
-      expect(workspaceTestStep).toContain(
-        `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"`,
-      );
-    }
-    for (const name of ['VITEST_MIN_THREADS', 'VITEST_MIN_FORKS']) {
-      expect(workspaceTestStep).toContain(
-        `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"`,
-      );
+    expect(scriptsTestStep).toContain('npm run test:scripts');
+    for (const cappedStep of [workspaceTestStep, scriptsTestStep]) {
+      for (const name of ['VITEST_MAX_THREADS', 'VITEST_MAX_FORKS']) {
+        expect(cappedStep).toContain(
+          `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && (vars.QWEN_CI_VITEST_MAX_WORKERS || '4') || '' }}"`,
+        );
+      }
+      for (const name of ['VITEST_MIN_THREADS', 'VITEST_MIN_FORKS']) {
+        expect(cappedStep).toContain(
+          `${name}: "\${{ startsWith(runner.name, 'ecs-qwen-') && '1' || '' }}"`,
+        );
+      }
     }
   });
 

scripts/tests/release-workflow.test.js

@@ -538,7 +538,10 @@ describe('release workflow', () => {
       (step) => step.name === 'Run Workspace Tests',
     );
     expect(testStep.run).toBe(
-      'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests',
+      'npm run test:release:workspaces -- --shard=${{ matrix.shard }}/3 --passWithNoTests --retry="${VITEST_RETRY}"',
+    );
+    expect(testStep.env.VITEST_RETRY).toBe(
+      "${{ (needs.prepare.outputs.is_nightly == 'true' || needs.prepare.outputs.is_preview == 'true') && '2' || '0' }}",
     );
 
     const workspacePackages = getTestCiWorkspaces();
@@ -1557,7 +1560,7 @@ describe('Live Host feed contract', () => {
 
 describe('release lane runner routing', () => {
   const ecsRunsOn =
-    '${{ (github.repository == \'QwenLM/qwen-code\' && vars.MAINTAINER_ECS_RUNNER_DISABLED != \'true\') && fromJSON(\'["self-hosted", "linux", "x64", "ecs-qwen"]\') || fromJSON(\'["ubuntu-latest"]\') }}';
+    '${{ (github.repository == \'QwenLM/qwen-code\' && vars.MAINTAINER_ECS_RUNNER_DISABLED != \'true\') && fromJSON(\'["self-hosted", "linux", "x64", "ecs-qwen-hk4-host"]\') || fromJSON(\'["ubuntu-latest"]\') }}';
 
   it('routes validation jobs to ECS with a hosted emergency fallback', () => {
     const validationJobs = [
@@ -1577,6 +1580,28 @@ describe('release lane runner routing', () => {
     }
   });
 
+  it('classifies each schedule cron by the exact string it fires with', () => {
+    // prepare tells nightly from preview by comparing github.event.schedule
+    // against the cron text; a cron edited here without its comparison
+    // silently turns that schedule into a no-op run.
+    const crons = releaseYaml.on.schedule.map((entry) => entry.cron);
+    expect(crons).toHaveLength(2);
+    const vars = releaseYaml.jobs.prepare.steps.find(
+      (step) => step.id === 'vars',
+    );
+    for (const cron of crons) {
+      expect(vars.run).toContain(`"\${CRON}" == "${cron}"`);
+    }
+  });
+
+  it('serializes scheduled release validation without coupling manual runs', () => {
+    expect(releaseYaml.concurrency).toEqual({
+      group:
+        "${{ github.event_name == 'schedule' && 'release-scheduled-validation' || format('release-{0}', github.run_id) }}",
+      'cancel-in-progress': false,
+    });
+  });
+
   it('passes the runner environment to integration test configuration', () => {
     for (const [name, expectedSteps] of [
       ['integration_none', 2],

scripts/tests/unit-vitest-configs.test.ts

@@ -105,10 +105,11 @@ describe('autofix gate load clamps', () => {
     vi.resetModules();
     // Re-imported under the stub: the configs read the env at import time,
     // and the static imports above already resolved the non-ECS branch.
-    const [core, cli, acpBridge] = await Promise.all([
+    const [core, cli, acpBridge, webShell] = await Promise.all([
       import('../../packages/core/vitest.config.js'),
       import('../../packages/cli/vitest.config.js'),
       import('../../packages/acp-bridge/vitest.config.js'),
+      import('../../packages/web-shell/vitest.config.js'),
     ]);
     vi.unstubAllEnvs();
 
@@ -135,8 +136,13 @@ describe('autofix gate load clamps', () => {
     );
 
     // 60_000 / 60_000 / '25%' on the ECS branch of core and cli;
-    // acp-bridge sets the two timeouts but defines no maxWorkers.
-    for (const config of [core.default, cli.default, acpBridge.default]) {
+    // acp-bridge and web-shell set the two timeouts but define no maxWorkers.
+    for (const config of [
+      core.default,
+      cli.default,
+      acpBridge.default,
+      webShell.default,
+    ]) {
       expect(String(config.test?.testTimeout)).toBe(clamps['testTimeout']);
       expect(String(config.test?.hookTimeout)).toBe(clamps['hookTimeout']);
     }