From 6a490386a4254d5b6be6f0830256670217a6de70 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 29 Aug 2026 07:38:59 +0800 Subject: [PATCH] ci(ecs): wait out npm publish propagation before updating the runner fleet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `npm publish --provenance` returns before the published version is resolvable — npm prints "Your package is being processed and may take a few minutes to become available" — and release.yml dispatches `npm-published` as soon as it returns. For v0.22.3 the package was published at 17:14Z and only became resolvable at 17:30Z, so the single un-retried `npm view` in "Resolve version" 404'd on ecs-update-sg, ecs-update-64c and ecs-update-hk-1. Only ecs-update-hk-2 succeeded, and only because its job queued for ~3 hours and started after the registry had caught up. That left the fleet split across two CLI versions with no signal: the review and triage workflows install qwen only when it is missing, so the three stale pools kept running 0.22.2 against PRs for a full day while hk-2 ran 0.22.3. Resolve the version once on a hosted runner, polling for up to 25 minutes, and feed the result to the matrix through `needs`. That keeps the registry wait off the ECS pools — which queue behind real review and triage work — and makes every pool install the same version even when their jobs start hours apart. --- .github/workflows/update-ecs-runner-qwen.yml | 70 ++++++--- .../update-ecs-runner-qwen-workflow.test.js | 142 +++++++++++++++++- 2 files changed, 189 insertions(+), 23 deletions(-) diff --git a/.github/workflows/update-ecs-runner-qwen.yml b/.github/workflows/update-ecs-runner-qwen.yml index 241ae8c0e38..fb7ae07d683 100644 --- a/.github/workflows/update-ecs-runner-qwen.yml +++ b/.github/workflows/update-ecs-runner-qwen.yml @@ -18,40 +18,74 @@ permissions: contents: 'read' jobs: - update: - name: 'Update Qwen on ${{ matrix.runner }}' + # Resolving on a hosted runner, once, keeps the registry wait below off the + # ECS pools (which queue behind real review/triage work) and guarantees every + # pool installs the SAME version even when their jobs start hours apart. + resolve: + name: 'Resolve version' if: "${{ github.repository == 'QwenLM/qwen-code' }}" - strategy: - matrix: - runner: ['ecs-update-sg', 'ecs-update-64c', 'ecs-update-hk-1', 'ecs-update-hk-2'] - fail-fast: false - runs-on: ['self-hosted', 'linux', 'x64', '${{ matrix.runner }}'] - concurrency: - group: 'update-ecs-runner-qwen-${{ matrix.runner }}' - cancel-in-progress: false - timeout-minutes: 10 + runs-on: 'ubuntu-latest' + timeout-minutes: 30 + outputs: + version: '${{ steps.version.outputs.version }}' steps: - name: 'Resolve version' id: 'version' env: INPUT_VERSION: '${{ inputs.version || github.event.client_payload.version }}' + # npm publishes asynchronously: `npm publish --provenance` returns + # "Your package is being processed and may take a few minutes to + # become available", and release.yml dispatches this workflow as soon + # as it returns. For v0.22.3 the gap was ~16 minutes (published + # 17:14Z, resolvable 17:30Z), so the single un-retried `npm view` this + # step used to run 404'd on 3 of the 4 pools and left the fleet split + # across two CLI versions until a maintainer noticed. Wait the + # registry out rather than losing the race. + RESOLVE_TIMEOUT_SECONDS: '1500' + RESOLVE_INTERVAL_SECONDS: '30' run: |- set -euo pipefail specifier="@qwen-code/qwen-code@${INPUT_VERSION#v}" if [[ "${specifier}" == '@qwen-code/qwen-code@' ]]; then specifier='@qwen-code/qwen-code@latest' fi - version="$(npm view "${specifier}" version | tail -n 1)" || true - if [[ -z "${version}" ]]; then - echo "::error::No published qwen version matches '${INPUT_VERSION:-latest}'." - exit 1 - fi + # Per-attempt stderr is held back so ~50 identical 404 blocks do not + # bury the log; the last one is replayed when the wait gives up. + err_log="$(mktemp)" + deadline=$(( SECONDS + RESOLVE_TIMEOUT_SECONDS )) + while :; do + version="$(npm view "${specifier}" version 2>"${err_log}" | tail -n 1)" || true + if [[ -n "${version}" ]]; then + break + fi + if (( SECONDS >= deadline )); then + cat "${err_log}" >&2 + echo "::error::No published qwen version matches '${INPUT_VERSION:-latest}' after ${RESOLVE_TIMEOUT_SECONDS}s." + exit 1 + fi + echo "'${specifier}' is not on the registry yet; retrying in ${RESOLVE_INTERVAL_SECONDS}s." + sleep "${RESOLVE_INTERVAL_SECONDS}" + done echo "version=${version}" >> "${GITHUB_OUTPUT}" echo "Resolved qwen version: ${version}" + update: + name: 'Update Qwen on ${{ matrix.runner }}' + needs: 'resolve' + if: "${{ github.repository == 'QwenLM/qwen-code' }}" + strategy: + matrix: + runner: ['ecs-update-sg', 'ecs-update-64c', 'ecs-update-hk-1', 'ecs-update-hk-2'] + fail-fast: false + runs-on: ['self-hosted', 'linux', 'x64', '${{ matrix.runner }}'] + concurrency: + group: 'update-ecs-runner-qwen-${{ matrix.runner }}' + cancel-in-progress: false + timeout-minutes: 10 + steps: - name: 'Update qwen' env: - VERSION: '${{ steps.version.outputs.version }}' + VERSION: '${{ needs.resolve.outputs.version }}' run: |- set -euo pipefail # The runner service resolves the system-wide qwen binary. Do not @@ -80,7 +114,7 @@ jobs: - name: 'Verify version' env: - VERSION: '${{ steps.version.outputs.version }}' + VERSION: '${{ needs.resolve.outputs.version }}' run: |- set -euo pipefail qwen_path="$(command -v qwen)" diff --git a/scripts/tests/update-ecs-runner-qwen-workflow.test.js b/scripts/tests/update-ecs-runner-qwen-workflow.test.js index e4c0e41772f..fad4dd15dfc 100644 --- a/scripts/tests/update-ecs-runner-qwen-workflow.test.js +++ b/scripts/tests/update-ecs-runner-qwen-workflow.test.js @@ -4,15 +4,93 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { readFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -describe('ECS runner qwen update workflow', () => { - const workflow = readFileSync( - '.github/workflows/update-ecs-runner-qwen.yml', - 'utf8', +const workflow = readFileSync( + '.github/workflows/update-ecs-runner-qwen.yml', + 'utf8', +); + +function step(name) { + const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const match = workflow.match( + new RegExp( + `\\n\\s+- name:\\s*(['"])${escaped}\\1[\\s\\S]*?(?=\\n\\s+- name:\\s*['"]|\\n\\s{2}[a-zA-Z0-9_-]+:|$)`, + ), ); + return match?.[0] ?? ''; +} + +// The body of a step's `run: |-` block, dedented to column zero. +function stepBody(name) { + const body = step(name).match(/run: \|-\n([\s\S]*)$/)?.[1] ?? ''; + return body.replace(/^ {10}/gm, ''); +} + +// Runs the 'Resolve version' step body against a stubbed `npm` that 404s for +// its first `failures` invocations and then reports `version`. +function runResolve({ failures = 0, version = '0.22.3', env = {} } = {}) { + const dir = mkdtempSync(join(tmpdir(), 'ecs-update-')); + try { + const counter = join(dir, 'attempts'); + const npmStub = join(dir, 'npm'); + writeFileSync( + npmStub, + [ + '#!/usr/bin/env bash', + `attempt=$(( $(cat ${counter} 2>/dev/null || echo 0) + 1 ))`, + `echo "$attempt" > ${counter}`, + `if (( attempt <= ${failures} )); then`, + ' echo "npm error code E404" >&2', + ' echo "npm error 404 No match found for version" >&2', + ' exit 1', + 'fi', + `echo '${version}'`, + ].join('\n'), + { mode: 0o755 }, + ); + chmodSync(npmStub, 0o755); + + const script = join(dir, 'resolve.sh'); + writeFileSync(script, stepBody('Resolve version')); + const ghOutput = join(dir, 'github-output'); + writeFileSync(ghOutput, ''); + const result = spawnSync('bash', [script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH ?? ''}`, + GITHUB_OUTPUT: ghOutput, + INPUT_VERSION: '0.22.3', + RESOLVE_TIMEOUT_SECONDS: '60', + RESOLVE_INTERVAL_SECONDS: '0', + ...env, + }, + }); + return { + status: result.status, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + output: readFileSync(ghOutput, 'utf8'), + attempts: Number(readFileSync(counter, 'utf8').trim()), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe('ECS runner qwen update workflow', () => { it('installs without the selected runner npm prefix', () => { expect(workflow).toContain('cd "${RUNNER_TEMP:?}"'); expect(workflow).toContain('sudo env -u NPM_CONFIG_PREFIX npm install -g'); @@ -37,4 +115,58 @@ describe('ECS runner qwen update workflow', () => { expect(workflow).toContain('if [[ "${attempt}" -lt 3 ]]; then'); expect(workflow).toContain('sudo rm -rf "${PKG_DIR}"/.qwen-code-*'); }); + + it('resolves once on a hosted runner and feeds every pool', () => { + // One resolution shared by the matrix is what keeps pools that start + // hours apart from installing different versions; it also keeps the + // registry wait off the ECS runners. + expect(workflow).toContain(" runs-on: 'ubuntu-latest'"); + expect(workflow).toContain( + " version: '${{ steps.version.outputs.version }}'", + ); + expect(workflow).toContain(" needs: 'resolve'"); + // Both consumers read the job output; a leftover step reference would + // silently expand to an empty version and install `@qwen-code/qwen-code@`. + const consumers = workflow.match( + /VERSION: '\$\{\{ needs\.resolve\.outputs\.version \}\}'/g, + ); + expect(consumers).toHaveLength(2); + expect(workflow).not.toContain( + "VERSION: '${{ steps.version.outputs.version }}'", + ); + }); + + it('waits out npm publish propagation instead of failing the race', () => { + // `npm publish --provenance` returns before the version is resolvable + // (~16 minutes for v0.22.3), and release.yml dispatches this workflow as + // soon as it returns. + const resolved = runResolve({ failures: 3 }); + expect(resolved.status).toBe(0); + expect(resolved.attempts).toBe(4); + expect(resolved.output.trim()).toBe('version=0.22.3'); + expect(resolved.stdout).toContain('is not on the registry yet'); + // The per-attempt 404 noise stays out of the log on the happy path. + expect(resolved.stderr).not.toContain('E404'); + }); + + it('fails with the registry error once the wait budget is spent', () => { + const resolved = runResolve({ + failures: 99, + env: { RESOLVE_TIMEOUT_SECONDS: '0' }, + }); + expect(resolved.status).toBe(1); + expect(resolved.output.trim()).toBe(''); + // The suppressed stderr is replayed, so the log still says *why*. + expect(resolved.stderr).toContain('npm error code E404'); + // The annotation stays on stdout, where Actions parses workflow commands. + expect(resolved.stdout).toContain( + "::error::No published qwen version matches '0.22.3' after 0s.", + ); + }); + + it('resolves the latest dist-tag when dispatched without a version', () => { + const resolved = runResolve({ env: { INPUT_VERSION: '' } }); + expect(resolved.status).toBe(0); + expect(resolved.output.trim()).toBe('version=0.22.3'); + }); });