ci(ecs): wait out npm publish propagation before updating the runner fleet

.github/workflows/update-ecs-runner-qwen.yml

@@ -18,40 +18,74 @@ permissions:
   contents: 'read'
 
 jobs:
-  update:
-    name: 'Update Qwen on ${{ matrix.runner }}'
+  # Resolving on a hosted runner, once, keeps the registry wait below off the
+  # ECS pools (which queue behind real review/triage work) and guarantees every
+  # pool installs the SAME version even when their jobs start hours apart.
+  resolve:
+    name: 'Resolve version'
     if: "${{ github.repository == 'QwenLM/qwen-code' }}"
-    strategy:
-      matrix:
-        runner: ['ecs-update-sg', 'ecs-update-64c', 'ecs-update-hk-1', 'ecs-update-hk-2']
-      fail-fast: false
-    runs-on: ['self-hosted', 'linux', 'x64', '${{ matrix.runner }}']
-    concurrency:
-      group: 'update-ecs-runner-qwen-${{ matrix.runner }}'
-      cancel-in-progress: false
-    timeout-minutes: 10
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 30
+    outputs:
+      version: '${{ steps.version.outputs.version }}'
     steps:
       - name: 'Resolve version'
         id: 'version'
         env:
           INPUT_VERSION: '${{ inputs.version || github.event.client_payload.version }}'
+          # npm publishes asynchronously: `npm publish --provenance` returns
+          # "Your package is being processed and may take a few minutes to
+          # become available", and release.yml dispatches this workflow as soon
+          # as it returns. For v0.22.3 the gap was ~16 minutes (published
+          # 17:14Z, resolvable 17:30Z), so the single un-retried `npm view` this
+          # step used to run 404'd on 3 of the 4 pools and left the fleet split
+          # across two CLI versions until a maintainer noticed. Wait the
+          # registry out rather than losing the race.
+          RESOLVE_TIMEOUT_SECONDS: '1500'
+          RESOLVE_INTERVAL_SECONDS: '30'
         run: |-
           set -euo pipefail
           specifier="@qwen-code/qwen-code@${INPUT_VERSION#v}"
           if [[ "${specifier}" == '@qwen-code/qwen-code@' ]]; then
             specifier='@qwen-code/qwen-code@latest'
           fi
-          version="$(npm view "${specifier}" version | tail -n 1)" || true
-          if [[ -z "${version}" ]]; then
-            echo "::error::No published qwen version matches '${INPUT_VERSION:-latest}'."
-            exit 1
-          fi
+          # Per-attempt stderr is held back so ~50 identical 404 blocks do not
+          # bury the log; the last one is replayed when the wait gives up.
+          err_log="$(mktemp)"
+          deadline=$(( SECONDS + RESOLVE_TIMEOUT_SECONDS ))
+          while :; do
+            version="$(npm view "${specifier}" version 2>"${err_log}" | tail -n 1)" || true
+            if [[ -n "${version}" ]]; then
+              break
+            fi
+            if (( SECONDS >= deadline )); then
+              cat "${err_log}" >&2
+              echo "::error::No published qwen version matches '${INPUT_VERSION:-latest}' after ${RESOLVE_TIMEOUT_SECONDS}s."
+              exit 1
+            fi
+            echo "'${specifier}' is not on the registry yet; retrying in ${RESOLVE_INTERVAL_SECONDS}s."
+            sleep "${RESOLVE_INTERVAL_SECONDS}"
+          done
           echo "version=${version}" >> "${GITHUB_OUTPUT}"
           echo "Resolved qwen version: ${version}"
 
+  update:
+    name: 'Update Qwen on ${{ matrix.runner }}'
+    needs: 'resolve'
+    if: "${{ github.repository == 'QwenLM/qwen-code' }}"
+    strategy:
+      matrix:
+        runner: ['ecs-update-sg', 'ecs-update-64c', 'ecs-update-hk-1', 'ecs-update-hk-2']
+      fail-fast: false
+    runs-on: ['self-hosted', 'linux', 'x64', '${{ matrix.runner }}']
+    concurrency:
+      group: 'update-ecs-runner-qwen-${{ matrix.runner }}'
+      cancel-in-progress: false
+    timeout-minutes: 10
+    steps:
       - name: 'Update qwen'
         env:
-          VERSION: '${{ steps.version.outputs.version }}'
+          VERSION: '${{ needs.resolve.outputs.version }}'
         run: |-
           set -euo pipefail
           # The runner service resolves the system-wide qwen binary. Do not
@@ -80,7 +114,7 @@ jobs:
 
       - name: 'Verify version'
         env:
-          VERSION: '${{ steps.version.outputs.version }}'
+          VERSION: '${{ needs.resolve.outputs.version }}'
         run: |-
           set -euo pipefail
           qwen_path="$(command -v qwen)"

scripts/tests/update-ecs-runner-qwen-workflow.test.js

@@ -4,15 +4,93 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { readFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import {
+  chmodSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { describe, expect, it } from 'vitest';
 
-describe('ECS runner qwen update workflow', () => {
-  const workflow = readFileSync(
-    '.github/workflows/update-ecs-runner-qwen.yml',
-    'utf8',
+const workflow = readFileSync(
+  '.github/workflows/update-ecs-runner-qwen.yml',
+  'utf8',
+);
+
+function step(name) {
+  const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const match = workflow.match(
+    new RegExp(
+      `\\n\\s+- name:\\s*(['"])${escaped}\\1[\\s\\S]*?(?=\\n\\s+- name:\\s*['"]|\\n\\s{2}[a-zA-Z0-9_-]+:|$)`,
+    ),
   );
+  return match?.[0] ?? '';
+}
+
+// The body of a step's `run: |-` block, dedented to column zero.
+function stepBody(name) {
+  const body = step(name).match(/run: \|-\n([\s\S]*)$/)?.[1] ?? '';
+  return body.replace(/^ {10}/gm, '');
+}
+
+// Runs the 'Resolve version' step body against a stubbed `npm` that 404s for
+// its first `failures` invocations and then reports `version`.
+function runResolve({ failures = 0, version = '0.22.3', env = {} } = {}) {
+  const dir = mkdtempSync(join(tmpdir(), 'ecs-update-'));
+  try {
+    const counter = join(dir, 'attempts');
+    const npmStub = join(dir, 'npm');
+    writeFileSync(
+      npmStub,
+      [
+        '#!/usr/bin/env bash',
+        `attempt=$(( $(cat ${counter} 2>/dev/null || echo 0) + 1 ))`,
+        `echo "$attempt" > ${counter}`,
+        `if (( attempt <= ${failures} )); then`,
+        '  echo "npm error code E404" >&2',
+        '  echo "npm error 404 No match found for version" >&2',
+        '  exit 1',
+        'fi',
+        `echo '${version}'`,
+      ].join('\n'),
+      { mode: 0o755 },
+    );
+    chmodSync(npmStub, 0o755);
+
+    const script = join(dir, 'resolve.sh');
+    writeFileSync(script, stepBody('Resolve version'));
+    const ghOutput = join(dir, 'github-output');
+    writeFileSync(ghOutput, '');
 
+    const result = spawnSync('bash', [script], {
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        PATH: `${dir}:${process.env.PATH ?? ''}`,
+        GITHUB_OUTPUT: ghOutput,
+        INPUT_VERSION: '0.22.3',
+        RESOLVE_TIMEOUT_SECONDS: '60',
+        RESOLVE_INTERVAL_SECONDS: '0',
+        ...env,
+      },
+    });
+    return {
+      status: result.status,
+      stdout: result.stdout ?? '',
+      stderr: result.stderr ?? '',
+      output: readFileSync(ghOutput, 'utf8'),
+      attempts: Number(readFileSync(counter, 'utf8').trim()),
+    };
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+describe('ECS runner qwen update workflow', () => {
   it('installs without the selected runner npm prefix', () => {
     expect(workflow).toContain('cd "${RUNNER_TEMP:?}"');
     expect(workflow).toContain('sudo env -u NPM_CONFIG_PREFIX npm install -g');
@@ -37,4 +115,58 @@ describe('ECS runner qwen update workflow', () => {
     expect(workflow).toContain('if [[ "${attempt}" -lt 3 ]]; then');
     expect(workflow).toContain('sudo rm -rf "${PKG_DIR}"/.qwen-code-*');
   });
+
+  it('resolves once on a hosted runner and feeds every pool', () => {
+    // One resolution shared by the matrix is what keeps pools that start
+    // hours apart from installing different versions; it also keeps the
+    // registry wait off the ECS runners.
+    expect(workflow).toContain("    runs-on: 'ubuntu-latest'");
+    expect(workflow).toContain(
+      "      version: '${{ steps.version.outputs.version }}'",
+    );
+    expect(workflow).toContain("    needs: 'resolve'");
+    // Both consumers read the job output; a leftover step reference would
+    // silently expand to an empty version and install `@qwen-code/qwen-code@`.
+    const consumers = workflow.match(
+      /VERSION: '\$\{\{ needs\.resolve\.outputs\.version \}\}'/g,
+    );
+    expect(consumers).toHaveLength(2);
+    expect(workflow).not.toContain(
+      "VERSION: '${{ steps.version.outputs.version }}'",
+    );
+  });
+
+  it('waits out npm publish propagation instead of failing the race', () => {
+    // `npm publish --provenance` returns before the version is resolvable
+    // (~16 minutes for v0.22.3), and release.yml dispatches this workflow as
+    // soon as it returns.
+    const resolved = runResolve({ failures: 3 });
+    expect(resolved.status).toBe(0);
+    expect(resolved.attempts).toBe(4);
+    expect(resolved.output.trim()).toBe('version=0.22.3');
+    expect(resolved.stdout).toContain('is not on the registry yet');
+    // The per-attempt 404 noise stays out of the log on the happy path.
+    expect(resolved.stderr).not.toContain('E404');
+  });
+
+  it('fails with the registry error once the wait budget is spent', () => {
+    const resolved = runResolve({
+      failures: 99,
+      env: { RESOLVE_TIMEOUT_SECONDS: '0' },
+    });
+    expect(resolved.status).toBe(1);
+    expect(resolved.output.trim()).toBe('');
+    // The suppressed stderr is replayed, so the log still says *why*.
+    expect(resolved.stderr).toContain('npm error code E404');
+    // The annotation stays on stdout, where Actions parses workflow commands.
+    expect(resolved.stdout).toContain(
+      "::error::No published qwen version matches '0.22.3' after 0s.",
+    );
+  });
+
+  it('resolves the latest dist-tag when dispatched without a version', () => {
+    const resolved = runResolve({ env: { INPUT_VERSION: '' } });
+    expect(resolved.status).toBe(0);
+    expect(resolved.output.trim()).toBe('version=0.22.3');
+  });
 });