fix(ci): retry sandbox image builds and file an issue when a release build fails

.github/scripts/ci/main-failure-signature.mjs

@@ -156,6 +156,10 @@ const ALSO_FAILING_HEADING = '## Also failing';
 // under it.
 const ALSO_FAILING_BLOCK = /\n*##\s+Also failing\s*\n+(?:- [^\n]*\n?)+/;
 
+// The same split/merge contract — head / recorded occurrences / tail around
+// the marker, human text kept verbatim, occurrences newest-first and capped —
+// is re-implemented in bash/awk by .github/scripts/image-build-failure-issue.sh
+// for the build-and-publish-image workflow; a fix to one must reach the other.
 function splitOccurrenceBlock(body) {
   const index = body.indexOf(OCCURRENCE_MARKER);
   if (index === -1) return { head: body.trimEnd(), lines: [], tail: '' };

.github/scripts/image-build-failure-issue.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# File (or update) one issue per version when the sandbox image build job
+# fails. The job gate is the WHOLE build job — checkout, version processing,
+# QEMU/buildx setup, metadata extraction, registry login, or either build
+# step — so the wording below must not assert which step failed.
+#
+# The body below is the 'File or update the image-build failure issue' step
+# of the file-failure-issue job in .github/workflows/build-and-publish-image.yml.
+# A released npm version without a matching GHCR sandbox image breaks every
+# sandbox-based CI lane (/resolve, sandboxed review, autofix) with
+# "manifest unknown", and nothing else surfaces that state — see #9898.
+set -euo pipefail
+
+# Tag pushes name the version through the tag; manual recovery dispatches
+# carry it in the version input.
+if [[ "${EVENT_NAME}" == 'push' ]]; then
+  version="${TAG_NAME}"
+else
+  version="${INPUT_VERSION}"
+fi
+# Both paths may carry a leading `v` (tag names always do; a dispatcher may
+# type one). Normalize once so the dedup marker and the image tag — which the
+# build job publishes without a `v` — always agree, instead of filing a
+# duplicate issue for a `v`-prefixed tag that can never exist.
+version="${version#v}"
+if [[ -z "${version}" ]]; then
+  echo "::error::No version resolved for the image-build failure issue."
+  exit 1
+fi
+marker="image-build-failure:${version}"
+marker_html="<!-- ${marker} -->"
+
+# Dedup by an exact body marker, matched CLIENT-side: GitHub search
+# tokenizes the colon out of the marker, so a search-based lookup
+# never finds the issues this job files.
+issues_file="${RUNNER_TEMP}/open-issues.json"
+gh issue list \
+  --repo "${REPO}" \
+  --state open \
+  --label "${DEDUP_LABEL}" \
+  --json number,body \
+  --limit 200 \
+  > "${issues_file}"
+existing="$(
+  jq -r --arg marker_html "${marker_html}" \
+    '.[] | select(.body | contains($marker_html)) | .number' \
+    "${issues_file}" \
+  | head -n 1
+)"
+
+# The machine-owned recurrence block: every recorded failed run is a bullet
+# under this marker, newest first. On recurrence ONLY this block is rebuilt —
+# hand-written annotations anywhere else in the body survive verbatim. This is
+# the same merge contract splitOccurrenceBlock()/renderIssueBody() in
+# .github/scripts/ci/main-failure-signature.mjs implements for
+# main-ci-failure-issue.yml; a fix to one must be applied to the other.
+runs_heading='## Failed runs'
+occurrences_marker='<!-- image-build-failure-occurrences -->'
+max_runs=10
+
+body_file="${RUNNER_TEMP}/image-build-failure.md"
+head_file="${RUNNER_TEMP}/body-head.md"
+runs_file="${RUNNER_TEMP}/body-runs.txt"
+
+# The backticks in these formats are literal markdown, not command
+# substitution, so shellcheck's SC2016 expansion warning is disabled.
+# shellcheck disable=SC2016
+write_prose() {
+  printf '%s\n' "${marker_html}"
+  printf '\n'
+  printf 'The release build job for `%s` failed before `ghcr.io/qwenlm/qwen-code:%s` could be published.\n' "${version}" "${version}"
+  printf '\n'
+  printf 'Until the image exists, every sandbox-based CI lane (`/resolve`, sandboxed review, autofix) crashes with `manifest unknown` when it installs the matching npm version.\n'
+  printf '\n'
+  printf 'Open the newest run below to see which step failed, then rerun the failed jobs (transient failures — for example buildx `ETXTBSY` races during the build steps — usually pass on retry), or dispatch `Build and Publish Docker Image` with `version=%s`, `publish=true`.\n' "${version}"
+}
+
+write_body() {
+  {
+    cat "${head_file}"
+    printf '\n%s\n\n%s\n' "${runs_heading}" "${occurrences_marker}"
+    cat "${runs_file}"
+  } > "${body_file}"
+}
+
+if [[ -z "${existing}" ]]; then
+  write_prose > "${head_file}"
+  printf -- '- %s\n' "${RUN_URL}" > "${runs_file}"
+  write_body
+  gh issue create \
+    --repo "${REPO}" \
+    --title "Sandbox image for ${version} not published: release build job failed" \
+    --body-file "${body_file}" \
+    --label 'type/bug' \
+    --label "${DEDUP_LABEL}"
+  exit 0
+fi
+
+# Recurrence: re-plan against the existing body instead of overwriting it.
+existing_body="${RUNNER_TEMP}/existing-body.md"
+gh issue view "${existing}" \
+  --repo "${REPO}" \
+  --json body \
+  --jq '.body' > "${existing_body}"
+
+tail_file="${RUNNER_TEMP}/body-tail.md"
+: > "${head_file}"
+: > "${runs_file}"
+: > "${tail_file}"
+# Split head / recorded runs / tail around the occurrences marker. Anything
+# that is not a recorded-run bullet below the marker was written by a human;
+# it lands in the tail and is re-emitted with the head prose.
+awk -v marker="${occurrences_marker}" \
+    -v head_f="${head_file}" -v runs_f="${runs_file}" -v tail_f="${tail_file}" '
+  BEGIN { state = "head" }
+  state == "head" {
+    if ($0 == marker) { state = "runs"; next }
+    print > head_f
+    next
+  }
+  state == "runs" {
+    line = $0
+    sub(/^[ \t]+/, "", line)
+    sub(/[ \t]+$/, "", line)
+    if (line == "") next
+    if (line ~ /^- https:\/\/[^ ]+\/actions\/runs\/[0-9]+$/) { print > runs_f; next }
+    state = "tail"
+  }
+  state == "tail" { print > tail_f; next }
+' "${existing_body}"
+
+# Drop trailing blank lines, and a stranded heading left behind if the
+# occurrences marker line was edited away — the rebuilt block re-emits
+# both. sed, not `head -n -1`: BSD head rejects negative line counts.
+printf '%s\n' "$(cat "${head_file}")" > "${head_file}"
+if [[ "$(tail -n 1 "${head_file}")" == "${runs_heading}" ]]; then
+  printf '%s\n' "$(sed '$d' "${head_file}")" > "${head_file}"
+fi
+# Re-check AFTER the strip, which can itself empty the head: fall back to
+# the generated prose so the narrative (and the dedup marker it carries)
+# is never lost.
+if [[ -z "$(cat "${head_file}")" ]]; then
+  write_prose > "${head_file}"
+fi
+
+if [[ -s "${tail_file}" ]]; then
+  printf '\n' >> "${head_file}"
+  cat "${tail_file}" >> "${head_file}"
+fi
+
+# Newest first; a re-run of the same run must not add a second line for it.
+# awk (not head) applies the cap so the pipeline never dies on SIGPIPE.
+{ printf -- '- %s\n' "${RUN_URL}"; cat "${runs_file}"; } \
+  | awk -v max="${max_runs}" '!seen[$0]++ && ++n <= max' \
+  > "${runs_file}.merged"
+mv "${runs_file}.merged" "${runs_file}"
+
+write_body
+gh issue edit "${existing}" \
+  --repo "${REPO}" \
+  --body-file "${body_file}"
+echo "Recorded this failure on issue #${existing}."

.github/workflows/.size-baseline

@@ -15,7 +15,7 @@
 2226 assign-issue-owner.yml
 3480 audio-capture-prebuilds.yml
 9023 auto-minimize-spam.yml
-4638 build-and-publish-image.yml
+9256 build-and-publish-image.yml
 49610 cd-cua-driver.yml
 2076 cd-mobile-mcp.yml
 74315 ci.yml

.github/workflows/build-and-publish-image.yml

@@ -25,8 +25,26 @@ jobs:
     permissions:
       contents: 'read'
       packages: 'write'
+    outputs:
+      # Job-level `if:` cannot read the env context, so PUSH_IMAGE leaves the
+      # build job through this output and file-failure-issue gates on it
+      # instead of restating the publish predicate (which would drift apart
+      # from PUSH_IMAGE the next time the predicate changes).
+      push_image: '${{ steps.publish-decision.outputs.push_image }}'
+    env:
+      # Whether this run publishes. Defined once at the job level so the
+      # login gate and both build steps cannot drift apart.
+      PUSH_IMAGE: |-
+        ${{ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true') }}
 
     steps:
+      # First on purpose: this output gates file-failure-issue, and it must
+      # exist even when every later step fails — a checkout or login failure
+      # on a publishing run is exactly a run the reporting job must cover.
+      - name: 'Export the publish decision'
+        id: 'publish-decision'
+        run: 'echo "push_image=${PUSH_IMAGE}" >> "$GITHUB_OUTPUT"'
+
       - name: 'Checkout repository'
         uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10'  # v6.0.3
         with:
@@ -97,23 +115,91 @@ jobs:
             type=sha,prefix=sha-,format=short,enable=${{ steps.version.outputs.clean == '' }}
 
       - name: 'Log in to the Container registry'
-        if: |-
-          ${{ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true') }}
+        if: "${{ env.PUSH_IMAGE == 'true' }}"
         uses: 'docker/login-action@v4' # ratchet:exclude
         with:
           registry: '${{ env.REGISTRY }}'
           username: '${{ github.actor }}'
           password: '${{ secrets.GITHUB_TOKEN }}'
 
+      # continue-on-error is load-bearing: a failed first attempt must not
+      # pre-fail the job, or a successful retry below would still leave the
+      # job red and file-failure-issue would report an image that WAS
+      # published. The job only fails when the retry fails too.
       - name: 'Build and push Docker image'
         id: 'build-and-push'
+        continue-on-error: true
+        uses: 'docker/build-push-action@v7' # ratchet:exclude
+        with:
+          context: '.'
+          platforms: 'linux/amd64,linux/arm64'
+          push: '${{ env.PUSH_IMAGE }}'
+          tags: '${{ steps.meta.outputs.tags }}'
+          labels: '${{ steps.meta.outputs.labels }}'
+          build-args: |
+            CLI_VERSION_ARG=${{ steps.version.outputs.clean || github.sha }}
+
+      # One bounded retry: the docker build hits transient buildx races such as
+      # ETXTBSY during `npm ci` (the v0.22.0 tag build died this way and the
+      # image was never published, breaking every sandbox-based CI lane — see
+      # issue #9898). A retry reuses the build cache and almost always passes;
+      # a genuine failure fails this step and with it the job.
+      - name: 'Build and push Docker image (retry)'
+        id: 'build-and-push-retry'
+        if: "${{ steps.build-and-push.outcome == 'failure' }}"
         uses: 'docker/build-push-action@v7' # ratchet:exclude
         with:
           context: '.'
           platforms: 'linux/amd64,linux/arm64'
-          push: |-
-            ${{ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true') }}
+          push: '${{ env.PUSH_IMAGE }}'
           tags: '${{ steps.meta.outputs.tags }}'
           labels: '${{ steps.meta.outputs.labels }}'
           build-args: |
             CLI_VERSION_ARG=${{ steps.version.outputs.clean || github.sha }}
+
+  # One issue per version when the release build job fails (the gate covers
+  # the whole job, not only the build steps — the issue wording stays
+  # step-agnostic for the same reason). A released npm version without a
+  # sandbox image silently breaks every sandbox-based CI lane. Covers
+  # publishing dispatches too: that is the recovery path the
+  # issue body recommends. Logic lives in the script named below. A
+  # publishing dispatch without a version input is skipped by design: the
+  # build job tags such runs with its branch/sha fallback, but this job
+  # files one issue per VERSION and cannot dedup a versionless failure.
+  # Known residual gap: a build job that fails before its first step runs
+  # at all (e.g. runner provisioning failure) never executes
+  # publish-decision, so push_image stays empty and this gate skips the
+  # job even though failure() is true — no issue gets filed. A "failed
+  # publish, no issue filed" investigation should start here; a scheduled
+  # npm-vs-GHCR reconciliation is the remaining backstop for this case.
+  # Second known gap: dedup needs the scope/ci-cd label the create call
+  # applies and the version marker in the issue body; if a human removes
+  # either, the lookup misses the tracked issue and the next failure files
+  # a duplicate. The edit call cannot repair either because the edit path
+  # is unreachable without them — same reconciliation backstop as above.
+  file-failure-issue:
+    needs: ['build-and-push-to-ghcr']
+    if: |-
+      ${{ failure() && github.repository == 'QwenLM/qwen-code' && needs.build-and-push-to-ghcr.outputs.push_image == 'true' && (github.event_name == 'push' || github.event.inputs.version != '') }}
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 5
+    permissions:
+      # checkout needs contents even though the job only files an issue.
+      contents: 'read'
+      issues: 'write'
+    steps:
+      - name: 'Checkout repository'
+        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10'  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: 'File or update the image-build failure issue'
+        env:
+          GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+          REPO: '${{ github.repository }}'
+          EVENT_NAME: '${{ github.event_name }}'
+          TAG_NAME: '${{ github.ref_name }}'
+          INPUT_VERSION: '${{ github.event.inputs.version }}'
+          DEDUP_LABEL: 'scope/ci-cd'
+          RUN_URL: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
+        run: 'bash .github/scripts/image-build-failure-issue.sh'