diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fc288a9cb739..6c32c12f5c52 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -126,16 +126,20 @@ jobs: name: Build Docker (${{ matrix.platform }}) needs: prepare if: needs.prepare.outputs.skip != 'true' - # amd64: the .113 omni-build pool (31 GB / 32 cores, two listeners). A - # next-build peaks at 14–16 GB; hosted ubuntu-24.04 is ~7 GB and dies - # ResourceExhausted (#11976). Falls back to hosted + webpack when - # USE_VPS_RUNNER is off. arm64: no ARM box — stay on GitHub's ubuntu-24.04-arm - # with webpack (Turbopack native RSS does not fit 7 GB). + # amd64: the .113 omni-build pool (31 GB / 32 cores, ONE listener since + # #12048). Hosted ubuntu-24.04 is ~7 GB and dies ResourceExhausted (#11976). + # Falls back to hosted when USE_VPS_RUNNER is off. arm64: no ARM box — stay + # on GitHub's ubuntu-24.04-arm. + # Webpack on BOTH arches: Turbopack on omniroute-113-6 hit + # TurbopackInternalError "there must be a path to a root" after 26 min + # (run 33253576569). The same tree's arm64 webpack build on hosted ARM + # succeeded (run 33264823398). Dockerfile already documents webpack as the + # Docker escape hatch (OMNIROUTE_USE_TURBOPACK=0). runs-on: ${{ matrix.arch == 'amd64' && (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-24.04') || 'ubuntu-24.04-arm' }} - # Share the 2-slot omni-build ceiling with ci.yml `Build` / npm-publish. - # Same group as main's Build so a :next publish waits beside the artefact - # instead of becoming the third heavy and getting kernel-OOM'd (2026-08-28). - # arm64 is hosted — its own group, cancelled by the workflow-level concurrency. + # Share the 1-slot omni-build ceiling (#12048) with ci.yml `Build` / + # npm-publish. Same group as main's Build so a :next publish waits beside + # the artefact instead of sitting next to it. arm64 is hosted — its own + # group, cancelled by the workflow-level concurrency. concurrency: group: ${{ matrix.arch == 'amd64' && 'heavy-build-main' || format('docker-publish-arm-{0}', github.ref) }} cancel-in-progress: ${{ matrix.arch != 'amd64' }} @@ -153,8 +157,6 @@ jobs: env: IMAGE_NAME: diegosouzapw/omniroute GHCR_IMAGE_NAME: ghcr.io/diegosouzapw/omniroute - # Turbopack on the 31 GB box; webpack on any hosted 7 GB fallback / ARM. - USE_TURBOPACK: ${{ (matrix.arch == 'amd64' && vars.USE_VPS_RUNNER == 'true') && '1' || '0' }} steps: - name: Checkout uses: actions/checkout@v7 @@ -190,7 +192,7 @@ jobs: target: runner-base platforms: ${{ matrix.platform }} build-args: | - OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} + OMNIROUTE_USE_TURBOPACK=0 outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} @@ -209,7 +211,7 @@ jobs: target: runner-web platforms: ${{ matrix.platform }} build-args: | - OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} + OMNIROUTE_USE_TURBOPACK=0 outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} @@ -236,7 +238,7 @@ jobs: target: runner-base platforms: ${{ matrix.platform }} build-args: | - OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} + OMNIROUTE_USE_TURBOPACK=0 outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} @@ -263,7 +265,7 @@ jobs: target: runner-web platforms: ${{ matrix.platform }} build-args: | - OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} + OMNIROUTE_USE_TURBOPACK=0 outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} diff --git a/changelog.d/maintenance/11976-docker-publish-webpack-omni-build.md b/changelog.d/maintenance/11976-docker-publish-webpack-omni-build.md new file mode 100644 index 000000000000..077671b64c6a --- /dev/null +++ b/changelog.d/maintenance/11976-docker-publish-webpack-omni-build.md @@ -0,0 +1 @@ +- `docker-publish.yml` builds with webpack (`OMNIROUTE_USE_TURBOPACK=0`) on **both** arches, including amd64 on the `.113` `omni-build` pool. Turbopack had the RAM (31 GB, `omniroute-113-6`) and still panicked (`TurbopackInternalError: there must be a path to a root`, run 33253576569, 26 min in). The same tree's arm64 webpack build on hosted `ubuntu-24.04-arm` succeeded (run 33264823398). Dockerfile already documents webpack as the Docker escape hatch (#11976). diff --git a/docs/ops/RUNNER_BOX.md b/docs/ops/RUNNER_BOX.md index 96b638a1ebbc..58e17414ed04 100644 --- a/docs/ops/RUNNER_BOX.md +++ b/docs/ops/RUNNER_BOX.md @@ -50,10 +50,19 @@ a time, only when idle**, with the idle check and the restart in the same comman ## Operating rules - **Heavy-build ceiling: ONE at a time — enforced by label (since 2026-08-29).** Every job - that runs a `next build` (`ci.yml` `build`, `npm-publish.yml` `publish`, both - `nightly-release-green` validations) targets `[self-hosted, omni-build]`, and only + that runs a full `next build` targets `[self-hosted, omni-build]`, and only **`omniroute-113-5`** carries that label (added through the runners API — no - re-registration). Two was the previous ceiling and it was wrong for 31 GB: on + re-registration): `ci.yml` `Build`, `npm-publish.yml` `publish`, both + `nightly-release-green` validations, and `docker-publish.yml` **amd64** (hosted + 7 GB ResourceExhausted this tree — #11976). The arm64 Docker leg stays on + `ubuntu-24.04-arm` (no ARM box) with webpack. Docker amd64 also uses webpack: + Turbopack on this tree panicked inside BuildKit (`TurbopackInternalError: + there must be a path to a root`, run 33253576569) even with 31 GB; the same + tree's arm64 webpack build on hosted ARM succeeded. `docker-publish` amd64 + shares the `heavy-build-main` concurrency group with `ci.yml` `Build` + (`cancel-in-progress: false`) so it queues on the one slot. Docker Engine + must be on `omniroute-113-5` (`docker info` is the first step of the publish + job). Two was the previous ceiling and it was wrong for 31 GB: on 2026-08-29 17:26 UTC two concurrent `next-build`s (15.4 GB + 17.2 GB RSS) drove the box to 5 GB free with 4 GB of swap in use and the kernel OOM-killed one of them — systemd booked the kill on the _other_ runner's unit, `runsvc.sh` SIGKILLed that listener, and