ci(cmux-next): implicit batch queue for feat-cmux-next

.github/workflows/cmux-next.yml

@@ -161,6 +161,9 @@ on:
       - scripts/lint_swift_namespaces.py
       - scripts/lint-namespace-types-*.txt
       - scripts/swift_source_mask.py
+  # Dispatch runs every tier on feat-cmux-next and on the next-batch/*
+  # integration branches that scripts/ci/next_batch.py validates (a stack of
+  # open pull requests); on any other ref the macOS jobs skip.
   workflow_dispatch:
 
 permissions:
@@ -300,7 +303,7 @@ jobs:
     # Mac jobs (needs plus !cancelled()) take their route without it. A re-run
     # needs no marker of its own: its Mac jobs take the fallback, and the
     # rescue's watch of a side lane ends at attempt 2.
-    if: ${{ needs.path_route.outputs.macos == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') && github.repository_owner == 'manaflow-ai' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CI_PR_POOL_OWNED == '1' && github.run_attempt == 1 }}
+    if: ${{ needs.path_route.outputs.macos == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) && github.repository_owner == 'manaflow-ai' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CI_PR_POOL_OWNED == '1' && github.run_attempt == 1 }}
     name: cmux-next macOS placement
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }}
     timeout-minutes: 3
@@ -366,7 +369,7 @@ jobs:
           retention-days: 1
 
   checks:
-    if: ${{ github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' }}
+    if: ${{ github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/') }}
     name: cmux-next checks (god files, concurrency, crash safety, l10n)
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }}
     timeout-minutes: 10
@@ -535,7 +538,7 @@ jobs:
   # timeout fails the gate (pin-cmux-tui.sh --help, wait).
   same-tree-cmux-tui:
     needs: [path_route, push-head-preflight]
-    if: ${{ !cancelled() && (needs.path_route.outputs.daemon == 'true' || needs.path_route.outputs.scheme == 'true') && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') }}
+    if: ${{ !cancelled() && (needs.path_route.outputs.daemon == 'true' || needs.path_route.outputs.scheme == 'true') && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) }}
     name: wait for the same-tree cmux-tui
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
     # actions: read lets the wait read the artifacts runs; contents: read the
@@ -575,7 +578,7 @@ jobs:
   # already published, and daemon-test runs them against it.
   swift-test:
     needs: [path_route, macos-placement, push-head-preflight]
-    if: ${{ !cancelled() && needs.path_route.outputs.swift == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') }}
+    if: ${{ !cancelled() && needs.path_route.outputs.swift == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) }}
     name: cmux-next swift test
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 2 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}
     permissions:
@@ -755,7 +758,7 @@ jobs:
   # daemon, Rust core, FFI and control paths, a full-ci PR and every push.
   daemon-test:
     needs: [path_route, macos-placement, push-head-preflight, same-tree-cmux-tui]
-    if: ${{ !cancelled() && needs.path_route.outputs.daemon == 'true' && needs.push-head-preflight.outputs.current == 'true' && needs.same-tree-cmux-tui.result == 'success' && needs.same-tree-cmux-tui.outputs.superseded != 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') }}
+    if: ${{ !cancelled() && needs.path_route.outputs.daemon == 'true' && needs.push-head-preflight.outputs.current == 'true' && needs.same-tree-cmux-tui.result == 'success' && needs.same-tree-cmux-tui.outputs.superseded != 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) }}
     name: cmux-next daemon tests
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 2 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}
     # actions: read lets the same-tree cmux-tui fetch read the artifacts runs.
@@ -860,7 +863,7 @@ jobs:
   # generated-autofix commits it to a same-repository pull request.
   generated-files:
     needs: [path_route, macos-placement, push-head-preflight]
-    if: ${{ !cancelled() && needs.path_route.outputs.generated == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') }}
+    if: ${{ !cancelled() && needs.path_route.outputs.generated == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) }}
     name: cmux-next generated files
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 2 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}
     permissions:
@@ -1063,7 +1066,7 @@ jobs:
 
   release-compile:
     needs: [path_route, macos-placement, push-head-preflight]
-    if: ${{ !cancelled() && needs.path_route.outputs.native == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') }}
+    if: ${{ !cancelled() && needs.path_route.outputs.native == 'true' && needs.push-head-preflight.outputs.current == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) }}
     name: cmux-next Release compile (Xcode 26)
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 2 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}
     # One-architecture Release (-O, whole-module) xcodebuild of CmuxNextApp
@@ -1158,7 +1161,7 @@ jobs:
 
   cmux-scheme-compile:
     needs: [path_route, macos-placement, push-head-preflight, same-tree-cmux-tui]
-    if: ${{ !cancelled() && needs.path_route.outputs.scheme == 'true' && needs.push-head-preflight.outputs.current == 'true' && needs.same-tree-cmux-tui.result == 'success' && needs.same-tree-cmux-tui.outputs.superseded != 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next') }}
+    if: ${{ !cancelled() && needs.path_route.outputs.scheme == 'true' && needs.push-head-preflight.outputs.current == 'true' && needs.same-tree-cmux-tui.result == 'success' && needs.same-tree-cmux-tui.outputs.superseded != 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/feat-cmux-next' || startsWith(github.ref, 'refs/heads/next-batch/')) }}
     name: cmux app scheme compile (Debug)
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 2 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}
     # actions: read lets the same-tree cmux-tui wait read the artifacts runs.

docs/ci/cmux-next-batch.md

@@ -0,0 +1,70 @@
+# cmux-next batch queue
+
+Open a PR into `feat-cmux-next` and let the queue land it. You don't stack, rebase or rerun anything by hand. `scripts/ci/next_batch.py` is the controller. It runs either as `.github/workflows/cmux-next-batch.yml` with a GitHub App, or as `next_batch.py serve` on a workstation under the operator's gh login (see [Serving locally](#serving-locally)).
+
+## What makes a PR eligible
+
+- It isn't a draft, and its branch is in this repository.
+- It's by a batch author (the `CMUX_NEXT_BATCH_AUTHORS` variable, default `teamleaderleo`), or anyone else labels it `batch-queue`.
+- It has none of the labels `hold`, `exploration`, `needs a call`, `default call`, `do-not-merge` or `wip`.
+- Its head commit is less than 5 days old.
+- It doesn't change `.github/workflows/`. Those PRs land by hand.
+- No check is red outside the heavy tier, except web formatting (`web / react-apps-check`, `web / Web status`) on a PR that changes `webviews/`. Before landing such a PR, the queue runs `bun run check:fix` (formatter and lint autofix) on its head in `regen-linux` and pushes the fix to its branch. gh-merge-green then waits for the rerun.
+- Pending checks are fine. The heavy tier is swift test, the Release and scheme compiles, daemon tests, generated files and the cmux-tui wait. The batch runs it once on the stack, so the PR's own copy may be red or still running.
+
+To keep a PR out, add `hold`. A PR the queue dropped at its current head stays out until you push.
+
+## One batch
+
+1. **Trigger.** A PR event or a feat-cmux-next push starts `debounce`. Each new event cancels the waiting run. The batch is dispatched after 2 minutes of quiet, and never later than 10 minutes after the first event. A running batch is never cancelled. A newer dispatch waits for it.
+2. **Stack.** Starting from the feat-cmux-next head, the batch merges up to 12 eligible PRs in PR-number order.
+   - A conflict in a generated file keeps the stack's copy, and the generator rebuilds it once after the stack is pushed:
+     - Web bundles: `scripts/cmux-next/regenerate-web-bundles.sh`, on Linux (`regen-linux`).
+     - SDK bindings: `cmux-tui/bindings/codegen/generate.py --write`, on Linux (`regen-linux`).
+     - Swift exports such as action contracts, the settings schema and MDM: `scripts/cmux-next/regenerate-swift-exports.sh`, on a mini (`regen`).
+
+     Generators run the stack's code, so they never run on the controller's host. Each job has a read-only token and returns its commit as a patch, which the controller applies and pushes.
+   - `cmux-tui/spec/*.json` merges key by key.
+   - String catalogs and the Xcode project merge as in `scripts/merge-main.sh`.
+   - Any other conflict drops that PR from the batch and comments on it.
+3. **Validate once.** The stack is pushed to `next-batch/<run>-<n>`. `cmux-next.yml` is dispatched there, and every tier runs. If the stack changes cmux-tui, `cmux-tui-artifacts.yml` is also dispatched on a `cmux-tui-pin-*` ref. At the same time, a mini submits a fleet `--production` build of the stack. A job that is red on feat-cmux-next itself doesn't count against the batch. Failed jobs get one rerun before any PR is blamed.
+4. **Land or bisect.**
+   - Green: each PR lands with `gh-merge-green --squash`, using main's copy, after a comment that links the batch. If the PR's own heavy check is red, `--override` names the batch run.
+   - Red: the batch tests prefixes of the stack by halving. The last PR of the smallest red prefix is the culprit. It gets a comment with the failing jobs, and the batch reruns without it, for at most 3 culprits per batch.
+5. **Report.** The job summary, and the sticky comment on `CMUX_NEXT_BATCH_STICKY` when set (an HQ issue), list each PR's outcome, every validation with its wall time, the heavy-tier run and the build link.
+
+In the workflow, pushes and merges use the `CMUX_NEXT_BATCH_APP_ID` App's token, which needs contents, workflows and pull-requests write. GitHub refuses the job token any branch based on feat-cmux-next. After landing, the batch dispatches `cmux-next.yml` and `cmux-tui-artifacts.yml` on feat-cmux-next, plus the next batch.
+
+## Serving locally
+
+Until the App exists, the queue runs on a workstation that is on the build tailnet, with a gh login that has `repo` and `workflow` scopes:
+
+```bash
+CMUX_NEXT_BATCH_STICKY=manaflow-ai/cmuxterm-hq#1392 \
+  python3 scripts/ci/next_batch.py serve --worktree ~/.cache/next-batch-stack
+```
+
+It polls the open PRs every minute and uses the debounce job's timing: 2 minutes of quiet, at most 10 minutes. It runs one batch at a time and never interrupts one. Its fleet build uses the host's `cmux-ci`. Regeneration and the heavy tier still run on Actions, and only the controller runs locally. Its pushes and merges start CI like anyone's, so nothing is re-dispatched after landing. Comments link the sticky issue. Keep `CMUX_NEXT_BATCH_ENABLED` unset while a local controller serves, so the two never race.
+
+By default `serve` merges nothing. It posts a receipt on each validated PR (the stack, the fleet job and the heavy jobs that passed), and the owner lands it with gh-merge-green. `--land` makes it merge. A receipt says when the heavy tier did not run, which happens when the stack's `cmux-next.yml` predates the next-batch gate.
+
+`run --local` runs one batch the same way, receipts included.
+
+## Run it by hand
+
+```bash
+gh workflow run cmux-next-batch.yml --ref feat-cmux-next -f mode=batch -f dry_run=true   # validate only, no writes
+gh workflow run cmux-next-batch.yml --ref feat-cmux-next -f mode=batch -f prs="17505 17508"
+python3 scripts/ci/next_batch.py select            # who is eligible, and why the rest aren't
+python3 scripts/ci/next_batch.py stack --prs "17505 17508" --no-regen --worktree /tmp/stack
+```
+
+## Repository settings
+
+| Setting | Use |
+| --- | --- |
+| `CMUX_NEXT_BATCH_ENABLED` (variable) | `1` turns the debounce dispatch on |
+| `CMUX_NEXT_BATCH_APP_ID` (variable), `CMUX_NEXT_BATCH_APP_KEY` (secret) | the App that pushes stacks and merges |
+| `CMUX_FLEET_CONTROLLER` (variable) | the build controller URL the mini's build job submits to |
+| `CMUX_NEXT_BATCH_STICKY` (variable) | `owner/repo#issue` for the sticky report |
+| `CMUX_NEXT_BATCH_AUTHORS` (variable) | authors whose PRs need no opt-in label |