Skip to content

ci: retry a refused owned job once on the fleet before Blacksmith #14325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
teamleaderleo merged 3 commits into from
Sep 25, 2026
Merged

ci: retry a refused owned job once on the fleet before Blacksmith #14325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0c9e8d7
ci: retry a refused owned job once on the fleet before Blacksmith
teamleaderleo Sep 25, 2026
773de5b
ci: pin the refused retry branch in the product consumer route check
teamleaderleo Sep 25, 2026
bd261f2
ci: retry a refused job on the fleet only when the rescue started the…
teamleaderleo Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions .github/workflows/ci-macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,15 @@ on:
required: false
default: ""
type: string
# Set only when pr_runner is persistent: the owned label again, for
# attempt 2 of a job in pr_owned_jobs. ci-owned-pool-rescue.yml re-runs
# the failed jobs of a run whose owned job was refused, and that re-run
# reuses attempt 1's outputs, so it tries the fleet once more before
# Blacksmith. A full re-run (a stuck job) picks again and gets none.
pr_refused_retry_runner:
required: false
default: ""
type: string
# Set only when pr_runner is persistent: the jobs of attempt 1 that take
# it, as " <key> <key> ". Every other job takes pr_retry_runner, so a
# run can use the owned machines that are free and overflow the rest
Expand Down Expand Up @@ -142,7 +151,7 @@ jobs:
# compile on the pool and Xcode seed-derived-data.yml builds with, so both
# can adopt its DerivedData seed below. Merge groups and dispatches on
# other branches keep the macos-15 lane.
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && ((github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
# A changed-suites run adds its tests after the compile: the same
# 30-minute batch ceiling the separate worker had.
timeout-minutes: ${{ inputs.unit_in_admission == 'true' && 105 || 75 }}
Expand Down Expand Up @@ -185,7 +194,7 @@ jobs:
# keys on the toolchain and the build path instead, so a product built
# here at the canonical root matches on any pool with the same Xcode.
# The macOS runner guard still requires it to track runs-on.
CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && ((github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
# What the changed-suites steps at the end of this job read, with the
# values `app-host unit tests` gives its changed-suites worker, shard 8.
# The compile reads none of them: it runs plain xcodebuild and bakes the
Expand Down Expand Up @@ -1243,7 +1252,7 @@ jobs:
# place there (no machine left, or CI_PR_POOL_OWNED_GUI=0), and a re-run
# of failed shards, take pr_retry_runner: Blacksmith on the lane's Xcode,
# which is the Xcode the owned label names.
runs-on: ${{ (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
runs-on: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard)) && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
timeout-minutes: 75
strategy:
# A pull request wants every shard's failures in one run. A merge group
Expand Down Expand Up @@ -1325,7 +1334,7 @@ jobs:
- name: Verify GitHub-hosted route
env:
RUNNER_ENVIRONMENT: ${{ runner.environment }}
REQUESTED_RUNNER: ${{ (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
REQUESTED_RUNNER: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard)) && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
RUNNER_CONTEXT_NAME: ${{ runner.name }}
run: |
set -euo pipefail
Expand Down Expand Up @@ -2239,7 +2248,7 @@ jobs:
# on the pool and Xcode that built it: the test bundle only loads under
# the Xcode that linked it. On an owned-pool run it may take
# pr_retry_runner instead, as the shards do.
runs-on: ${{ (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
runs-on: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-product ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
timeout-minutes: 40
env:
CMUX_NODE_PRODUCT_CACHE_ROOT: ${{ vars.CMUX_NODE_PRODUCT_CACHE_ROOT }}
Expand All @@ -2255,7 +2264,7 @@ jobs:
- name: Verify GitHub-hosted route
env:
RUNNER_ENVIRONMENT: ${{ runner.environment }}
REQUESTED_RUNNER: ${{ (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
REQUESTED_RUNNER: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-product ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
RUNNER_CONTEXT_NAME: ${{ runner.name }}
run: |
set -euo pipefail
Expand Down Expand Up @@ -3006,7 +3015,7 @@ jobs:
# full-suite dispatch follows admission onto the pull-request pool and
# Xcode. The product consumer guard in the CI change-area tests fails when
# the two drift.
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && ((github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' lag ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
timeout-minutes: 75
env:
CMUX_NODE_PRODUCT_CACHE_ROOT: ${{ vars.CMUX_NODE_PRODUCT_CACHE_ROOT }}
Expand All @@ -3019,7 +3028,7 @@ jobs:
steps:
- name: Validate display runner identity
env:
REQUESTED_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && ((github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
REQUESTED_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' lag ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
RUNNER_CONTEXT_NAME: ${{ runner.name }}
run: |
set -euo pipefail
Expand Down
9 changes: 8 additions & 1 deletion .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ jobs:
# Set only when the pool is persistent: the jobs of attempt 1 that take
# it (" admission shard-1 ... "); every other job takes the retry runner.
macos_pr_owned_jobs: ${{ steps.macos-pool.outputs.owned_jobs }}
# Set only when the pool is persistent: the owned label, which attempt 2
# of an owned job takes after a refusal (ci-owned-pool-rescue.yml
# re-runs failed jobs once on the fleet before Blacksmith).
macos_pr_refused_retry_runner: ${{ steps.macos-pool.outputs.refused_retry_runner }}
permissions:
actions: read
contents: read
Expand Down Expand Up @@ -926,6 +930,7 @@ jobs:
pr_runner: ${{ needs.changes.outputs.macos_pr_runner }}
pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
pr_owned_jobs: ${{ needs.changes.outputs.macos_pr_owned_jobs }}
pr_refused_retry_runner: ${{ needs.changes.outputs.macos_pr_refused_retry_runner }}
native_tests: ${{ needs.changes.outputs.remote_daemon_native == 'true' || contains(github.event.pull_request.labels.*.name, 'full-ci') }}

cli:
Expand All @@ -936,6 +941,7 @@ jobs:
pr_runner: ${{ needs.changes.outputs.macos_pr_runner }}
pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
pr_owned_jobs: ${{ needs.changes.outputs.macos_pr_owned_jobs }}
pr_refused_retry_runner: ${{ needs.changes.outputs.macos_pr_refused_retry_runner }}
pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }}

web:
Expand All @@ -952,7 +958,7 @@ jobs:
name: Claude wrapper regressions
needs: [changes, static-preflight]
if: ${{ !cancelled() && needs.changes.result == 'success' && needs.static-preflight.result == 'success' && (needs.changes.outputs.claude_wrapper == 'true' || (needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true')) }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && needs.changes.outputs.macos_pr_refused_retry_runner || github.event_name == 'pull_request' && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
timeout-minutes: 10
steps:
- name: Checkout wrapper and test inputs
Expand Down Expand Up @@ -1088,6 +1094,7 @@ jobs:
pr_runner: ${{ needs.changes.outputs.macos_pr_runner }}
pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
pr_owned_jobs: ${{ needs.changes.outputs.macos_pr_owned_jobs }}
pr_refused_retry_runner: ${{ needs.changes.outputs.macos_pr_refused_retry_runner }}
pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }}

tests:
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/cli-pipe-regressions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ on:
required: false
default: ""
type: string
# Set only when pr_runner is persistent: the owned label again, for
# attempt 2 of a job in pr_owned_jobs. ci-owned-pool-rescue.yml re-runs
# the failed jobs of a run whose owned job was refused, and that re-run
# reuses attempt 1's outputs, so it tries the fleet once more before
# Blacksmith. A full re-run (a stuck job) picks again and gets none.
pr_refused_retry_runner:
required: false
default: ""
type: string
# Set only when pr_runner is persistent: the jobs of attempt 1 that take
# it, as " <key> <key> ". Every other job takes pr_retry_runner, so a
# run can use the owned machines that are free and overflow the rest
Expand All @@ -41,7 +50,7 @@ concurrency:

jobs:
cli-pipe-regressions:
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && ((github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-pipe ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-pipe ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-pipe ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
timeout-minutes: 30
env:
CMUX_CI_XCODE_APP: ${{ github.event_name == 'pull_request' && (inputs.pr_xcode_app || github.event.pull_request.head.repo.full_name == github.repository && vars.CMUX_CI_XCODE_APP_PR || vars.CMUX_CI_XCODE_APP_MACOS_15) || vars.CMUX_CI_XCODE_APP_MACOS_15 }}
Expand Down
Loading
Loading