From b4dd4fd7cac56ef51a02b3bb80af6ad5dd049941 Mon Sep 17 00:00:00 2001 From: Robert Yan Date: Sat, 4 Jul 2026 17:38:30 +0000 Subject: [PATCH 1/8] ci: sync green main to staging --- .github/workflows/sync-main-to-staging.yml | 210 +++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 .github/workflows/sync-main-to-staging.yml diff --git a/.github/workflows/sync-main-to-staging.yml b/.github/workflows/sync-main-to-staging.yml new file mode 100644 index 00000000000..88f5a79acd7 --- /dev/null +++ b/.github/workflows/sync-main-to-staging.yml @@ -0,0 +1,210 @@ +name: Sync main to staging + +on: + workflow_dispatch: + schedule: + - cron: "17,47 * * * *" + workflow_run: + workflows: + - Code Coverage + - Code Style + - Docker Image + - Live Canary + - Platform & Compat + - Reborn Coverage + - Reborn E2E + - Release-plz + - Replay Snapshot Gate + - Tests (Reborn) + branches: + - main + types: + - completed + +permissions: + contents: write + +concurrency: + group: sync-main-to-staging + cancel-in-progress: false + +jobs: + check-main: + name: Check latest main status + runs-on: ubuntu-latest + outputs: + green: ${{ steps.rollup.outputs.green }} + main_sha: ${{ steps.main.outputs.sha }} + short_sha: ${{ steps.main.outputs.short_sha }} + steps: + - name: Resolve latest main + id: main + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + sha="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq '.object.sha')" + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + echo "short_sha=${sha:0:12}" >> "$GITHUB_OUTPUT" + + - name: Verify latest main has green checks + id: rollup + env: + GH_TOKEN: ${{ github.token }} + MAIN_SHA: ${{ steps.main.outputs.sha }} + run: | + set -euo pipefail + + query=' + query($owner: String!, $repo: String!, $oid: GitObjectID!, $after: String) { + repository(owner: $owner, name: $repo) { + object(oid: $oid) { + ... on Commit { + statusCheckRollup { + contexts(first: 100, after: $after) { + pageInfo { + hasNextPage + endCursor + } + nodes { + __typename + ... on CheckRun { + name + status + conclusion + } + ... on StatusContext { + context + state + } + } + } + } + } + } + } + } + ' + + owner="${GITHUB_REPOSITORY_OWNER}" + repo="${GITHUB_REPOSITORY#*/}" + after="" + bad=0 + pending=0 + total=0 + + while :; do + args=( + -f "query=${query}" + -f "owner=${owner}" + -f "repo=${repo}" + -f "oid=${MAIN_SHA}" + ) + if [[ -n "${after}" ]]; then + args+=(-f "after=${after}") + fi + + page="$(gh api graphql "${args[@]}")" + contexts="$(jq -c '.data.repository.object.statusCheckRollup.contexts.nodes[]?' <<<"${page}")" + + while IFS= read -r context; do + [[ -n "${context}" ]] || continue + type="$(jq -r '.__typename' <<<"${context}")" + if [[ "${type}" == "CheckRun" ]]; then + name="$(jq -r '.name' <<<"${context}")" + status="$(jq -r '.status' <<<"${context}")" + conclusion="$(jq -r '.conclusion // ""' <<<"${context}")" + + # Ignore this workflow's own jobs. They are attached to the + # same main commit while this guard is running, so including + # them would make the guard self-block forever. + case "${name}" in + "Check latest main status"|"Fast-forward staging") + continue + ;; + esac + + total=$((total + 1)) + if [[ "${status}" != "COMPLETED" ]]; then + echo "::notice::Pending check on main ${MAIN_SHA}: ${name} (${status})" + pending=$((pending + 1)) + continue + fi + case "${conclusion}" in + SUCCESS|SKIPPED|NEUTRAL) + ;; + *) + echo "::notice::Non-green check on main ${MAIN_SHA}: ${name} (${conclusion})" + bad=$((bad + 1)) + ;; + esac + elif [[ "${type}" == "StatusContext" ]]; then + name="$(jq -r '.context' <<<"${context}")" + state="$(jq -r '.state' <<<"${context}")" + total=$((total + 1)) + if [[ "${state}" != "SUCCESS" ]]; then + echo "::notice::Non-green status on main ${MAIN_SHA}: ${name} (${state})" + bad=$((bad + 1)) + fi + fi + done <<<"${contexts}" + + has_next="$(jq -r '.data.repository.object.statusCheckRollup.contexts.pageInfo.hasNextPage // false' <<<"${page}")" + if [[ "${has_next}" != "true" ]]; then + break + fi + after="$(jq -r '.data.repository.object.statusCheckRollup.contexts.pageInfo.endCursor' <<<"${page}")" + done + + if (( total == 0 )); then + echo "::notice::No status checks found for main ${MAIN_SHA}; not syncing staging." + echo "green=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if (( pending > 0 || bad > 0 )); then + echo "::notice::main ${MAIN_SHA} is not green: total=${total}, pending=${pending}, non_green=${bad}." + echo "green=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "::notice::main ${MAIN_SHA} is green across ${total} checks/statuses." + echo "green=true" >> "$GITHUB_OUTPUT" + + sync-staging: + name: Fast-forward staging + needs: check-main + if: needs.check-main.outputs.green == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Fast-forward staging to green main + env: + GH_TOKEN: ${{ github.token }} + MAIN_SHA: ${{ needs.check-main.outputs.main_sha }} + run: | + set -euo pipefail + + git fetch origin main staging + + if git merge-base --is-ancestor "origin/main" "origin/staging"; then + echo "::notice::staging already contains main ${MAIN_SHA}; nothing to sync." + exit 0 + fi + + if ! git merge-base --is-ancestor "origin/staging" "origin/main"; then + echo "::error::staging has commits that are not in main; refusing to overwrite or merge automatically." + exit 1 + fi + + gh api \ + -X PATCH \ + "repos/${GITHUB_REPOSITORY}/git/refs/heads/staging" \ + -f "sha=${MAIN_SHA}" \ + -F "force=false" >/dev/null + + echo "::notice::Fast-forwarded staging to green main ${MAIN_SHA}." From 5d9f290c9fceb1d6c9a3efe3a77e911d9ca4e0a8 Mon Sep 17 00:00:00 2001 From: Robert Yan Date: Sat, 11 Jul 2026 02:07:06 +0800 Subject: [PATCH 2/8] Merge main CI checks workflows --- .github/workflows/README.md | 17 +- .github/workflows/main-ci-checks.yml | 228 +++++++++++++++++++++ .github/workflows/main-ci-slack-alerts.yml | 105 ---------- .github/workflows/sync-main-to-staging.yml | 210 ------------------- 4 files changed, 242 insertions(+), 318 deletions(-) create mode 100644 .github/workflows/main-ci-checks.yml delete mode 100644 .github/workflows/main-ci-slack-alerts.yml delete mode 100644 .github/workflows/sync-main-to-staging.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 066b444406e..de1792d2320 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -97,9 +97,9 @@ trail: the former in-run alert jobs and `nightly-alert-issue.sh` were removed in favor of this single external check, because an in-run alert dies with its own run on a startup_failure and can never see a cron that didn't fire. -### Main branch alerting +### Main CI checks -`main-ci-slack-alerts.yml` watches completed `workflow_run` events for the +`main-ci-checks.yml` watches completed `workflow_run` events for the current `push` to `main` workflows: Code Style, Tests (Reborn), Reborn E2E, Platform & Compat, Replay Snapshot Gate, Code Coverage, nearai-bench dispatcher tests, and Release-plz. Any watched run that concludes @@ -111,8 +111,19 @@ Alerts go to `secrets.MAIN_CI_SLACK_WEBHOOK_URLS`; the value may be a single webhook URL or multiple URLs separated by newlines or commas. This is intentionally separate from the canary/nightly `SLACK_WEBHOOK_URL` so main CI alerts can target dedicated channels. + +The same workflow fast-forwards `staging-release` to the main commit after all +required push-to-main CI workflows for that SHA have succeeded. Required +workflows are Code Style, Tests (Reborn), Platform & Compat, Replay Snapshot +Gate, Code Coverage, and Release-plz. Path-filtered workflows — Reborn E2E and +nearai-bench dispatcher tests — block staging-release promotion when they ran +for the SHA, but are ignored when their path filters skip them. The +staging-release update is a normal `git push` to +`refs/heads/staging-release`, so it fails instead of forcing when +`staging-release` is not a fast-forward from `main`. + When adding a new workflow that runs on `push` to `main`, add its workflow -`name:` to the watched list in `main-ci-slack-alerts.yml`. +`name:` to the watched list in `main-ci-checks.yml`. ## Known accepted gaps (deliberate, revisit as needed) diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml new file mode 100644 index 00000000000..4d0f968629e --- /dev/null +++ b/.github/workflows/main-ci-checks.yml @@ -0,0 +1,228 @@ +name: Main CI Checks + +on: + workflow_run: + workflows: + - Code Style + - Tests (Reborn) + - Reborn E2E + - Platform & Compat + - Replay Snapshot Gate + - Code Coverage + - nearai-bench dispatcher tests + - Release-plz + branches: + - main + types: + - completed + +permissions: + actions: read + contents: read + +jobs: + alert: + name: Post Slack alert + if: >- + github.event.workflow_run.event == 'push' && + contains(fromJSON('["failure","timed_out","action_required","startup_failure"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-latest + steps: + - name: Post main CI failure to Slack + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + ACTOR: ${{ github.event.workflow_run.actor.login }} + MAIN_CI_SLACK_WEBHOOK_URLS: ${{ secrets.MAIN_CI_SLACK_WEBHOOK_URLS }} + run: | + set -euo pipefail + + short_sha="${HEAD_SHA:0:10}" + failed_jobs="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100" \ + --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") | "\(.name) (\(.conclusion))"] | .[0:12] | join(", ")' \ + 2>/dev/null || true)" + if [ -z "$failed_jobs" ]; then + failed_jobs="No failed jobs listed; inspect the workflow run for startup or workflow-level errors." + fi + + text="Main branch CI failed: ${WORKFLOW_NAME} concluded ${CONCLUSION} on ${short_sha}" + payload="$(jq -n \ + --arg text "$text" \ + --arg workflow "$WORKFLOW_NAME" \ + --arg conclusion "$CONCLUSION" \ + --arg branch "$HEAD_BRANCH" \ + --arg sha "$short_sha" \ + --arg actor "$ACTOR" \ + --arg failed_jobs "$failed_jobs" \ + --arg url "$RUN_URL" \ + '{ + text: $text, + blocks: [ + { + type: "header", + text: {type: "plain_text", text: "Main branch CI failed"} + }, + { + type: "section", + text: { + type: "mrkdwn", + text: "*Workflow:* \($workflow)\n*Conclusion:* `\($conclusion)`\n*Branch:* `\($branch)`\n*Commit:* `\($sha)`\n*Actor:* `\($actor)`\n*Failed jobs:* \($failed_jobs)\n<\($url)|Open workflow run>" + } + } + ] + }')" + + webhooks="${MAIN_CI_SLACK_WEBHOOK_URLS:-}" + if [ -z "$webhooks" ]; then + echo "::error::Set MAIN_CI_SLACK_WEBHOOK_URLS to receive main CI failure alerts." + exit 1 + fi + + webhook_file="$(mktemp)" + trap 'rm -f "$webhook_file"' EXIT + printf '%s\n' "$webhooks" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;/^$/d' > "$webhook_file" + + posted=0 + while IFS= read -r webhook; do + if curl -fsS --max-time 30 -X POST -H 'Content-type: application/json' \ + --data "$payload" "$webhook" > /dev/null; then + posted=$((posted + 1)) + else + echo "::warning::Failed to post alert to webhook (HTTP error or timeout)." + fi + done < "$webhook_file" + + if [ "$posted" -eq 0 ]; then + echo "::error::Failed to post alert to any Slack webhook." + exit 1 + fi + + echo "Posted main CI failure alert to ${posted} Slack webhook(s)." + + fast-forward-staging-release: + name: Fast-forward staging-release to main + if: >- + github.event.workflow_run.event == 'push' && + github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + concurrency: + group: main-ci-checks-staging-release + cancel-in-progress: false + permissions: + actions: read + contents: write + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + STAGING_BRANCH: staging-release + steps: + - name: Check main CI run set + id: ci + run: | + set -euo pipefail + + if [ "$HEAD_BRANCH" != "main" ]; then + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "Ignoring non-main workflow_run branch: ${HEAD_BRANCH}" + exit 0 + fi + + required_workflows=( + "Code Style" + "Tests (Reborn)" + "Platform & Compat" + "Replay Snapshot Gate" + "Code Coverage" + "Release-plz" + ) + optional_workflows=( + "Reborn E2E" + "nearai-bench dispatcher tests" + ) + + missing=() + pending=() + failed=() + + check_workflow() { + local workflow="$1" + local required="$2" + local run_json status conclusion url + + run_json="$(gh run list \ + --repo "$GITHUB_REPOSITORY" \ + --workflow "$workflow" \ + --branch main \ + --event push \ + --commit "$HEAD_SHA" \ + --limit 1 \ + --json conclusion,status,url \ + --jq '.[0] // {}')" + status="$(jq -r '.status // "missing"' <<<"$run_json")" + conclusion="$(jq -r '.conclusion // "missing"' <<<"$run_json")" + url="$(jq -r '.url // ""' <<<"$run_json")" + + if [ "$status" = "missing" ]; then + if [ "$required" = "required" ]; then + missing+=("$workflow") + else + echo "Optional workflow did not run for this SHA: ${workflow}" + fi + return + fi + + if [ "$status" != "completed" ]; then + pending+=("${workflow} (${status})") + return + fi + + if [ "$conclusion" != "success" ]; then + failed+=("${workflow} (${conclusion}) ${url}") + fi + } + + for workflow in "${required_workflows[@]}"; do + check_workflow "$workflow" required + done + for workflow in "${optional_workflows[@]}"; do + check_workflow "$workflow" optional + done + + if [ "${#failed[@]}" -gt 0 ]; then + printf '::error::Main CI failed for %s; refusing to fast-forward staging-release.\n' "$HEAD_SHA" + printf 'Failed workflow: %s\n' "${failed[@]}" + exit 1 + fi + + if [ "${#missing[@]}" -gt 0 ] || [ "${#pending[@]}" -gt 0 ]; then + echo "ready=false" >> "$GITHUB_OUTPUT" + if [ "${#missing[@]}" -gt 0 ]; then + printf 'Required workflow has not appeared yet: %s\n' "${missing[@]}" + fi + if [ "${#pending[@]}" -gt 0 ]; then + printf 'Workflow is still pending: %s\n' "${pending[@]}" + fi + exit 0 + fi + + echo "ready=true" >> "$GITHUB_OUTPUT" + echo "All required main CI workflows passed for ${HEAD_SHA}." + + - name: Checkout main commit + if: steps.ci.outputs.ready == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.workflow_run.head_sha }} + fetch-depth: 0 + + - name: Fast-forward staging-release + if: steps.ci.outputs.ready == 'true' + run: | + set -euo pipefail + git push origin "HEAD:refs/heads/${STAGING_BRANCH}" diff --git a/.github/workflows/main-ci-slack-alerts.yml b/.github/workflows/main-ci-slack-alerts.yml deleted file mode 100644 index 138624e70e8..00000000000 --- a/.github/workflows/main-ci-slack-alerts.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Main CI Slack Alerts - -on: - workflow_run: - workflows: - - Code Style - - Tests (Reborn) - - Reborn E2E - - Platform & Compat - - Replay Snapshot Gate - - Code Coverage - - nearai-bench dispatcher tests - - Release-plz - branches: - - main - types: - - completed - -permissions: - actions: read - contents: read - -jobs: - alert: - name: Post Slack alert - if: >- - github.event.workflow_run.event == 'push' && - contains(fromJSON('["failure","timed_out","action_required","startup_failure"]'), github.event.workflow_run.conclusion) - runs-on: ubuntu-latest - steps: - - name: Post main CI failure to Slack - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_URL: ${{ github.event.workflow_run.html_url }} - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} - CONCLUSION: ${{ github.event.workflow_run.conclusion }} - HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - ACTOR: ${{ github.event.workflow_run.actor.login }} - MAIN_CI_SLACK_WEBHOOK_URLS: ${{ secrets.MAIN_CI_SLACK_WEBHOOK_URLS }} - run: | - set -euo pipefail - - short_sha="${HEAD_SHA:0:10}" - failed_jobs="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100" \ - --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") | "\(.name) (\(.conclusion))"] | .[0:12] | join(", ")' \ - 2>/dev/null || true)" - if [ -z "$failed_jobs" ]; then - failed_jobs="No failed jobs listed; inspect the workflow run for startup or workflow-level errors." - fi - - text="Main branch CI failed: ${WORKFLOW_NAME} concluded ${CONCLUSION} on ${short_sha}" - payload="$(jq -n \ - --arg text "$text" \ - --arg workflow "$WORKFLOW_NAME" \ - --arg conclusion "$CONCLUSION" \ - --arg branch "$HEAD_BRANCH" \ - --arg sha "$short_sha" \ - --arg actor "$ACTOR" \ - --arg failed_jobs "$failed_jobs" \ - --arg url "$RUN_URL" \ - '{ - text: $text, - blocks: [ - { - type: "header", - text: {type: "plain_text", text: "Main branch CI failed"} - }, - { - type: "section", - text: { - type: "mrkdwn", - text: "*Workflow:* \($workflow)\n*Conclusion:* `\($conclusion)`\n*Branch:* `\($branch)`\n*Commit:* `\($sha)`\n*Actor:* `\($actor)`\n*Failed jobs:* \($failed_jobs)\n<\($url)|Open workflow run>" - } - } - ] - }')" - - webhooks="${MAIN_CI_SLACK_WEBHOOK_URLS:-}" - if [ -z "$webhooks" ]; then - echo "::error::Set MAIN_CI_SLACK_WEBHOOK_URLS to receive main CI failure alerts." - exit 1 - fi - - webhook_file="$(mktemp)" - trap 'rm -f "$webhook_file"' EXIT - printf '%s\n' "$webhooks" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;/^$/d' > "$webhook_file" - - posted=0 - while IFS= read -r webhook; do - if curl -fsS --max-time 30 -X POST -H 'Content-type: application/json' \ - --data "$payload" "$webhook" > /dev/null; then - posted=$((posted + 1)) - else - echo "::warning::Failed to post alert to webhook (HTTP error or timeout)." - fi - done < "$webhook_file" - - if [ "$posted" -eq 0 ]; then - echo "::error::Failed to post alert to any Slack webhook." - exit 1 - fi - - echo "Posted main CI failure alert to ${posted} Slack webhook(s)." diff --git a/.github/workflows/sync-main-to-staging.yml b/.github/workflows/sync-main-to-staging.yml deleted file mode 100644 index 88f5a79acd7..00000000000 --- a/.github/workflows/sync-main-to-staging.yml +++ /dev/null @@ -1,210 +0,0 @@ -name: Sync main to staging - -on: - workflow_dispatch: - schedule: - - cron: "17,47 * * * *" - workflow_run: - workflows: - - Code Coverage - - Code Style - - Docker Image - - Live Canary - - Platform & Compat - - Reborn Coverage - - Reborn E2E - - Release-plz - - Replay Snapshot Gate - - Tests (Reborn) - branches: - - main - types: - - completed - -permissions: - contents: write - -concurrency: - group: sync-main-to-staging - cancel-in-progress: false - -jobs: - check-main: - name: Check latest main status - runs-on: ubuntu-latest - outputs: - green: ${{ steps.rollup.outputs.green }} - main_sha: ${{ steps.main.outputs.sha }} - short_sha: ${{ steps.main.outputs.short_sha }} - steps: - - name: Resolve latest main - id: main - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - sha="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq '.object.sha')" - echo "sha=${sha}" >> "$GITHUB_OUTPUT" - echo "short_sha=${sha:0:12}" >> "$GITHUB_OUTPUT" - - - name: Verify latest main has green checks - id: rollup - env: - GH_TOKEN: ${{ github.token }} - MAIN_SHA: ${{ steps.main.outputs.sha }} - run: | - set -euo pipefail - - query=' - query($owner: String!, $repo: String!, $oid: GitObjectID!, $after: String) { - repository(owner: $owner, name: $repo) { - object(oid: $oid) { - ... on Commit { - statusCheckRollup { - contexts(first: 100, after: $after) { - pageInfo { - hasNextPage - endCursor - } - nodes { - __typename - ... on CheckRun { - name - status - conclusion - } - ... on StatusContext { - context - state - } - } - } - } - } - } - } - } - ' - - owner="${GITHUB_REPOSITORY_OWNER}" - repo="${GITHUB_REPOSITORY#*/}" - after="" - bad=0 - pending=0 - total=0 - - while :; do - args=( - -f "query=${query}" - -f "owner=${owner}" - -f "repo=${repo}" - -f "oid=${MAIN_SHA}" - ) - if [[ -n "${after}" ]]; then - args+=(-f "after=${after}") - fi - - page="$(gh api graphql "${args[@]}")" - contexts="$(jq -c '.data.repository.object.statusCheckRollup.contexts.nodes[]?' <<<"${page}")" - - while IFS= read -r context; do - [[ -n "${context}" ]] || continue - type="$(jq -r '.__typename' <<<"${context}")" - if [[ "${type}" == "CheckRun" ]]; then - name="$(jq -r '.name' <<<"${context}")" - status="$(jq -r '.status' <<<"${context}")" - conclusion="$(jq -r '.conclusion // ""' <<<"${context}")" - - # Ignore this workflow's own jobs. They are attached to the - # same main commit while this guard is running, so including - # them would make the guard self-block forever. - case "${name}" in - "Check latest main status"|"Fast-forward staging") - continue - ;; - esac - - total=$((total + 1)) - if [[ "${status}" != "COMPLETED" ]]; then - echo "::notice::Pending check on main ${MAIN_SHA}: ${name} (${status})" - pending=$((pending + 1)) - continue - fi - case "${conclusion}" in - SUCCESS|SKIPPED|NEUTRAL) - ;; - *) - echo "::notice::Non-green check on main ${MAIN_SHA}: ${name} (${conclusion})" - bad=$((bad + 1)) - ;; - esac - elif [[ "${type}" == "StatusContext" ]]; then - name="$(jq -r '.context' <<<"${context}")" - state="$(jq -r '.state' <<<"${context}")" - total=$((total + 1)) - if [[ "${state}" != "SUCCESS" ]]; then - echo "::notice::Non-green status on main ${MAIN_SHA}: ${name} (${state})" - bad=$((bad + 1)) - fi - fi - done <<<"${contexts}" - - has_next="$(jq -r '.data.repository.object.statusCheckRollup.contexts.pageInfo.hasNextPage // false' <<<"${page}")" - if [[ "${has_next}" != "true" ]]; then - break - fi - after="$(jq -r '.data.repository.object.statusCheckRollup.contexts.pageInfo.endCursor' <<<"${page}")" - done - - if (( total == 0 )); then - echo "::notice::No status checks found for main ${MAIN_SHA}; not syncing staging." - echo "green=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - if (( pending > 0 || bad > 0 )); then - echo "::notice::main ${MAIN_SHA} is not green: total=${total}, pending=${pending}, non_green=${bad}." - echo "green=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "::notice::main ${MAIN_SHA} is green across ${total} checks/statuses." - echo "green=true" >> "$GITHUB_OUTPUT" - - sync-staging: - name: Fast-forward staging - needs: check-main - if: needs.check-main.outputs.green == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Fast-forward staging to green main - env: - GH_TOKEN: ${{ github.token }} - MAIN_SHA: ${{ needs.check-main.outputs.main_sha }} - run: | - set -euo pipefail - - git fetch origin main staging - - if git merge-base --is-ancestor "origin/main" "origin/staging"; then - echo "::notice::staging already contains main ${MAIN_SHA}; nothing to sync." - exit 0 - fi - - if ! git merge-base --is-ancestor "origin/staging" "origin/main"; then - echo "::error::staging has commits that are not in main; refusing to overwrite or merge automatically." - exit 1 - fi - - gh api \ - -X PATCH \ - "repos/${GITHUB_REPOSITORY}/git/refs/heads/staging" \ - -f "sha=${MAIN_SHA}" \ - -F "force=false" >/dev/null - - echo "::notice::Fast-forwarded staging to green main ${MAIN_SHA}." From afb549d92eb31079f0a282502753b968b17b0fe6 Mon Sep 17 00:00:00 2001 From: Robert Yan Date: Sat, 11 Jul 2026 02:22:37 +0800 Subject: [PATCH 3/8] Handle stale staging-release promotions --- .github/workflows/README.md | 5 +++-- .github/workflows/main-ci-checks.yml | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index de1792d2320..4c12a5eba44 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -119,8 +119,9 @@ Gate, Code Coverage, and Release-plz. Path-filtered workflows — Reborn E2E and nearai-bench dispatcher tests — block staging-release promotion when they ran for the SHA, but are ignored when their path filters skip them. The staging-release update is a normal `git push` to -`refs/heads/staging-release`, so it fails instead of forcing when -`staging-release` is not a fast-forward from `main`. +`refs/heads/staging-release`: it skips when `staging-release` already contains +the SHA, and fails only when `staging-release` has commits that are not +ancestors of `main`. When adding a new workflow that runs on `push` to `main`, add its workflow `name:` to the watched list in `main-ci-checks.yml`. diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml index 4d0f968629e..661f886e0b4 100644 --- a/.github/workflows/main-ci-checks.yml +++ b/.github/workflows/main-ci-checks.yml @@ -225,4 +225,16 @@ jobs: if: steps.ci.outputs.ready == 'true' run: | set -euo pipefail + git fetch origin main "${STAGING_BRANCH}" + + if git merge-base --is-ancestor "$HEAD_SHA" "origin/${STAGING_BRANCH}"; then + echo "::notice::${STAGING_BRANCH} already contains main ${HEAD_SHA}; nothing to sync." + exit 0 + fi + + if ! git merge-base --is-ancestor "origin/${STAGING_BRANCH}" "origin/main"; then + echo "::error::${STAGING_BRANCH} has commits that are not in main; refusing to overwrite or merge automatically." + exit 1 + fi + git push origin "HEAD:refs/heads/${STAGING_BRANCH}" From 7ace70ad205acdd1a526b8cec01a266dfffd53e0 Mon Sep 17 00:00:00 2001 From: Robert Yan Date: Sat, 11 Jul 2026 02:25:33 +0800 Subject: [PATCH 4/8] Clarify staging-release promotion target --- .github/workflows/main-ci-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml index 661f886e0b4..e31957897e2 100644 --- a/.github/workflows/main-ci-checks.yml +++ b/.github/workflows/main-ci-checks.yml @@ -120,6 +120,7 @@ jobs: GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + # Dedicated branch used by staging deployment. STAGING_BRANCH: staging-release steps: - name: Check main CI run set From f1df91f6b944e45756491e33929e864f6022106c Mon Sep 17 00:00:00 2001 From: aiworkbot Date: Sun, 12 Jul 2026 01:49:50 +0000 Subject: [PATCH 5/8] fix(main-ci-checks): harden workflow check ordering and retry handling --- .github/workflows/README.md | 15 +++--- .github/workflows/main-ci-checks.yml | 76 ++++++++++++++++++---------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4c12a5eba44..ac054a9213b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -116,12 +116,15 @@ The same workflow fast-forwards `staging-release` to the main commit after all required push-to-main CI workflows for that SHA have succeeded. Required workflows are Code Style, Tests (Reborn), Platform & Compat, Replay Snapshot Gate, Code Coverage, and Release-plz. Path-filtered workflows — Reborn E2E and -nearai-bench dispatcher tests — block staging-release promotion when they ran -for the SHA, but are ignored when their path filters skip them. The -staging-release update is a normal `git push` to -`refs/heads/staging-release`: it skips when `staging-release` already contains -the SHA, and fails only when `staging-release` has commits that are not -ancestors of `main`. +nearai-bench dispatcher tests — block staging-release promotion when they ran for +the SHA, but are ignored when their path filters skip them. + +The `staging-release` update is a normal `git push` to +`refs/heads/staging-release`. It skips when `staging-release` already contains +the SHA, fails when required checks are hard-blocking (`staging-release` has +commits not in `main`), and can also be delayed or fail for other operational +reasons such as missing/pending CI runs (at trigger time), insufficient push +permissions, concurrent workflow updates, or checkout/push errors. When adding a new workflow that runs on `push` to `main`, add its workflow `name:` to the watched list in `main-ci-checks.yml`. diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml index e31957897e2..d77826dac62 100644 --- a/.github/workflows/main-ci-checks.yml +++ b/.github/workflows/main-ci-checks.yml @@ -2,6 +2,7 @@ name: Main CI Checks on: workflow_run: + # Keep this list aligned with the required/optional workflow lists in `jobs.fast-forward-staging-release.steps.ci`. workflows: - Code Style - Tests (Reborn) @@ -156,7 +157,7 @@ jobs: local required="$2" local run_json status conclusion url - run_json="$(gh run list \ + if ! run_json="$(gh run list \ --repo "$GITHUB_REPOSITORY" \ --workflow "$workflow" \ --branch main \ @@ -164,7 +165,11 @@ jobs: --commit "$HEAD_SHA" \ --limit 1 \ --json conclusion,status,url \ - --jq '.[0] // {}')" + --jq '.[0] // {}' \ + 2>/dev/null)"; then + echo "::warning::Failed to query workflow status for ${workflow}; treating as missing." + run_json='{}' + fi status="$(jq -r '.status // "missing"' <<<"$run_json")" conclusion="$(jq -r '.conclusion // "missing"' <<<"$run_json")" url="$(jq -r '.url // ""' <<<"$run_json")" @@ -188,29 +193,48 @@ jobs: fi } - for workflow in "${required_workflows[@]}"; do - check_workflow "$workflow" required - done - for workflow in "${optional_workflows[@]}"; do - check_workflow "$workflow" optional - done + attempts=1 + max_attempts=3 + while true; do + missing=() + pending=() + failed=() - if [ "${#failed[@]}" -gt 0 ]; then - printf '::error::Main CI failed for %s; refusing to fast-forward staging-release.\n' "$HEAD_SHA" - printf 'Failed workflow: %s\n' "${failed[@]}" - exit 1 - fi + for workflow in "${required_workflows[@]}"; do + check_workflow "$workflow" required + done + for workflow in "${optional_workflows[@]}"; do + check_workflow "$workflow" optional + done - if [ "${#missing[@]}" -gt 0 ] || [ "${#pending[@]}" -gt 0 ]; then - echo "ready=false" >> "$GITHUB_OUTPUT" - if [ "${#missing[@]}" -gt 0 ]; then - printf 'Required workflow has not appeared yet: %s\n' "${missing[@]}" + if [ "${#failed[@]}" -gt 0 ]; then + echo "::error::Main CI failed for ${HEAD_SHA}; refusing to fast-forward staging-release." + for workflow in "${failed[@]}"; do + printf 'Failed workflow: %s\n' "$workflow" + done + exit 1 fi - if [ "${#pending[@]}" -gt 0 ]; then - printf 'Workflow is still pending: %s\n' "${pending[@]}" + + if [ "${#missing[@]}" -gt 0 ] || [ "${#pending[@]}" -gt 0 ]; then + if [ "$attempts" -lt "$max_attempts" ]; then + attempts=$((attempts + 1)) + echo "::warning::Required workflow checks for ${HEAD_SHA} are incomplete (attempt ${attempts}/${max_attempts}). Retrying in 30 seconds." + sleep 30 + continue + fi + + echo "ready=false" >> "$GITHUB_OUTPUT" + if [ "${#missing[@]}" -gt 0 ]; then + printf 'Required workflow has not appeared yet: %s\n' "${missing[@]}" + fi + if [ "${#pending[@]}" -gt 0 ]; then + printf 'Workflow is still pending: %s\n' "${pending[@]}" + fi + exit 0 fi - exit 0 - fi + + break + done echo "ready=true" >> "$GITHUB_OUTPUT" echo "All required main CI workflows passed for ${HEAD_SHA}." @@ -228,14 +252,14 @@ jobs: set -euo pipefail git fetch origin main "${STAGING_BRANCH}" - if git merge-base --is-ancestor "$HEAD_SHA" "origin/${STAGING_BRANCH}"; then - echo "::notice::${STAGING_BRANCH} already contains main ${HEAD_SHA}; nothing to sync." - exit 0 - fi - if ! git merge-base --is-ancestor "origin/${STAGING_BRANCH}" "origin/main"; then echo "::error::${STAGING_BRANCH} has commits that are not in main; refusing to overwrite or merge automatically." exit 1 fi + if git merge-base --is-ancestor "$HEAD_SHA" "origin/${STAGING_BRANCH}"; then + echo "::notice::${STAGING_BRANCH} already contains main ${HEAD_SHA}; nothing to sync." + exit 0 + fi + git push origin "HEAD:refs/heads/${STAGING_BRANCH}" From c7d16df87b3778f247f984d68dee86ebca84ed4e Mon Sep 17 00:00:00 2001 From: aiworkbot Date: Sun, 12 Jul 2026 02:09:12 +0000 Subject: [PATCH 6/8] fix(main-ci-checks): reduce duplicate triggers and clean up workflow checks --- .github/workflows/main-ci-checks.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml index d77826dac62..ead623e15cf 100644 --- a/.github/workflows/main-ci-checks.yml +++ b/.github/workflows/main-ci-checks.yml @@ -109,7 +109,8 @@ jobs: name: Fast-forward staging-release to main if: >- github.event.workflow_run.event == 'push' && - github.event.workflow_run.conclusion == 'success' + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.name == 'Code Style' runs-on: ubuntu-latest concurrency: group: main-ci-checks-staging-release @@ -121,7 +122,7 @@ jobs: GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - # Dedicated branch used by staging deployment. + # Dedicated branch used by staging deployment (`staging-release`). STAGING_BRANCH: staging-release steps: - name: Check main CI run set @@ -217,8 +218,8 @@ jobs: if [ "${#missing[@]}" -gt 0 ] || [ "${#pending[@]}" -gt 0 ]; then if [ "$attempts" -lt "$max_attempts" ]; then - attempts=$((attempts + 1)) echo "::warning::Required workflow checks for ${HEAD_SHA} are incomplete (attempt ${attempts}/${max_attempts}). Retrying in 30 seconds." + attempts=$((attempts + 1)) sleep 30 continue fi @@ -245,6 +246,7 @@ jobs: with: ref: ${{ github.event.workflow_run.head_sha }} fetch-depth: 0 + persist-credentials: true - name: Fast-forward staging-release if: steps.ci.outputs.ready == 'true' From 1c7c411055aab0ec20741251f4b99e543edf01c6 Mon Sep 17 00:00:00 2001 From: aiworkbot Date: Sun, 12 Jul 2026 17:09:01 +0000 Subject: [PATCH 7/8] fix(main-ci-checks): allow trigger on any successful watched workflow --- .github/workflows/main-ci-checks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml index ead623e15cf..cd6e1a915a3 100644 --- a/.github/workflows/main-ci-checks.yml +++ b/.github/workflows/main-ci-checks.yml @@ -109,8 +109,7 @@ jobs: name: Fast-forward staging-release to main if: >- github.event.workflow_run.event == 'push' && - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.name == 'Code Style' + github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest concurrency: group: main-ci-checks-staging-release From ed9a15f5685fc7cc2cc2eac15ec1771ac0b36e60 Mon Sep 17 00:00:00 2001 From: aiworkbot Date: Sun, 12 Jul 2026 17:13:22 +0000 Subject: [PATCH 8/8] fix(main-ci-checks): treat workflow query failures as pending --- .github/workflows/main-ci-checks.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main-ci-checks.yml b/.github/workflows/main-ci-checks.yml index cd6e1a915a3..81855d75da5 100644 --- a/.github/workflows/main-ci-checks.yml +++ b/.github/workflows/main-ci-checks.yml @@ -156,6 +156,7 @@ jobs: local workflow="$1" local required="$2" local run_json status conclusion url + local api_error=false if ! run_json="$(gh run list \ --repo "$GITHUB_REPOSITORY" \ @@ -168,12 +169,18 @@ jobs: --jq '.[0] // {}' \ 2>/dev/null)"; then echo "::warning::Failed to query workflow status for ${workflow}; treating as missing." - run_json='{}' + api_error=true + run_json='{"status":"pending","conclusion":"pending","url":""}' fi status="$(jq -r '.status // "missing"' <<<"$run_json")" conclusion="$(jq -r '.conclusion // "missing"' <<<"$run_json")" url="$(jq -r '.url // ""' <<<"$run_json")" + if [ "$api_error" = true ]; then + pending+=("${workflow} (query error)") + return + fi + if [ "$status" = "missing" ]; then if [ "$required" = "required" ]; then missing+=("$workflow")