Alert live-canary Slack channel on merge queue failures

.github/workflows/README.md

@@ -213,20 +213,23 @@ trail: the former in-run alert jobs and `nightly-alert-issue.sh` were removed
 in favor of this single external check, because an in-run alert dies with its
 own run on a startup_failure and can never see a cron that didn't fire.
 
-### Main branch alerting
+### Main branch and merge-queue alerting
 
 `main-ci-slack-alerts.yml` watches completed `workflow_run` events for the
-current `push` to `main` workflows: Code Style, Tests (Reborn), Reborn E2E,
-Platform & Compat, Replay Snapshot Gate, Code Coverage,
+current `push` to `main` and `merge_group` workflows: Code Style, Tests
+(Reborn), Reborn E2E, Platform & Compat, Replay Snapshot Gate, Code Coverage,
 nearai-bench dispatcher tests, and Release-plz. Any watched run that concludes
 `failure`, `timed_out`, `action_required`, or `startup_failure` posts a Slack
-message with the workflow, conclusion, failed job names, commit, actor, and run
-link.
-
-Alerts go to `secrets.MAIN_CI_SLACK_WEBHOOK_URLS`; the value may be a single
-webhook URL or multiple URLs separated by newlines or commas. This is
-intentionally separate from the canary/nightly `SLACK_WEBHOOK_URL` so main CI
-alerts can target dedicated channels.
+message with the workflow, conclusion, failed job and step names, available
+failure annotations, commit, actor, and run link. Merge-queue alerts also
+resolve the PR number from GitHub's `gh-readonly-queue/main/pr-<number>-...`
+ref and include the PR title, author, and link.
+
+Main-branch alerts go to `secrets.MAIN_CI_SLACK_WEBHOOK_URLS`; the value may be
+a single webhook URL or multiple URLs separated by newlines or commas.
+Merge-queue alerts go to `secrets.SLACK_WEBHOOK_URL`, the existing live-canary
+channel. This keeps post-merge CI alerts in their dedicated channels while
+making queue bounces visible alongside live-canary failures.
 When adding a new workflow that runs on `push` to `main`, add its workflow
 `name:` to the watched list in `main-ci-slack-alerts.yml`.
 

.github/workflows/code_style.yml

@@ -62,7 +62,7 @@ jobs:
         run: |
           CHANGED_FILES="$(git diff --name-only "$BASE_SHA"..."$HEAD_SHA")"
 
-          if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(crates/|tests/|migrations/|Cargo\.toml$|Cargo\.lock$|Dockerfile|\.gitignore$|scripts/ci/|\.githooks/|scripts/check_no_panics\.py$|scripts/no_panics_reborn_baseline\.txt$|\.github/scripts/(pr-labeler|test-pr-labeler)\.sh$|\.github/workflows/code_style\.yml$)'; then
+          if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(crates/|tests/|migrations/|Cargo\.toml$|Cargo\.lock$|Dockerfile|\.gitignore$|scripts/ci/|\.githooks/|scripts/check_no_panics\.py$|scripts/no_panics_reborn_baseline\.txt$|\.github/scripts/(pr-labeler|test-pr-labeler)\.sh$|\.github/workflows/(code_style|main-ci-slack-alerts)\.yml$)'; then
             echo "has_code=true" >> "$GITHUB_OUTPUT"
           else
             echo "has_code=false" >> "$GITHUB_OUTPUT"
@@ -127,6 +127,7 @@ jobs:
           scripts/ci/test-classify-test-scope.sh
           scripts/ci/test-package-feature-flags.sh
           scripts/ci/test-reborn-crate-test-buckets.sh
+          scripts/ci/test-main-ci-slack-alerts.sh
           python3 scripts/ci/test_ws12_suite_shards.py
           python3 scripts/ci/test_ws12_workflow_contracts.py
           scripts/ci/test-hermetic-test-process.sh

.github/workflows/main-ci-slack-alerts.yml

@@ -13,22 +13,25 @@ on:
       - Release-plz
     branches:
       - main
+      - gh-readonly-queue/main/**
     types:
       - completed
 
 permissions:
   actions: read
+  checks: read
   contents: read
+  pull-requests: read
 
 jobs:
   alert:
     name: Post Slack alert
     if: >-
-      github.event.workflow_run.event == 'push' &&
+      contains(fromJSON('["push","merge_group"]'), github.event.workflow_run.event) &&
       contains(fromJSON('["failure","timed_out","action_required","startup_failure"]'), github.event.workflow_run.conclusion)
     runs-on: ubuntu-latest
     steps:
-      - name: Post main CI failure to Slack
+      - name: Post CI failure to Slack
         env:
           GH_TOKEN: ${{ github.token }}
           RUN_ID: ${{ github.event.workflow_run.id }}
@@ -38,48 +41,131 @@ jobs:
           HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
           HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
           ACTOR: ${{ github.event.workflow_run.actor.login }}
+          EVENT_NAME: ${{ github.event.workflow_run.event }}
           MAIN_CI_SLACK_WEBHOOK_URLS: ${{ secrets.MAIN_CI_SLACK_WEBHOOK_URLS }}
+          LIVE_CANARY_SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
           set -euo pipefail
 
+          slack_escape() {
+            sed -e 's/&/\&amp;/g' -e 's/</\&lt;/g' -e 's/>/\&gt;/g'
+          }
+
           short_sha="${HEAD_SHA:0:10}"
-          failed_jobs="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100" \
-            --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") | "\(.name) (\(.conclusion))"] | .[0:12] | join(", ")' \
-            2>/dev/null || true)"
-          if [ -z "$failed_jobs" ]; then
+          jobs_json="$(gh api --paginate --slurp \
+            "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100" \
+            2>/dev/null | jq '[.[].jobs[]]' || printf '[]')"
+          failed_jobs="$(jq -r '
+            [.[]
+              | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure")
+              | . as $job
+              | ([.steps[]? | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") | .name] | unique) as $steps
+              | if ($steps | length) > 0
+                then "\($job.name) (\($job.conclusion)) — \($steps | join(", "))"
+                else "\($job.name) (\($job.conclusion))"
+                end]
+            | .[0:8]
+            | join("\n• ")' <<< "$jobs_json")"
+          if [ -n "$failed_jobs" ]; then
+            failed_jobs="• ${failed_jobs}"
+          else
             failed_jobs="No failed jobs listed; inspect the workflow run for startup or workflow-level errors."
           fi
 
-          text="Main branch CI failed: ${WORKFLOW_NAME} concluded ${CONCLUSION} on ${short_sha}"
+          annotations=""
+          while IFS= read -r check_run_id; do
+            [ -n "$check_run_id" ] || continue
+            job_annotations="$(gh api --paginate \
+              "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}/annotations?per_page=100" \
+              --jq '.[]
+                | select(.annotation_level == "failure")
+                | ((if (.title // "") != "" then (.title + ": ") else "" end) + (.message // ""))
+                | gsub("[\\r\\n\\t]+"; " ")
+                | select(test("^Process completed with exit code [0-9]+\\.$") | not)' \
+              2>/dev/null || true)"
+            if [ -n "$job_annotations" ]; then
+              annotations="${annotations}${annotations:+$'\n'}${job_annotations}"
+            fi
+          done < <(jq -r '[.[]
+            | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure")]
+            | .[0:8][]
+            | .check_run_url
+            | split("/")[-1]' <<< "$jobs_json")
+          annotations="$(printf '%s\n' "$annotations" | awk 'NF && !seen[$0]++ && count < 8 { print; count++ }')"
+          if [ -z "$annotations" ]; then
+            annotations="Not reported by the failing checks."
+          fi
+
+          if [ "$EVENT_NAME" = "merge_group" ]; then
+            alert_title="Merge queue CI failed"
+            webhooks="${LIVE_CANARY_SLACK_WEBHOOK_URL:-}"
+            pr_number=""
+            if [[ "$HEAD_BRANCH" =~ ^gh-readonly-queue/main/pr-([0-9]+)- ]]; then
+              pr_number="${BASH_REMATCH[1]}"
+            fi
+
+            pr_display="Unknown PR (queue ref: ${HEAD_BRANCH})"
+            if [ -n "$pr_number" ]; then
+              pr_json="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" 2>/dev/null || true)"
+              if [ -n "$pr_json" ]; then
+                pr_title="$(jq -r '.title | gsub("[\\r\\n\\t]+"; " ")' <<< "$pr_json" | slack_escape)"
+                pr_url="$(jq -r '.html_url' <<< "$pr_json")"
+                pr_author="$(jq -r '.user.login' <<< "$pr_json" | slack_escape)"
+                pr_display="<${pr_url}|#${pr_number} ${pr_title}> by \`@${pr_author}\`"
+              else
+                pr_display="#${pr_number} (details unavailable)"
+              fi
+            fi
+            context_line="*PR:* ${pr_display}\n*Queue commit:* \`${short_sha}\`"
+          else
+            alert_title="Main branch CI failed"
+            webhooks="${MAIN_CI_SLACK_WEBHOOK_URLS:-}"
+            escaped_branch="$(printf '%s' "$HEAD_BRANCH" | slack_escape)"
+            context_line="*Branch:* \`${escaped_branch}\`\n*Commit:* \`${short_sha}\`"
+          fi
+
+          escaped_workflow="$(printf '%s' "$WORKFLOW_NAME" | slack_escape)"
+          escaped_conclusion="$(printf '%s' "$CONCLUSION" | slack_escape)"
+          escaped_actor="$(printf '%s' "$ACTOR" | slack_escape)"
+          escaped_failed_jobs="$(printf '%s' "$failed_jobs" | slack_escape)"
+          escaped_annotations="$(printf '%s' "$annotations" | slack_escape)"
+          escaped_failed_jobs="${escaped_failed_jobs:0:1200}"
+          escaped_annotations="${escaped_annotations:0:800}"
+
+          text="${alert_title}: ${WORKFLOW_NAME} concluded ${CONCLUSION} on ${short_sha}"
           payload="$(jq -n \
             --arg text "$text" \
-            --arg workflow "$WORKFLOW_NAME" \
-            --arg conclusion "$CONCLUSION" \
-            --arg branch "$HEAD_BRANCH" \
-            --arg sha "$short_sha" \
-            --arg actor "$ACTOR" \
-            --arg failed_jobs "$failed_jobs" \
+            --arg alert_title "$alert_title" \
+            --arg workflow "$escaped_workflow" \
+            --arg conclusion "$escaped_conclusion" \
+            --arg context_line "$context_line" \
+            --arg actor "$escaped_actor" \
+            --arg failed_jobs "$escaped_failed_jobs" \
+            --arg annotations "$escaped_annotations" \
             --arg url "$RUN_URL" \
             '{
               text: $text,
               blocks: [
                 {
                   type: "header",
-                  text: {type: "plain_text", text: "Main branch CI failed"}
+                  text: {type: "plain_text", text: $alert_title}
                 },
                 {
                   type: "section",
                   text: {
                     type: "mrkdwn",
-                    text: "*Workflow:* \($workflow)\n*Conclusion:* `\($conclusion)`\n*Branch:* `\($branch)`\n*Commit:* `\($sha)`\n*Actor:* `\($actor)`\n*Failed jobs:* \($failed_jobs)\n<\($url)|Open workflow run>"
+                    text: "*Workflow:* \($workflow)\n*Conclusion:* `\($conclusion)`\n\($context_line)\n*Actor:* `\($actor)`\n*Failed jobs / steps:*\n\($failed_jobs)\n*Failure annotations (when available):*\n\($annotations)\n<\($url)|Open workflow run>"
                   }
                 }
               ]
             }')"
 
-          webhooks="${MAIN_CI_SLACK_WEBHOOK_URLS:-}"
           if [ -z "$webhooks" ]; then
-            echo "::error::Set MAIN_CI_SLACK_WEBHOOK_URLS to receive main CI failure alerts."
+            if [ "$EVENT_NAME" = "merge_group" ]; then
+              echo "::error::Set SLACK_WEBHOOK_URL to receive merge-queue failure alerts in the live-canary channel."
+            else
+              echo "::error::Set MAIN_CI_SLACK_WEBHOOK_URLS to receive main CI failure alerts."
+            fi
             exit 1
           fi
 
@@ -102,4 +188,4 @@ jobs:
             exit 1
           fi
 
-          echo "Posted main CI failure alert to ${posted} Slack webhook(s)."
+          echo "Posted ${EVENT_NAME} CI failure alert to ${posted} Slack webhook(s)."

scripts/ci/test-main-ci-slack-alerts.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+workflow="${repo_root}/.github/workflows/main-ci-slack-alerts.yml"
+code_style_workflow="${repo_root}/.github/workflows/code_style.yml"
+
+assert_contains() {
+  local expected="$1"
+  if ! grep -Fq -- "$expected" "$workflow"; then
+    echo "Expected main-ci-slack-alerts.yml to contain: ${expected}" >&2
+    exit 1
+  fi
+}
+
+assert_contains "- gh-readonly-queue/main/**"
+assert_contains "contains(fromJSON('[\"push\",\"merge_group\"]'), github.event.workflow_run.event)"
+assert_contains "checks: read"
+assert_contains "pull-requests: read"
+assert_contains 'LIVE_CANARY_SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}'
+assert_contains 'if [[ "$HEAD_BRANCH" =~ ^gh-readonly-queue/main/pr-([0-9]+)- ]]; then'
+assert_contains '"repos/${GITHUB_REPOSITORY}/pulls/${pr_number}"'
+assert_contains '*Failed jobs / steps:*'
+assert_contains '*Failure annotations (when available):*'
+
+if ! grep -Fq -- '.github/workflows/(code_style|main-ci-slack-alerts)\.yml$' "$code_style_workflow"; then
+  echo "Expected code_style.yml to run CI alert workflow contract tests when the alert changes" >&2
+  exit 1
+fi
+
+echo "main-ci-slack-alerts workflow contract passed"