From e9ee94e0d68a74ca52c92ee429567608fe4936b8 Mon Sep 17 00:00:00 2001 From: Firat Sertgoz Date: Sat, 1 Aug 2026 12:12:16 +0300 Subject: [PATCH] Alert Slack on merge queue failures --- .github/workflows/README.md | 23 ++-- .github/workflows/code_style.yml | 3 +- .github/workflows/main-ci-slack-alerts.yml | 122 ++++++++++++++++++--- scripts/ci/test-main-ci-slack-alerts.sh | 31 ++++++ 4 files changed, 150 insertions(+), 29 deletions(-) create mode 100755 scripts/ci/test-main-ci-slack-alerts.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c53f4411ab9..f7cc0b8db68 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -213,20 +213,23 @@ trail: the former in-run alert jobs and `nightly-alert-issue.sh` were removed in favor of this single external check, because an in-run alert dies with its own run on a startup_failure and can never see a cron that didn't fire. -### Main branch alerting +### Main branch and merge-queue alerting `main-ci-slack-alerts.yml` watches completed `workflow_run` events for the -current `push` to `main` workflows: Code Style, Tests (Reborn), Reborn E2E, -Platform & Compat, Replay Snapshot Gate, Code Coverage, +current `push` to `main` and `merge_group` workflows: Code Style, Tests +(Reborn), Reborn E2E, Platform & Compat, Replay Snapshot Gate, Code Coverage, nearai-bench dispatcher tests, and Release-plz. Any watched run that concludes `failure`, `timed_out`, `action_required`, or `startup_failure` posts a Slack -message with the workflow, conclusion, failed job names, commit, actor, and run -link. - -Alerts go to `secrets.MAIN_CI_SLACK_WEBHOOK_URLS`; the value may be a single -webhook URL or multiple URLs separated by newlines or commas. This is -intentionally separate from the canary/nightly `SLACK_WEBHOOK_URL` so main CI -alerts can target dedicated channels. +message with the workflow, conclusion, failed job and step names, available +failure annotations, commit, actor, and run link. Merge-queue alerts also +resolve the PR number from GitHub's `gh-readonly-queue/main/pr--...` +ref and include the PR title, author, and link. + +Main-branch alerts go to `secrets.MAIN_CI_SLACK_WEBHOOK_URLS`; the value may be +a single webhook URL or multiple URLs separated by newlines or commas. +Merge-queue alerts go to `secrets.SLACK_WEBHOOK_URL`, the existing live-canary +channel. This keeps post-merge CI alerts in their dedicated channels while +making queue bounces visible alongside live-canary failures. When adding a new workflow that runs on `push` to `main`, add its workflow `name:` to the watched list in `main-ci-slack-alerts.yml`. diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 075014934c1..242470b0a1b 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -62,7 +62,7 @@ jobs: run: | CHANGED_FILES="$(git diff --name-only "$BASE_SHA"..."$HEAD_SHA")" - if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(crates/|tests/|migrations/|Cargo\.toml$|Cargo\.lock$|Dockerfile|\.gitignore$|scripts/ci/|\.githooks/|scripts/check_no_panics\.py$|scripts/no_panics_reborn_baseline\.txt$|\.github/scripts/(pr-labeler|test-pr-labeler)\.sh$|\.github/workflows/code_style\.yml$)'; then + if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(crates/|tests/|migrations/|Cargo\.toml$|Cargo\.lock$|Dockerfile|\.gitignore$|scripts/ci/|\.githooks/|scripts/check_no_panics\.py$|scripts/no_panics_reborn_baseline\.txt$|\.github/scripts/(pr-labeler|test-pr-labeler)\.sh$|\.github/workflows/(code_style|main-ci-slack-alerts)\.yml$)'; then echo "has_code=true" >> "$GITHUB_OUTPUT" else echo "has_code=false" >> "$GITHUB_OUTPUT" @@ -127,6 +127,7 @@ jobs: scripts/ci/test-classify-test-scope.sh scripts/ci/test-package-feature-flags.sh scripts/ci/test-reborn-crate-test-buckets.sh + scripts/ci/test-main-ci-slack-alerts.sh python3 scripts/ci/test_ws12_suite_shards.py python3 scripts/ci/test_ws12_workflow_contracts.py scripts/ci/test-hermetic-test-process.sh diff --git a/.github/workflows/main-ci-slack-alerts.yml b/.github/workflows/main-ci-slack-alerts.yml index 138624e70e8..06908eeab67 100644 --- a/.github/workflows/main-ci-slack-alerts.yml +++ b/.github/workflows/main-ci-slack-alerts.yml @@ -13,22 +13,25 @@ on: - Release-plz branches: - main + - gh-readonly-queue/main/** types: - completed permissions: actions: read + checks: read contents: read + pull-requests: read jobs: alert: name: Post Slack alert if: >- - github.event.workflow_run.event == 'push' && + contains(fromJSON('["push","merge_group"]'), github.event.workflow_run.event) && contains(fromJSON('["failure","timed_out","action_required","startup_failure"]'), github.event.workflow_run.conclusion) runs-on: ubuntu-latest steps: - - name: Post main CI failure to Slack + - name: Post CI failure to Slack env: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ github.event.workflow_run.id }} @@ -38,48 +41,131 @@ jobs: HEAD_SHA: ${{ github.event.workflow_run.head_sha }} HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} ACTOR: ${{ github.event.workflow_run.actor.login }} + EVENT_NAME: ${{ github.event.workflow_run.event }} MAIN_CI_SLACK_WEBHOOK_URLS: ${{ secrets.MAIN_CI_SLACK_WEBHOOK_URLS }} + LIVE_CANARY_SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} run: | set -euo pipefail + slack_escape() { + sed -e 's/&/\&/g' -e 's//\>/g' + } + short_sha="${HEAD_SHA:0:10}" - failed_jobs="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100" \ - --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") | "\(.name) (\(.conclusion))"] | .[0:12] | join(", ")' \ - 2>/dev/null || true)" - if [ -z "$failed_jobs" ]; then + jobs_json="$(gh api --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100" \ + 2>/dev/null | jq '[.[].jobs[]]' || printf '[]')" + failed_jobs="$(jq -r ' + [.[] + | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") + | . as $job + | ([.steps[]? | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure") | .name] | unique) as $steps + | if ($steps | length) > 0 + then "\($job.name) (\($job.conclusion)) — \($steps | join(", "))" + else "\($job.name) (\($job.conclusion))" + end] + | .[0:8] + | join("\n• ")' <<< "$jobs_json")" + if [ -n "$failed_jobs" ]; then + failed_jobs="• ${failed_jobs}" + else failed_jobs="No failed jobs listed; inspect the workflow run for startup or workflow-level errors." fi - text="Main branch CI failed: ${WORKFLOW_NAME} concluded ${CONCLUSION} on ${short_sha}" + annotations="" + while IFS= read -r check_run_id; do + [ -n "$check_run_id" ] || continue + job_annotations="$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/check-runs/${check_run_id}/annotations?per_page=100" \ + --jq '.[] + | select(.annotation_level == "failure") + | ((if (.title // "") != "" then (.title + ": ") else "" end) + (.message // "")) + | gsub("[\\r\\n\\t]+"; " ") + | select(test("^Process completed with exit code [0-9]+\\.$") | not)' \ + 2>/dev/null || true)" + if [ -n "$job_annotations" ]; then + annotations="${annotations}${annotations:+$'\n'}${job_annotations}" + fi + done < <(jq -r '[.[] + | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required" or .conclusion == "startup_failure")] + | .[0:8][] + | .check_run_url + | split("/")[-1]' <<< "$jobs_json") + annotations="$(printf '%s\n' "$annotations" | awk 'NF && !seen[$0]++ && count < 8 { print; count++ }')" + if [ -z "$annotations" ]; then + annotations="Not reported by the failing checks." + fi + + if [ "$EVENT_NAME" = "merge_group" ]; then + alert_title="Merge queue CI failed" + webhooks="${LIVE_CANARY_SLACK_WEBHOOK_URL:-}" + pr_number="" + if [[ "$HEAD_BRANCH" =~ ^gh-readonly-queue/main/pr-([0-9]+)- ]]; then + pr_number="${BASH_REMATCH[1]}" + fi + + pr_display="Unknown PR (queue ref: ${HEAD_BRANCH})" + if [ -n "$pr_number" ]; then + pr_json="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" 2>/dev/null || true)" + if [ -n "$pr_json" ]; then + pr_title="$(jq -r '.title | gsub("[\\r\\n\\t]+"; " ")' <<< "$pr_json" | slack_escape)" + pr_url="$(jq -r '.html_url' <<< "$pr_json")" + pr_author="$(jq -r '.user.login' <<< "$pr_json" | slack_escape)" + pr_display="<${pr_url}|#${pr_number} ${pr_title}> by \`@${pr_author}\`" + else + pr_display="#${pr_number} (details unavailable)" + fi + fi + context_line="*PR:* ${pr_display}\n*Queue commit:* \`${short_sha}\`" + else + alert_title="Main branch CI failed" + webhooks="${MAIN_CI_SLACK_WEBHOOK_URLS:-}" + escaped_branch="$(printf '%s' "$HEAD_BRANCH" | slack_escape)" + context_line="*Branch:* \`${escaped_branch}\`\n*Commit:* \`${short_sha}\`" + fi + + escaped_workflow="$(printf '%s' "$WORKFLOW_NAME" | slack_escape)" + escaped_conclusion="$(printf '%s' "$CONCLUSION" | slack_escape)" + escaped_actor="$(printf '%s' "$ACTOR" | slack_escape)" + escaped_failed_jobs="$(printf '%s' "$failed_jobs" | slack_escape)" + escaped_annotations="$(printf '%s' "$annotations" | slack_escape)" + escaped_failed_jobs="${escaped_failed_jobs:0:1200}" + escaped_annotations="${escaped_annotations:0:800}" + + text="${alert_title}: ${WORKFLOW_NAME} concluded ${CONCLUSION} on ${short_sha}" payload="$(jq -n \ --arg text "$text" \ - --arg workflow "$WORKFLOW_NAME" \ - --arg conclusion "$CONCLUSION" \ - --arg branch "$HEAD_BRANCH" \ - --arg sha "$short_sha" \ - --arg actor "$ACTOR" \ - --arg failed_jobs "$failed_jobs" \ + --arg alert_title "$alert_title" \ + --arg workflow "$escaped_workflow" \ + --arg conclusion "$escaped_conclusion" \ + --arg context_line "$context_line" \ + --arg actor "$escaped_actor" \ + --arg failed_jobs "$escaped_failed_jobs" \ + --arg annotations "$escaped_annotations" \ --arg url "$RUN_URL" \ '{ text: $text, blocks: [ { type: "header", - text: {type: "plain_text", text: "Main branch CI failed"} + text: {type: "plain_text", text: $alert_title} }, { type: "section", text: { type: "mrkdwn", - text: "*Workflow:* \($workflow)\n*Conclusion:* `\($conclusion)`\n*Branch:* `\($branch)`\n*Commit:* `\($sha)`\n*Actor:* `\($actor)`\n*Failed jobs:* \($failed_jobs)\n<\($url)|Open workflow run>" + text: "*Workflow:* \($workflow)\n*Conclusion:* `\($conclusion)`\n\($context_line)\n*Actor:* `\($actor)`\n*Failed jobs / steps:*\n\($failed_jobs)\n*Failure annotations (when available):*\n\($annotations)\n<\($url)|Open workflow run>" } } ] }')" - webhooks="${MAIN_CI_SLACK_WEBHOOK_URLS:-}" if [ -z "$webhooks" ]; then - echo "::error::Set MAIN_CI_SLACK_WEBHOOK_URLS to receive main CI failure alerts." + if [ "$EVENT_NAME" = "merge_group" ]; then + echo "::error::Set SLACK_WEBHOOK_URL to receive merge-queue failure alerts in the live-canary channel." + else + echo "::error::Set MAIN_CI_SLACK_WEBHOOK_URLS to receive main CI failure alerts." + fi exit 1 fi @@ -102,4 +188,4 @@ jobs: exit 1 fi - echo "Posted main CI failure alert to ${posted} Slack webhook(s)." + echo "Posted ${EVENT_NAME} CI failure alert to ${posted} Slack webhook(s)." diff --git a/scripts/ci/test-main-ci-slack-alerts.sh b/scripts/ci/test-main-ci-slack-alerts.sh new file mode 100755 index 00000000000..01a4ffa5b8b --- /dev/null +++ b/scripts/ci/test-main-ci-slack-alerts.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +workflow="${repo_root}/.github/workflows/main-ci-slack-alerts.yml" +code_style_workflow="${repo_root}/.github/workflows/code_style.yml" + +assert_contains() { + local expected="$1" + if ! grep -Fq -- "$expected" "$workflow"; then + echo "Expected main-ci-slack-alerts.yml to contain: ${expected}" >&2 + exit 1 + fi +} + +assert_contains "- gh-readonly-queue/main/**" +assert_contains "contains(fromJSON('[\"push\",\"merge_group\"]'), github.event.workflow_run.event)" +assert_contains "checks: read" +assert_contains "pull-requests: read" +assert_contains 'LIVE_CANARY_SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}' +assert_contains 'if [[ "$HEAD_BRANCH" =~ ^gh-readonly-queue/main/pr-([0-9]+)- ]]; then' +assert_contains '"repos/${GITHUB_REPOSITORY}/pulls/${pr_number}"' +assert_contains '*Failed jobs / steps:*' +assert_contains '*Failure annotations (when available):*' + +if ! grep -Fq -- '.github/workflows/(code_style|main-ci-slack-alerts)\.yml$' "$code_style_workflow"; then + echo "Expected code_style.yml to run CI alert workflow contract tests when the alert changes" >&2 + exit 1 +fi + +echo "main-ci-slack-alerts workflow contract passed"