Fix main CI failure attribution and classification

.github/workflows/analyze-ci-failure-candidates.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+
+set -euo pipefail
+
+if [ "$#" -ne 5 ]; then
+  echo "Usage: $0 <repo> <last-successful-sha> <failed-sha> <candidates-file> <status-file>" >&2
+  exit 1
+fi
+
+REPO="$1"
+LAST_SUCCESSFUL_SHA="$2"
+FAILED_SHA="$3"
+CANDIDATES_FILE="$4"
+STATUS_FILE="$5"
+
+printf '%s\n' '[]' > "$CANDIDATES_FILE"
+printf '%s\n' '{"state":"unavailable"}' > "$STATUS_FILE"
+
+if [ -z "$LAST_SUCCESSFUL_SHA" ] || [ -z "$FAILED_SHA" ]; then
+  exit 0
+fi
+
+COMPARISON_PAGES=$(mktemp)
+COMPARISON=$(mktemp)
+CANDIDATES_TMP="${CANDIDATES_FILE}.tmp"
+trap 'rm -f "$COMPARISON_PAGES" "$COMPARISON" "$CANDIDATES_TMP"' EXIT
+
+if ! gh api --paginate --slurp \
+    "repos/${REPO}/compare/${LAST_SUCCESSFUL_SHA}...${FAILED_SHA}?per_page=100" \
+    > "$COMPARISON_PAGES" 2>/dev/null; then
+  echo "::warning::Unable to compare the last successful main commit with the failed commit."
+  exit 0
+fi
+
+if ! jq -e '
+    type == "array" and
+    length > 0 and
+    .[0].status == "ahead" and
+    (.[0].total_commits | type) == "number" and
+    .[0].total_commits > 0 and
+    .[0].total_commits == (.[0].total_commits | floor) and
+    all(.[]; type == "object" and (.commits | type) == "array") and
+    all(.[].commits[];
+      type == "object" and
+      (.sha | type) == "string" and
+      (.sha | length) > 0 and
+      (.commit | type) == "object" and
+      (.commit.message | type) == "string" and
+      (.html_url | type) == "string")
+  ' "$COMPARISON_PAGES" >/dev/null; then
+  echo "::warning::GitHub returned invalid comparison metadata."
+  exit 0
+fi
+
+jq '{
+  total_commits: .[0].total_commits,
+  commits: (
+    reduce [.[].commits[]][] as $commit
+      ([]; if any(.[]; .sha == $commit.sha) then . else . + [$commit] end)
+  )
+}' "$COMPARISON_PAGES" > "$COMPARISON"
+
+RECEIVED_COMMIT_COUNT=$(jq '.commits | length' "$COMPARISON")
+TOTAL_COMMIT_COUNT=$(jq '.total_commits' "$COMPARISON")
+if [ "$RECEIVED_COMMIT_COUNT" -ne "$TOTAL_COMMIT_COUNT" ]; then
+  echo "::warning::GitHub returned ${RECEIVED_COMMIT_COUNT} of ${TOTAL_COMMIT_COUNT} unique commits in the comparison."
+  printf '%s\n' '{"state":"incomplete"}' > "$STATUS_FILE"
+else
+  printf '%s\n' '{"state":"available"}' > "$STATUS_FILE"
+fi
+
+jq -c '.commits[]? | {sha, message: .commit.message, html_url}' "$COMPARISON" |
+  while IFS= read -r COMMIT; do
+    COMMIT_SHA=$(jq -r '.sha' <<< "${COMMIT}")
+    # --slurp wraps paginated response arrays as [[page 1], [page 2]].
+    if ! MERGE_PR=$(
+      gh api --paginate --slurp \
+        "repos/${REPO}/commits/${COMMIT_SHA}/pulls?per_page=100" 2>/dev/null |
+        jq -c --arg repo "$REPO" \
+        '[.[][] | select(.base.repo.full_name == $repo and .base.ref == "main" and .merged_at != null)] |
+         unique_by(.number) |
+         if length == 1 then .[0] else null end'
+    ); then
+      echo "::warning::Unable to associate commit ${COMMIT_SHA} with a merged pull request."
+      printf '%s\n' '{"state":"incomplete"}' > "$STATUS_FILE"
+      continue
+    fi
+    if [ "${MERGE_PR}" = "null" ]; then
+      echo "::warning::Commit ${COMMIT_SHA} does not have exactly one merged pull request association."
+      printf '%s\n' '{"state":"incomplete"}' > "$STATUS_FILE"
+      continue
+    fi
+    jq --argjson commit "${COMMIT}" --argjson pr "${MERGE_PR}" \
+      '. + [$commit + {pull_request: {
+        number: $pr.number,
+        title: $pr.title,
+        url: $pr.html_url,
+        merged_at: $pr.merged_at
+      }}]' "$CANDIDATES_FILE" > "$CANDIDATES_TMP"
+    mv "$CANDIDATES_TMP" "$CANDIDATES_FILE"
+  done

.github/workflows/analyze-ci-failure-comment.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+
+set -euo pipefail
+
+if [ "$#" -ne 3 ]; then
+  echo "Usage: $0 <analysis-file> <trusted-failed-jobs-file> <run-url>" >&2
+  exit 1
+fi
+
+ANALYSIS_FILE="$1"
+TRUSTED_FAILED_JOBS_FILE="$2"
+RUN_URL="$3"
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+SANITIZED_ANALYSIS_FILE=$(mktemp)
+SANITIZED_TRUSTED_FAILED_JOBS_FILE=$(mktemp)
+trap 'rm -f "$SANITIZED_ANALYSIS_FILE" "$SANITIZED_TRUSTED_FAILED_JOBS_FILE"' EXIT
+bash "$SCRIPT_DIR/analyze-ci-failure-persistence.sh" \
+  sanitize-analysis "$ANALYSIS_FILE" "$SANITIZED_ANALYSIS_FILE"
+ANALYSIS_FILE="$SANITIZED_ANALYSIS_FILE"
+bash "$SCRIPT_DIR/analyze-ci-failure-persistence.sh" \
+  sanitize-trusted-failed-jobs "$TRUSTED_FAILED_JOBS_FILE" "$SANITIZED_TRUSTED_FAILED_JOBS_FILE"
+TRUSTED_FAILED_JOBS_FILE="$SANITIZED_TRUSTED_FAILED_JOBS_FILE"
+
+jq -r --arg run_url "$RUN_URL" --slurpfile trusted_jobs "$TRUSTED_FAILED_JOBS_FILE" '
+  ($trusted_jobs[0]) as $trusted_jobs |
+  (.failed_jobs | map({key: (.id | tostring), value: .}) | from_entries) as $analysis_jobs |
+  def code_span:
+    gsub("[\r\n\t]+"; " ") as $value |
+    (([ $value | scan("`+") | length ] | max // 0) + 1) as $delimiter_length |
+    ("`" * $delimiter_length) + " " + $value + " " + ("`" * $delimiter_length);
+  def indented_block($spaces):
+    (" " * $spaces) as $indent |
+    split("\n") | map($indent + .) | join("\n");
+  def job_list:
+    [$trusted_jobs[] |
+      . as $trusted_job |
+      ($analysis_jobs[($trusted_job.id | tostring)]) as $analysis_job |
+      "- " + ($trusted_job.name | code_span) + " — " +
+      (($analysis_job.reason // "") | code_span) +
+      " (\($analysis_job.classification))"]
+    | join("\n");
+  def trusted_job_suffix($reported_name):
+    ($trusted_jobs | map(select(.name == $reported_name)) | first) as $trusted_job |
+    if $trusted_job == null then "" else " in job " + ($trusted_job.name | code_span) end;
+  def test_list:
+    [.failed_tests[] | select(.classification == "flaky") |
+      "- " + (.name | code_span) + trusted_job_suffix(.job) +
+      "\n  - **Error**:\n\n" + (.error | indented_block(8)) + "\n" +
+      (if (.stack_trace // "") != "" then
+        "  - **Stack Trace** (first frames):\n\n" +
+        (.stack_trace | split("\n") | .[0:5] | join("\n") | indented_block(8)) + "\n"
+      else "" end) +
+      "  - **Why likely flaky**: " + (.reason | code_span)]
+    | join("\n");
+  def test_section:
+    test_list as $tests |
+    if $tests == "" then "" else "\n\n**Suspected flaky test(s):**\n" + $tests end;
+
+  "<!-- analyze-ci-failure -->\n" +
+  if .verdict == "transient-infra" then
+    "🔍 **CI Failure Analysis: Transient Infrastructure Failure**\n\nThe CI build failed due to transient infrastructure issues.\n\n**Failed jobs:**\n" + job_list + "\n\nIf a rerun was not already requested automatically, visit the [workflow run page](" + $run_url + ") to rerun the failed jobs manually.\n"
+  elif .verdict == "flaky-test" then
+    "⚠️ **CI Failure Analysis: Possible Flaky Test(s)**\n\nThe CI build failed due to test failure(s) that appear unrelated to the PR changes. These may be flaky tests.\n\n**Failed jobs:**\n" + job_list + test_section + "\n\n**Suggested actions:**\n- Re-run the failed CI jobs to confirm if the failure is intermittent\n- If the test continues to fail, consider [quarantining it](https://github.com/microsoft/aspire/blob/main/docs/quarantined-tests.md) using `/quarantine-test <test name> <issue URL>`\n- Search [existing issues](https://github.com/microsoft/aspire/issues?q=is%3Aissue+label%3Atest-failure) to see if this test is already known to be flaky\n\nYou can re-run the failed jobs from the [workflow run page](" + $run_url + ").\n"
+  elif .verdict == "code-issue" then
+    "❌ **CI Failure Analysis: Code Issue Detected**\n\nThe CI build failed due to issue(s) caused by changes in this PR.\n\n**Failed jobs:**\n" + job_list + "\n\nThe CI will not be automatically rerun. Please fix the issue and push an updated commit.\n"
+  else
+    "⚠️ **CI Failure Analysis: Mixed Failures**\n\nThe CI build contains both transient and non-transient failures.\n\n**Failed jobs:**\n" + job_list + test_section + "\n\nThe CI will not be automatically rerun. Please review the failures above.\n"
+  end
+' "$ANALYSIS_FILE"

.github/workflows/analyze-ci-failure-history.sh

@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+
+set -euo pipefail
+
+REPO="${1:?repository is required}"
+WORKFLOW_ID="${2:?workflow ID is required}"
+FAILED_RUN_CREATED_AT="${3:?failed run creation time is required}"
+FAILED_RUN_ID="${4:?failed run ID is required}"
+OUTPUT_FILE="${5:?output file is required}"
+
+if [[ ! "$FAILED_RUN_ID" =~ ^[0-9]+$ ]]; then
+  echo "::error::Failed run ID must be numeric." >&2
+  exit 1
+fi
+
+TEMP_DIRECTORY=$(mktemp -d)
+trap 'rm -rf "$TEMP_DIRECTORY"' EXIT
+
+format_epoch()
+{
+  jq -nr --argjson epoch "$1" '$epoch | strftime("%Y-%m-%dT%H:%M:%SZ")'
+}
+
+query_window()
+{
+  local start_epoch="$1"
+  local end_epoch="$2"
+  local result_file="$3"
+  local start_time
+  local end_time
+  local first_page
+  local total_count
+  local page_size
+  local received_run_count
+
+  start_time=$(format_epoch "$start_epoch")
+  end_time=$(format_epoch "$end_epoch")
+  first_page="$TEMP_DIRECTORY/page-${start_epoch}-${end_epoch}-1.json"
+
+  gh api --method GET "repos/${REPO}/actions/workflows/${WORKFLOW_ID}/runs" \
+    -f branch=main \
+    -f event=push \
+    -f status=success \
+    -f per_page=100 \
+    -f page=1 \
+    -f "created=${start_time}..${end_time}" > "$first_page"
+
+  if ! jq -e '
+      type == "object" and
+      (.total_count | type) == "number" and
+      .total_count >= 0 and
+      .total_count == (.total_count | floor) and
+      (.workflow_runs | type) == "array" and
+      all(.workflow_runs[];
+        type == "object" and
+        (.id | type) == "number" and
+        .id >= 0 and
+        .id == (.id | floor) and
+        (.created_at | type) == "string" and
+        (.head_sha | type) == "string" and
+        (.head_sha | length) > 0)
+    ' "$first_page" >/dev/null; then
+    echo "::error::GitHub returned invalid workflow-run metadata." >&2
+    return 1
+  fi
+  total_count=$(jq -r '.total_count' "$first_page")
+
+  # GitHub caps filtered workflow-run searches at 1,000 results. Search the
+  # newer half first so a dense window can be subdivided without scanning
+  # older history after the nearest successful run has been found.
+  # https://docs.github.com/rest/actions/workflow-runs#list-workflow-runs-for-a-workflow
+  if [ "$total_count" -ge 1000 ]; then
+    if [ $((end_epoch - start_epoch)) -le 1 ]; then
+      echo "::error::A one-second workflow-run window reached GitHub's 1,000-result cap." >&2
+      return 1
+    fi
+
+    local midpoint=$((start_epoch + (end_epoch - start_epoch) / 2))
+    query_window "$midpoint" "$end_epoch" "$result_file"
+    if [ "$(jq -r 'has("id")' "$result_file")" = "true" ]; then
+      return 0
+    fi
+
+    query_window "$start_epoch" "$midpoint" "$result_file"
+    return
+  fi
+
+  local runs_file="$TEMP_DIRECTORY/runs-${start_epoch}-${end_epoch}.jsonl"
+  jq -c '.workflow_runs[]?' "$first_page" > "$runs_file"
+
+  page_size=$(jq '.workflow_runs | length' "$first_page")
+  local page=2
+  while [ "$page_size" -eq 100 ]; do
+    local page_file="$TEMP_DIRECTORY/page-${start_epoch}-${end_epoch}-${page}.json"
+    gh api --method GET "repos/${REPO}/actions/workflows/${WORKFLOW_ID}/runs" \
+      -f branch=main \
+      -f event=push \
+      -f status=success \
+      -f per_page=100 \
+      -f "page=${page}" \
+      -f "created=${start_time}..${end_time}" > "$page_file"
+    if ! jq -e '
+        type == "object" and
+        (.workflow_runs | type) == "array" and
+        all(.workflow_runs[];
+          type == "object" and
+          (.id | type) == "number" and
+          .id >= 0 and
+          .id == (.id | floor) and
+          (.created_at | type) == "string" and
+          (.head_sha | type) == "string" and
+          (.head_sha | length) > 0)
+      ' "$page_file" >/dev/null; then
+      echo "::error::GitHub returned invalid workflow-run page metadata." >&2
+      return 1
+    fi
+    jq -c '.workflow_runs[]?' "$page_file" >> "$runs_file"
+    page_size=$(jq '.workflow_runs | length' "$page_file")
+    page=$((page + 1))
+  done
+
+  received_run_count=$(jq -s '[.[] | select((.id | type) == "number") | .id] | unique | length' "$runs_file")
+  if [ "$received_run_count" -lt "$total_count" ]; then
+    echo "::error::GitHub returned only ${received_run_count} of ${total_count} unique workflow runs." >&2
+    return 1
+  fi
+
+  # The API's range syntax includes both boundaries. Apply the intended
+  # half-open [start, end) contract locally before selecting the newest run.
+  jq -s \
+    --arg start_time "$start_time" \
+    --arg end_time "$end_time" \
+    --arg failed_time "$FAILED_RUN_CREATED_AT" \
+    --argjson failed_run_id "$FAILED_RUN_ID" \
+    '
+      map(select(
+        (.id | type) == "number" and
+        (.created_at | type) == "string" and
+        .created_at >= $start_time and
+        (
+          .created_at < $end_time or
+          ($end_time == $failed_time and
+            .created_at == $failed_time and
+            .id < $failed_run_id)
+        )
+      ))
+      | sort_by([.id, .created_at])
+      | unique_by(.id)
+      | sort_by([.created_at, .id])
+      | last // {}
+    ' "$runs_file" > "$result_file"
+}
+
+FAILED_EPOCH=$(jq -nr --arg timestamp "$FAILED_RUN_CREATED_AT" '$timestamp | fromdateiso8601')
+WINDOW_END="$FAILED_EPOCH"
+WINDOW_SPAN=86400
+
+while [ "$WINDOW_END" -gt 0 ]; do
+  WINDOW_START=$((WINDOW_END - WINDOW_SPAN))
+  if [ "$WINDOW_START" -lt 0 ]; then
+    WINDOW_START=0
+  fi
+
+  query_window "$WINDOW_START" "$WINDOW_END" "$OUTPUT_FILE"
+  if [ "$(jq -r 'has("id")' "$OUTPUT_FILE")" = "true" ]; then
+    exit 0
+  fi
+
+  WINDOW_END="$WINDOW_START"
+  WINDOW_SPAN=$((WINDOW_SPAN * 4))
+done
+
+echo "{}" > "$OUTPUT_FILE"