From 322609dedfd655398e014e33ed00dcd86921ccad Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Wed, 4 Mar 2026 11:08:21 -0500 Subject: [PATCH 1/4] Enhance install failure skill with thorough bootstrap log analysis Bootstrap failures are varied and complex. The previous guidance was too thin ("check for bootkube errors") and led to misdiagnoses when the agent didn't read actual error output from crashed processes. Replace with systematic 4-step guidance: build a full timeline from bootkube.log, pursue every error by reading context and following causation chains, cross-reference supporting logs, and check OS-level problems. Also add TechPreview job awareness and a general process crash pattern. Co-Authored-By: Claude Opus 4.6 --- .../prow-job-analyze-install-failure/SKILL.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md b/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md index 6c4df6200..b011c3189 100644 --- a/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md +++ b/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md @@ -315,13 +315,15 @@ OpenShift installations exhibit "eventual consistency" behavior, which means: 2. **Analyze based on failure mode from junit_install.xml** **For "cluster bootstrap" failures:** - - Check `bootstrap/journals/bootkube.log` for bootkube errors - - Check `bootstrap/journals/kubelet.log` for kubelet issues - - Check `clusterapi/kube-apiserver.log` for API server startup issues - - Check `clusterapi/etcd.log` for etcd cluster formation issues - - Check `serial/{cluster-name}-bootstrap-serial.log` for bootstrap VM boot issues - - Look for temporary control plane startup problems - - This is an early failure - focus on bootstrap node and initial control plane + + Bootstrap failures are varied and complex. You MUST thoroughly examine the log bundle to build a complete timeline — do not guess the root cause from a single error. + + - Read `bootstrap/journals/bootkube.log` thoroughly. Identify every process that started, crashed, or errored, noting timestamps to build a chronological sequence. + - For any crashed process (non-zero exit status, ContainerDied), read its stderr/stdout in the surrounding lines. Exit codes tell you *that* it crashed; the error output tells you *why*. Treat a crash as a potential bug but validate the termination reason (OOM, host restart, killed by signal, resource limits, etc.) by examining container exit status, kernel messages, and host metrics before assigning root cause. Consult surrounding logs and infra signals — exit codes, ContainerDied event details, dmesg/journal entries, and resource utilization — to distinguish software defects from infra/resource-induced terminations. + - Pursue errors: read surrounding context, follow references to other components' logs, and trace the chain of causation back to the originating failure. The first error is often a symptom, not the cause. + - Check supporting logs: `clusterapi/kube-apiserver.log`, `clusterapi/etcd.log`, `bootstrap/journals/kubelet.log`. Cross-reference timestamps with bootkube.log. + - Check `serial/{cluster-name}-bootstrap-serial.log` for kernel panics, ignition failures, disk errors. + - Check `failed-units.txt` for failed systemd units. **For "infrastructure" failures:** - Primary focus on installer log, not log bundle (failure happens before bootstrap) @@ -609,6 +611,7 @@ See that skill's documentation for details on dev-scripts, libvirt logs, sosrepo - Pay attention to job name clues: fips, ipv6, dualstack, metal, single-node, upgrade - IPv6 jobs are often disconnected and use mirror registries - Only suggest must-gather if the .tar file exists; if not, cluster was too unstable +- **TechPreview jobs** (names containing "techpreview") enable additional feature gates not active in Default clusters. Bootstrap failures in TechPreview jobs may be in TechPreview-gated code paths (e.g., on-cluster layering, OS image management) that won't reproduce in Default clusters. Note this in your analysis when relevant. ## Important Notes From 425a98b474ea2f057f570ea8a5d1214e88474226 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 5 Mar 2026 08:10:49 -0500 Subject: [PATCH 2/4] Improve payload analysis: handle accepted payloads, full lookback, thorough investigation - Analyze accepted payloads instead of skipping them (force-accepts may have failed blocking jobs) - Scan entire lookback window regardless of payload phase, tracking full pass/fail pattern (F F S F F) instead of stopping at first pass - Add failure_pattern field to history, HTML report, and JSON schema - Require full analysis (no --fast) for test failure subagents - Always perform thorough investigation including must-gather extraction Co-Authored-By: Claude Opus 4.6 --- plugins/ci/commands/analyze-payload.md | 4 +-- plugins/ci/skills/analyze-payload/SKILL.md | 34 ++++++++++++---------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/plugins/ci/commands/analyze-payload.md b/plugins/ci/commands/analyze-payload.md index 01d5dab16..b59687e58 100644 --- a/plugins/ci/commands/analyze-payload.md +++ b/plugins/ci/commands/analyze-payload.md @@ -1,5 +1,5 @@ --- -description: Analyze a rejected or in-progress payload with historical lookback to identify root causes of blocking job failures +description: Analyze a payload (rejected, accepted, or in-progress) with historical lookback to identify root causes of blocking job failures argument-hint: " [--lookback N]" --- @@ -17,7 +17,7 @@ ci:analyze-payload The `ci:analyze-payload` command analyzes a specific payload tag, investigates every failed blocking job, and produces a self-contained HTML report summarizing what went wrong. -It supports both **Rejected** payloads (full analysis) and **Ready** payloads (early analysis of blocking jobs that have already failed, to determine if the payload is on track for rejection). +It supports **Rejected** payloads (full analysis), **Ready** payloads (early analysis of blocking jobs that have already failed, to determine if the payload is on track for rejection), and **Accepted** payloads (which may have been force-accepted despite blocking job failures). It performs **historical lookback** through consecutive rejected payloads to determine when each failure first appeared. For each originating payload (where a job first started failing), it fetches the new PRs introduced in that payload as likely culprits. This distinguishes new failures from persistent/permafailing jobs and helps identify the root cause commits. diff --git a/plugins/ci/skills/analyze-payload/SKILL.md b/plugins/ci/skills/analyze-payload/SKILL.md index 5830bcdce..9cb879ccf 100644 --- a/plugins/ci/skills/analyze-payload/SKILL.md +++ b/plugins/ci/skills/analyze-payload/SKILL.md @@ -1,19 +1,20 @@ --- name: Analyze Payload -description: Analyze a rejected or in-progress payload with historical lookback to identify root causes of blocking job failures and produce an HTML report +description: Analyze a payload (rejected, accepted, or in-progress) with historical lookback to identify root causes of blocking job failures and produce an HTML report --- # Analyze Payload This skill analyzes a payload for a given OCP version, walks back through consecutive rejected payloads to determine when each failure started, correlates failures with newly introduced PRs, investigates each failed job in parallel, and produces a comprehensive HTML report. -It supports both **Rejected** payloads (full analysis of all failed blocking jobs) and **Ready** payloads (early analysis of blocking jobs that have already failed, with a determination of whether the payload is on track for rejection). +It supports **Rejected** payloads (full analysis of all failed blocking jobs), **Ready** payloads (early analysis of blocking jobs that have already failed, with a determination of whether the payload is on track for rejection), and **Accepted** payloads (payloads can be force-accepted despite blocking failures, so any failed blocking jobs are still analyzed). ## When to Use This Skill Use this skill when you need to: - Understand why a payload was rejected +- Investigate failures in a force-accepted payload (Accepted payloads may still have failed blocking jobs) - Assess whether an in-progress ("Ready") payload is likely to be rejected based on already-failed blocking jobs - Determine whether failures are new or persistent (permafailing) - Identify which PRs likely caused new failures @@ -63,26 +64,23 @@ Find the **target payload** (the tag from Step 1) in the fetched list. Based on - **Rejected**: Extract all failed blocking job names and their Prow URLs. Proceed with full analysis. - **Ready**: Extract blocking jobs that have already **failed** (with their Prow URLs). These are jobs that will not pass — they indicate the payload is on track for rejection. Proceed with analysis of those failed jobs and note in the report that the payload is still in progress. -- **Accepted**: Report "Payload was accepted, no analysis needed" and exit. +- **Accepted**: Extract any failed blocking job names and their Prow URLs. Payloads can be force-accepted despite blocking job failures, so do NOT assume all blocking jobs passed. If there are failed blocking jobs, proceed with full analysis and note in the report that the payload was accepted despite these failures. If there are truly no failed blocking jobs, report "Payload was accepted with all blocking jobs passing, no analysis needed" and exit. ### Step 3: Build Failure History (Lookback) -The goal is to determine **when each failing job first started failing** in the chain of consecutive rejected payloads. +The goal is to determine **when each failing job first started failing** and understand its failure pattern across recent payloads. Using the full payload list from Step 2 (which includes all phases): 1. Starting from the target payload, collect the set of failed blocking jobs. -2. Walk backwards through consecutive rejected payloads (up to `lookback` limit). -3. For each failed job in the target payload, check whether it also failed in the previous rejected payload. -4. Continue until either: - - The job was NOT failing in an earlier payload (meaning you found the originating payload) - - You reach a non-rejected (Accepted) payload - - You exhaust the lookback window +2. Walk backwards through **all** previous payloads in the lookback window (up to `lookback` limit), regardless of phase (Rejected, Accepted, or Ready). Accepted payloads can have failed blocking jobs (force-accepted), so check every payload. +3. For each failed job in the target payload, record whether it passed or failed in each previous payload across the entire lookback window. Do NOT stop at the first pass — a job may show an intermittent pattern like F-F-S-F-F due to flaky behavior, and understanding this pattern is important. For each failed job, record: -- **streak_length**: How many consecutive rejected payloads it has been failing in -- **originating_payload**: The first payload in the streak where this job started failing -- **is_new_failure**: Whether the job first started failing in the latest payload +- **streak_length**: How many consecutive payloads (counting backwards from the target) this job has been failing in (stops counting at the first pass) +- **originating_payload**: The first payload in the current consecutive failure streak where this job started failing +- **is_new_failure**: Whether the job first started failing in the target payload (streak_length == 1) +- **failure_pattern**: The full pass/fail history across the lookback window (e.g., "F F F S F F"). This helps contextualize whether the failure is a solid regression or intermittent. Intermittent failures are still fully investigated — the pattern is informational context, not a reason to skip analysis or discount the failure. ### Step 4: Fetch New PRs in Originating Payloads @@ -111,7 +109,7 @@ Instruct each subagent as follows: > > Based on the failure type, use the appropriate skill: > - **Install failure**: Use the `ci:prow-job-analyze-install-failure` skill. **You MUST download and examine the actual installer log bundle** — do NOT skip this step or make assessments based only on high-level metadata like pass rates or job names. The log bundle contains the actual error messages that reveal the root cause. For metal/bare-metal jobs (job name contains "metal"), perform additional analysis using the `ci:prow-job-analyze-metal-install-failure` skill as needed for dev-scripts, Metal3/Ironic, and BareMetalHost-specific diagnostics. -> - **Test failure**: Use the `ci:prow-job-analyze-test-failure` skill. +> - **Test failure**: Use the `ci:prow-job-analyze-test-failure` skill. Do NOT use `--fast` — always perform the full analysis including must-gather extraction and analysis. > > **IMPORTANT — Classify failures based on log evidence, not assumptions.** You must examine the actual logs (installer log, log bundle, bootstrap journals, kube-apiserver logs) before classifying a failure. A bootstrap timeout could be infrastructure, a product bug, or a race condition — the logs will tell you which. Cite specific error messages in your assessment. > @@ -240,6 +238,7 @@ A table showing ALL blocking jobs with columns: - Job Name - Status (color-coded: green for passed, red for failed) - Streak (how many consecutive payloads it has been failing; "N/A" for passed jobs) +- History (the failure_pattern across the lookback window, e.g., "F F F S F F", showing most recent first; use color-coded markers — red for F, green for S) - First Failed In (originating payload tag, linked to release controller) #### 7.3: Failed Job Details @@ -518,6 +517,7 @@ The filename **must** end with `-autodl.json`: `payload-analysis- "failure_type": "string", "streak_length": "int64", "is_new_failure": "int64", + "failure_pattern": "string", "originating_payload_tag": "string", "failure_analysis": "string", "revert_pr_url": "string", @@ -546,6 +546,7 @@ The filename **must** end with `-autodl.json`: `payload-analysis- "failure_type": "test", "streak_length": "5", "is_new_failure": "0", + "failure_pattern": "F F F F F S S", "originating_payload_tag": "4.22.0-0.nightly-2026-02-20-150000", "failure_analysis": "Root cause summary from subagent...", "revert_pr_url": "https://github.com/openshift/cno/pull/2037", @@ -572,6 +573,7 @@ The filename **must** end with `-autodl.json`: `payload-analysis- "failure_type": "install", "streak_length": "2", "is_new_failure": "0", + "failure_pattern": "F F S S S S S", "originating_payload_tag": "4.22.0-0.nightly-2026-02-23-080000", "failure_analysis": "Install timeout waiting for etcd quorum...", "revert_pr_url": "", @@ -645,7 +647,7 @@ If the release controller or Sippy API is unreachable, report the error clearly ## Notes -- The lookback only examines **consecutive** rejected payloads. If an Accepted payload breaks the chain, the lookback stops there. -- Subagents should run in **fast mode** (skip optional prompts like must-gather extraction) to keep analysis time reasonable. +- The lookback examines the job across **all payloads in the lookback window**, regardless of phase. A job may have a pattern like F-F-F-S-F-F-F due to flaky behavior — the lookback captures the full history so the report can distinguish persistent/intermittent failures from new regressions. +- Subagents should perform a **thorough analysis** — do not skip steps like must-gather extraction to save time. A proper root cause analysis is more important than speed. - The HTML report is fully self-contained — no external CSS/JS dependencies. - For very large numbers of failed jobs (>8), consider whether some share the same underlying failure and group them in the report. From bf330c0cbd1471c28b019d3426d3aae10a3f24c5 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 5 Mar 2026 08:14:26 -0500 Subject: [PATCH 3/4] `make update` --- .claude-plugin/marketplace.json | 2 +- PLUGINS.md | 2 +- docs/data.json | 6 +++--- plugins/ci/.claude-plugin/plugin.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 565f2ae93..78a535d2b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -26,7 +26,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.18" + "version": "0.0.19" }, { "name": "teams", diff --git a/PLUGINS.md b/PLUGINS.md index d06863a59..8df327a1a 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -58,7 +58,7 @@ Tools for working with OpenShift CI and analyzing Prow job results **Commands:** - **`/ci:add-debug-wait` ` [timeout]`** - Add a wait step to a CI workflow for debugging test failures -- **`/ci:analyze-payload` ` [--lookback N]`** - Analyze a rejected or in-progress payload with historical lookback to identify root causes of blocking job failures +- **`/ci:analyze-payload` ` [--lookback N]`** - Analyze a payload (rejected, accepted, or in-progress) with historical lookback to identify root causes of blocking job failures - **`/ci:analyze-pr-reverts` `[limit]`** - Analyze recent PR reverts to identify patterns and recommend preventive measures - **`/ci:analyze-prow-job-install-failure` ``** - Analyze OpenShift installation failures in Prow CI jobs - **`/ci:analyze-prow-job-resource` `prowjob-url resource-name`** - Analyze Kubernetes resource lifecycle in Prow job artifacts diff --git a/docs/data.json b/docs/data.json index d0bf73c96..c98218c3f 100644 --- a/docs/data.json +++ b/docs/data.json @@ -290,7 +290,7 @@ }, { "argument_hint": " [--lookback N]", - "description": "Analyze a rejected or in-progress payload with historical lookback to identify root causes of blocking job failures", + "description": "Analyze a payload (rejected, accepted, or in-progress) with historical lookback to identify root causes of blocking job failures", "name": "analyze-payload", "synopsis": "/ci:analyze-payload [--lookback N]" }, @@ -415,7 +415,7 @@ "name": "ci", "skills": [ { - "description": "Analyze a rejected or in-progress payload with historical lookback to identify root causes of blocking job failures and produce an HTML report", + "description": "Analyze a payload (rejected, accepted, or in-progress) with historical lookback to identify root causes of blocking job failures and produce an HTML report", "id": "analyze-payload", "name": "Analyze Payload" }, @@ -530,7 +530,7 @@ "name": "Triage Regression" } ], - "version": "0.0.18" + "version": "0.0.19" }, { "commands": [ diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index 34fa51e3a..f0b8a78fb 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.18", + "version": "0.0.19", "author": { "name": "openshift" } From a17fb9be359fcc6a6488033931d8dd6351a6f6ba Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 5 Mar 2026 09:58:59 -0500 Subject: [PATCH 4/4] Fetch payloads script assumes accepted = all jobs passed --- .claude-plugin/marketplace.json | 2 +- docs/data.json | 2 +- plugins/ci/.claude-plugin/plugin.json | 2 +- .../skills/fetch-payloads/fetch_payloads.py | 60 ++++++++----------- 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 78a535d2b..17f74b71b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -26,7 +26,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.19" + "version": "0.0.20" }, { "name": "teams", diff --git a/docs/data.json b/docs/data.json index c98218c3f..530772480 100644 --- a/docs/data.json +++ b/docs/data.json @@ -530,7 +530,7 @@ "name": "Triage Regression" } ], - "version": "0.0.19" + "version": "0.0.20" }, { "commands": [ diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index f0b8a78fb..ed4fb2d88 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.19", + "version": "0.0.20", "author": { "name": "openshift" } diff --git a/plugins/ci/skills/fetch-payloads/fetch_payloads.py b/plugins/ci/skills/fetch-payloads/fetch_payloads.py index f1be1bf5f..54fbdde4f 100644 --- a/plugins/ci/skills/fetch-payloads/fetch_payloads.py +++ b/plugins/ci/skills/fetch-payloads/fetch_payloads.py @@ -113,42 +113,34 @@ def format_payload(tag: dict, details: dict, architecture: str, stream_name: str if not blocking: return "\n".join(lines) - if phase == "Rejected": - failed = {k: v for k, v in blocking.items() if v.get("state") == "Failed"} - succeeded = len([v for v in blocking.values() if v.get("state") == "Succeeded"]) - if failed: - lines.append(f" Blocking: {succeeded}/{len(blocking)} succeeded, {len(failed)} failed:") - for job_name, info in sorted(failed.items()): - retries = info.get("retries", 0) - retry_str = f" ({retries} retries)" if retries else "" - prow_url = info.get("url", "") - lines.append(f" FAILED {job_name}{retry_str}") - if prow_url: - lines.append(f" {prow_url}") - elif phase == "Ready": - pending_jobs = {k: v for k, v in blocking.items() if v.get("state") == "Pending"} - succeeded_jobs = {k: v for k, v in blocking.items() if v.get("state") == "Succeeded"} - failed_jobs = {k: v for k, v in blocking.items() if v.get("state") == "Failed"} - parts = [] - if succeeded_jobs: - parts.append(f"{len(succeeded_jobs)} succeeded") - if pending_jobs: - parts.append(f"{len(pending_jobs)} pending") - if failed_jobs: - parts.append(f"{len(failed_jobs)} failed") + # Categorize jobs by state + succeeded_jobs = {k: v for k, v in blocking.items() if v.get("state") == "Succeeded"} + failed_jobs = {k: v for k, v in blocking.items() if v.get("state") == "Failed"} + pending_jobs = {k: v for k, v in blocking.items() if v.get("state") == "Pending"} + + # Build summary line + parts = [] + if succeeded_jobs: + parts.append(f"{len(succeeded_jobs)} succeeded") + if pending_jobs: + parts.append(f"{len(pending_jobs)} pending") + if failed_jobs: + parts.append(f"{len(failed_jobs)} failed") + + if parts: lines.append(f" Blocking: {', '.join(parts)} (of {len(blocking)})") - if failed_jobs: - lines.append(f" Failed blocking jobs:") - for job_name, info in sorted(failed_jobs.items()): - retries = info.get("retries", 0) - retry_str = f" ({retries} retries)" if retries else "" - prow_url = info.get("url", "") - lines.append(f" FAILED {job_name}{retry_str}") - if prow_url: - lines.append(f" {prow_url}") else: - # Accepted - brief summary - lines.append(f" Blocking: {len(blocking)}/{len(blocking)} succeeded") + lines.append(f" Blocking: {len(blocking)} jobs") + + # Show failed jobs with details + if failed_jobs: + for job_name, info in sorted(failed_jobs.items()): + retries = info.get("retries", 0) + retry_str = f" ({retries} retries)" if retries else "" + prow_url = info.get("url", "") + lines.append(f" FAILED {job_name}{retry_str}") + if prow_url: + lines.append(f" {prow_url}") return "\n".join(lines)