diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9472a2708..6181b9843 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -49,7 +49,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.43", + "version": "0.0.44", "category": "ci", "keywords": [ "prow", diff --git a/.skillsaw.yaml b/.skillsaw.yaml index fcf636a5f..42c76e742 100644 --- a/.skillsaw.yaml +++ b/.skillsaw.yaml @@ -98,9 +98,10 @@ custom-rules: # Exclude patterns (glob format) # Use exclude: [] to disable all excludes including defaults exclude: - # - "**/template/**" - # - "**/templates/**" - # - "**/_template/**" + - "node_modules/**" + - "**/template/**" + - "**/templates/**" + - "**/_template/**" # Additional markdown files to run content rules on (glob format) content-paths: [] diff --git a/docs/index.html b/docs/index.html index b16025c22..c489f9f9f 100644 --- a/docs/index.html +++ b/docs/index.html @@ -487,7 +487,7 @@

ai-helpers

{ "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.43", + "version": "0.0.44", "has_readme": true, "commands": [ { @@ -626,21 +626,29 @@

ai-helpers

"synopsis": "/ci:list-unstable-tests \u003cversion\u003e \u003ckeywords\u003e [sippy-url]", "body_html": "\u003cp\u003eThe \u003ccode\u003eci:list-unstable-tests\u003c/code\u003e command queries OpenShift CI test results from Sippy and lists all tests matching the keywords that have a pass rate below 95%. This is useful for quickly identifying unstable tests that need attention.\u003c/p\u003e\u003cp\u003eBy default, it queries the production Sippy instance at \u003ccode\u003esippy.dptools.openshift.org\u003c/code\u003e. You can optionally specify a different Sippy instance URL to query alternative environments (e.g., QE component readiness).\u003c/p\u003e\u003cp\u003eThis command is useful for:\u003cbr\u003e- Identifying unstable tests with inconsistent pass rates\u003cbr\u003e- Finding regression candidates for investigation\u003cbr\u003e- Generating reports of unstable test cases\u003cbr\u003e- Prioritizing test stabilization efforts\u003cbr\u003e- Quality gate checks before releases\u003c/p\u003e" }, + { + "name": "payload-analysis", + "full_name": "ci:payload-analysis", + "description": "Analyze a payload snapshot to identify root causes of blocking job failures and produce an HTML report with revert recommendations", + "description_html": "Analyze a payload snapshot to identify root causes of blocking job failures and produce an HTML report with revert recommendations", + "synopsis": "/ci:payload-analysis \u003cpayload-tag\u003e [--snapshot-dir DIR]", + "body_html": "\u003cp\u003eThe \u003ccode\u003eci:payload-analysis\u003c/code\u003e command analyzes a specific payload tag using a local snapshot (from the \u003ccode\u003epayload-snapshot\u003c/code\u003e skill), investigates every failed blocking job, and produces a self-contained HTML report summarizing what went wrong.\u003c/p\u003e\u003cp\u003eIt supports \u003cstrong\u003eRejected\u003c/strong\u003e payloads (full analysis), \u003cstrong\u003eReady\u003c/strong\u003e payloads (early analysis of blocking jobs that have already failed), and \u003cstrong\u003eAccepted\u003c/strong\u003e payloads (which may have been force-accepted despite blocking failures).\u003c/p\u003e\u003cp\u003eThe analysis reads from a pre-built snapshot that contains all release controller, GitHub, and CI data frozen at the time of snapshot creation. This means:\u003c/p\u003e\u003cp\u003e- \u003cstrong\u003eReproducible\u003c/strong\u003e: Re-running the analysis on the same snapshot produces consistent results\u003cbr\u003e- \u003cstrong\u003eHistorical replay\u003c/strong\u003e: Analyze old payloads against their original data, even after the world has moved on\u003cbr\u003e- \u003cstrong\u003eModel-friendly\u003c/strong\u003e: Less capable models can perform analysis without orchestrating complex multi-skill data gathering\u003c/p\u003e\u003cp\u003eIf no snapshot exists for the requested payload, one is created automatically.\u003c/p\u003e\u003cp\u003eThe analysis performs \u003cstrong\u003ehistorical lookback\u003c/strong\u003e through the snapshot's payload chain to determine when each failure first appeared. For each originating payload, it identifies the PRs introduced as likely culprits. When a candidate PR can be correlated with high confidence (>= 85 rubric score), the report recommends it for \u003cstrong\u003eimmediate revert\u003c/strong\u003e per OCP policy.\u003c/p\u003e\u003cp\u003eAn \u003cstrong\u003eadversarial review\u003c/strong\u003e subagent checks the analysis for weak correlations, misattributions, and logical gaps before finalizing the report.\u003c/p\u003e\u003cp\u003eThe payload results YAML output (\u003ccode\u003epayload-results-{tag}.yaml\u003c/code\u003e) can be consumed by composable downstream commands: \u003ccode\u003e/ci:payload-revert\u003c/code\u003e stages reverts for high-confidence candidates, and \u003ccode\u003e/ci:payload-experiment\u003c/code\u003e opens draft revert PRs for medium-confidence candidates.\u003c/p\u003e\u003cp\u003e### Key Features\u003c/p\u003e\u003cp\u003e- \u003cstrong\u003eSnapshot-based\u003c/strong\u003e: All data pre-gathered locally — no live API orchestration during analysis\u003cbr\u003e- \u003cstrong\u003eHistorical lookback\u003c/strong\u003e: Uses the snapshot's chain data to identify when each job first started failing\u003cbr\u003e- \u003cstrong\u003ePR correlation\u003c/strong\u003e: Reads local PR diffs from the snapshot to match error messages with code changes\u003cbr\u003e- \u003cstrong\u003eParallel investigation\u003c/strong\u003e: Kicks off subagents for each failed blocking job using the appropriate CI analysis skill\u003cbr\u003e- \u003cstrong\u003eAdversarial review\u003c/strong\u003e: A dedicated reviewer subagent checks conclusions before finalizing\u003cbr\u003e- \u003cstrong\u003eRevert recommendations\u003c/strong\u003e: Proposes specific PRs to revert when evidence strongly links them to a failure\u003cbr\u003e- \u003cstrong\u003eHTML report\u003c/strong\u003e: Self-contained HTML report with collapsible sections, color-coded severity, and executive summary\u003c/p\u003e" + }, { "name": "payload-experiment", "full_name": "ci:payload-experiment", "description": "Open draft revert PRs for medium-confidence payload candidates and trigger payload jobs to experimentally determine which PR is causing failures", "description_html": "Open draft revert PRs for medium-confidence payload candidates and trigger payload jobs to experimentally determine which PR is causing failures", "synopsis": "/ci:payload-experiment \u003cpayload-tag\u003e", - "body_html": "\u003cp\u003eThe \u003ccode\u003eci:payload-experiment\u003c/code\u003e command opens draft revert PRs for medium-confidence payload candidates (confidence score 60-84) and triggers payload jobs to experimentally determine which PR is causing failures. It operates in two phases separated by a CI wait period.\u003c/p\u003e\u003cp\u003e\u003cstrong\u003ePhase 1\u003c/strong\u003e: Reads the payload results YAML, filters medium-confidence candidates, opens draft revert PRs, triggers payload jobs, and appends action entries (\u003ccode\u003etype: "experiment"\u003c/code\u003e, \u003ccode\u003estatus: "pending"\u003c/code\u003e) to each candidate's \u003ccode\u003eactions\u003c/code\u003e array.\u003c/p\u003e\u003cp\u003e\u003cstrong\u003ePhase 2\u003c/strong\u003e: Detects candidates with a \u003ccode\u003estatus: "pending"\u003c/code\u003e action entry in the results YAML, checks job results, promotes confirmed causes to real revert PRs (with TRT JIRA bugs), and closes innocent draft PRs.\u003c/p\u003e\u003cp\u003eAll state is tracked in the payload results YAML via the \u003ccode\u003epayload-results-yaml\u003c/code\u003e skill — no separate tracking file is created.\u003c/p\u003e\u003cp\u003eThis command is one of three composable stages in the payload triage pipeline:\u003cbr\u003e1. \u003ccode\u003e/ci:analyze-payload\u003c/code\u003e — produces the payload results YAML\u003cbr\u003e2. \u003ccode\u003e/ci:payload-revert\u003c/code\u003e — stages reverts for HIGH confidence candidates\u003cbr\u003e3. \u003ccode\u003e/ci:payload-experiment\u003c/code\u003e — experimentally tests MEDIUM confidence candidates (this command)\u003c/p\u003e\u003cp\u003e### Job Triggering Limits\u003c/p\u003e\u003cp\u003e- \u003cstrong\u003eNon-aggregated jobs\u003c/strong\u003e: Up to 5 total across all candidates\u003cbr\u003e- \u003cstrong\u003eAggregated jobs\u003c/strong\u003e: Up to 1 total\u003c/p\u003e" + "body_html": "\u003cp\u003eThe \u003ccode\u003eci:payload-experiment\u003c/code\u003e command opens draft revert PRs for medium-confidence payload candidates (confidence score 60-84) and triggers payload jobs to experimentally determine which PR is causing failures. It operates in two phases separated by a CI wait period.\u003c/p\u003e\u003cp\u003e\u003cstrong\u003ePhase 1\u003c/strong\u003e: Reads the payload results YAML, filters medium-confidence candidates, opens draft revert PRs, triggers payload jobs, and appends action entries (\u003ccode\u003etype: "experiment"\u003c/code\u003e, \u003ccode\u003estatus: "pending"\u003c/code\u003e) to each candidate's \u003ccode\u003eactions\u003c/code\u003e array.\u003c/p\u003e\u003cp\u003e\u003cstrong\u003ePhase 2\u003c/strong\u003e: Detects candidates with a \u003ccode\u003estatus: "pending"\u003c/code\u003e action entry in the results YAML, checks job results, promotes confirmed causes to real revert PRs (with TRT JIRA bugs), and closes innocent draft PRs.\u003c/p\u003e\u003cp\u003eAll state is tracked in the payload results YAML via the \u003ccode\u003epayload-results-yaml\u003c/code\u003e skill — no separate tracking file is created.\u003c/p\u003e\u003cp\u003eThis command is one of three composable stages in the payload triage pipeline:\u003cbr\u003e1. \u003ccode\u003e/ci:payload-analysis\u003c/code\u003e — produces the payload results YAML\u003cbr\u003e2. \u003ccode\u003e/ci:payload-revert\u003c/code\u003e — stages reverts for HIGH confidence candidates\u003cbr\u003e3. \u003ccode\u003e/ci:payload-experiment\u003c/code\u003e — experimentally tests MEDIUM confidence candidates (this command)\u003c/p\u003e\u003cp\u003e### Job Triggering Limits\u003c/p\u003e\u003cp\u003e- \u003cstrong\u003eNon-aggregated jobs\u003c/strong\u003e: Up to 5 total across all candidates\u003cbr\u003e- \u003cstrong\u003eAggregated jobs\u003c/strong\u003e: Up to 1 total\u003c/p\u003e" }, { "name": "payload-revert", "full_name": "ci:payload-revert", - "description": "Stage reverts for high-confidence payload candidates identified by analyze-payload", - "description_html": "Stage reverts for high-confidence payload candidates identified by analyze-payload", + "description": "Stage reverts for high-confidence payload candidates identified by payload-analysis", + "description_html": "Stage reverts for high-confidence payload candidates identified by payload-analysis", "synopsis": "/ci:payload-revert \u003cpayload-tag\u003e", - "body_html": "\u003cp\u003eThe \u003ccode\u003eci:payload-revert\u003c/code\u003e command reads the payload results YAML produced by \u003ccode\u003e/ci:analyze-payload\u003c/code\u003e and stages reverts for all high-confidence candidates (confidence score >= 85) that have not already been reverted.\u003c/p\u003e\u003cp\u003eFor each qualifying candidate, it creates a TRT JIRA bug, opens a revert PR, and triggers payload validation jobs using the \u003ccode\u003estage-payload-reverts\u003c/code\u003e skill.\u003c/p\u003e\u003cp\u003eThis command is one of three composable stages in the payload triage pipeline:\u003cbr\u003e1. \u003ccode\u003e/ci:analyze-payload\u003c/code\u003e — produces the payload results YAML\u003cbr\u003e2. \u003ccode\u003e/ci:payload-revert\u003c/code\u003e — stages reverts for HIGH confidence candidates (this command)\u003cbr\u003e3. \u003ccode\u003e/ci:payload-experiment\u003c/code\u003e — opens draft revert PRs for MEDIUM confidence candidates\u003c/p\u003e\u003cp\u003e### Job Triggering Limits\u003c/p\u003e\u003cp\u003e- \u003cstrong\u003eNon-aggregated jobs\u003c/strong\u003e: Up to 5 total across all candidates\u003cbr\u003e- \u003cstrong\u003eAggregated jobs\u003c/strong\u003e: Up to 1 total\u003c/p\u003e\u003cp\u003eWhen the number of failing jobs across all candidates exceeds these limits, prioritize jobs from higher-confidence candidates first.\u003c/p\u003e" + "body_html": "\u003cp\u003eThe \u003ccode\u003eci:payload-revert\u003c/code\u003e command reads the payload results YAML produced by \u003ccode\u003e/ci:payload-analysis\u003c/code\u003e and stages reverts for all high-confidence candidates (confidence score >= 85) that have not already been reverted.\u003c/p\u003e\u003cp\u003eFor each qualifying candidate, it creates a TRT JIRA bug, opens a revert PR, and triggers payload validation jobs using the \u003ccode\u003estage-payload-reverts\u003c/code\u003e skill.\u003c/p\u003e\u003cp\u003eThis command is one of three composable stages in the payload triage pipeline:\u003cbr\u003e1. \u003ccode\u003e/ci:payload-analysis\u003c/code\u003e — produces the payload results YAML\u003cbr\u003e2. \u003ccode\u003e/ci:payload-revert\u003c/code\u003e — stages reverts for HIGH confidence candidates (this command)\u003cbr\u003e3. \u003ccode\u003e/ci:payload-experiment\u003c/code\u003e — opens draft revert PRs for MEDIUM confidence candidates\u003c/p\u003e\u003cp\u003e### Job Triggering Limits\u003c/p\u003e\u003cp\u003e- \u003cstrong\u003eNon-aggregated jobs\u003c/strong\u003e: Up to 5 total across all candidates\u003cbr\u003e- \u003cstrong\u003eAggregated jobs\u003c/strong\u003e: Up to 1 total\u003c/p\u003e\u003cp\u003eWhen the number of failing jobs across all candidates exceeds these limits, prioritize jobs from higher-confidence candidates first.\u003c/p\u003e" }, { "name": "query-job-status", @@ -776,10 +784,16 @@

ai-helpers

"description_html": "Helper skill to retrieve OAuth tokens from the correct OpenShift cluster context when multiple clusters are configured", "meta": "" }, + { + "name": "payload-analysis", + "description": "Analyze a payload snapshot to identify root causes of blocking job failures, score candidate PRs, and produce an HTML report with revert recommendations", + "description_html": "Analyze a payload snapshot to identify root causes of blocking job failures, score candidate PRs, and produce an HTML report with revert recommendations", + "meta": "" + }, { "name": "payload-autodl-json", - "description": "Schema for the autodl JSON data file produced by analyze-payload for database ingestion — you must use this skill whenever generating the autodl JSON file", - "description_html": "Schema for the autodl JSON data file produced by analyze-payload for database ingestion — you must use this skill whenever generating the autodl JSON file", + "description": "Schema for the autodl JSON data file produced by payload-analysis for database ingestion — you must use this skill whenever generating the autodl JSON file", + "description_html": "Schema for the autodl JSON data file produced by payload-analysis for database ingestion — you must use this skill whenever generating the autodl JSON file", "meta": "" }, { @@ -794,6 +808,12 @@

ai-helpers

"description_html": "State management for agentic payload triage actions — you must use this skill whenever reading or writing the payload results YAML file", "meta": "" }, + { + "name": "payload-snapshot", + "description": "Snapshot OpenShift payload data (release controller, PR diffs, comments, CI jobs, JUnit results, regression tracking) to a local directory for offline analysis", + "description_html": "Snapshot OpenShift payload data (release controller, PR diffs, comments, CI jobs, JUnit results, regression tracking) to a local directory for offline analysis", + "meta": "" + }, { "name": "prow-job-analyze-install-failure", "description": "Analyze OpenShift installation failures in Prow CI jobs by examining installer logs, log bundles, and sosreports. Use when CI job fails \"install should succeed\" tests at bootstrap, cluster creation or other stages.", diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index c4d6211a5..0710ea470 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.43", + "version": "0.0.44", "author": { "name": "github.com/openshift-eng" } diff --git a/plugins/ci/commands/payload-analysis.md b/plugins/ci/commands/payload-analysis.md new file mode 100644 index 000000000..c572231b5 --- /dev/null +++ b/plugins/ci/commands/payload-analysis.md @@ -0,0 +1,102 @@ +--- +description: Analyze a payload snapshot to identify root causes of blocking job failures and produce an HTML report with revert recommendations +argument-hint: " [--snapshot-dir DIR]" +--- + +## Name + +ci:payload-analysis + +## Synopsis + +``` +/ci:payload-analysis [--snapshot-dir DIR] +``` + +## Description + +The `ci:payload-analysis` command analyzes a specific payload tag using a local snapshot (from the `payload-snapshot` skill), investigates every failed blocking job, and produces a self-contained HTML report summarizing what went wrong. + +It supports **Rejected** payloads (full analysis), **Ready** payloads (early analysis of blocking jobs that have already failed), and **Accepted** payloads (which may have been force-accepted despite blocking failures). + +The analysis reads from a pre-built snapshot that contains all release controller, GitHub, and CI data frozen at the time of snapshot creation. This means: + +- **Reproducible**: Re-running the analysis on the same snapshot produces consistent results +- **Historical replay**: Analyze old payloads against their original data, even after the world has moved on +- **Model-friendly**: Less capable models can perform analysis without orchestrating complex multi-skill data gathering + +If no snapshot exists for the requested payload, one is created automatically. + +The analysis performs **historical lookback** through the snapshot's payload chain to determine when each failure first appeared. For each originating payload, it identifies the PRs introduced as likely culprits. When a candidate PR can be correlated with high confidence (>= 85 rubric score), the report recommends it for **immediate revert** per OCP policy. + +An **adversarial review** subagent checks the analysis for weak correlations, misattributions, and logical gaps before finalizing the report. + +The payload results YAML output (`payload-results-{tag}.yaml`) can be consumed by composable downstream commands: `/ci:payload-revert` stages reverts for high-confidence candidates, and `/ci:payload-experiment` opens draft revert PRs for medium-confidence candidates. + +### Key Features + +- **Snapshot-based**: All data pre-gathered locally — no live API orchestration during analysis +- **Historical lookback**: Uses the snapshot's chain data to identify when each job first started failing +- **PR correlation**: Reads local PR diffs from the snapshot to match error messages with code changes +- **Parallel investigation**: Kicks off subagents for each failed blocking job using the appropriate CI analysis skill +- **Adversarial review**: A dedicated reviewer subagent checks conclusions before finalizing +- **Revert recommendations**: Proposes specific PRs to revert when evidence strongly links them to a failure +- **HTML report**: Self-contained HTML report with collapsible sections, color-coded severity, and executive summary + +## Implementation + +Load the "payload-analysis" skill and follow its implementation steps. The skill orchestrates: + +1. Locating or creating a payload snapshot +2. Reading failure data, streaks, and candidate PRs from the snapshot +3. Launching parallel subagents to investigate each failed job +4. Scoring candidates and identifying revert recommendations +5. Adversarial review of conclusions +6. Generating the final HTML report, YAML, and JSON outputs + +## Return Value + +- **Format**: Self-contained HTML file + payload results YAML + JSON data file saved to the current working directory +- **Filenames**: + - `payload-analysis-{tag}-summary.html` — HTML report + - `payload-analysis-{tag}-autodl.json` — JSON data for database ingestion + - `payload-results-{tag}.yaml` — Scored candidates for downstream commands +- **Contents** (all `` links must use `target="_blank"` to open in a new tab): + - Executive summary with overall payload health + - Summary table of all blocking jobs (pass/fail with streaks and failure patterns) + - Per-job failure analysis with root cause, error messages, and logs + - Originating payload and candidate PRs for each failure + - Recommended reverts section with PR links, rationale, and copy-paste revert text + - Adversarial review notes + - Color-coded severity and collapsible detail sections + +## Examples + +1. **Analyze an amd64 nightly payload** (auto-creates snapshot if needed): + ``` + /ci:payload-analysis 4.22.0-0.nightly-2026-02-25-152806 + ``` + +2. **Analyze using an existing snapshot directory**: + ``` + /ci:payload-analysis 4.22.0-0.nightly-2026-02-25-152806 --snapshot-dir payload/4.22/nightly + ``` + +3. **Analyze an arm64 payload** (architecture inferred from tag): + ``` + /ci:payload-analysis 4.22.0-0.nightly-arm64-2026-02-25-152806 + ``` + +## Arguments + +- $1: A full payload tag (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Version, stream, and architecture are parsed from the tag automatically. Tags without an architecture suffix are amd64. (required) +- `--snapshot-dir DIR`: Path to an existing snapshot directory containing `summary.json` (optional). If not provided, searches standard locations or creates a new snapshot. + +## Skills Used + +- `payload-snapshot`: Creates the local data snapshot (auto-invoked if needed) +- `payload-analysis`: Orchestrates the full analysis workflow +- `payload-results-yaml`: Schema for the results YAML output +- `payload-autodl-json`: Schema for the JSON data output +- `prow-job-analyze-install-failure`: Analyzes install failures (used by subagents) +- `prow-job-analyze-test-failure`: Analyzes test failures (used by subagents) diff --git a/plugins/ci/commands/payload-experiment.md b/plugins/ci/commands/payload-experiment.md index 5dbeef9a3..9c877341f 100644 --- a/plugins/ci/commands/payload-experiment.md +++ b/plugins/ci/commands/payload-experiment.md @@ -24,7 +24,7 @@ The `ci:payload-experiment` command opens draft revert PRs for medium-confidence All state is tracked in the payload results YAML via the `payload-results-yaml` skill — no separate tracking file is created. This command is one of three composable stages in the payload triage pipeline: -1. `/ci:analyze-payload` — produces the payload results YAML +1. `/ci:payload-analysis` — produces the payload results YAML 2. `/ci:payload-revert` — stages reverts for HIGH confidence candidates 3. `/ci:payload-experiment` — experimentally tests MEDIUM confidence candidates (this command) @@ -35,12 +35,12 @@ This command is one of three composable stages in the payload triage pipeline: ## Implementation -1. **Parse the payload tag** from the argument. Extract `version`, `stream`, and `architecture` from the tag (see `analyze-payload` Step 1 for parsing rules). +1. **Parse the payload tag** from the argument. Extract `version`, `stream`, and `architecture` from the tag (see `payload-analysis` Step 1 for parsing rules). 2. **Read the payload results YAML** using the `payload-results-yaml` skill: Look for `payload-results-{tag}.yaml` in the current working directory. If not found, print an error and exit: ``` Error: Payload results YAML not found for {payload_tag}. - Run `/ci:analyze-payload {payload_tag}` first to generate it. + Run `/ci:payload-analysis {payload_tag}` first to generate it. ``` 3. **Detect Phase 2 resume**: If the results YAML contains any action entry with `type: "experiment"` and `status: "pending"`, jump to Phase 2 (step 5). Phase 2 processes only pending experiments — candidates with other statuses are left unchanged. @@ -60,7 +60,7 @@ This command is one of three composable stages in the payload triage pipeline: 1. **Start experiments after analysis**: ``` - /ci:analyze-payload 4.22.0-0.nightly-2026-02-25-152806 + /ci:payload-analysis 4.22.0-0.nightly-2026-02-25-152806 /ci:payload-experiment 4.22.0-0.nightly-2026-02-25-152806 ``` @@ -71,7 +71,7 @@ This command is one of three composable stages in the payload triage pipeline: ## Arguments -- $1: A full payload tag (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Must match the tag used with `/ci:analyze-payload`. (required) +- $1: A full payload tag (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Must match the tag used with `/ci:payload-analysis`. (required) ## Skills Used diff --git a/plugins/ci/commands/payload-revert.md b/plugins/ci/commands/payload-revert.md index ceb0c759e..2ef85ec0b 100644 --- a/plugins/ci/commands/payload-revert.md +++ b/plugins/ci/commands/payload-revert.md @@ -1,5 +1,5 @@ --- -description: Stage reverts for high-confidence payload candidates identified by analyze-payload +description: Stage reverts for high-confidence payload candidates identified by payload-analysis argument-hint: "" --- @@ -15,12 +15,12 @@ ci:payload-revert ## Description -The `ci:payload-revert` command reads the payload results YAML produced by `/ci:analyze-payload` and stages reverts for all high-confidence candidates (confidence score >= 85) that have not already been reverted. +The `ci:payload-revert` command reads the payload results YAML produced by `/ci:payload-analysis` and stages reverts for all high-confidence candidates (confidence score >= 85) that have not already been reverted. For each qualifying candidate, it creates a TRT JIRA bug, opens a revert PR, and triggers payload validation jobs using the `stage-payload-reverts` skill. This command is one of three composable stages in the payload triage pipeline: -1. `/ci:analyze-payload` — produces the payload results YAML +1. `/ci:payload-analysis` — produces the payload results YAML 2. `/ci:payload-revert` — stages reverts for HIGH confidence candidates (this command) 3. `/ci:payload-experiment` — opens draft revert PRs for MEDIUM confidence candidates @@ -33,12 +33,12 @@ When the number of failing jobs across all candidates exceeds these limits, prio ## Implementation -1. **Parse the payload tag** from the argument. Extract `version`, `stream`, and `architecture` from the tag (see `analyze-payload` Step 1 for parsing rules). +1. **Parse the payload tag** from the argument. Extract `version`, `stream`, and `architecture` from the tag (see `payload-analysis` Step 1 for parsing rules). 2. **Read the payload results YAML** using the `payload-results-yaml` skill: Look for `payload-results-{tag}.yaml` in the current working directory. If not found, print an error and exit: ``` Error: Payload results YAML not found for {payload_tag}. - Run `/ci:analyze-payload {payload_tag}` first to generate it. + Run `/ci:payload-analysis {payload_tag}` first to generate it. ``` 3. **Filter candidates**: Select candidates with `confidence_score >= 85`. Exclude any that already have an action with `status` of `"open"` or `"merged"` (pre-existing revert). @@ -56,13 +56,13 @@ When the number of failing jobs across all candidates exceeds these limits, prio 1. **Stage reverts after analysis**: ``` - /ci:analyze-payload 4.22.0-0.nightly-2026-02-25-152806 + /ci:payload-analysis 4.22.0-0.nightly-2026-02-25-152806 /ci:payload-revert 4.22.0-0.nightly-2026-02-25-152806 ``` ## Arguments -- $1: A full payload tag (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Must match the tag used with `/ci:analyze-payload`. (required) +- $1: A full payload tag (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Must match the tag used with `/ci:payload-analysis`. (required) ## Skills Used diff --git a/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/annotations.yaml new file mode 100644 index 000000000..4513e9e6c --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/annotations.yaml @@ -0,0 +1,37 @@ +expected_phase: Rejected +expected_failed_job_count: 3 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/cluster-authentication-operator/pull/839" + component: "cluster-authentication-operator" + min_confidence: 85 + expected_confidence: 92 + description: "Ginkgo v2 migration broke test binary stdout output, corrupting JSON test listing in openshift-tests" + expected_failing_jobs: + - "metal-ipi-ovn-ipv4" + - "metal-ipi-ovn-ipv6" + +expected_ci_config_issues: + - pr_url: "https://github.com/openshift/release/pull/76852" + repo: "openshift/release" + description: "Changed vSphere VCM default lease to multi-tenant, breaking UPI workflows that require single-tenant leases" + expected_failing_jobs: + - "install-analysis-all" + +notes: > + Mixed-root-cause case combining a product regression and a CI configuration + issue. Two metal-ipi jobs fail because cluster-authentication-operator#839 + (Ginkgo v2 migration) introduced a warning message to stdout that corrupts + JSON test listing in openshift-tests. The install-analysis-all job fails + because openshift/release#76852 changed the default vSphere VCM lease type + from single-tenant to multi-tenant, breaking UPI workflows. + + The CI config issue is the key test: openshift/release PRs do NOT appear in + payload changelogs. The agent must recognize that install-analysis-all is a + CI step failure (not a product bug), dig into the Prow job to find the + underlying vsphere-upi error ("SUBNETS.JSON does not contain enough + addresses... expected to be a single-tenant lease"), then search + openshift/release for recent commits matching the error. This tests the + "look beyond the snapshot" capability via the prow-job-analyze sub-skills. diff --git a/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/input.yaml new file mode 100644 index 000000000..e5b0b6ba0 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.22.0-0.nightly-2026-03-26-231124" diff --git a/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/annotations.yaml new file mode 100644 index 000000000..5f0d10f7e --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/annotations.yaml @@ -0,0 +1,32 @@ +expected_phase: Rejected +expected_failed_job_count: 13 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/origin/pull/31006" + component: "tests" + min_confidence: 85 + expected_confidence: 95 + description: "add oc-mirror to extensionBinaries — caused widespread test failures across serial, techpreview, metal, and upgrade jobs" + expected_failing_jobs: + - "aws-ovn-serial-1of2" + - "aws-ovn-serial-2of2" + - "aws-ovn-techpreview" + - "aws-ovn-techpreview-serial-1of3" + - "aws-ovn-techpreview-serial-2of3" + - "aws-ovn-techpreview-serial-3of3" + - "metal-ipi-ovn-ipv4" + - "metal-ipi-ovn-ipv6" + +notes: > + All 13 blocking jobs are new failures (streak=1), all originating from the + same payload with 63 candidate PRs. openshift/origin#31006 ("add oc-mirror + to extensionBinaries") caused widespread test failures across multiple job + families — serial, techpreview-serial, metal-ipi, and potentially upgrade + jobs. The agent must identify this PR as the root cause despite the large + candidate pool (63 PRs). Key signals: cross-job correlation (same PR blamed + for failures in many independent jobs), error messages referencing test + binaries/extensions, and new failure mode not present in prior payloads. + force_accept_expected is false because hours_since_baseline is 11.7h (under + the 18h threshold) and failures are product regressions, not infrastructure. diff --git a/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/input.yaml new file mode 100644 index 000000000..a11a378e0 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.nightly-2026-05-30-072431" diff --git a/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/annotations.yaml new file mode 100644 index 000000000..f28f57c48 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/annotations.yaml @@ -0,0 +1,25 @@ +expected_phase: Rejected +expected_failed_job_count: 2 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/cluster-network-operator/pull/2959" + component: "cluster-network-operator" + min_confidence: 80 + expected_confidence: 90 + description: "Added remaining CNO NetworkPolicies that blocked egress from cloud-network-config-controller, causing CrashLoopBackOff due to API server timeout" + expected_failing_jobs: + - "e2e-aws-ovn" + - "e2e-azure-ovn" + - "e2e-gcp-ovn" + +notes: > + Textbook true positive. CNO #2959 added NetworkPolicies that blocked egress + from cloud-network-config-controller, causing CrashLoopBackOff across all + platforms (AWS, Azure, GCP) due to API server connection timeouts. Evidence + quality is excellent: 0% pass rate vs 100% historical, cross-platform + consistency, precise mechanism match. Reverted by #2999 on 2026-05-08. + Tests the agent's ability to identify infrastructure-level networking + restrictions causing application failures and to correlate + cross-platform failures to a single root cause. diff --git a/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/input.yaml new file mode 100644 index 000000000..d172d9ea0 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.ci-2026-05-07-142711" diff --git a/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/annotations.yaml new file mode 100644 index 000000000..d7d3a762d --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/annotations.yaml @@ -0,0 +1,30 @@ +expected_phase: Rejected +expected_failed_job_count: 3 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/cluster-ingress-operator/pull/1354" + component: "cluster-ingress-operator" + min_confidence: 85 + expected_confidence: 95 + description: "Replaced OLM-based Istio install with Sail Library, causing GatewayAPIController failures" + expected_failing_jobs: + - "e2e-aws-ovn-techpreview" + - "e2e-aws-ovn-techpreview-serial" + - pr_url: "https://github.com/openshift/oc/pull/2232" + component: "oc" + min_confidence: 85 + expected_confidence: 90 + description: "Changed manifest reading to in-memory, causing nil pointer dereference in oc adm release mirror" + expected_failing_jobs: + - "metal-ipi-ovn-ipv6" + +notes: > + Payload 4.22.0-0.nightly-2026-03-20-053450 was rejected with 3 failed + blocking jobs: aws-ovn-techpreview (test failure: GatewayAPI/OSSM Sail + Library migration), aws-ovn-techpreview-serial-3of3 (same root cause), + and metal-ipi-ovn-ipv6 (install failure: oc mirror nil pointer crash). + Two high-confidence revert candidates: cluster-ingress-operator#1354 + (score 95, caused GatewayAPI failures) and oc#2232 (score 90, caused + mirror crash). diff --git a/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/input.yaml new file mode 100644 index 000000000..4c52ac2be --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.22.0-0.nightly-2026-03-20-053450" diff --git a/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/annotations.yaml new file mode 100644 index 000000000..a3c538455 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/annotations.yaml @@ -0,0 +1,22 @@ +expected_phase: Rejected +expected_failed_job_count: 3 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/cloud-credential-operator/pull/978" + component: "cloud-credential-operator" + min_confidence: 85 + expected_confidence: 95 + description: "Set operator condition to Progressing when pod identity webhook pods are updating, causing CCO Progressing=True during MCO rolling phase" + expected_failing_jobs: + - "aggregated-aws-ovn-upgrade-4.22-minor" + - "aggregated-gcp-ovn-upgrade-4.22-micro" + +notes: > + Payload 4.22.0-0.ci-2026-03-31-050515 was rejected with 3 failed blocking + jobs: aggregated-aws-ovn-upgrade (CCO Progressing invariant violation), + aggregated-gcp-ovn-upgrade (same root cause), and hypershift-e2e-aws + (infra teardown timeout, not a code regression). One high-confidence + revert candidate: cloud-credential-operator#978 (score 95). The hypershift + failure should be classified as infrastructure, not product. diff --git a/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/input.yaml new file mode 100644 index 000000000..7d4ad2650 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.22.0-0.ci-2026-03-31-050515" diff --git a/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/annotations.yaml new file mode 100644 index 000000000..ce90ad2ee --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/annotations.yaml @@ -0,0 +1,28 @@ +expected_phase: Rejected +expected_failed_job_count: 2 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/hypershift/pull/7790" + component: "hypershift" + min_confidence: 85 + expected_confidence: 95 + description: "Added guest cluster metrics forwarder feature and EnsureMetricsForwarderWorking test; HCCO silently skips deployment creation when prerequisites missing" + expected_failing_jobs: + - "periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aks" + - "periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aws-ovn" + - pr_url: "https://github.com/openshift/hypershift/pull/8138" + component: "hypershift" + min_confidence: 40 + expected_confidence: 50 + description: "Reverted AWS guest resource cleanup timeout extension, causing intermittent teardown timeouts" + expected_failing_jobs: + - "periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aws-ovn" + +notes: > + Payload 4.22.0-0.ci-2026-03-31-170515 was rejected with 2 failed + hypershift blocking jobs. Primary revert candidate: hypershift#7790 + (score 95) introduced the metrics forwarder feature and its test. + Secondary candidate: hypershift#8138 (score 50) reduced cleanup + timeout causing intermittent teardown failures. diff --git a/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/input.yaml new file mode 100644 index 000000000..764b8e3ba --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.22.0-0.ci-2026-03-31-170515" diff --git a/plugins/ci/evals/cases/payload-analysis/case-007-4.22-cvo-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-007-4.22-cvo-revert/annotations.yaml new file mode 100644 index 000000000..288eb9db2 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-007-4.22-cvo-revert/annotations.yaml @@ -0,0 +1,23 @@ +expected_phase: Rejected +expected_failed_job_count: 3 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/cluster-version-operator/pull/1309" + component: "cluster-version-operator" + min_confidence: 85 + expected_confidence: 95 + description: "CVO test binary emits klog INFO lines to stdout, corrupting JSON stream in openshift-tests test discovery" + expected_failing_jobs: + - "e2e-metal-ipi-ovn-ipv4" + - "e2e-metal-ipi-ovn-ipv6" + +notes: > + Payload 4.22.0-0.nightly-2026-03-18-161724 was rejected with 3 failed + blocking jobs: aggregated-hypershift-ovn-conformance (infra failure: + AWS lease exhaustion), metal-ipi-ovn-ipv4 (test failure: JSON parse + error from klog corruption), and metal-ipi-ovn-ipv6 (same root cause). + One high-confidence revert candidate: cluster-version-operator#1309 + (score 95). The hypershift failure is infrastructure-related and should + NOT be attributed to any PR. diff --git a/plugins/ci/evals/cases/payload-analysis/case-007-4.22-cvo-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-007-4.22-cvo-revert/input.yaml new file mode 100644 index 000000000..687db7221 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-007-4.22-cvo-revert/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.22.0-0.nightly-2026-03-18-161724" diff --git a/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/annotations.yaml new file mode 100644 index 000000000..1741ec525 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/annotations.yaml @@ -0,0 +1,19 @@ +expected_phase: "" +expected_failed_job_count: 2 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/cluster-monitoring-operator/pull/2814" + component: "cluster-monitoring-operator" + min_confidence: 85 + expected_confidence: 95 + description: "Minimal and telemetry CP monitors change caused monitoring test regressions" + expected_failing_jobs: + - "e2e-aws-ovn" + +notes: > + True positive from the CMO monitoring regression series. CMO #2814 + (MON-4517: Minimal and telemetry CP monitors) introduced monitoring + test failures. Reverted by #2901 on 2026-04-28. Expected phase is + empty because the payload was not formally Rejected. diff --git a/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/input.yaml new file mode 100644 index 000000000..867dce763 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.nightly-2026-04-27-183150" diff --git a/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/annotations.yaml new file mode 100644 index 000000000..1713bd703 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/annotations.yaml @@ -0,0 +1,17 @@ +expected_phase: Rejected +expected_failed_job_count: 1 +has_revert_candidates: false +force_accept_expected: false + +expected_candidates: [] + +notes: > + False positive test case. The agent originally flagged hypershift #8194 + (builder image update from 4.22 to 4.23) with score 92, constructing a + plausible but incorrect causal chain: builder image SDK changes leading + to auth library changes leading to Azure Graph API authentication + failure. The actual failure was an Azure-side Graph API auth issue that + resolved on its own without any code revert. The PR was NOT reverted. + This case tests the agent's ability to distinguish cloud-provider-side + platform issues from code regressions. The correct outcome is zero + revert candidates. diff --git a/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/input.yaml new file mode 100644 index 000000000..4f96beabc --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.ci-2026-04-14-085906" diff --git a/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/annotations.yaml new file mode 100644 index 000000000..57211e2df --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/annotations.yaml @@ -0,0 +1,9 @@ +expected_phase: Rejected +expected_failed_job_count: 4 +has_revert_candidates: false +force_accept_expected: false +notes: > + Complex case: 4 failed blocking jobs (aws-ovn-serial, aws-ovn-single-node-upgrade, + aws-ovn-upgrade, and one more). Tests the skill's ability to launch parallel + subagents, correlate multiple failures, and identify patterns across job families. + Previous payload was accepted, so these may be new failures. diff --git a/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/input.yaml new file mode 100644 index 000000000..f5dc61174 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.18.0-0.nightly-2026-05-14-085133" diff --git a/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/annotations.yaml new file mode 100644 index 000000000..ea069d352 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/annotations.yaml @@ -0,0 +1,13 @@ +expected_phase: Rejected +expected_failed_job_count: 1 +has_revert_candidates: false +force_accept_expected: false + +expected_candidates: [] + +notes: > + Infrastructure-only failure test case. This payload was rejected due to + Insights API Gateway returning HTTP 500 errors, which is an infrastructure + issue unrelated to any code change. The correct outcome is zero revert + candidates. This case tests the agent's ability to classify and dismiss + infrastructure issues without producing false positive revert recommendations. diff --git a/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/input.yaml new file mode 100644 index 000000000..87e3c3ace --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.ci-2026-05-14-181709" diff --git a/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/annotations.yaml new file mode 100644 index 000000000..12509e289 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/annotations.yaml @@ -0,0 +1,11 @@ +expected_phase: Rejected +expected_failed_job_count: 3 +has_revert_candidates: false +force_accept_expected: false +notes: > + Rejection streak case: this payload is part of a multi-payload rejection streak + (4 consecutive rejections: 2026-05-11-185554, 2026-05-12-035058, 2026-05-12-130101, + 2026-05-12-225204) before an accepted payload on 2026-05-13. Failed jobs include + fips-scan, hypershift-ovn-conformance, and microshift-ovn-conformance-serial. + Tests the skill's lookback/streak detection and whether it can distinguish + persistent failures from new regressions within a streak. diff --git a/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/input.yaml new file mode 100644 index 000000000..6e966cc4b --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.20.0-0.nightly-2026-05-12-225204" diff --git a/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/annotations.yaml new file mode 100644 index 000000000..a5b1236ab --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/annotations.yaml @@ -0,0 +1,10 @@ +expected_phase: Accepted +expected_failed_job_count: 1 +has_revert_candidates: false +force_accept_expected: false +notes: > + Edge case: payload was Accepted despite 1 failed blocking job + (hypershift-ovn-conformance-4.20). Tests that the skill correctly handles + force-accepted payloads — it should still analyze the failed job and note + that the payload was accepted despite the failure. Should NOT recommend + force-accept since the payload was already accepted. diff --git a/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/input.yaml new file mode 100644 index 000000000..e6eae53ae --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/input.yaml @@ -0,0 +1 @@ +payload_tag: "4.20.0-0.nightly-2026-05-13-064706" diff --git a/plugins/ci/evals/eval-payload-analysis.yaml b/plugins/ci/evals/eval-payload-analysis.yaml new file mode 100644 index 000000000..b19dd4c50 --- /dev/null +++ b/plugins/ci/evals/eval-payload-analysis.yaml @@ -0,0 +1,400 @@ +name: ci-payload-analysis-eval +description: Evaluate the payload-analysis skill's ability to analyze OCP payload snapshots, trace root causes, correlate with PRs, and produce actionable HTML reports with revert recommendations +skill: ci:payload-analysis + +execution: + mode: case + arguments: "{payload_tag} --snapshot-dir $EVAL_SNAPSHOT_DIR/{payload_tag}" + timeout: 3600 + max_budget_usd: 25.0 + env: + EVAL_SNAPSHOT_DIR: $EVAL_SNAPSHOT_DIR + +runner: + type: claude-code + plugin_dirs: + - plugins/ci + system_prompt: | + You are a diligent senior OpenShift release engineer triaging failures. + + **CRITICAL**: You have many ci, must-gather, and jira skills at your + disposal. You MUST load the relevant skills using the Skill tool BEFORE + you begin any work. Do NOT improvise or guess. This applies equally to + subagents: instruct every subagent to review its available skills and + load the appropriate ones before beginning its investigation. A subagent + that does not load a skill will produce shallow, unreliable analysis. + + After completing your analysis, you MUST use the Skill tool to invoke + ci:payload-results-yaml and ci:payload-autodl-json to generate the + structured output files. NEVER write these files directly — the skills + enforce the canonical schema. + + When searching external sources (GitHub, Prow, GCS) for CI configuration + changes or PRs not found in the snapshot changelog, constrain searches to + changes merged before the payload creation timestamp. The payload tag + encodes its creation date (e.g., 4.22.0-0.nightly-2026-03-26-231124 was + created on 2026-03-26). + +models: + judge: claude-opus-4-6 + +permissions: + allow: + - "Skill" + - "Bash" + - "Agent" + - "WebFetch" + - "Write" + - "Read" + deny: [] + +mlflow: + experiment: ci-payload-analysis-eval + +dataset: + path: plugins/ci/evals/cases/payload-analysis + schema: | + Each case directory contains: + - input.yaml: YAML file with fields: + - 'payload_tag' — a real payload tag from amd64.ocp.releases.ci.openshift.org + (e.g., '5.0.0-0.nightly-2026-05-30-072431') + - annotations.yaml: Expected outcomes and metadata for scoring: + - 'expected_phase': Rejected|Ready|Accepted + - 'expected_failed_job_count': integer + - 'has_revert_candidates': boolean + - 'force_accept_expected': boolean + - 'expected_candidates': list of expected revert candidates with pr_url, + component, min_confidence, expected_confidence, description, + expected_failing_jobs + - 'notes': free text context + + Snapshots are stored as tar.gz archives in + plugins/ci/evals/snapshots/payload-analysis/ and must be extracted before + running evals. Run: + export EVAL_SNAPSHOT_DIR=$(plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh ) + +outputs: + - path: "output" + schema: | + The skill writes three files to the workspace root (not a subdirectory): + 1. payload-analysis-{sanitized_tag}-summary.html — self-contained HTML report + with executive summary, blocking jobs summary table, recommended reverts + (or no-revert verdict), optional force-accept recommendation, and per-job + collapsible failure details. Uses GitHub dark mode styling with embedded CSS. + 2. payload-results-{sanitized_tag}.yaml — structured YAML with: + - metadata: payload_tag, version, stream, architecture, release_controller_url, + analyzed_at, force_accept_recommended + - failing_jobs[]: job_name, prow_url, is_aggregated, underlying_job_name, + failure_type, root_cause_summary, streak_length, originating_payload_tag, + failure_pattern + - candidates[]: pr_url, pr_number, component, title, confidence_score, + rationale, failing_jobs[], actions[] + 3. payload-analysis-{sanitized_tag}-autodl.json — flat denormalized JSON object + with table_name, schema, and rows array. One row per (failed blocking job, + candidate PR) pair. All row values are strings. Fields include payload_tag, + version, stream, architecture, phase, job_name, prow_url, failure_type, + root_cause_summary, candidate_pr_url, candidate_confidence_score. + + Judges should check outputs["files"] and outputs["modified_files"] for files + matching these naming patterns. + +traces: + stdout: true + stderr: true + events: true + metrics: true + +judges: + - name: output_files_exist + description: | + Verify all three required output files are produced: HTML report, + payload results YAML, and autodl JSON. + check: | + import os + files = outputs.get("files", {}) + modified = outputs.get("modified_files", {}) + all_f = {**files, **modified} + html = [k for k in all_f if k.endswith("-summary.html")] + yaml_f = [k for k in all_f if os.path.basename(k).startswith("payload-results-") and k.endswith(".yaml")] + json_f = [k for k in all_f if k.endswith("-autodl.json")] + missing = [] + if not html: + missing.append("HTML report (*-summary.html)") + if not yaml_f: + missing.append("payload results YAML (payload-results-*.yaml)") + if not json_f: + missing.append("autodl JSON (*-autodl.json)") + if missing: + return (False, f"Missing: {', '.join(missing)}") + return (True, f"All 3 files found: {html[0]}, {yaml_f[0]}, {json_f[0]}") + + - name: yaml_results_valid + description: | + Verify the payload results YAML has the required schema: metadata block + with payload_tag, version, stream, architecture; failing_jobs array; + and candidates array with properly structured entries. + check: | + import yaml, os + files = outputs.get("files", {}) + modified = outputs.get("modified_files", {}) + all_f = {**files, **modified} + yaml_files = {k: v for k, v in all_f.items() if os.path.basename(k).startswith("payload-results-") and k.endswith(".yaml")} + if not yaml_files: + return (False, "No payload results YAML found") + content = list(yaml_files.values())[0] + try: + data = yaml.safe_load(content) + except Exception as e: + return (False, f"Invalid YAML: {e}") + if not isinstance(data, dict): + return (False, "YAML root is not a dict") + meta = data.get("metadata", {}) + required_meta = ["payload_tag", "version", "stream", "architecture"] + missing = [f for f in required_meta if f not in meta] + if missing: + return (False, f"Missing metadata: {', '.join(missing)}") + if "failing_jobs" not in data: + return (False, "Missing failing_jobs array") + if "candidates" not in data: + return (False, "Missing candidates array") + jobs = data.get("failing_jobs", []) + cands = data.get("candidates", []) + if jobs: + required_job_fields = ["job_name", "failure_type", "root_cause_summary"] + j0 = jobs[0] + jmissing = [f for f in required_job_fields if f not in j0] + if jmissing: + return (False, f"failing_jobs[0] missing: {', '.join(jmissing)}") + if cands: + required_cand_fields = ["pr_url", "confidence_score", "failing_jobs"] + c0 = cands[0] + cmissing = [f for f in required_cand_fields if f not in c0] + if cmissing: + return (False, f"candidates[0] missing: {', '.join(cmissing)}") + return (True, f"Valid: {len(jobs)} failing jobs, {len(cands)} candidates") + + - name: json_data_valid + description: | + Verify the autodl JSON has the expected structure: table_name, schema dict, + rows array with required fields, and all row values are strings. + check: | + import json + files = outputs.get("files", {}) + modified = outputs.get("modified_files", {}) + all_f = {**files, **modified} + json_files = {k: v for k, v in all_f.items() if k.endswith("-autodl.json")} + if not json_files: + return (False, "No autodl JSON found") + content = list(json_files.values())[0] + try: + data = json.loads(content) + except Exception as e: + return (False, f"Invalid JSON: {e}") + if not isinstance(data, dict): + return (False, "JSON root is not a dict (expected object with table_name, schema, rows)") + if "table_name" not in data: + return (False, "Missing 'table_name' field") + if "schema" not in data or not isinstance(data["schema"], dict): + return (False, "Missing or invalid 'schema' field") + rows = data.get("rows", []) + if not isinstance(rows, list): + return (False, "'rows' is not an array") + if len(rows) == 0: + return (False, "rows array is empty") + required = ["payload_tag", "job_name", "failure_type", "root_cause_summary"] + row = rows[0] + missing = [f for f in required if f not in row] + if missing: + return (False, f"Missing fields in rows[0]: {', '.join(missing)}") + non_string = [f for f in row if not isinstance(row[f], str)] + if non_string: + return (False, f"Non-string values in rows[0]: {', '.join(non_string)} (all values must be strings)") + return (True, f"Valid JSON: {len(rows)} rows, table_name={data['table_name']}, all values are strings") + + - name: html_report_structure + description: | + Verify the HTML report contains required sections: executive summary, + blocking jobs table, revert/no-revert verdict, embedded CSS, and + per-job details. + check: | + files = outputs.get("files", {}) + modified = outputs.get("modified_files", {}) + all_f = {**files, **modified} + html_files = {k: v for k, v in all_f.items() if k.endswith("-summary.html")} + if not html_files: + return (False, "No HTML report found") + html = list(html_files.values())[0] + checks = { + "executive summary": "executive" in html.lower() or "summary" in html.lower(), + "blocking jobs table": "" in html, + } + failed = [k for k, v in checks.items() if not v] + if failed: + return (False, f"Missing: {', '.join(failed)}") + return (True, f"All {len(checks)} structural checks passed") + + - name: required_skill_invocations + description: | + Verify that payload-results-yaml and payload-autodl-json skills were + invoked via the Skill tool during execution. + check: | + import json + stdout = outputs.get("stdout", "") + invoked = set() + for line in stdout.splitlines(): + try: + entry = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if entry.get("type") != "assistant": + continue + for block in entry.get("message", {}).get("content", []): + if block.get("type") == "tool_use" and block.get("name") == "Skill": + skill = block.get("input", {}).get("skill", "") + invoked.add(skill) + required = ["ci:payload-results-yaml", "ci:payload-autodl-json"] + found = [s for s in required if s in invoked] + missing = [s for s in required if s not in invoked] + if missing: + return (False, f"Missing Skill calls: {', '.join(missing)}. Found: {', '.join(found)}") + return (True, f"Both required skills invoked: {', '.join(found)}") + + - name: analysis_quality + description: | + LLM judge to assess root cause analysis depth, PR correlation accuracy, + and overall report quality. Uses only YAML results and HTML report, not + the full output tree (which includes the snapshot and exceeds context). + prompt: | + You are evaluating the output of an OCP payload analysis skill. The skill + analyzes failed blocking jobs in an OpenShift CI payload using a local + snapshot, traces root causes by examining Prow job logs and artifacts, + correlates failures with candidate PRs using a weighted scoring rubric, + and produces a comprehensive HTML report with revert recommendations. + + Review the skill's YAML results file: + + {% for path, content in outputs.files.items() if path.endswith('.yaml') and 'payload-results' in path %} + ### {{ path }} + + {{ content }} + {% endfor %} + + Review the skill's HTML report: + + {% for path, content in outputs.files.items() if path.endswith('-summary.html') %} + ### {{ path }} + + {{ content }} + {% endfor %} + + Expected outcomes for this test case: + + {{ annotations }} + + Evaluate the YAML results file and HTML report on a 1-5 scale: + + Score 1: Analysis is missing or completely wrong — no root causes identified, + no PR correlation, report is empty or broken. + Score 2: Superficial analysis — root causes restate symptoms without explanation + (e.g., "node not ready" without tracing WHY), PR correlations are + absent or random, report structure is incomplete. + Score 3: Adequate analysis — root causes go one level deeper than symptoms, + some PR correlations are plausible, report has required sections + but may have gaps in coverage or depth. + Score 4: Good analysis — root causes cite specific error messages from logs, + PR correlations reference actual code/components changed, revert + recommendations have clear rationale, report is well-structured. + Score 5: Excellent analysis — root causes traced to specific code paths with + log excerpts, PR correlations demonstrate causal reasoning (not just + temporal coincidence), rubric scores are itemized, report is + comprehensive and immediately actionable. + + Key evaluation criteria: + - Does the number of failing jobs approximately match expected_failed_job_count? + - Are the root cause summaries specific (citing error messages, code paths) + rather than generic (just restating test names)? + - If expected_candidates are listed in annotations, are they identified with + evidence-based rationale? + - Does the HTML report contain actionable information? + + - name: revert_scoring_accuracy + description: | + LLM judge to assess whether the confidence scoring rubric was correctly + applied and expected revert candidates identified. Uses only YAML results + and autodl JSON, not the full output tree. + prompt: | + You are evaluating whether the payload analysis skill correctly identified + revert candidates and applied its confidence scoring rubric. + + The rubric awards points for: + - New failure mode: +30 (this specific failure wasn't present in prior payloads) + - Component exclusivity: +10 to +30 (fewer PRs touching same component = higher) + - Error message match: +40 (errors directly reference code changed by the PR) + - Multi-job correlation: +10 (same PR is candidate for multiple failed jobs) + - Presubmit coverage gap: +10 (failing scenario not in PR's presubmit tests) + - Single candidate: +10 (only one PR touches the affected component) + Maximum: 130 (capped at 100). Revert threshold: >= 85. + + Review the YAML results file: + + {% for path, content in outputs.files.items() if path.endswith('.yaml') and 'payload-results' in path %} + ### {{ path }} + + {{ content }} + {% endfor %} + + Review the autodl JSON file: + + {% for path, content in outputs.files.items() if path.endswith('-autodl.json') %} + ### {{ path }} + + {{ content }} + {% endfor %} + + Expected outcomes for this test case: + + {{ annotations }} + + If expected_candidates are listed in annotations, those PRs should be + identified with confidence >= min_confidence and linked to the + expected_failing_jobs. If expected_candidates is empty, no false positive + revert recommendations should appear. + + Evaluate on a 1-5 scale: + + Score 1: No scoring applied, or scores are arbitrary. Expected revert + candidates (from annotations) are completely missed. + Score 2: Some expected candidates identified but confidence scores are way + off (>20 points from expected) or key failing job linkages are + missing. + Score 3: All expected high-confidence candidates (min_confidence >= 85) are + identified above threshold, but scores may be off by 10-20 points + or rubric components not itemized. + Score 4: All expected candidates identified with scores within 10 points of + expected_confidence. Rubric correctly applied with itemized + breakdown. Most expected failing jobs correctly linked. + Infrastructure failures correctly distinguished from product + failures. + Score 5: All expected candidates identified with scores within 5 points of + expected_confidence. Rubric meticulously applied — each component + score justified with specific evidence. All expected failing jobs + correctly linked. No false positive revert recommendations for + unrelated PRs. Cross-job correlation signal clearly documented. + +thresholds: + output_files_exist: + min_pass_rate: 1.0 + yaml_results_valid: + min_pass_rate: 1.0 + json_data_valid: + min_pass_rate: 1.0 + html_report_structure: + min_pass_rate: 1.0 + analysis_quality: + min_mean: 3.5 + revert_scoring_accuracy: + min_mean: 3.0 + required_skill_invocations: + min_pass_rate: 1.0 diff --git a/plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh b/plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh new file mode 100755 index 000000000..af59d9471 --- /dev/null +++ b/plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Fetch payload-analysis snapshot data for eval runs. +# +# Clones historical-payload-data repo and makes snapshots available +# at .work/eval-payload-snapshots// +# +# Usage: ./plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh +# Or: SNAPSHOT_DIR=$(./plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh 5.0.0-0.nightly-2026-05-30-072431) + +set -euo pipefail + +REPO_URL="https://github.com/stbenjam/historical-payload-data.git" +REPO_ROOT="$(cd "$(dirname "$0")/../../../.." && pwd)" +BASE_DIR="$REPO_ROOT/.work/eval-payload-snapshots" + +tag="${1:-}" + +if [[ -d "$BASE_DIR/.git" ]]; then + echo "Snapshot repo already cloned at $BASE_DIR" >&2 + git -C "$BASE_DIR" pull --ff-only >&2 2>/dev/null || true +else + echo "Cloning snapshot data from $REPO_URL..." >&2 + git clone --depth 1 "$REPO_URL" "$BASE_DIR" >&2 +fi + +if [[ -n "$tag" ]]; then + if [[ -d "$BASE_DIR/$tag" ]]; then + echo "$BASE_DIR/$tag" + else + echo "Snapshot not found for tag: $tag" >&2 + exit 1 + fi +else + echo "$BASE_DIR" +fi diff --git a/plugins/ci/skills/payload-analysis/SKILL.md b/plugins/ci/skills/payload-analysis/SKILL.md new file mode 100644 index 000000000..9c6399480 --- /dev/null +++ b/plugins/ci/skills/payload-analysis/SKILL.md @@ -0,0 +1,501 @@ +--- +name: payload-analysis +description: Analyze a payload snapshot to identify root causes of blocking job failures, score candidate PRs, and produce an HTML report with revert recommendations +--- + +# Payload Analysis + +This skill analyzes a payload using a local snapshot (produced by `payload-snapshot`) to identify root causes of blocking job failures and produce a comprehensive HTML report. The snapshot pre-gathers all release controller, GitHub, and CI data so this skill can focus purely on analysis — no live API orchestration required. + +It supports **Rejected** payloads (full analysis of all failed blocking jobs), **Ready** payloads (early analysis of blocking jobs that have already failed), and **Accepted** payloads (which may have been force-accepted despite blocking failures). + +## When to Use This Skill + +Use this skill when you need to: + +- Understand why a payload was rejected +- Investigate failures in a force-accepted payload +- Assess whether an in-progress ("Ready") payload is likely to be rejected +- Determine whether failures are new or persistent +- Identify which PRs likely caused new failures +- Get a comprehensive overview of payload health with actionable root cause analysis +- Re-analyze a historical payload against its original snapshot data + +## Required Skills + +Before starting, you **MUST** load the following skills (they define output schemas used in Steps 6 and 8): + +1. **`payload-results-yaml`** — schema for the payload results YAML file +2. **`payload-autodl-json`** — schema for the autodl JSON data file + +## Prerequisites + +1. **Python 3** (3.10 or later) — for running the snapshot script if needed +2. **gcloud CLI** — for subagent artifact download (must-gather, pod logs) +3. **GitHub CLI (`gh`)** — for checking existing revert PRs (Step 6.3) + +## Implementation Steps + +### Step 1: Parse Arguments + +The first argument is a **full payload tag** (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Parse from it: +- `tag`: The specific payload tag to analyze +- `version`: Extract from the tag (e.g., `4.22` from `4.22.0-0.nightly-...`) +- `stream`: Extract from the tag (e.g., `nightly` from `4.22.0-0.nightly-...`) +- `architecture`: Inferred from the tag. The tag format is `-0.[-]-`. If no architecture is present between the stream and timestamp, it is `amd64`. Otherwise, the architecture is the segment between the stream and timestamp. Examples: + - `4.22.0-0.nightly-2026-02-25-152806` → `amd64` + - `4.22.0-0.nightly-arm64-2026-02-25-152806` → `arm64` + - `4.22.0-0.nightly-ppc64le-2026-02-25-152806` → `ppc64le` + +### Step 2: Locate or Create Snapshot + +The analysis requires a local snapshot produced by the `payload-snapshot` skill. Search for an existing snapshot in this order: + +1. **Explicit `--snapshot-dir DIR`**: If provided, look for `DIR/summary.json`. If not found, exit with an error. +2. **Current directory**: Check if `./summary.json` exists and its `payload_tag` field matches the requested tag. +3. **Standard relative path**: Check if `payload///summary.json` exists and matches the tag. + +If no matching snapshot is found, create one: + +```bash +SNAPSHOT_SCRIPT="${CLAUDE_PLUGIN_ROOT}/skills/payload-snapshot/scripts/payload_snapshot.py" +if [ ! -f "$SNAPSHOT_SCRIPT" ]; then + SNAPSHOT_SCRIPT=$(find ~/.claude/plugins -type f -path "*/ci/skills/payload-snapshot/scripts/payload_snapshot.py" 2>/dev/null | sort | head -1) +fi +if [ -z "$SNAPSHOT_SCRIPT" ] || [ ! -f "$SNAPSHOT_SCRIPT" ]; then echo "ERROR: payload_snapshot.py not found" >&2; exit 2; fi +python3 "$SNAPSHOT_SCRIPT" +``` + +After locating `summary.json`, set `SNAPSHOT_DIR` to the directory containing it. All relative paths in `summary.json` (e.g., `job_json`, `junit_results`, `build_log`, PR paths) resolve from this directory. + +### Step 3: Extract Failure Data from Snapshot + +Read `summary.json` to extract all data needed for analysis. The snapshot has already done the work of fetching payloads, building the chain, tracking streaks, and collecting PR data. + +#### 3.1: Payload Metadata + +From `summary.json` top-level fields: +- `payload_tag`, `phase`, `release_url`, `architecture`, `stream`, `version` +- `chain_length`, `baseline_tag`, `hours_since_baseline` + +#### 3.2: Failed Blocking Jobs + +From `summary.json` → `blocking_jobs.failed_jobs[]`, each entry contains: +- `name`, `state`, `prow_url`, `gcs_url`, `is_aggregated`, `retries` +- `streak`: `streak_length`, `originating_payload`, `is_new_failure`, `failure_pattern` +- `build_log_errors`, `test_failure_count` +- Paths: `job_json`, `junit_results`, `build_log` + +For each failed job, read its `job.json` (at `SNAPSHOT_DIR/` path) to get `previousAttemptURLs`. + +#### 3.3: Candidate PRs + +For each failed job's `streak.originating_payload`, find the matching entry in `summary.json` → `payloads[]`. Its `prs[]` array contains the PRs introduced in that payload: +- `url`, `component`, `number`, `description` +- Paths to local artifacts: `diff`, `comments`, `jobs` + +These PRs are the **candidates** for failures that started in that originating payload. + +#### 3.4: Test Failure Details + +From `summary.json` → `test_failures.blocking[]`: +- `test_name`, `jobs`, `first_failed_in`, `payloads_failing` +- `failure_message`, `failure_text` (full, not truncated) + +#### 3.5: Build Log Errors + +For deeper context, read `build_log.json` (at the `build_log` path) for any failed job. It contains `error_warning_lines[]` with `line_number` and `text`, plus `tail_lines[]` (last 20% of the log). + +### Step 4: Investigate Each Failed Job in Parallel + +For each failed blocking job in the **target payload**, launch a **parallel subagent** to investigate the failure. Pass the subagent the Prow URL and all previous attempt URLs from Step 3.2. + +Each subagent should determine whether the failure is an install failure or a test failure by checking the JUnit results (e.g., look for `install should succeed*` test failures), then use the appropriate analysis skill. Almost all blocking jobs install a cluster and then run tests, so the job name alone does not tell you the failure type. + +You MUST use the following prompt verbatim (substituting the placeholder values) when launching each subagent. Do NOT paraphrase, shorten, or write your own prompt — the specific instructions below are critical for analysis quality: + +> Analyze the failure at . This job had retries. The previous attempt URLs are: . +> +> **Aggregated jobs**: If this is an aggregated job (has `aggregated-` prefix or an `aggregator` step), retries only re-run the aggregation analysis — they do NOT re-run the underlying test jobs. Therefore, only examine the most recent attempt; previous attempts contain the same underlying results and do not provide additional signal. +> +> **Non-aggregated jobs**: **Examine the final attempt first**, then compare with previous attempts to determine whether all retries failed the same way. If retries show different failure modes, note this — it distinguishes consistent regressions from intermittent/infrastructure issues. Consistent failures across all attempts strongly indicate a product regression rather than flakiness. +> +> First, check the JUnit results or build log to determine whether this is an install failure (look for `install should succeed: overall` or similar install-related test failures) or a test failure (install passed, specific tests failed). +> +> Based on the failure type, use the appropriate skill: +> - **Install failure**: Use the `ci:prow-job-analyze-install-failure` skill. For metal/bare-metal jobs (job name contains "metal"), also perform analysis using the `ci:prow-job-analyze-metal-install-failure` skill for dev-scripts, Metal3/Ironic, and BareMetalHost-specific diagnostics. +> - **Test failure**: Use the `ci:prow-job-analyze-test-failure` skill. Do NOT use `--fast` — always perform the full analysis including must-gather extraction and analysis. +> +> **IMPORTANT** — Trace every failure to its specific root cause by examining actual logs. Never stop at high-level symptoms like "0 nodes ready", "operator degraded", or "containers are crash-looping". Download and read the actual log bundles, pod logs, and container previous logs. Cite specific error messages. The root cause must be actionable, not a restatement of the symptom. +> +> **Do NOT classify a failure as "infrastructure flake" or "transient" unless you have affirmative evidence** of an infrastructure problem (cloud API errors, quota exceeded, network timeouts from the cloud provider, Boskos lease failures, CI platform outages). The absence of an obvious code-level explanation does NOT make something infrastructure — it means you need to investigate deeper. Default to treating failures as potential product regressions until evidence proves otherwise. +> +> Return a concise summary including: failure type (install vs test), root cause, key error messages, and any relevant log excerpts. Do not ask user questions. Keep the output concise for inclusion in a summary report. +> +> If the job is an aggregated job (has `aggregated-` prefix in the name or an `aggregator` container/step), also return the **underlying job name** (e.g., `periodic-ci-openshift-release-main-ci-4.22-e2e-aws-upgrade-ovn-single-node`). This is found in the junit-aggregated.xml artifacts — each `` has `` YAML data with a `humanurl` field linking to individual runs whose URL path contains the underlying job name. The underlying job name cannot be derived from the aggregated job name — it must be extracted from the artifacts. + +**Structured Return Format**: Instruct each subagent to include an `ANALYSIS_RESULT` block at the end of its response: + +``` +ANALYSIS_RESULT: +- failure_type: install|test|upgrade|infra +- root_cause_summary: +- affected_components: +- key_error_patterns: +- known_symptoms: +- underlying_job_name: +- retries_consistent: yes|no|no_retries|only_final_examined +- retry_summary: +``` + +**Note for aggregated jobs**: Since only the final attempt is examined (retries re-run aggregation only), set `retries_consistent: only_final_examined` and `retry_summary: "Aggregated job — only final attempt examined (retries re-run aggregation only)"`. + +**Important**: Launch ALL subagents in parallel for maximum speed. Do NOT set the `model` parameter — let subagents inherit the parent model, as these analysis tasks require a capable model. + +#### Cross-Platform and Cross-Job Failure Pattern Recognition + +After collecting subagent results, look for patterns across multiple jobs: + +- **Same failure across a job family** (e.g., all `techpreview` jobs, all `fips` jobs, all `upgrade` jobs): This often indicates a failure specific to that feature set or configuration. +- **Same failure across multiple platforms**: This often points to a product bug in shared code. + +### Step 4b: Consult Previous Claude Analyses + +Read the target payload's `payload.json` (at `SNAPSHOT_DIR/`) and check if a `claude-payload-agent` async job exists with state `Succeeded`. If so, fetch the HTML report from its Prow artifacts: + +``` +{prow_artifacts_url}/artifacts/claude-payload-agent/openshift-release-analysis-claude-payload-agent/artifacts/payload-analysis-{tag}-summary.html +``` + +Convert the Prow URL to a gcsweb URL and use WebFetch to read it. + +**Important**: Previous analyses are a secondary input. Always complete your own analysis first, then compare. Use previous findings to bolster confidence, challenge assumptions, or fill gaps — never adopt conclusions without verifying against the snapshot data. + +### Step 5: Validate Failure Streaks + +After collecting all subagent results, verify that consecutive failures across payloads share the same root cause. A consecutive failure streak does NOT automatically mean the same root cause. + +Compare the subagent's root cause analysis for the target payload against previous payload analyses (from Step 4b) or the failure signatures in the snapshot's streak data. + +If a job fails in two consecutive payloads but for **different reasons**, treat each as a separate streak=1 failure with its own originating payload and candidate PRs. Re-split the streak and re-assign originating payloads before proceeding to scoring. + +### Step 6: Collect Investigation Results and Identify Revert Candidates + +Wait for all subagents to complete and collect their analysis results. For each failed job, you now have: + +- **Job name** and **Prow URL** (from snapshot) +- **Failure analysis** (from subagent) +- **Streak data** (from snapshot: `streak_length`, `originating_payload`, `failure_pattern`) +- **Candidate PRs** (from snapshot: originating payload's `prs[]`) + +#### 6.1: Correlate Failures with Candidate PRs + +For each failed job, cross-reference the failure analysis from the subagent with the candidate PRs from the originating payload. Read the PR's `code.diff` file (at the path from `summary.json` → `payloads[].prs[].diff`) to check for code-level correlation. + +If a subagent traced the root cause to a PR outside the payload (e.g., an `openshift/release` PR that modified a CI step registry script), include that PR as a candidate. + +Score each (failed job, candidate PR) pair using the following weighted rubric: + +| Signal | Weight | Criteria | +|--------|--------|----------| +| New failure mode | +30 | The specific failure mode was not present in previous payloads | +| Component exclusivity | +10 to +30 | The failure involves a component modified by this PR. Sole modifier = +30, 2-3 PRs = +20, 4+ PRs = +10 | +| Error message match | +40 | Error messages or stack traces directly reference code, packages, or functionality changed by this PR | +| Multi-job correlation | +10 | The same PR is a candidate for failures in multiple independent jobs | +| Presubmit coverage gap | +10 | The failing job tests a scenario not covered by the PR's presubmit tests | +| Single candidate | +10 | Only one PR landed in the originating payload that touches the affected component | + +Maximum possible score is 130, capped at 100. Record the numeric score alongside qualitative rationale. + +**Apply the rubric mechanically.** Calculate the score by summing the weights for each signal that fires based on concrete evidence. Do NOT adjust the score downward based on speculative counter-arguments like "if this were the sole cause, other jobs would also fail" or "this could be a coincidence." If the error messages reference the PR's changes, that's +40 — the fact that some other jobs didn't fail doesn't negate the match. If the failure is new (streak=1) and the PR is the only one touching the component, those signals fire regardless of theoretical alternatives. Trust the rubric — it exists to prevent both over- and under-attribution. + +#### 6.2: Propose Revert Candidates + +For each candidate PR with a rubric score of **>= 85**, mark it as a **revert candidate**. A PR qualifies when: + +1. The failure clearly maps to the PR's changes +2. The timing is exact — the job was passing before the originating payload +3. No other plausible explanation — infrastructure flakiness and platform problems have been ruled out + +Per OCP policy, PRs that break payloads MUST be reverted. When confidence is high, the report must clearly state that a revert is required — not optional. + +For each revert candidate, record: PR URL, description, component, confidence score with rationale. + +**Do NOT propose reverts for**: Infrastructure failures, flaky tests that also fail on accepted payloads, jobs where analysis is inconclusive. + +#### 6.3: Check if Revert Candidates Were Already Reverted + +For each revert candidate: + +```bash +gh pr list --repo / --search "revert " --json number,title,url,state,mergedAt --limit 5 +``` + +If a revert PR is found: +- **Merged**: Note when it merged relative to the payload. If after the payload was cut, the fix is expected in the next payload. Do not recommend reverting again. +- **Open**: Mention the existing revert PR and link to it. +- **Closed (not merged)**: Ignore. + +#### 6.4: Determine Force-Accept Recommendation + +Recommend force-accepting when **all** of the following are true: + +1. All failures are temporary infrastructure issues (`failure_type: "infra"`) +2. No more than 2 blocking jobs failed +3. `hours_since_baseline` from `summary.json` is >= 18 (or null) + +#### 6.5: Write Payload Results YAML + +Use the `payload-results-yaml` skill to create: `payload-results-{tag}.yaml` + +This file contains ALL scored candidates across all confidence tiers (HIGH, MEDIUM, LOW), enabling downstream commands to filter by their own criteria. + +### Step 7: Generate HTML Report + +Create a self-contained HTML file named `payload-analysis--summary.html` in the current working directory. The tag should be sanitized for use as a filename. + +The report must include the following sections: + +#### 7.1: Header and Executive Summary + +```html +

Payload Analysis: {payload_tag}

+
+

Architecture: {architecture} | Stream: {stream} | Generated: {timestamp}

+

Release Controller: {payload_tag}

+

Snapshot: {snapshot_dir}

+
+ +
+

Executive Summary

+

{total_blocking} blocking jobs: {succeeded} passed, {failed} failed

+

{new_failures} new failure(s), {persistent_failures} persistent failure(s)

+

Chain: {chain_length} payloads, {hours_since_baseline}h since baseline

+
+``` + +#### 7.2: Blocking Jobs Summary Table + +A table showing ALL blocking jobs with columns: +- Job Name +- Status (color-coded: green for passed, red for failed) +- Streak (consecutive failing payloads; "N/A" for passed) +- History (the `failure_pattern` from the snapshot, e.g., "F F F S F F", with color-coded markers) +- First Failed In (originating payload tag, linked to release controller) + +#### 7.3: Failed Job Details + +For each failed job, a collapsible section containing: + +```html +
+ + {job_name} + {New Failure|Failing for N payloads} + +
+

Prow Job

+

{prow_url} | GCS Artifacts

+ +

Failure Analysis

+
{analysis_from_subagent}
+ +

Known Symptoms Seen

+

{comma-separated symptom summaries, or omit if "none"}

+ +

First Failed In

+

{originating_payload_tag}

+ +

Candidate PRs (introduced in {originating_payload_tag})

+ + +
ComponentPRDescriptionScore
+
+
+``` + +#### 7.4: Recommended Reverts + +Include this section **before** the per-job details, immediately after the executive summary. + +If revert candidates were identified (score >= 85): + +```html +
+

Recommended Reverts

+

OCP Policy: PRs that break payloads MUST be reverted.

+ + +
PRComponentDescriptionCaused Failure InFailing SinceRationale
+

Automated Reverts

+
+ + 
/ci:payload-revert {payload_tag}
+
+
+``` + +If no revert candidates: + +```html +
+ No Recommended Reverts +

No PRs were identified with sufficient confidence for revert recommendation.

+
+``` + +#### 7.5: Force-Accept Recommendation + +If recommended (Step 6.4): + +```html +
+ Force-Accept Recommended +

All blocking job failures are temporary infrastructure issues and no payload has been + accepted in this stream for more than 18 hours.

+

Baseline: {baseline_tag} ({hours_since_baseline}h ago)

+
+``` + +#### 7.6: Review Notes + +Include this section at the end of the report, before the footer: + +```html +
+

Adversarial Review

+

{review_summary}

+ +

Issues Found

+
    +
  • {issue_description} — {action_taken}
    • +
+
+``` + +#### 7.7: Styling + +The HTML must be fully self-contained with embedded CSS. Use a GitHub-inspired dark mode design. Use CSS variables for the color palette: + +```css +:root { + --bg: #0d1117; --surface: #161b22; --border: #30363d; + --text: #e6edf3; --text-muted: #8b949e; + --green: #3fb950; --red: #f85149; --orange: #d29922; + --blue: #58a6ff; --purple: #bc8cff; +} +``` + +Follow the styling conventions from the existing report format. All `` links must use `target="_blank"`. + +### Step 8: Generate JSON Data File + +Use the `payload-autodl-json` skill to produce `payload-analysis--autodl.json`. + +See the `payload-autodl-json` skill for the complete schema, row cardinality rules, and field rules. + +### Step 9: Completeness Review + +After generating the initial report and output files, launch a **dedicated subagent** to check that the analysis is complete and well-supported. The reviewer catches lazy or shallow work — it does NOT challenge or re-score rubric-based confidence scores. + +The reviewer should receive **only** the following (NOT the full conversation history): + +1. The `summary.json` snapshot data (payload metadata, failed jobs, streaks, test regressions) +2. The scored candidate list with per-component rubric breakdowns from Step 6 +3. The `ANALYSIS_RESULT` blocks from all subagents in Step 4 +4. The revert recommendations (if any) + +Use this prompt for the reviewer: + +> You are a completeness reviewer for a payload failure analysis. Your job is to catch gaps in coverage and shallow analysis — NOT to challenge correct conclusions or lower confidence scores. +> +> **Snapshot data**: {summary.json contents — metadata, failed jobs with streaks, test regressions} +> +> **Subagent analyses**: {ANALYSIS_RESULT blocks for each failed job} +> +> **Scored candidates**: {list of (job, PR, score, rubric breakdown) tuples} +> +> **Revert recommendations**: {list of PRs recommended for revert, or "none"} +> +> Check for these specific problems: +> +> 1. **Missing skill invocations**: Were `prow-job-analyze-install-failure` and `prow-job-analyze-test-failure` skills actually loaded and used? A subagent that improvises without loading the appropriate skill produces shallow analysis. +> +> 2. **Shallow root causes**: Do root cause summaries cite specific error messages, code paths, or log excerpts? Or do they just restate test names and job status? "Test X failed" is not a root cause. "Test X failed because pod Y OOMKilled at 512Mi limit after PR Z increased memory usage in function F" is a root cause. +> +> 3. **Incomplete coverage**: Are there failed jobs with no subagent analysis or with only a one-line summary? Every failed blocking job deserves a thorough investigation. +> +> 4. **Wrong skill for failure type**: Was an install failure analyzed with the test failure skill or vice versa? +> +> **Rules**: +> - Do NOT suggest lowering confidence scores. If the rubric signals fired (error message match, new failure, component exclusivity), the score is correct. Period. +> - Do NOT suggest that a failure "might be infrastructure" when there is positive evidence linking it to a PR. Infrastructure classification requires affirmative evidence (cloud API errors, quota limits, network timeouts) — not just uncertainty about the code change. +> - Do NOT second-guess revert recommendations. When confidence >= 85 based on the rubric, the revert is warranted per OCP policy. +> +> For each issue found, provide: +> - **Issue**: One-line description +> - **Affected job(s)**: Which jobs are affected +> - **Recommendation**: Re-run subagent with correct skill, deepen analysis, or add missing coverage +> +> If the analysis is thorough, say so: "Analysis is complete — all jobs investigated with appropriate skills and specific root causes identified." + +After receiving the reviewer's response: + +- If coverage gaps are found (missing skill invocation, shallow analysis, wrong skill): re-run the affected subagent analyses, then re-score. Update the HTML report and YAML/JSON files. +- If the analysis is already thorough: note this in the report. +- **Never lower rubric-based confidence scores** based on the reviewer's response. The rubric is mechanical — if the signals fired, the score stands. +- Populate the "Adversarial Review" section (Step 7.6) in the HTML report with the reviewer's findings and any actions taken. + +### Step 10: Save and Present + +1. Save all output files to the current working directory: + - HTML report: `payload-analysis--summary.html` + - JSON data file: `payload-analysis--autodl.json` + - Payload results YAML: `payload-results-.yaml` + +2. Tell the user: + - Path to each saved file + - Brief text summary (number of failures, new vs persistent, key candidate PRs) + - Whether the adversarial review changed any conclusions + - Mention that `/ci:payload-revert` and `/ci:payload-experiment` can consume the YAML for automated actions + +## Error Handling + +### No Snapshot Available + +If no snapshot is found and the snapshot script fails to create one: +``` +Error: Could not locate or create a snapshot for {tag}. Run the payload-snapshot skill manually first. +``` + +### Subagent Failure + +If a subagent fails to analyze a job, include the job in the report with: +``` +Analysis unavailable: {error_message} +``` +Do not let one failed subagent block the entire report. + +### Missing PR Data + +If the snapshot was created without `gh` authentication, PR diffs/comments will be absent. Note this in the report: +``` +Note: PR diff data not available in snapshot. Scoring based on component match and timing only. +``` + +## Notes + +- The snapshot is a **frozen archive** — it captures release controller, GitHub, and CI data as it was when the snapshot was taken. This enables re-analysis of historical payloads and provides reproducible results. +- Subagents still download artifacts from GCS (must-gather, pod logs, step logs) because these are not included in the snapshot. The snapshot provides the data scaffolding; subagents provide deep investigation. +- The adversarial review adds one subagent call but catches misattributions before they reach the report. +- For very large numbers of failed jobs (>8), consider whether some share the same underlying failure and group them in the report. + +## See Also + +- Related Skill: `payload-snapshot` — creates the snapshot data this skill consumes +- Related Skill: `payload-results-yaml` — schema for the results YAML +- Related Skill: `payload-autodl-json` — schema for the autodl JSON data file +- Related Skill: `prow-job-analyze-test-failure` — deep test failure investigation (used by subagents) +- Related Skill: `prow-job-analyze-install-failure` — deep install failure investigation (used by subagents) +- Related Command: `/ci:payload-revert` — stages reverts for high-confidence candidates +- Related Command: `/ci:payload-experiment` — tests medium-confidence candidates experimentally diff --git a/plugins/ci/skills/payload-autodl-json/SKILL.md b/plugins/ci/skills/payload-autodl-json/SKILL.md index 034d0bf7e..0e09b0f65 100644 --- a/plugins/ci/skills/payload-autodl-json/SKILL.md +++ b/plugins/ci/skills/payload-autodl-json/SKILL.md @@ -1,6 +1,6 @@ --- name: payload-autodl-json -description: Schema for the autodl JSON data file produced by analyze-payload for database ingestion — you must use this skill whenever generating the autodl JSON file +description: Schema for the autodl JSON data file produced by payload-analysis for database ingestion — you must use this skill whenever generating the autodl JSON file --- # Payload Autodl JSON @@ -9,7 +9,7 @@ This skill defines the schema for the `payload-analysis-{tag}-autodl.json` file. ## When to Use This Skill -Use this skill when you need to generate the autodl JSON file during `analyze-payload` (Step 8). +Use this skill when you need to generate the autodl JSON file during `payload-analysis` (Step 8). ## File Location @@ -180,7 +180,7 @@ The filename **must** end with `-autodl.json`. Sanitize the tag for filename saf ## Operations -### Create (used by `analyze-payload`) +### Create (used by `payload-analysis`) Generate the full autodl JSON file with all rows populated from the analysis results. Each failed blocking job produces at least one row. Candidate fields are populated when a PR is correlated to the failure, otherwise they are empty strings / `"0"`. @@ -202,7 +202,7 @@ After staging reverts, find rows matching `candidate_pr_url` and set: ## See Also -- Related Skill: `analyze-payload` — creates this file in Step 8 +- Related Skill: `payload-analysis` — creates this file in Step 8 - Related Skill: `stage-payload-reverts` — updates revert fields after staging reverts - Related Skill: `payload-experimental-reverts` — updates revert fields after experiments - Related Skill: `payload-results-yaml` — the YAML results file for downstream agentic actions diff --git a/plugins/ci/skills/payload-autodl-json/scripts/test_validate.py b/plugins/ci/skills/payload-autodl-json/scripts/test_validate.py new file mode 100644 index 000000000..04756ab0a --- /dev/null +++ b/plugins/ci/skills/payload-autodl-json/scripts/test_validate.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Tests for payload-autodl JSON validator.""" + +import os +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(__file__)) +from validate import validate + +TESTDATA = os.path.join(os.path.dirname(__file__), "testdata") + + +def test(name, path, expected_exit): + result = validate(path) + status = "PASS" if result == expected_exit else "FAIL" + if status == "FAIL": + print(f" {status}: {name} (expected exit {expected_exit}, got {result})") + else: + print(f" {status}: {name}") + return status == "PASS" + + +if __name__ == "__main__": + passed = 0 + failed = 0 + + cases = [ + ("valid", f"{TESTDATA}/valid.json", 0), + ("invalid no table_name", f"{TESTDATA}/invalid_no_table_name.json", 1), + ("invalid non-string values", f"{TESTDATA}/invalid_non_string_values.json", 1), + ("invalid empty rows", f"{TESTDATA}/invalid_empty_rows.json", 1), + ("file not found", f"{TESTDATA}/nonexistent.json", 1), + ] + + with tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) as f: + f.write("{bad json") + tmp_path = f.name + cases.append(("invalid JSON syntax", tmp_path, 1)) + + for name, path, expected in cases: + if test(name, path, expected): + passed += 1 + else: + failed += 1 + + os.unlink(tmp_path) + + print(f"\n{passed}/{passed + failed} passed") + sys.exit(1 if failed else 0) diff --git a/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_empty_rows.json b/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_empty_rows.json new file mode 100644 index 000000000..85b5513c6 --- /dev/null +++ b/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_empty_rows.json @@ -0,0 +1,5 @@ +{ + "table_name": "payload_triage", + "schema": {"payload_tag": "string"}, + "rows": [] +} diff --git a/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_no_table_name.json b/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_no_table_name.json new file mode 100644 index 000000000..43a12a4ab --- /dev/null +++ b/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_no_table_name.json @@ -0,0 +1,10 @@ +{ + "rows": [ + { + "payload_tag": "4.22.0-0.nightly-2026-02-25-152806", + "job_name": "some-job", + "failure_type": "test", + "root_cause_summary": "something broke" + } + ] +} diff --git a/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_non_string_values.json b/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_non_string_values.json new file mode 100644 index 000000000..56b78af82 --- /dev/null +++ b/plugins/ci/skills/payload-autodl-json/scripts/testdata/invalid_non_string_values.json @@ -0,0 +1,19 @@ +{ + "table_name": "payload_triage", + "schema": {"payload_tag": "string"}, + "rows": [ + { + "payload_tag": "4.22.0-0.nightly-2026-02-25-152806", + "version": "4.22", + "stream": "nightly", + "architecture": "amd64", + "phase": "Rejected", + "job_name": "some-job", + "prow_url": "https://prow.ci.openshift.org/...", + "failure_type": "test", + "root_cause_summary": "something broke", + "streak_length": 5, + "confidence_score": 95 + } + ] +} diff --git a/plugins/ci/skills/payload-autodl-json/scripts/testdata/valid.json b/plugins/ci/skills/payload-autodl-json/scripts/testdata/valid.json new file mode 100644 index 000000000..3310bedb1 --- /dev/null +++ b/plugins/ci/skills/payload-autodl-json/scripts/testdata/valid.json @@ -0,0 +1,22 @@ +{ + "table_name": "payload_triage", + "schema": { + "payload_tag": "string", + "job_name": "string", + "failure_type": "string", + "root_cause_summary": "string" + }, + "rows": [ + { + "payload_tag": "4.22.0-0.nightly-2026-02-25-152806", + "version": "4.22", + "stream": "nightly", + "architecture": "amd64", + "phase": "Rejected", + "job_name": "periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn", + "prow_url": "https://prow.ci.openshift.org/view/gs/...", + "failure_type": "test", + "root_cause_summary": "OVN gateway regression" + } + ] +} diff --git a/plugins/ci/skills/payload-autodl-json/scripts/validate.py b/plugins/ci/skills/payload-autodl-json/scripts/validate.py new file mode 100644 index 000000000..fe9c367cb --- /dev/null +++ b/plugins/ci/skills/payload-autodl-json/scripts/validate.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Validate a payload-analysis autodl JSON file against the canonical schema.""" + +import json +import sys + +REQUIRED_ROW_FIELDS = [ + "payload_tag", "version", "stream", "architecture", "phase", + "job_name", "prow_url", "failure_type", "root_cause_summary", +] + + +def validate(path): + errors = [] + + try: + with open(path) as f: + data = json.load(f) + except FileNotFoundError: + print(f"FAIL: file not found: {path}") + return 1 + except json.JSONDecodeError as e: + print(f"FAIL: invalid JSON: {e}") + return 1 + + if not isinstance(data, dict): + print("FAIL: root is not an object") + return 1 + + if "table_name" not in data: + errors.append("missing 'table_name'") + + if "schema" not in data or not isinstance(data.get("schema"), dict): + errors.append("missing or invalid 'schema'") + + rows = data.get("rows", []) + if not isinstance(rows, list): + errors.append("'rows' is not an array") + elif len(rows) == 0: + errors.append("'rows' is empty") + else: + for i, row in enumerate(rows): + if not isinstance(row, dict): + errors.append(f"rows[{i}] is not an object") + continue + for field in REQUIRED_ROW_FIELDS: + if field not in row: + errors.append(f"rows[{i}] missing '{field}'") + non_string = [k for k, v in row.items() if not isinstance(v, str)] + if non_string: + errors.append(f"rows[{i}] has non-string values: {', '.join(non_string)}") + + if errors: + print(f"FAIL: {len(errors)} error(s)") + for e in errors: + print(f" - {e}") + return 1 + + print(f"OK: {len(rows)} rows, table_name={data.get('table_name')}") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} <*-autodl.json>") + sys.exit(2) + sys.exit(validate(sys.argv[1])) diff --git a/plugins/ci/skills/payload-results-yaml/SKILL.md b/plugins/ci/skills/payload-results-yaml/SKILL.md index ef8cff935..c6ce64202 100644 --- a/plugins/ci/skills/payload-results-yaml/SKILL.md +++ b/plugins/ci/skills/payload-results-yaml/SKILL.md @@ -10,7 +10,7 @@ This skill defines the schema for the `payload-results-{tag}.yaml` file and prov ## When to Use This Skill Use this skill whenever you need to: -- **Create** a new results file (during `analyze-payload`) +- **Create** a new results file (during `payload-analysis`) - **Read** candidates or their actions (during `payload-revert`, `payload-experiment`) - **Append an action** to a candidate (during `stage-payload-reverts`, `payload-experimental-reverts`) - **Update an action's status** (during `payload-experimental-reverts` Phase 2) @@ -82,7 +82,7 @@ candidates: ### `metadata` -Written once by `analyze-payload`. Never modified by downstream skills. +Written once by `payload-analysis`. Never modified by downstream skills. | Field | Type | Description | |-------|------|-------------| @@ -92,11 +92,11 @@ Written once by `analyze-payload`. Never modified by downstream skills. | `architecture` | string | `"amd64"`, `"arm64"`, `"multi"`, etc. | | `release_controller_url` | string | URL to the payload on the release controller | | `analyzed_at` | string | ISO 8601 timestamp of when the analysis was performed | -| `force_accept_recommended` | bool | `true` when all failures are temporary infrastructure issues, no more than 2 blocking jobs failed, and no payload has been accepted in the stream for 18+ hours. Determined by `analyze-payload` Step 6.4. | +| `force_accept_recommended` | bool | `true` when all failures are temporary infrastructure issues, no more than 2 blocking jobs failed, and no payload has been accepted in the stream for 18+ hours. Determined by `payload-analysis` Step 6.4. | ### `failing_jobs[]` -All failed blocking jobs in the payload. Written once by `analyze-payload`. Never modified by downstream skills. This is the authoritative list of failures — every failed blocking job appears here regardless of whether a candidate PR has been identified. +All failed blocking jobs in the payload. Written once by `payload-analysis`. Never modified by downstream skills. This is the authoritative list of failures — every failed blocking job appears here regardless of whether a candidate PR has been identified. | Field | Type | Description | |-------|------|-------------| @@ -112,7 +112,7 @@ All failed blocking jobs in the payload. Written once by `analyze-payload`. Neve ### `candidates[]` -Each entry represents a PR identified as a candidate cause of payload failures. Top-level candidate fields are written once by `analyze-payload` and are read-only to downstream skills. The `actions` sub-array is mutable (see below). +Each entry represents a PR identified as a candidate cause of payload failures. Top-level candidate fields are written once by `payload-analysis` and are read-only to downstream skills. The `actions` sub-array is mutable (see below). Candidates reference failing jobs by `job_name` via the `failing_jobs` string array, linking back to the top-level `failing_jobs[]` entries. @@ -168,7 +168,7 @@ Payload validation jobs triggered against the revert PR. ## Operations -### Create (used by `analyze-payload`) +### Create (used by `payload-analysis`) Write a new `payload-results-{tag}.yaml` with `metadata`, `failing_jobs`, and `candidates` populated. All failed blocking jobs are recorded in `failing_jobs`. Candidates with no pre-existing revert start with `actions: []`. If a pre-existing revert PR is discovered during analysis, append an action with `type: "revert"` and `status: "open"` or `"merged"`. @@ -190,7 +190,7 @@ Scan all candidates. If any candidate has an action with `type: "experiment"` an ## See Also -- Related Skill: `analyze-payload` — creates the results file +- Related Skill: `payload-analysis` — creates the results file - Related Skill: `stage-payload-reverts` — appends `type: "revert"` actions - Related Skill: `payload-experimental-reverts` — appends `type: "experiment"` actions, updates status in Phase 2 - Related Command: `/ci:payload-revert` — stages reverts for high-confidence candidates diff --git a/plugins/ci/skills/payload-results-yaml/scripts/test_validate.py b/plugins/ci/skills/payload-results-yaml/scripts/test_validate.py new file mode 100644 index 000000000..e1d2f793a --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/test_validate.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Tests for payload-results YAML validator.""" + +import os +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(__file__)) +from validate import validate + +TESTDATA = os.path.join(os.path.dirname(__file__), "testdata") + + +def test(name, path, expected_exit): + result = validate(path) + status = "PASS" if result == expected_exit else "FAIL" + if status == "FAIL": + print(f" {status}: {name} (expected exit {expected_exit}, got {result})") + else: + print(f" {status}: {name}") + return status == "PASS" + + +if __name__ == "__main__": + passed = 0 + failed = 0 + + cases = [ + ("valid full schema", f"{TESTDATA}/valid.yaml", 0), + ("valid no candidates", f"{TESTDATA}/valid_no_candidates.yaml", 0), + ("invalid flat schema (no metadata wrapper)", f"{TESTDATA}/invalid_flat_schema.yaml", 1), + ("invalid metadata is string", f"{TESTDATA}/invalid_metadata_string.yaml", 1), + ("invalid missing job/candidate fields", f"{TESTDATA}/invalid_missing_job_fields.yaml", 1), + ("file not found", f"{TESTDATA}/nonexistent.yaml", 1), + ] + + # Test with a non-YAML file + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w", delete=False) as f: + f.write("{{invalid yaml") + tmp_path = f.name + cases.append(("invalid YAML syntax", tmp_path, 1)) + + for name, path, expected in cases: + if test(name, path, expected): + passed += 1 + else: + failed += 1 + + os.unlink(tmp_path) + + print(f"\n{passed}/{passed + failed} passed") + sys.exit(1 if failed else 0) diff --git a/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_flat_schema.yaml b/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_flat_schema.yaml new file mode 100644 index 000000000..d3fe7bde6 --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_flat_schema.yaml @@ -0,0 +1,9 @@ +payload_tag: "5.0.0-0.nightly-2026-05-01-200018" +phase: "Rejected" +release_stream: "5.0.0-0.nightly" +architecture: "amd64" + +failed_jobs: + - job_name: "aws-ovn-techpreview-serial" + failure_type: "test_failure" + streak_length: 4 diff --git a/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_metadata_string.yaml b/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_metadata_string.yaml new file mode 100644 index 000000000..a528022ae --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_metadata_string.yaml @@ -0,0 +1,3 @@ +metadata: "not a mapping" +failing_jobs: [] +candidates: [] diff --git a/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_missing_job_fields.yaml b/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_missing_job_fields.yaml new file mode 100644 index 000000000..2d3bc4f40 --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/testdata/invalid_missing_job_fields.yaml @@ -0,0 +1,12 @@ +metadata: + payload_tag: "4.22.0-0.nightly-2026-02-25-152806" + version: "4.22" + stream: "nightly" + architecture: "amd64" + +failing_jobs: + - job_name: "some-job" + - prow_url: "https://prow.ci.openshift.org/..." + +candidates: + - pr_url: "https://github.com/openshift/foo/pull/1" diff --git a/plugins/ci/skills/payload-results-yaml/scripts/testdata/valid.yaml b/plugins/ci/skills/payload-results-yaml/scripts/testdata/valid.yaml new file mode 100644 index 000000000..240ea6d5c --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/testdata/valid.yaml @@ -0,0 +1,25 @@ +metadata: + payload_tag: "4.22.0-0.nightly-2026-02-25-152806" + version: "4.22" + stream: "nightly" + architecture: "amd64" + release_controller_url: "https://amd64.ocp.releases.ci.openshift.org/..." + analyzed_at: "2026-02-26T10:30:00Z" + force_accept_recommended: false + +failing_jobs: + - job_name: "periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn" + prow_url: "https://prow.ci.openshift.org/view/gs/..." + failure_type: "test" + root_cause_summary: "OVN gateway mode regression" + streak_length: 3 + originating_payload_tag: "4.22.0-0.nightly-2026-02-20-150000" + +candidates: + - pr_url: "https://github.com/openshift/cno/pull/2037" + pr_number: 2037 + component: "cluster-network-operator" + confidence_score: 95 + failing_jobs: + - "periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn" + actions: [] diff --git a/plugins/ci/skills/payload-results-yaml/scripts/testdata/valid_no_candidates.yaml b/plugins/ci/skills/payload-results-yaml/scripts/testdata/valid_no_candidates.yaml new file mode 100644 index 000000000..ff9b2a941 --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/testdata/valid_no_candidates.yaml @@ -0,0 +1,12 @@ +metadata: + payload_tag: "5.0.0-0.ci-2026-05-01-212308" + version: "5.0" + stream: "ci" + architecture: "amd64" + +failing_jobs: + - job_name: "periodic-ci-openshift-release-main-ci-5.0-e2e-gcp" + failure_type: "infra" + root_cause_summary: "GCP quota exceeded" + +candidates: [] diff --git a/plugins/ci/skills/payload-results-yaml/scripts/validate.py b/plugins/ci/skills/payload-results-yaml/scripts/validate.py new file mode 100644 index 000000000..90c959eef --- /dev/null +++ b/plugins/ci/skills/payload-results-yaml/scripts/validate.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Validate a payload-results YAML file against the canonical schema.""" + +import sys +import yaml + +REQUIRED_METADATA = ["payload_tag", "version", "stream", "architecture"] +REQUIRED_JOB_FIELDS = ["job_name", "failure_type", "root_cause_summary"] +REQUIRED_CANDIDATE_FIELDS = ["pr_url", "confidence_score", "failing_jobs"] + + +def validate(path): + errors = [] + + try: + with open(path) as f: + data = yaml.safe_load(f) + except FileNotFoundError: + print(f"FAIL: file not found: {path}") + return 1 + except yaml.YAMLError as e: + print(f"FAIL: invalid YAML: {e}") + return 1 + + if not isinstance(data, dict): + print("FAIL: root is not a mapping") + return 1 + + meta = data.get("metadata") + if not isinstance(meta, dict): + errors.append("'metadata' is not a mapping") + meta = {} + for field in REQUIRED_METADATA: + if field not in meta: + errors.append(f"metadata missing '{field}'") + + if "failing_jobs" not in data: + errors.append("missing 'failing_jobs' key") + elif not isinstance(data["failing_jobs"], list): + errors.append("'failing_jobs' is not a list") + else: + for i, job in enumerate(data["failing_jobs"]): + if not isinstance(job, dict): + errors.append(f"failing_jobs[{i}] is not an object") + continue + for field in REQUIRED_JOB_FIELDS: + if field not in job: + errors.append(f"failing_jobs[{i}] missing '{field}'") + + if "candidates" not in data: + errors.append("missing 'candidates' key") + elif not isinstance(data["candidates"], list): + errors.append("'candidates' is not a list") + else: + for i, cand in enumerate(data["candidates"]): + if not isinstance(cand, dict): + errors.append(f"candidates[{i}] is not an object") + continue + for field in REQUIRED_CANDIDATE_FIELDS: + if field not in cand: + errors.append(f"candidates[{i}] missing '{field}'") + + if errors: + print(f"FAIL: {len(errors)} error(s)") + for e in errors: + print(f" - {e}") + return 1 + + jobs = len(data.get("failing_jobs", [])) + cands = len(data.get("candidates", [])) + print(f"OK: {jobs} failing jobs, {cands} candidates") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(2) + sys.exit(validate(sys.argv[1])) diff --git a/plugins/ci/skills/payload-snapshot/SKILL.md b/plugins/ci/skills/payload-snapshot/SKILL.md new file mode 100644 index 000000000..5b78835c2 --- /dev/null +++ b/plugins/ci/skills/payload-snapshot/SKILL.md @@ -0,0 +1,244 @@ +--- +name: payload-snapshot +description: Snapshot OpenShift payload data (release controller, PR diffs, comments, CI jobs, JUnit results, regression tracking) to a local directory for offline analysis +--- + +# Payload Snapshot + +This skill downloads all data needed to analyze an OpenShift payload into a local directory tree. The resulting snapshot can be navigated entirely via file reads — no live API calls required during analysis. + +## When to Use This Skill + +Use this skill when you need to: + +- Analyze a rejected payload and want all data available locally before starting +- Create a reproducible snapshot of payload state at a point in time +- Track test failure regressions across multiple payloads +- Work offline or reduce API calls during payload analysis + +## Prerequisites + +1. **Python 3** (3.10 or later) + - Uses only standard library (no external dependencies) + +2. **GitHub CLI (`gh`)** — for PR diff, comment, and job data + - Install: `brew install gh` (macOS) or see https://cli.github.com + - Authenticate: `gh auth login` + - Without `gh`, release controller data is still fetched; PR data is skipped + +3. **Google Cloud SDK (`gcloud`)** — for JUnit test result download + - Install: `brew install google-cloud-sdk` (macOS) or see https://cloud.google.com/sdk + - Authenticate: `gcloud auth login` + - Without `gcloud`, JUnit data is skipped; job directories still created + +4. **Network access** to: + - `*.ocp.releases.ci.openshift.org` (release controller) + - `api.github.com` (via `gh` CLI) + - `storage.googleapis.com` (via `gcloud` CLI) + +## Implementation Steps + +### Step 1: Run the Snapshot Script + +```bash +script_path="plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py" + +# Snapshot a specific payload +python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 + +# Custom output directory +python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 --output-dir .work/snapshot + +# Limit chain depth +python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 --max-chain 5 + +# Skip JUnit download (faster, still generates job structure and summary) +python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 --no-junit +``` + +The script will: +1. Parse the payload tag to determine version, stream, and architecture +2. Probe all available streams for the version (nightly, ci, across architectures) +3. Chain backwards through previous payloads until finding one where all blocking jobs passed +4. For each payload in the chain, download release controller data and the changelog (PR diff) +5. Split jobs into blocking/informing directories with metadata and GCS browser links +6. For each failed blocking job, download and parse JUnit XML test results +7. For each failed blocking job, download build-log.txt from GCS and extract error/warning lines + log tail +8. Track test failure regressions — when did each failure first appear? +9. Track per-job failure streaks — consecutive failures, originating payload, failure pattern +10. For each unique PR across all changelogs, fetch the git diff, comments, and CI jobs via `gh` +11. Generate summary.json with comprehensive triage data, plus AGENTS.md/CLAUDE.md for agent orientation + +### Step 2: Navigate the Snapshot + +The output directory is structured for easy navigation: + +```text +payload/ + / + / + summary.json # START HERE — full triage data + CLAUDE.md # Imports AGENTS.md for Claude Code + AGENTS.md # Dynamic snapshot orientation doc + streams.json # All streams for this version + / # Each payload in the chain + payload.json # Release controller API response + changelog.json # PRs that changed vs. previous payload + regressions.json # Test failure regression tracking + jobs/ + blocking/ + / + job.json # Job metadata (state, URLs, GCS link, retries) + build_log.json # Error/warning lines + log tail (failed only) + junit/ # Only for failed jobs + junit_operator.xml # CI phase results + junit-aggregated.xml # Aggregated jobs only + results.json # Parsed test failures (full output) + informing/ + / + job.json # Job metadata only (no JUnit/build log) + / # e.g., machine-config-operator + prs/ + / + code.diff # Git diff of the PR + comments.json # PR comments and reviews + jobs.json # CI check runs +``` + +### Step 3: Use the Data + +**Find failed blocking jobs (with streaks):** +```bash +jq '.blocking_jobs.failed_jobs[] | {name, state, streak: .streak.streak_length, pattern: .streak.failure_pattern}' payload///summary.json +``` + +**Check test failures and when they started:** +```bash +jq '.[] | {test: .test_name, first_failed: .first_failed_in, payloads: .payloads_failing, jobs: .jobs}' payload////regressions.json +``` + +**List PRs in a payload:** +```bash +jq '.changeLogJson.updatedImages[].commits[] | {component: .name, pr: .pullURL, subject: .subject}' payload////changelog.json +``` + +**Read a specific PR's diff:** +```bash +cat payload/////prs//code.diff +``` + +**Check JUnit failures for a specific job:** +```bash +jq '.[].name' payload////jobs/blocking//junit/results.json +``` + +## CLI Reference + +```text +python3 payload_snapshot.py [OPTIONS] + +Positional: + payload_tag Payload tag (e.g., 4.22.0-0.nightly-2026-02-25-152806) + +Options: + --output-dir DIR Base output directory (default: payload) + --max-chain N Maximum backward chain depth (default: 20) + --workers N Parallel workers for API calls (default: 8) + --no-junit Skip JUnit download and regression tracking +``` + +## Output Files + +### `streams.json` + +Lists all available streams for the payload's version. + +### `summary.json` + +Comprehensive stream-level triage data — start here. Contains: +- Payload metadata: `payload_tag`, `phase`, `release_url`, `architecture`, `stream`, `version` +- Chain data: `chain_length`, `baseline_tag`, `hours_since_baseline` +- `blocking_jobs.failed_jobs[]` — detailed objects with `name`, `state`, `prow_url`, `gcs_url`, `streak` (streak_length, originating_payload, is_new_failure, failure_pattern), `build_log_errors`, `test_failure_count`, and relative paths to `job_json`, `junit_results`, `build_log` +- `informing_jobs.failed_jobs[]` — job name strings +- `test_failures.blocking[]` — `test_name`, `jobs`, `first_failed_in`, `payloads_failing`, `failure_message`, `failure_text` (full, not truncated) +- `payloads[]` — per-payload entries with `tag`, `phase`, relative file paths, and `prs[]` with component/diff/comments paths + +### `AGENTS.md` / `CLAUDE.md` + +Dynamic orientation document generated at snapshot time. Contains the specific payload tag, chain, failed jobs, file layout, key concepts, and summary.json schema. `CLAUDE.md` imports `AGENTS.md` via `@AGENTS.md`. + +### `payload.json` + +Full release controller response including `blockingJobs`, `informingJobs`, and `asyncJobs` with their states, Prow URLs, and retry attempt URLs. + +### `changelog.json` + +Release controller diff response with `changeLogJson.updatedImages` listing every PR that changed between this payload and its predecessor. + +### `regressions.json` + +Per-payload regression tracking data. For each failing test in the target payload: +- `test_name`: the failing test +- `jobs`: which jobs it fails in +- `first_failed_in`: the earliest payload in the chain where it was failing +- `payloads_failing`: how many consecutive payloads it has been failing +- `failure_message`: the error message +- `failure_text`: full failure output + +### `job.json` + +Per-job metadata including name, state, lifecycle (blocking/informing), Prow URL, GCS browser URL (`gcs_url`), retry count, whether it's an aggregated job, and GCS bucket path. + +### `build_log.json` (failed blocking jobs only) + +Extracted from `build-log.txt` in GCS (handles gzip decompression). Contains: +- `total_lines`: total line count of the build log +- `error_warning_count`: number of lines matching error/warning patterns +- `error_warning_lines[]`: each with `line_number` and `text` +- `tail_start_line`, `tail_lines[]`: last 20% of the log for context + +### `results.json` (in junit/ subdirectory) + +Parsed JUnit test failures for a specific job. Only includes failed/error tests. For aggregated jobs, includes per-run pass/fail/skip data with Prow URLs for each run. + +### `code.diff`, `comments.json`, `jobs.json` + +PR artifacts from GitHub (unchanged from previous version). + +## Chain Logic + +The script chains backwards from the target payload until it finds a payload where **all blocking jobs succeeded**. This is stricter than the `Accepted` phase — a payload can be force-accepted with failed blocking jobs, which does not count as a stop point. + +For terminal payloads (Accepted/Rejected), jobs showing `Pending` on the release controller are cross-checked against the actual Prow `prowjob.json` artifact to get their real state. + +## Aggregated Jobs + +Aggregated jobs run the same underlying test multiple times with statistical analysis. The script: +- Detects aggregated jobs by the `aggregated-` name prefix +- Downloads `junit-aggregated.xml` which contains per-run pass/fail/skip data +- Parses the YAML in `` to extract individual run URLs + +## Error Handling + +- **Tag not found**: Exits with code 2 and a descriptive error +- **Release controller unreachable**: Exits with code 1 +- **`gh` not authenticated**: Prints a warning and continues without PR data +- **`gcloud` not available**: Prints a warning and skips JUnit download +- **Individual job/PR fetch failure**: Logs a warning and continues +- **Idempotent**: Re-running skips files that already exist + +## Notes + +- The script uses only Python standard library — no pip dependencies +- PR data is deduplicated across payloads — each PR is fetched once +- JUnit and build-log download are scoped to failed blocking jobs only (informing jobs get `job.json` but no JUnit or build log) +- The `--workers` flag controls parallelism for all subprocess calls (default 8) +- Summary is always regenerated on re-run (not skipped like other files) +- Progress is printed to stderr; the script produces no stdout output + +## See Also + +- Related Skill: `fetch-payloads` (fetches recent payloads from the release controller) +- Related Skill: `fetch-new-prs-in-payload` (fetches PRs new in a specific payload) +- Related Skill: `payload-analysis` (analyzes a payload snapshot for revert candidates) + diff --git a/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py b/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py new file mode 100755 index 000000000..0c6272ccc --- /dev/null +++ b/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py @@ -0,0 +1,2154 @@ +#!/usr/bin/env python3 +"""Snapshot OpenShift payload data to a local directory for offline analysis. + +Downloads release controller data, PR diffs, comments, and CI job links +for a payload and its predecessors, building a complete local archive +that an AI agent can navigate without live API calls. +""" + +import argparse +import gzip +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from abc import ABC, abstractmethod +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +KNOWN_ARCHITECTURES = ("amd64", "arm64", "ppc64le", "s390x", "multi") +STREAM_TYPES = ("nightly", "ci") + +GCSWEB_BASE = "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs" +PROW_VIEW_PREFIX = "https://prow.ci.openshift.org/view/gs/" + +PROW_STATE_MAP = { + "success": "Succeeded", + "failure": "Failed", + "aborted": "Failed", + "error": "Failed", +} + + +# --------------------------------------------------------------------------- +# PayloadTag — immutable value object for parsed payload tags +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class PayloadTag: + """Parsed representation of an OpenShift payload tag. + + Example tags: + 4.22.0-0.nightly-2026-02-25-152806 -> amd64 + 4.22.0-0.nightly-arm64-2026-02-25-152806 -> arm64 + 4.18.0-0.ci-2026-01-15-114134 -> amd64, ci stream + """ + + raw: str + version: str + stream: str + architecture: str + stream_name: str + timestamp: str + + @classmethod + def parse(cls, tag: str) -> "PayloadTag": + """Parse a payload tag string into its components.""" + m = re.match(r"^(.+)-(\d{4}-\d{2}-\d{2}-\d{6})$", tag) + if not m: + raise ValueError(f"Cannot parse payload tag: {tag}") + + stream_name = m.group(1) + timestamp = m.group(2) + + sm = re.match( + r"^(\d+\.\d+)\.0-0\.(\w+?)(?:-(arm64|ppc64le|s390x|multi))?$", + stream_name, + ) + if not sm: + raise ValueError(f"Cannot parse stream name: {stream_name}") + + version = sm.group(1) + stream = sm.group(2) + architecture = sm.group(3) or "amd64" + + return cls( + raw=tag, + version=version, + stream=stream, + architecture=architecture, + stream_name=stream_name, + timestamp=timestamp, + ) + + +# --------------------------------------------------------------------------- +# JobInfo — metadata for a single CI job +# --------------------------------------------------------------------------- + +@dataclass +class JobInfo: + """Metadata for a CI job extracted from payload.json.""" + + name: str + state: str + lifecycle: str # "blocking" or "informing" + url: str + retries: int + previous_attempt_urls: list[str] + is_aggregated: bool + gcs_bucket_path: str + gcs_url: str = "" + + +# --------------------------------------------------------------------------- +# HTTP helpers +# --------------------------------------------------------------------------- + +def fetch_json(url: str, timeout: int = 30) -> dict: + """Fetch JSON from a URL. Raises on error.""" + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def try_fetch_json(url: str, timeout: int = 10) -> Optional[dict]: + """Fetch JSON from a URL, returning None on any failure.""" + try: + return fetch_json(url, timeout=timeout) + except Exception: + return None + + +# --------------------------------------------------------------------------- +# ReleaseController — API client +# --------------------------------------------------------------------------- + +class ReleaseController: + """Client for the OpenShift release controller API.""" + + def __init__(self, architecture: str = "amd64"): + self.architecture = architecture + self.domain = f"{architecture}.ocp.releases.ci.openshift.org" + self._base = f"https://{self.domain}/api/v1" + + def fetch_tags(self, stream_name: str) -> list[dict]: + """Fetch all tags for a release stream, newest first.""" + url = f"{self._base}/releasestream/{urllib.parse.quote(stream_name)}/tags" + data = fetch_json(url) + return data.get("tags", []) + + def fetch_release(self, stream_name: str, tag: str) -> dict: + """Fetch full release details for a specific tag.""" + url = ( + f"{self._base}/releasestream/{urllib.parse.quote(stream_name)}" + f"/release/{urllib.parse.quote(tag)}" + ) + return fetch_json(url, timeout=60) + + def fetch_changelog( + self, stream_name: str, tag: str, from_tag: str + ) -> dict: + """Fetch the diff/changelog between two tags.""" + url = ( + f"{self._base}/releasestream/{urllib.parse.quote(stream_name)}" + f"/release/{urllib.parse.quote(tag)}" + f"?from={urllib.parse.quote(from_tag)}" + ) + return fetch_json(url, timeout=60) + + def release_url(self, stream_name: str, tag: str) -> str: + """Build the human-readable release controller page URL.""" + return ( + f"https://{self.domain}" + f"/releasestream/{urllib.parse.quote(stream_name)}" + f"/release/{urllib.parse.quote(tag)}" + ) + + def resolve_prow_state(self, prow_url: str) -> Optional[str]: + """Cross-check a Prow job's actual state via its GCS artifact.""" + if not prow_url or not prow_url.startswith(PROW_VIEW_PREFIX): + return None + gcs_path = prow_url[len(PROW_VIEW_PREFIX):] + prowjob_url = f"{GCSWEB_BASE}/{gcs_path}/prowjob.json" + data = try_fetch_json(prowjob_url) + if not data: + return None + prow_state = data.get("status", {}).get("state", "") + return PROW_STATE_MAP.get(prow_state) + + +# --------------------------------------------------------------------------- +# SippyClient — fallback API client for historical payloads +# --------------------------------------------------------------------------- + +class SippyClient: + """Client for Sippy APIs, used when release controller data is unavailable.""" + + SIPPY_BASE = "https://sippy.dptools.openshift.org/api" + + def __init__(self, release: str): + self.release = release + self._tags_cache: Optional[list] = None + + def _get_tags(self) -> list[dict]: + if self._tags_cache is None: + url = f"{self.SIPPY_BASE}/releases/tags?release={urllib.parse.quote(self.release)}" + self._tags_cache = fetch_json(url, timeout=60) + return self._tags_cache + + def find_tag(self, tag_name: str) -> Optional[dict]: + for t in self._get_tags(): + if t.get("release_tag") == tag_name: + return t + return None + + def fetch_job_runs(self, tag_name: str) -> list[dict]: + filter_json = json.dumps({"items": [ + {"columnField": "release_tag", "operatorValue": "equals", + "value": tag_name} + ]}) + url = (f"{self.SIPPY_BASE}/releases/job_runs" + f"?filter={urllib.parse.quote(filter_json)}" + f"&sortField=kind&sort=asc&limit=200") + return fetch_json(url, timeout=60) + + def fetch_changelog(self, tag_name: str, from_tag: Optional[str] = None) -> list[dict]: + params = f"toPayload={urllib.parse.quote(tag_name)}" + if not from_tag: + tag_meta = self.find_tag(tag_name) + if tag_meta: + from_tag = tag_meta.get("previous_release_tag", "") + if from_tag: + params += f"&fromPayload={urllib.parse.quote(from_tag)}" + url = f"{self.SIPPY_BASE}/payloads/diff?{params}" + try: + return fetch_json(url, timeout=60) + except Exception: + return [] + + def build_synthetic_payload(self, tag_name: str, tag: "PayloadTag") -> dict: + tag_meta = self.find_tag(tag_name) + job_runs = self.fetch_job_runs(tag_name) + + phase = tag_meta.get("phase", "Unknown") if tag_meta else "Unknown" + + blocking_jobs: dict = {} + informing_jobs: dict = {} + for jr in job_runs: + name = jr.get("job_name", "") + if not name: + continue + prow_url = jr.get("url", "") + entry = { + "state": jr.get("state", ""), + "url": prow_url, + "retries": jr.get("retries", 0), + } + if jr.get("kind") == "Blocking": + blocking_jobs[name] = entry + else: + informing_jobs[name] = entry + + rc = ReleaseController(tag.architecture) + return { + "phase": phase, + "results": { + "blockingJobs": blocking_jobs, + "informingJobs": informing_jobs, + }, + "_release_url": rc.release_url(tag.stream_name, tag_name), + "_source": "sippy", + } + + def build_synthetic_changelog(self, tag_name: str, from_tag: Optional[str] = None) -> dict: + prs = self.fetch_changelog(tag_name, from_tag=from_tag) + if not isinstance(prs, list): + return {"changeLogJson": {"updatedImages": []}, "_source": "sippy"} + + by_component: dict = {} + for pr in prs: + comp = pr.get("name", "unknown") + if comp not in by_component: + by_component[comp] = {"name": comp, "commits": []} + pr_url = pr.get("url", "") + pr_id = pr.get("pull_request_id", "") + by_component[comp]["commits"].append({ + "pullURL": pr_url, + "pullID": int(pr_id) if pr_id and str(pr_id).isdigit() else 0, + "subject": pr.get("description", ""), + }) + + return { + "changeLogJson": { + "updatedImages": list(by_component.values()), + }, + "_source": "sippy", + } + + +# --------------------------------------------------------------------------- +# PayloadChain — backward walk to find all-green baseline +# --------------------------------------------------------------------------- + +class PayloadChain: + """Walks backwards through payloads to find the all-green baseline.""" + + def __init__(self, rc: ReleaseController, stream_name: str, max_depth: int = 20): + self.rc = rc + self.stream_name = stream_name + self.max_depth = max_depth + + def build(self, start_tag: str) -> list[str]: + """Return an ordered list of tags from start_tag back to the baseline. + + The baseline (last element) is the first payload where all blocking + jobs succeeded. The start_tag is always the first element. + """ + all_tags = self.rc.fetch_tags(self.stream_name) + tag_names = [t["name"] for t in all_tags] + + try: + start_idx = tag_names.index(start_tag) + except ValueError: + raise ValueError( + f"Tag {start_tag} not found in stream {self.stream_name}" + ) + + chain = [] + for i in range(start_idx, min(start_idx + self.max_depth, len(tag_names))): + tag = tag_names[i] + chain.append(tag) + + details = self.rc.fetch_release(self.stream_name, tag) + if self._all_blocking_passed(details): + break + + return chain + + def _all_blocking_passed(self, details: dict) -> bool: + """Check if every blocking job in a payload succeeded.""" + phase = details.get("phase", "") + blocking = details.get("results", {}).get("blockingJobs", {}) + if not blocking: + return True + + for job_name, job_info in blocking.items(): + state = job_info.get("state", "") + if state == "Pending" and phase in ("Accepted", "Rejected"): + resolved = self.rc.resolve_prow_state(job_info.get("url", "")) + if resolved: + state = resolved + if state != "Succeeded": + return False + return True + + +class SippyPayloadChain: + """Walks backwards through payloads using Sippy tag data.""" + + def __init__(self, sippy: SippyClient, max_depth: int = 20): + self.sippy = sippy + self.max_depth = max_depth + + def _has_blocking_failures(self, tag_name: str) -> bool: + """Check whether a payload has any failed blocking jobs.""" + runs = self.sippy.fetch_job_runs(tag_name) + return any( + r.get("kind") == "Blocking" and r.get("state") == "Failed" + for r in runs + ) + + def build(self, start_tag: str) -> list[str]: + chain = [start_tag] + current = start_tag + for _ in range(self.max_depth - 1): + tag_meta = self.sippy.find_tag(current) + if not tag_meta: + break + prev = tag_meta.get("previous_release_tag", "") + if not prev: + break + chain.append(prev) + if not self._has_blocking_failures(prev): + # First payload with all blocking jobs green — include + # one more predecessor so this payload gets a changelog. + prev_meta = self.sippy.find_tag(prev) + if prev_meta: + anchor = prev_meta.get("previous_release_tag", "") + if anchor: + chain.append(anchor) + break + current = prev + return chain + + +# --------------------------------------------------------------------------- +# Collector base class +# --------------------------------------------------------------------------- + +class Collector(ABC): + """Base class for artifact collectors. + + Subclasses implement ``_fetch()`` to retrieve data and return it. + The base ``collect()`` handles idempotency (skip if file exists) and + writing the result to disk. + """ + + def __init__(self, output_path: str): + self.output_path = output_path + + def collect(self) -> bool: + """Fetch and write the artifact. Returns True if new data was written.""" + if os.path.exists(self.output_path): + _log(f" skip (exists): {self.output_path}") + return False + + data = self._fetch() + if data is None: + return False + + os.makedirs(os.path.dirname(self.output_path), exist_ok=True) + if isinstance(data, (dict, list)): + _write_json(self.output_path, data) + else: + _write_text(self.output_path, str(data)) + return True + + @abstractmethod + def _fetch(self) -> Any: + """Retrieve the artifact data. Return None to skip writing.""" + + +# --------------------------------------------------------------------------- +# Concrete collectors +# --------------------------------------------------------------------------- + +class StreamsCollector(Collector): + """Probes all stream/architecture combinations for a version.""" + + def __init__(self, output_path: str, version: str): + super().__init__(output_path) + self.version = version + + def _fetch(self) -> list[dict]: + _log("Fetching streams list...") + streams = [] + for arch in KNOWN_ARCHITECTURES: + for stream_type in STREAM_TYPES: + if stream_type == "ci" and arch != "amd64": + continue + stream_name = _build_stream_name(self.version, stream_type, arch) + rc = ReleaseController(arch) + url = f"{rc._base}/releasestream/{urllib.parse.quote(stream_name)}/tags" + data = try_fetch_json(url) + if data and data.get("tags"): + latest = data["tags"][0] if data["tags"] else {} + streams.append({ + "architecture": arch, + "stream": stream_type, + "stream_name": stream_name, + "tag_count": len(data["tags"]), + "latest_tag": latest.get("name", ""), + "latest_phase": latest.get("phase", ""), + }) + return streams + + +class PayloadDetailCollector(Collector): + """Fetches release controller details for a single payload tag.""" + + def __init__(self, output_path: str, rc: ReleaseController, + stream_name: str, tag: str): + super().__init__(output_path) + self.rc = rc + self.stream_name = stream_name + self.tag = tag + + def _fetch(self) -> dict: + _log(f" Fetching payload details: {self.tag}") + details = self.rc.fetch_release(self.stream_name, self.tag) + details["_release_url"] = self.rc.release_url(self.stream_name, self.tag) + return details + + +class ChangelogCollector(Collector): + """Fetches the changelog (PR diff) between two consecutive payload tags.""" + + def __init__(self, output_path: str, rc: ReleaseController, + stream_name: str, tag: str, from_tag: str): + super().__init__(output_path) + self.rc = rc + self.stream_name = stream_name + self.tag = tag + self.from_tag = from_tag + + def _fetch(self) -> dict: + _log(f" Fetching changelog: {self.tag} from {self.from_tag}") + return self.rc.fetch_changelog(self.stream_name, self.tag, self.from_tag) + + +class PullRequestCollector(Collector): + """Fetches diff, comments, and job data for a single GitHub PR. + + Unlike other collectors that produce a single file, this writes three + files into a directory. ``output_path`` is the PR directory. + """ + + def __init__(self, output_path: str, org: str, repo: str, pr_number: int): + super().__init__(output_path) + self.org = org + self.repo = repo + self.pr_number = pr_number + + def collect(self) -> bool: + """Fetch all three PR artifacts independently.""" + os.makedirs(self.output_path, exist_ok=True) + wrote_any = False + + artifacts = [ + ( + "code.diff", + ["gh", "pr", "diff", str(self.pr_number), + "--repo", f"{self.org}/{self.repo}"], + ), + ( + "comments.json", + ["gh", "pr", "view", str(self.pr_number), + "--repo", f"{self.org}/{self.repo}", + "--json", "comments,reviews"], + ), + ( + "jobs.json", + ["gh", "pr", "view", str(self.pr_number), + "--repo", f"{self.org}/{self.repo}", + "--json", "statusCheckRollup"], + ), + ] + + for filename, cmd in artifacts: + path = os.path.join(self.output_path, filename) + if os.path.exists(path): + continue + result = _run_gh(cmd) + if result is not None: + _write_text(path, result) + wrote_any = True + elif filename == "code.diff": + result = self._git_diff_fallback() + if result is not None: + _write_text(path, result) + wrote_any = True + + return wrote_any + + def _git_diff_fallback(self) -> Optional[str]: + """Clone repo and generate diff locally when gh pr diff fails.""" + meta_str = _run_gh( + ["gh", "pr", "view", str(self.pr_number), + "--repo", f"{self.org}/{self.repo}", + "--json", "baseRefName"], + timeout=30, + ) + if not meta_str: + return None + try: + base_ref = json.loads(meta_str).get("baseRefName", "main") + except json.JSONDecodeError: + return None + + repo_url = f"https://github.com/{self.org}/{self.repo}.git" + tmpdir = tempfile.mkdtemp(prefix="snapshot-diff-") + try: + _log(f" fallback: cloning {self.org}/{self.repo} " + f"for PR #{self.pr_number}") + + # Blobless clone — fetches commit/tree objects only, blobs + # are pulled on demand when git diff needs them. + clone = subprocess.run( + ["git", "clone", "--filter=blob:none", "--bare", + "--single-branch", "--branch", base_ref, + "--no-tags", repo_url, tmpdir], + capture_output=True, text=True, timeout=300, + ) + if clone.returncode != 0: + return None + + fetch = subprocess.run( + ["git", "-C", tmpdir, "fetch", "origin", + f"refs/pull/{self.pr_number}/head:refs/heads/pr"], + capture_output=True, text=True, timeout=120, + ) + if fetch.returncode != 0: + return None + + # Use merge-base for a correct diff against where the PR + # branched off, not the current branch tip. + mb = subprocess.run( + ["git", "-C", tmpdir, "merge-base", base_ref, "pr"], + capture_output=True, text=True, timeout=30, + ) + if mb.returncode != 0: + return None + + result = subprocess.run( + ["git", "-C", tmpdir, "diff", + mb.stdout.strip(), "pr"], + capture_output=True, text=True, timeout=120, + ) + if result.returncode == 0: + return result.stdout + except (subprocess.TimeoutExpired, FileNotFoundError): + return None + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + return None + + def _fetch(self) -> Any: + pass # not used — collect() is overridden + + +class JobCollector(Collector): + """Writes job metadata to an individual job.json file.""" + + def __init__(self, output_path: str, job: JobInfo): + super().__init__(output_path) + self.job = job + + def _fetch(self) -> dict: + return { + "name": self.job.name, + "state": self.job.state, + "lifecycle": self.job.lifecycle, + "url": self.job.url, + "gcs_url": self.job.gcs_url, + "retries": self.job.retries, + "previousAttemptURLs": self.job.previous_attempt_urls, + "is_aggregated": self.job.is_aggregated, + "gcs_bucket_path": self.job.gcs_bucket_path, + } + + +class JUnitCollector(Collector): + """Downloads and parses JUnit artifacts for a failed CI job.""" + + def __init__(self, output_dir: str, job: JobInfo, payload_tag: str): + super().__init__(os.path.join(output_dir, "results.json")) + self.output_dir = output_dir + self.job = job + self.payload_tag = payload_tag + + def collect(self) -> bool: + if os.path.exists(self.output_path): + _log(f" skip (exists): {self.output_path}") + return False + + if not self.job.gcs_bucket_path: + return False + + os.makedirs(self.output_dir, exist_ok=True) + all_results: list[_TestResult] = [] + + junit_files = self._list_junit_files() + for gcs_uri in junit_files: + filename = os.path.basename(gcs_uri) + local_path = os.path.join(self.output_dir, filename) + if not os.path.exists(local_path): + data = _run_gcloud_bytes( + ["gcloud", "storage", "cat", gcs_uri], timeout=60 + ) + if data: + with open(local_path, "wb") as f: + f.write(data) + + if os.path.exists(local_path): + results = _parse_junit_xml(local_path, source_name=filename) + all_results.extend(results) + + failures = _test_results_to_json(all_results) + _write_json(self.output_path, failures) + + underlying = self._detect_underlying_job_name() + if underlying: + job_json_path = os.path.join( + os.path.dirname(self.output_dir), "job.json" + ) + job_data = _read_json(job_json_path) + if job_data: + job_data["underlying_job_name"] = underlying + _write_json(job_json_path, job_data) + + return True + + def _list_junit_files(self) -> list[str]: + """Discover JUnit XML files in GCS for this job.""" + bucket_path = self.job.gcs_bucket_path + base = f"gs://{bucket_path}" + + output = _run_gcloud( + ["gcloud", "storage", "ls", f"{base}/artifacts/**/junit*.xml"], + timeout=30, + ) + if not output: + return [] + + files = [l.strip() for l in output.strip().splitlines() if l.strip()] + + if self.job.is_aggregated: + # For aggregated jobs, keep junit_operator.xml and the + # junit-aggregated.xml. Filter out other junit files that are + # less useful. + keep = [] + for f in files: + bn = os.path.basename(f) + if bn == "junit_operator.xml" or bn == "junit-aggregated.xml": + keep.append(f) + return keep if keep else files + + # For regular jobs, keep junit_operator.xml and the main test + # results file (largest non-operator junit file). + operator = [f for f in files if f.endswith("/junit_operator.xml")] + others = [f for f in files if not f.endswith("/junit_operator.xml")] + # Prefer files with "e2e" or "analysis" in the name + preferred = [f for f in others + if "e2e" in os.path.basename(f) + or "analysis" in os.path.basename(f)] + keep = operator + (preferred if preferred else others[:1]) + return keep if keep else files + + def _detect_underlying_job_name(self) -> Optional[str]: + """For aggregated jobs, extract the underlying job name from GCS paths.""" + if not self.job.is_aggregated: + return None + agg_xml = os.path.join(self.output_dir, "junit-aggregated.xml") + if not os.path.exists(agg_xml): + # Try to find it from GCS path structure + bucket_path = self.job.gcs_bucket_path + output = _run_gcloud( + ["gcloud", "storage", "ls", + f"gs://{bucket_path}/artifacts/release-analysis-aggregator/" + f"openshift-release-analysis-aggregator/artifacts/" + f"release-analysis-aggregator/*/"], + timeout=30, + ) + if output: + for line in output.strip().splitlines(): + parts = line.rstrip("/").split("/") + if parts: + return parts[-1] + return None + + def _fetch(self) -> Any: + pass # not used — collect() is overridden + + +class BuildLogCollector(Collector): + """Downloads build-log.txt from GCS and extracts error/warning lines.""" + + _ERROR_RE = re.compile( + r"(?:^|\s)(?:error|ERROR|Error|warning|WARNING|Warning" + r"|FATAL|fatal|panic|PANIC)[:\s\[]", + ) + + def __init__(self, output_path: str, job: JobInfo): + super().__init__(output_path) + self.job = job + + def _fetch(self) -> Optional[dict]: + if not self.job.gcs_bucket_path: + return None + + gcs_uri = f"gs://{self.job.gcs_bucket_path}/build-log.txt" + raw = _run_gcloud_bytes( + ["gcloud", "storage", "cat", gcs_uri], timeout=120 + ) + if not raw: + return None + + try: + text = gzip.decompress(raw).decode("utf-8", errors="replace") + except (gzip.BadGzipFile, OSError): + text = raw.decode("utf-8", errors="replace") + + lines = text.splitlines() + total = len(lines) + if total == 0: + return {"error_warning_lines": [], "tail_lines": [], + "total_lines": 0} + + error_warning = [] + for i, line in enumerate(lines): + if self._ERROR_RE.search(line): + error_warning.append({ + "line_number": i + 1, + "text": line.rstrip(), + }) + + tail_start = max(0, total - total // 5) + tail_lines = [l.rstrip() for l in lines[tail_start:]] + + return { + "total_lines": total, + "error_warning_count": len(error_warning), + "error_warning_lines": error_warning, + "tail_start_line": tail_start + 1, + "tail_lines": tail_lines, + } + + +# --------------------------------------------------------------------------- +# RegressionTracker — traces test failures across the payload chain +# --------------------------------------------------------------------------- + +class RegressionTracker: + """Identifies when each test failure first appeared in the payload chain.""" + + def __init__(self, base_dir: str, chain: list[str], target_tag: str): + self.base_dir = base_dir + self.chain = chain + self.target_tag = target_tag + + def track(self) -> list[dict]: + """Analyze failures in the target payload and trace their origins.""" + target_dir = os.path.join(self.base_dir, self.target_tag) + target_failures = self._load_all_failures(target_dir) + + if not target_failures: + return [] + + regressions = [] + for test_name, info in sorted(target_failures.items()): + first_failed_in = self.target_tag + for i in range(1, len(self.chain)): + prior_tag = self.chain[i] + prior_dir = os.path.join(self.base_dir, prior_tag) + prior_failures = self._load_all_failures(prior_dir) + if test_name in prior_failures: + first_failed_in = prior_tag + else: + break + + chain_idx_first = self.chain.index(first_failed_in) + regressions.append({ + "test_name": test_name, + "jobs": info["jobs"], + "lifecycle": info["lifecycle"], + "first_failed_in": first_failed_in, + "payloads_failing": chain_idx_first + 1, + "failure_message": info.get("failure_message", ""), + "failure_text": info.get("failure_text", ""), + }) + return regressions + + def _load_all_failures(self, tag_dir: str) -> dict[str, dict]: + """Load all test failures from all jobs in a payload directory.""" + failures: dict[str, dict] = {} + jobs_dir = os.path.join(tag_dir, "jobs") + if not os.path.isdir(jobs_dir): + return failures + + for lifecycle in ("blocking",): + lifecycle_dir = os.path.join(jobs_dir, lifecycle) + if not os.path.isdir(lifecycle_dir): + continue + for job_name in os.listdir(lifecycle_dir): + results_path = os.path.join( + lifecycle_dir, job_name, "junit", "results.json" + ) + results = _read_json(results_path) + if not results: + continue + for test in results: + name = test.get("name", "") + if not name: + continue + if name in failures: + if job_name not in failures[name]["jobs"]: + failures[name]["jobs"].append(job_name) + else: + failures[name] = { + "jobs": [job_name], + "lifecycle": lifecycle, + "failure_message": test.get("failure_message", "") + or test.get("error_message", ""), + "failure_text": test.get("failure_text", "") + or test.get("error_text", ""), + } + return failures + + +# --------------------------------------------------------------------------- +# JobStreakTracker — per-job failure streaks across the payload chain +# --------------------------------------------------------------------------- + +class JobStreakTracker: + """Tracks per-job failure streaks across the payload chain.""" + + def __init__(self, base_dir: str, chain: list[str]): + self.base_dir = base_dir + self.chain = chain + + def track(self) -> dict[str, dict]: + """Return a dict keyed by job name with streak data. + + Only tracks failed blocking jobs in the target (first) payload. + """ + if not self.chain: + return {} + + target_tag = self.chain[0] + target_dir = os.path.join(self.base_dir, target_tag) + + failed_jobs = self._get_failed_jobs(target_dir) + if not failed_jobs: + return {} + + streaks: dict[str, dict] = {} + for job_name in failed_jobs: + pattern = [] + for tag in self.chain: + tag_dir = os.path.join(self.base_dir, tag) + state = self._get_job_state(tag_dir, job_name) + if state == "Succeeded": + pattern.append("S") + elif state: + pattern.append("F") + else: + pattern.append("?") + + streak = 0 + for p in pattern: + if p == "F": + streak += 1 + else: + break + + originating_idx = min(streak - 1, len(self.chain) - 1) + originating_tag = self.chain[originating_idx] + + streaks[job_name] = { + "streak_length": streak, + "originating_payload": originating_tag, + "is_new_failure": streak == 1, + "failure_pattern": " ".join(pattern), + } + + return streaks + + def _get_failed_jobs(self, tag_dir: str) -> list[str]: + blocking_dir = os.path.join(tag_dir, "jobs", "blocking") + if not os.path.isdir(blocking_dir): + return [] + failed = [] + for job_name in os.listdir(blocking_dir): + job_data = _read_json( + os.path.join(blocking_dir, job_name, "job.json") + ) + if job_data and job_data.get("state") != "Succeeded": + failed.append(job_name) + return sorted(failed) + + def _get_job_state(self, tag_dir: str, job_name: str) -> Optional[str]: + job_path = os.path.join( + tag_dir, "jobs", "blocking", job_name, "job.json" + ) + job_data = _read_json(job_path) + if job_data: + return job_data.get("state", "") + return None + + +# --------------------------------------------------------------------------- +# SummaryGenerator — produces stream-level roll-up +# --------------------------------------------------------------------------- + +class SummaryGenerator: + """Generates summary.json and summary.md for the stream.""" + + def __init__(self, base_dir: str, chain: list[str], target_tag: str, + tag: "PayloadTag", streaks: Optional[dict] = None): + self.base_dir = base_dir + self.chain = chain + self.target_tag = target_tag + self.tag = tag + self.streaks = streaks or {} + + def generate(self) -> None: + target_dir = os.path.join(self.base_dir, self.target_tag) + payload_data = _read_json(os.path.join(target_dir, "payload.json")) + regressions = _read_json( + os.path.join(target_dir, "regressions.json") + ) or [] + + phase = payload_data.get("phase", "Unknown") if payload_data else "Unknown" + release_url = (payload_data.get("_release_url", "") + if payload_data else "") + results = payload_data.get("results", {}) if payload_data else {} + + blocking = results.get("blockingJobs", {}) or {} + informing = results.get("informingJobs", {}) or {} + + blocking_passed = sum( + 1 for v in blocking.values() if v.get("state") == "Succeeded" + ) + informing_passed = sum( + 1 for v in informing.values() if v.get("state") == "Succeeded" + ) + + hours_since = self._compute_hours_since_baseline() + + failed_blocking_detail = self._build_failed_job_details( + "blocking" + ) + failed_informing_names = sorted( + k for k, v in informing.items() if v.get("state") != "Succeeded" + ) + + payloads = self._build_payload_entries() + + summary = { + "payload_tag": self.target_tag, + "phase": phase, + "release_url": release_url, + "architecture": self.tag.architecture, + "stream": self.tag.stream, + "version": self.tag.version, + "chain_length": len(self.chain), + "baseline_tag": self.chain[-1] if self.chain else "", + "hours_since_baseline": hours_since, + "blocking_jobs": { + "total": len(blocking), + "passed": blocking_passed, + "failed": len(blocking) - blocking_passed, + "failed_jobs": failed_blocking_detail, + }, + "informing_jobs": { + "total": len(informing), + "passed": informing_passed, + "failed": len(informing) - informing_passed, + "failed_jobs": failed_informing_names, + }, + "test_failures": { + "blocking": [ + r for r in regressions if r.get("lifecycle") == "blocking" + ], + }, + "payloads": payloads, + } + + _write_json(os.path.join(self.base_dir, "summary.json"), summary) + self._write_agents_md(summary) + _log(" Generated summary.json, AGENTS.md, and CLAUDE.md") + + def _write_agents_md(self, summary: dict) -> None: + """Write AGENTS.md and CLAUDE.md into the snapshot directory.""" + tag = summary["payload_tag"] + phase = summary["phase"] + version = summary["version"] + stream = summary["stream"] + arch = summary["architecture"] + chain_len = summary["chain_length"] + baseline = summary["baseline_tag"] + hours = summary.get("hours_since_baseline") + bj = summary["blocking_jobs"] + ij = summary["informing_jobs"] + + failed_names = [ + j["name"] if isinstance(j, dict) else j + for j in bj.get("failed_jobs", []) + ] + + hours_str = f" ({hours}h ago)" if hours is not None else "" + chain_tags = "\n".join(f" - {t}" for t in self.chain) + + lines = [ + f"# Payload Snapshot: {tag}", + "", + f"OpenShift {version} {stream} ({arch}) — **{phase}**", + "", + "## Quick Start", + "", + "Read `summary.json` first. It contains everything you need for", + "triage: job states, failure streaks, test regressions, build-log", + "error counts, and relative paths to all detailed data files.", + "Only drill into per-job or per-PR files when you need specifics.", + "", + "## This Snapshot", + "", + f"- **Target payload**: `{tag}`", + f"- **Phase**: {phase}", + f"- **Blocking jobs**: {bj['failed']}/{bj['total']} failed", + f"- **Informing jobs**: {ij['failed']}/{ij['total']} failed", + f"- **Chain depth**: {chain_len} payloads back to baseline", + f"- **Baseline**: `{baseline}`{hours_str}", + "", + "### Payload chain (newest first)", + "", + chain_tags, + "", + ] + + if failed_names: + lines.append("### Failed blocking jobs") + lines.append("") + for n in failed_names: + lines.append(f" - `{n}`") + lines.append("") + + lines.extend([ + "## File Layout", + "", + "```", + f"{version}/{stream}/", + " summary.json # START HERE — full triage data", + " CLAUDE.md # This file", + " streams.json # All streams for this OCP version", + f" {tag}/ # Target payload", + " payload.json # Release controller API response", + " changelog.json # PRs changed vs previous payload", + " regressions.json # Test failure regression tracking", + " jobs/", + " blocking/", + " /", + " job.json # State, prow URL, GCS URL, retries", + " build_log.json # Error/warning lines + log tail", + " junit/", + " results.json # Parsed test failures", + " *.xml # Raw JUnit XML", + " informing/", + " /", + " job.json # State and URLs only (no junit)", + " /", + " prs/", + " /", + " code.diff # Full git diff", + " comments.json # PR comments and reviews", + " jobs.json # CI check runs", + " / # Each prior payload in the chain", + " ... # Same structure", + "```", + "", + "## Key Concepts", + "", + "- **Blocking vs informing**: Only blocking job failures prevent", + " payload acceptance. Informing jobs are tracked but don't block.", + "- **Chain**: The sequence of payloads walking backwards from the", + " target until one where all blocking jobs passed (the baseline).", + "- **Streaks**: Per-job consecutive failure count from the target", + " backwards. `failure_pattern` shows the full history (F=fail,", + " S=succeed) across the chain.", + "- **Regressions**: Per-test tracking — when did each test failure", + " first appear? `first_failed_in` identifies the originating", + " payload, `payloads_failing` counts how many payloads it spans.", + "- **Build log**: Error/warning lines extracted from the Prow", + " build-log.txt, plus the last 20% of the log for context.", + "", + "## summary.json Schema", + "", + "Top-level fields:", + "- `payload_tag`, `phase`, `release_url`, `architecture`,", + " `stream`, `version`", + "- `chain_length`, `baseline_tag`, `hours_since_baseline`", + "- `blocking_jobs.failed_jobs[]` — each entry has: `name`,", + " `state`, `prow_url`, `gcs_url`, `streak` (with", + " `streak_length`, `originating_payload`, `is_new_failure`,", + " `failure_pattern`), `build_log_errors`, `test_failure_count`,", + " and relative paths to `job_json`, `junit_results`, `build_log`", + "- `informing_jobs.failed_jobs[]` — job name strings only", + "- `test_failures.blocking[]` — `test_name`, `jobs`,", + " `first_failed_in`, `payloads_failing`, `failure_message`,", + " `failure_text`", + "- `payloads[]` — per-payload entries with `tag`, `phase`,", + " relative paths, and `prs[]` with component/diff/comments paths", + "", + ]) + + _write_text( + os.path.join(self.base_dir, "AGENTS.md"), + "\n".join(lines), + ) + _write_text( + os.path.join(self.base_dir, "CLAUDE.md"), + "@AGENTS.md\n", + ) + + def _compute_hours_since_baseline(self) -> Optional[float]: + """Compute hours between target and baseline payload timestamps.""" + if len(self.chain) < 2: + return None + try: + target_ts = _parse_tag_timestamp(self.chain[0]) + baseline_ts = _parse_tag_timestamp(self.chain[-1]) + if target_ts and baseline_ts: + delta = target_ts - baseline_ts + return round(delta.total_seconds() / 3600, 1) + except (ValueError, IndexError): + pass + return None + + def _build_failed_job_details(self, lifecycle: str) -> list[dict]: + """Build detailed entries for each failed job.""" + target_dir = os.path.join(self.base_dir, self.target_tag) + lifecycle_dir = os.path.join(target_dir, "jobs", lifecycle) + if not os.path.isdir(lifecycle_dir): + return [] + + details = [] + for job_name in sorted(os.listdir(lifecycle_dir)): + job_data = _read_json( + os.path.join(lifecycle_dir, job_name, "job.json") + ) + if not job_data or job_data.get("state") == "Succeeded": + continue + + tag_rel = self.target_tag + entry: dict = { + "name": job_name, + "state": job_data.get("state", ""), + "prow_url": job_data.get("url", ""), + "gcs_url": job_data.get("gcs_url", ""), + "is_aggregated": job_data.get("is_aggregated", False), + "retries": job_data.get("retries", 0), + "job_json": ( + f"{tag_rel}/jobs/{lifecycle}/{job_name}/job.json" + ), + } + + streak_data = self.streaks.get(job_name) + if streak_data: + entry["streak"] = streak_data + + results_path = os.path.join( + lifecycle_dir, job_name, "junit", "results.json" + ) + if os.path.exists(results_path): + entry["junit_results"] = ( + f"{tag_rel}/jobs/{lifecycle}/{job_name}/junit/results.json" + ) + results_data = _read_json(results_path) or [] + entry["test_failure_count"] = len(results_data) + + build_log_path = os.path.join( + lifecycle_dir, job_name, "build_log.json" + ) + if os.path.exists(build_log_path): + entry["build_log"] = ( + f"{tag_rel}/jobs/{lifecycle}/{job_name}/build_log.json" + ) + bl_data = _read_json(build_log_path) + if bl_data: + entry["build_log_errors"] = bl_data.get( + "error_warning_count", 0 + ) + + details.append(entry) + return details + + def _build_payload_entries(self) -> list[dict]: + """Build the payloads array with all path references.""" + payloads = [] + for tag_name in self.chain: + tag_rel = tag_name + entry: dict = { + "tag": tag_name, + "payload": f"{tag_rel}/payload.json", + } + + pd = _read_json( + os.path.join(self.base_dir, tag_name, "payload.json") + ) + if pd: + entry["phase"] = pd.get("phase", "") + + changelog_path = os.path.join( + self.base_dir, tag_name, "changelog.json" + ) + if os.path.exists(changelog_path): + entry["changelog"] = f"{tag_rel}/changelog.json" + changelog = _read_json(changelog_path) + prs = _extract_prs(changelog) if changelog else [] + if prs: + entry["prs"] = [ + { + "url": p["url"], + "component": p["component"], + "number": p["number"], + "description": p["description"], + "diff": f"{tag_rel}/{p['component']}/prs/{p['number']}/code.diff", + "comments": f"{tag_rel}/{p['component']}/prs/{p['number']}/comments.json", + "jobs": f"{tag_rel}/{p['component']}/prs/{p['number']}/jobs.json", + } + for p in prs + ] + regressions_path = os.path.join( + self.base_dir, tag_name, "regressions.json" + ) + if os.path.exists(regressions_path): + entry["regressions"] = f"{tag_rel}/regressions.json" + + job_paths = self._collect_job_paths(tag_name) + if job_paths: + entry["jobs"] = job_paths + + payloads.append(entry) + return payloads + + def _collect_job_paths(self, tag_name: str) -> dict: + """Build path references for jobs in a payload.""" + tag_rel = tag_name + job_paths: dict = {"blocking": [], "informing": []} + jobs_dir = os.path.join(self.base_dir, tag_name, "jobs") + if not os.path.isdir(jobs_dir): + return job_paths + + for lifecycle in ("blocking", "informing"): + lifecycle_dir = os.path.join(jobs_dir, lifecycle) + if not os.path.isdir(lifecycle_dir): + continue + for job_name in sorted(os.listdir(lifecycle_dir)): + job_entry: dict = { + "name": job_name, + "job_json": f"{tag_rel}/jobs/{lifecycle}/{job_name}/job.json", + } + results_path = os.path.join( + lifecycle_dir, job_name, "junit", "results.json" + ) + if os.path.exists(results_path): + job_entry["junit_results"] = ( + f"{tag_rel}/jobs/{lifecycle}/{job_name}/junit/results.json" + ) + build_log_path = os.path.join( + lifecycle_dir, job_name, "build_log.json" + ) + if os.path.exists(build_log_path): + job_entry["build_log"] = ( + f"{tag_rel}/jobs/{lifecycle}/{job_name}/build_log.json" + ) + job_data = _read_json( + os.path.join(lifecycle_dir, job_name, "job.json") + ) + if job_data: + job_entry["state"] = job_data.get("state", "") + job_entry["gcs_url"] = job_data.get("gcs_url", "") + job_paths[lifecycle].append(job_entry) + + return job_paths + + + +# --------------------------------------------------------------------------- +# Snapshotter — top-level orchestrator +# --------------------------------------------------------------------------- + +class Snapshotter: + """Orchestrates the full payload snapshot.""" + + def __init__(self, tag: PayloadTag, output_dir: str = "payload", + max_chain: int = 20, workers: int = 8, + collect_junit: bool = True, use_sippy: bool = False): + self.tag = tag + self.output_dir = output_dir + self.max_chain = max_chain + self.workers = workers + self.collect_junit = collect_junit + self.use_sippy = use_sippy + self.rc = ReleaseController(tag.architecture) + self.sippy: Optional[SippyClient] = None + if use_sippy: + self.sippy = SippyClient(tag.version) + + def run(self) -> None: + """Execute the full snapshot.""" + base_dir = os.path.join( + self.output_dir, self.tag.version, self.tag.stream + ) + os.makedirs(base_dir, exist_ok=True) + + self._collect_streams(base_dir) + + chain = self._build_chain() + _log(f"Payload chain: {len(chain)} payloads") + for t in chain: + _log(f" {t}") + + pr_tasks = self._collect_payloads(base_dir, chain) + + self._collect_jobs(base_dir, chain) + + self._collect_prs(pr_tasks) + + if self.collect_junit: + self._collect_junit(base_dir, chain) + self._collect_build_logs(base_dir, chain) + self._track_regressions(base_dir, chain) + + streaks = self._track_job_streaks(base_dir, chain) + + self._generate_summary(base_dir, chain, streaks) + + _log(f"\nSnapshot complete: {base_dir}/") + + def _collect_streams(self, base_dir: str) -> None: + """Collect the streams list for this version.""" + path = os.path.join(base_dir, "streams.json") + if self.sippy: + if os.path.exists(path): + return + _log("Skipping streams collection in Sippy mode (RC-only feature)") + _write_json(path, []) + return + StreamsCollector(path, self.tag.version).collect() + + def _build_chain(self) -> list[str]: + """Build the backward payload chain.""" + if self.sippy: + chain_builder = SippyPayloadChain(self.sippy, self.max_chain) + return chain_builder.build(self.tag.raw) + chain_builder = PayloadChain( + self.rc, self.tag.stream_name, self.max_chain + ) + return chain_builder.build(self.tag.raw) + + def _collect_payloads( + self, base_dir: str, chain: list[str] + ) -> list[PullRequestCollector]: + """Collect payload details and changelogs; return PR collectors.""" + seen_prs: set[tuple[str, str, int]] = set() + pr_collectors: list[PullRequestCollector] = [] + + for i, tag_name in enumerate(chain): + tag_dir = os.path.join(base_dir, tag_name) + _log(f"\nProcessing payload: {tag_name}") + + payload_path = os.path.join(tag_dir, "payload.json") + if self.sippy: + if not os.path.exists(payload_path): + _log(f" Fetching payload details from Sippy: {tag_name}") + tag_obj = PayloadTag.parse(tag_name) + data = self.sippy.build_synthetic_payload(tag_name, tag_obj) + os.makedirs(os.path.dirname(payload_path), exist_ok=True) + _write_json(payload_path, data) + else: + _log(f" skip (exists): {payload_path}") + else: + PayloadDetailCollector( + payload_path, + self.rc, self.tag.stream_name, tag_name, + ).collect() + + if i >= len(chain) - 1: + continue + + prev_tag = chain[i + 1] + changelog_path = os.path.join(tag_dir, "changelog.json") + if self.sippy: + if not os.path.exists(changelog_path): + _log(f" Fetching changelog from Sippy: {tag_name}") + data = self.sippy.build_synthetic_changelog(tag_name, from_tag=prev_tag) + os.makedirs(os.path.dirname(changelog_path), exist_ok=True) + _write_json(changelog_path, data) + else: + _log(f" skip (exists): {changelog_path}") + else: + ChangelogCollector( + changelog_path, self.rc, + self.tag.stream_name, tag_name, prev_tag, + ).collect() + + changelog = _read_json(changelog_path) + if not changelog: + continue + + prs = _extract_prs(changelog) + for pr in prs: + key = (pr["org"], pr["repo"], pr["number"]) + if key in seen_prs: + continue + seen_prs.add(key) + pr_dir = os.path.join( + tag_dir, pr["component"], "prs", str(pr["number"]) + ) + pr_collectors.append( + PullRequestCollector(pr_dir, pr["org"], pr["repo"], pr["number"]) + ) + + return pr_collectors + + def _collect_prs(self, collectors: list[PullRequestCollector]) -> None: + """Fetch PR artifacts in parallel.""" + if not collectors: + _log("\nNo PRs to fetch.") + return + + _log(f"\nFetching {len(collectors)} unique PRs ({self.workers} workers)...") + + with ThreadPoolExecutor(max_workers=self.workers) as pool: + futures = { + pool.submit(c.collect): c for c in collectors + } + done = 0 + for future in as_completed(futures): + done += 1 + collector = futures[future] + label = f"{collector.org}/{collector.repo}#{collector.pr_number}" + try: + future.result() + _log(f" [{done}/{len(collectors)}] {label}") + except Exception as e: + _log(f" [{done}/{len(collectors)}] {label}: error: {e}") + + def _collect_jobs(self, base_dir: str, chain: list[str]) -> None: + """Split payload.json jobs into individual job directories.""" + _log("\nSplitting jobs into directories...") + for tag_name in chain: + tag_dir = os.path.join(base_dir, tag_name) + payload_path = os.path.join(tag_dir, "payload.json") + payload_data = _read_json(payload_path) + if not payload_data: + continue + + jobs = _extract_jobs(payload_data) + for job in jobs: + job_path = os.path.join( + tag_dir, "jobs", job.lifecycle, job.name, "job.json" + ) + JobCollector(job_path, job).collect() + + blocking_count = sum(1 for j in jobs if j.lifecycle == "blocking") + informing_count = sum(1 for j in jobs if j.lifecycle == "informing") + _log(f" {tag_name}: {blocking_count} blocking, " + f"{informing_count} informing") + + def _find_failed_jobs( + self, base_dir: str, chain: list[str], + lifecycles: tuple[str, ...] = ("blocking",), + ) -> list[tuple[str, str, JobInfo]]: + """Find failed jobs across the chain for the given lifecycles. + + Returns (tag_name, job_dir, JobInfo) tuples. + """ + results = [] + for tag_name in chain: + tag_dir = os.path.join(base_dir, tag_name) + for lifecycle in lifecycles: + jobs_dir = os.path.join(tag_dir, "jobs", lifecycle) + if not os.path.isdir(jobs_dir): + continue + for job_name in os.listdir(jobs_dir): + job_json_path = os.path.join( + jobs_dir, job_name, "job.json" + ) + job_data = _read_json(job_json_path) + if not job_data: + continue + if job_data.get("state") == "Succeeded": + continue + job_info = JobInfo( + name=job_data["name"], + state=job_data["state"], + lifecycle=job_data["lifecycle"], + url=job_data["url"], + retries=job_data.get("retries", 0), + previous_attempt_urls=job_data.get( + "previousAttemptURLs", [] + ), + is_aggregated=job_data.get("is_aggregated", False), + gcs_bucket_path=job_data.get("gcs_bucket_path", ""), + gcs_url=job_data.get("gcs_url", ""), + ) + job_dir = os.path.join(jobs_dir, job_name) + results.append((tag_name, job_dir, job_info)) + return results + + def _collect_junit(self, base_dir: str, chain: list[str]) -> None: + """Download and parse JUnit for failed jobs across the chain. + + Blocking jobs: all payloads in chain (for streak/regression tracking). + Informing jobs: target payload only (first in chain). + """ + collectors: list[JUnitCollector] = [] + + for tag_name, job_dir, job_info in self._find_failed_jobs( + base_dir, chain, ("blocking",) + ): + junit_dir = os.path.join(job_dir, "junit") + collectors.append( + JUnitCollector(junit_dir, job_info, tag_name) + ) + + for tag_name, job_dir, job_info in self._find_failed_jobs( + base_dir, chain[:1], ("informing",) + ): + junit_dir = os.path.join(job_dir, "junit") + collectors.append( + JUnitCollector(junit_dir, job_info, tag_name) + ) + + if not collectors: + _log("\nNo failed jobs to fetch JUnit for.") + return + + _log(f"\nFetching JUnit for {len(collectors)} failed jobs " + f"({self.workers} workers)...") + + with ThreadPoolExecutor(max_workers=self.workers) as pool: + futures = { + pool.submit(c.collect): c for c in collectors + } + done = 0 + for future in as_completed(futures): + done += 1 + collector = futures[future] + try: + future.result() + _log(f" [{done}/{len(collectors)}] {collector.job.name}") + except Exception as e: + _log(f" [{done}/{len(collectors)}] " + f"{collector.job.name}: error: {e}") + + def _track_regressions(self, base_dir: str, chain: list[str]) -> None: + """Track test failure regressions across the payload chain.""" + _log("\nTracking test failure regressions...") + target_tag = chain[0] + regressions_path = os.path.join( + base_dir, target_tag, "regressions.json" + ) + + if os.path.exists(regressions_path): + _log(f" skip (exists): {regressions_path}") + return + + tracker = RegressionTracker(base_dir, chain, target_tag) + regressions = tracker.track() + _write_json(regressions_path, regressions) + + new_count = sum(1 for r in regressions if r["payloads_failing"] == 1) + persistent_count = len(regressions) - new_count + _log(f" Found {len(regressions)} failing tests: " + f"{new_count} new, {persistent_count} persistent") + + def _collect_build_logs(self, base_dir: str, chain: list[str]) -> None: + """Download and parse build-log.txt for failed jobs. + + Blocking jobs: all payloads in chain. + Informing jobs: target payload only (first in chain). + """ + collectors: list[BuildLogCollector] = [] + + for _tag_name, job_dir, job_info in self._find_failed_jobs( + base_dir, chain, ("blocking",) + ): + build_log_path = os.path.join(job_dir, "build_log.json") + collectors.append( + BuildLogCollector(build_log_path, job_info) + ) + + for _tag_name, job_dir, job_info in self._find_failed_jobs( + base_dir, chain[:1], ("informing",) + ): + build_log_path = os.path.join(job_dir, "build_log.json") + collectors.append( + BuildLogCollector(build_log_path, job_info) + ) + + if not collectors: + _log("\nNo failed jobs for build-log extraction.") + return + + _log(f"\nExtracting build logs for {len(collectors)} failed " + f"jobs ({self.workers} workers)...") + + with ThreadPoolExecutor(max_workers=self.workers) as pool: + futures = { + pool.submit(c.collect): c for c in collectors + } + done = 0 + for future in as_completed(futures): + done += 1 + collector = futures[future] + try: + future.result() + _log(f" [{done}/{len(collectors)}] {collector.job.name}") + except Exception as e: + _log(f" [{done}/{len(collectors)}] " + f"{collector.job.name}: error: {e}") + + def _track_job_streaks( + self, base_dir: str, chain: list[str] + ) -> dict[str, dict]: + """Track per-job failure streaks across the payload chain.""" + _log("\nTracking job failure streaks...") + tracker = JobStreakTracker(base_dir, chain) + streaks = tracker.track() + new_count = sum(1 for s in streaks.values() if s["is_new_failure"]) + _log(f" {len(streaks)} failed jobs: {new_count} new, " + f"{len(streaks) - new_count} persistent") + return streaks + + def _generate_summary( + self, base_dir: str, chain: list[str], + streaks: Optional[dict] = None, + ) -> None: + """Generate the stream-level summary.""" + _log("\nGenerating summary...") + summary_path = os.path.join(base_dir, "summary.json") + if os.path.exists(summary_path): + os.remove(summary_path) + + generator = SummaryGenerator( + base_dir, chain, chain[0], self.tag, streaks=streaks + ) + generator.generate() + + +# --------------------------------------------------------------------------- +# Utility functions +# --------------------------------------------------------------------------- + +def _log(msg: str) -> None: + """Print a progress message to stderr.""" + print(msg, file=sys.stderr) + + +def _write_json(path: str, data: Any) -> None: + """Write JSON data to a file.""" + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + f.write("\n") + + +def _write_text(path: str, text: str) -> None: + """Write text to a file.""" + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(text) + + +def _read_json(path: str) -> Optional[dict]: + """Read a JSON file, returning None if it doesn't exist.""" + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return None + + +def _parse_tag_timestamp(tag: str) -> Optional[datetime]: + """Extract and parse the timestamp from a payload tag name.""" + m = re.search(r"(\d{4}-\d{2}-\d{2}-\d{6})$", tag) + if not m: + return None + try: + return datetime.strptime(m.group(1), "%Y-%m-%d-%H%M%S").replace( + tzinfo=timezone.utc + ) + except ValueError: + return None + + +def _build_stream_name(version: str, stream: str, architecture: str) -> str: + """Build the release stream name from components.""" + name = f"{version}.0-0.{stream}" + if architecture != "amd64": + name += f"-{architecture}" + return name + + +def _run_gh(args: list[str], timeout: int = 60) -> Optional[str]: + """Run a gh CLI command, returning stdout or None on error.""" + try: + result = subprocess.run( + args, capture_output=True, text=True, timeout=timeout + ) + if result.returncode != 0: + _log(f" warning: {' '.join(args[:5])}... failed: " + f"{result.stderr.strip()[:200]}") + return None + return result.stdout + except subprocess.TimeoutExpired: + _log(f" warning: {' '.join(args[:5])}... timed out") + return None + except FileNotFoundError: + _log(" error: 'gh' CLI not found — install it or run 'gh auth login'") + return None + + +def _extract_prs(changelog: dict) -> list[dict]: + """Extract PR info from a release controller changelog response.""" + prs = [] + images = changelog.get("changeLogJson", {}).get("updatedImages", []) or [] + for image in images: + component = image.get("name", "") + for commit in image.get("commits", []): + pull_url = commit.get("pullURL", "") + parsed = _parse_pr_url(pull_url) + if parsed: + org, repo, number = parsed + prs.append({ + "org": org, + "repo": repo, + "number": number, + "component": component, + "description": commit.get("subject", ""), + "url": pull_url, + }) + return prs + + +def _parse_pr_url(url: str) -> Optional[tuple[str, str, int]]: + """Parse a GitHub PR URL into (org, repo, pr_number).""" + m = re.match(r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)", url) + if m: + return m.group(1), m.group(2), int(m.group(3)) + return None + + +def _check_gh_auth() -> bool: + """Verify that gh CLI is authenticated.""" + try: + result = subprocess.run( + ["gh", "auth", "status"], + capture_output=True, text=True, timeout=10, + ) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +def _run_gcloud(args: list[str], timeout: int = 120) -> Optional[str]: + """Run a gcloud CLI command, returning stdout or None on error.""" + try: + result = subprocess.run( + args, capture_output=True, text=True, timeout=timeout + ) + if result.returncode != 0: + return None + return result.stdout + except (subprocess.TimeoutExpired, FileNotFoundError): + return None + + +def _run_gcloud_bytes(args: list[str], timeout: int = 120) -> Optional[bytes]: + """Run a gcloud CLI command, returning raw stdout bytes or None.""" + try: + result = subprocess.run( + args, capture_output=True, timeout=timeout + ) + if result.returncode != 0: + return None + return result.stdout + except (subprocess.TimeoutExpired, FileNotFoundError): + return None + + +def _check_gcloud() -> bool: + """Verify that gcloud CLI is available.""" + try: + result = subprocess.run( + ["gcloud", "--version"], + capture_output=True, text=True, timeout=10, + ) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +def _prow_url_to_gcs_bucket_path(prow_url: str) -> Optional[str]: + """Extract the GCS bucket path from a Prow URL. + + Returns 'test-platform-results/logs/{job}/{build_id}' or None. + """ + if not prow_url or not prow_url.startswith(PROW_VIEW_PREFIX): + return None + return prow_url[len(PROW_VIEW_PREFIX):] + + +def _extract_jobs(payload_data: dict) -> list[JobInfo]: + """Extract job metadata from a payload.json response.""" + jobs = [] + for lifecycle, key in [("blocking", "blockingJobs"), + ("informing", "informingJobs")]: + job_dict = payload_data.get("results", {}).get(key, {}) or {} + for name, info in job_dict.items(): + url = info.get("url", "") + gcs_path = _prow_url_to_gcs_bucket_path(url) or "" + gcs_url = f"{GCSWEB_BASE}/{gcs_path}/" if gcs_path else "" + jobs.append(JobInfo( + name=name, + state=info.get("state", ""), + lifecycle=lifecycle, + url=url, + retries=info.get("retries", 0), + previous_attempt_urls=info.get("previousAttemptURLs", []) or [], + is_aggregated=name.startswith("aggregated-"), + gcs_bucket_path=gcs_path, + gcs_url=gcs_url, + )) + return jobs + + +# --------------------------------------------------------------------------- +# JUnit XML parsing (embedded from parse_junit.py) +# --------------------------------------------------------------------------- + +@dataclass +class _TestResult: + """Parsed test result from JUnit XML.""" + + name: str + status: str # passed, failed, error, skipped + suite_name: str = "" + failure_message: str = "" + failure_text: str = "" + error_message: str = "" + error_text: str = "" + system_out: str = "" + agg_passes: list = field(default_factory=list) + agg_failures: list = field(default_factory=list) + agg_skips: list = field(default_factory=list) + + +def _parse_system_out_yaml(text: str) -> dict: + """Parse YAML-like system-out from aggregated JUnit XML.""" + result = {"passes": [], "failures": [], "skips": []} + if not text: + return result + + current_section = None + current_entry: dict = {} + + for line in text.strip().splitlines(): + stripped = line.strip() + if stripped in ("passes:", "failures:", "skips:"): + if current_entry and current_section: + result[current_section].append(current_entry) + current_entry = {} + current_section = stripped.rstrip(":") + continue + if current_section is None: + continue + if stripped.startswith("- "): + if current_entry: + result[current_section].append(current_entry) + current_entry = {} + kv = stripped[2:] + if ":" in kv: + key, val = kv.split(":", 1) + current_entry[key.strip()] = val.strip().strip('"') + elif ":" in stripped: + key, val = stripped.split(":", 1) + current_entry[key.strip()] = val.strip().strip('"') + + if current_entry and current_section: + result[current_section].append(current_entry) + return result + + +def _parse_junit_xml(source, source_name: str = "") -> list[_TestResult]: + """Parse JUnit XML, returning a list of _TestResult.""" + try: + tree = ET.parse(source) + except (ET.ParseError, OSError): + return [] + + root = tree.getroot() + if root.tag == "testsuites": + suites = root.findall("testsuite") + elif root.tag == "testsuite": + suites = [root] + else: + suites = root.findall(".//testsuite") + if not suites: + suites = [root] + + results = [] + for suite in suites: + suite_name = suite.get("name", "") + for tc in suite.findall("testcase"): + name = tc.get("name", "") + failure_el = tc.find("failure") + error_el = tc.find("error") + skipped_el = tc.find("skipped") + + status = "passed" + failure_message = failure_text = "" + error_message = error_text = "" + + if error_el is not None: + status = "error" + error_message = error_el.get("message", "") + error_text = error_el.text or "" + elif failure_el is not None: + status = "failed" + failure_message = failure_el.get("message", "") + failure_text = failure_el.text or "" + elif skipped_el is not None: + status = "skipped" + + sysout_el = tc.find("system-out") + system_out = (sysout_el.text or "") if sysout_el is not None else "" + + agg_passes = [] + agg_failures = [] + agg_skips = [] + if system_out and re.search( + r"^(?:passes|failures|skips):", system_out, re.MULTILINE + ): + parsed = _parse_system_out_yaml(system_out) + agg_passes = parsed["passes"] + agg_failures = parsed["failures"] + agg_skips = parsed["skips"] + + results.append(_TestResult( + name=name, + status=status, + suite_name=suite_name, + failure_message=failure_message, + failure_text=failure_text, + error_message=error_message, + error_text=error_text, + system_out=system_out, + agg_passes=agg_passes, + agg_failures=agg_failures, + agg_skips=agg_skips, + )) + return results + + +def _test_results_to_json(results: list[_TestResult]) -> list[dict]: + """Convert _TestResult list to JSON-serializable dicts (failures only).""" + output = [] + for r in results: + if r.status not in ("failed", "error"): + continue + d: dict = { + "name": r.name, + "status": r.status, + "suite_name": r.suite_name, + } + if r.failure_message: + d["failure_message"] = r.failure_message + if r.failure_text: + d["failure_text"] = r.failure_text + if r.error_message: + d["error_message"] = r.error_message + if r.error_text: + d["error_text"] = r.error_text + if r.agg_passes or r.agg_failures or r.agg_skips: + d["aggregated"] = { + "passes": r.agg_passes, + "failures": r.agg_failures, + "skips": r.agg_skips, + } + output.append(d) + return output + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Snapshot OpenShift payload data for offline analysis.", + epilog=( + "Examples:\n" + " python3 snapshot_payload.py 4.22.0-0.nightly-2026-02-25-152806\n" + " python3 snapshot_payload.py 4.18.0-0.ci-2026-01-15-114134 " + "--output-dir .work/snapshot\n" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "payload_tag", + help="Payload tag to snapshot (e.g., 4.22.0-0.nightly-2026-02-25-152806)", + ) + parser.add_argument( + "--output-dir", default="payload", + help="Base output directory (default: payload)", + ) + parser.add_argument( + "--max-chain", type=int, default=20, + help="Maximum backward chain depth (default: 20)", + ) + parser.add_argument( + "--workers", type=int, default=8, + help="Parallel workers for GitHub API calls (default: 8)", + ) + parser.add_argument( + "--no-junit", action="store_true", + help="Skip JUnit download, regression tracking", + ) + parser.add_argument( + "--sippy", action="store_true", + help="Use Sippy APIs instead of release controller (for historical payloads)", + ) + + args = parser.parse_args() + + try: + tag = PayloadTag.parse(args.payload_tag) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) + + _log(f"Payload: {tag.raw}") + _log(f"Version: {tag.version}") + _log(f"Stream: {tag.stream} ({tag.stream_name})") + _log(f"Arch: {tag.architecture}") + _log("") + + if not _check_gh_auth(): + _log("Warning: 'gh' CLI is not authenticated. PR data will not be fetched.") + _log("Run 'gh auth login' to enable PR diff/comment/job collection.\n") + + collect_junit = not args.no_junit + if collect_junit and not _check_gcloud(): + _log("Warning: 'gcloud' CLI not found. JUnit data will not be fetched.") + _log("Install gcloud SDK to enable JUnit download and regression tracking.\n") + collect_junit = False + + if args.sippy: + _log("Using Sippy APIs for release controller data.\n") + + try: + snapshotter = Snapshotter( + tag=tag, + output_dir=args.output_dir, + max_chain=args.max_chain, + workers=args.workers, + collect_junit=collect_junit, + use_sippy=args.sippy, + ) + snapshotter.run() + except urllib.error.HTTPError as e: + print(f"Error: HTTP {e.code}: {e.reason}", file=sys.stderr) + sys.exit(1) + except urllib.error.URLError as e: + print(f"Error: Cannot connect to release controller: {e.reason}", + file=sys.stderr) + sys.exit(1) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/plugins/ci/skills/stage-payload-reverts/SKILL.md b/plugins/ci/skills/stage-payload-reverts/SKILL.md index ef0aa39b7..ef7fc6b44 100644 --- a/plugins/ci/skills/stage-payload-reverts/SKILL.md +++ b/plugins/ci/skills/stage-payload-reverts/SKILL.md @@ -9,7 +9,7 @@ This skill automates the full revert-staging workflow for payload regressions: c ## When to Use This Skill -Use this skill when revert candidates have already been identified with high confidence by the `analyze-payload` skill. The caller passes all required context in-memory — this skill does not perform its own analysis. +Use this skill when revert candidates have already been identified with high confidence by the `payload-analysis` skill. The caller passes all required context in-memory — this skill does not perform its own analysis. **Inputs** (passed in-context by the caller): @@ -151,7 +151,7 @@ Find the existing "Recommended Reverts" section in the HTML. For each candidate - **Payload Jobs**: Link to the pr-payload-tests URL (e.g., `Payload Test`) - **Status**: Badge showing `Revert Staged` (use the `badge-rejected` class for visual consistency) -If the report has no "Recommended Reverts" section (all candidates scored below 85 during analysis), add one before the per-job details section, using the same HTML structure as described in `analyze-payload` Step 7.4. +If the report has no "Recommended Reverts" section (all candidates scored below 85 during analysis), add one before the per-job details section, using the same HTML structure as described in `payload-analysis` Step 7.4. ### Update autodl JSON @@ -171,5 +171,5 @@ Return results to the caller for inclusion in the report. - Related Skill: `payload-results-yaml` - Schema and operations for the payload results YAML - Related Skill: `revert-pr` - The git revert workflow (`plugins/ci/skills/revert-pr/SKILL.md`) - Related Skill: `trigger-payload-job` - Triggers payload jobs and collects URLs (`plugins/ci/skills/trigger-payload-job/SKILL.md`) -- Related Skill: `analyze-payload` - Identifies revert candidates (`plugins/ci/skills/analyze-payload/SKILL.md`) +- Related Skill: `payload-analysis` - Identifies revert candidates (`plugins/ci/skills/payload-analysis/SKILL.md`) - Related Command: `/ci:payload-revert` - Command for staging reverts (`plugins/ci/commands/payload-revert.md`)