From e0ca9b0a73406852c6e42632a21711cc6080a6d8 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 14 May 2026 23:06:57 -0400 Subject: [PATCH] Improve analyze-payload scoring precision with three targeted signals - Add test-introduction match signal (+40) to rubric: near-certain causation when a PR introduces a test that immediately fails - Add sub-component resolution guidance: score component exclusivity at sub-component level (e.g., control-plane-operator/ vs test/) for repos with 3+ PRs, fixing misidentification in Case 008 - Add cloud platform infrastructure triage with -20 adjustment for platform-specific failures lacking a causal mechanism from PR changes, reducing false positive revert recommendations - Bump ci plugin version to 0.0.40 Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/marketplace.json | 6 ++++-- docs/data.json | 2 +- plugins/ci/.claude-plugin/plugin.json | 2 +- plugins/ci/skills/analyze-payload/SKILL.md | 12 +++++++++++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index bd587969a..0a9fda5ce 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -3,7 +3,9 @@ "owner": { "name": "openshift-eng" }, - "allowCrossMarketplaceDependenciesOn": ["claude-plugins-official"], + "allowCrossMarketplaceDependenciesOn": [ + "claude-plugins-official" + ], "plugins": [ { "name": "git", @@ -27,7 +29,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.39" + "version": "0.0.40" }, { "name": "teams", diff --git a/docs/data.json b/docs/data.json index fb62d802e..7824d115f 100644 --- a/docs/data.json +++ b/docs/data.json @@ -607,7 +607,7 @@ "name": "Trigger Payload Job" } ], - "version": "0.0.39" + "version": "0.0.40" }, { "commands": [ diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index 9d5c6014e..5803b85cd 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.39", + "version": "0.0.40", "author": { "name": "openshift" } diff --git a/plugins/ci/skills/analyze-payload/SKILL.md b/plugins/ci/skills/analyze-payload/SKILL.md index 78c0616e6..c5427faf1 100644 --- a/plugins/ci/skills/analyze-payload/SKILL.md +++ b/plugins/ci/skills/analyze-payload/SKILL.md @@ -199,6 +199,13 @@ Wait for all subagents to complete and collect their analysis results. For each For each failed job, cross-reference the failure analysis from the subagent with the candidate PRs from the originating payload. Additionally, if a subagent traced the root cause to a PR outside the payload (e.g., an `openshift/release` PR that modified a CI step registry script), include that PR as a candidate — it is a regression like any other and should be scored and treated the same way as payload PRs. +**Infrastructure and platform-specific failure triage**: Before scoring candidate PRs, check whether the failure is caused by cloud provider infrastructure rather than code changes. Key indicators: +- **Cloud provider API errors**: Azure Graph API auth failures, AWS STS failures, GCP quota exhaustion — these are provider-side issues that resolve without code changes +- **Single-platform failures**: If a failure occurs on only one cloud platform (e.g., only AKS) and the candidate PR doesn't directly modify platform-specific code, the causal chain requires stronger evidence +- **Transient resolution**: If the same test passed on the same platform in a very recent payload with no relevant PR changes between the pass and failure + +Apply a negative adjustment: **Platform-specific failure without mechanism**: -20 (failure occurs only on one cloud platform and the causal chain from PR to platform-specific behavior requires multiple speculative steps) + Score each (failed job, candidate PR) pair using the following weighted rubric: | Signal | Weight | Criteria | @@ -209,8 +216,11 @@ Score each (failed job, candidate PR) pair using the following weighted rubric: | Multi-job correlation | +10 | The same PR is a candidate for failures in multiple independent jobs — the more jobs that point to the same PR, the stronger the signal | | Presubmit coverage gap | +10 | The failing job tests a scenario (upgrade, FIPS, SNO, techpreview, etc.) that wasn't covered by the PR's presubmit tests | | Single candidate | +10 | Only one PR landed in the originating payload that touches the affected component | +| Test-introduction match | +40 | The candidate PR introduced a new test (visible in its diff) and that specific test is now failing. A PR that both creates a feature and its test, where the test immediately fails, is near-certain evidence of causation. | + +The maximum possible score is 170, but scores above 100 should be capped at 100. Record the numeric score for each (job, candidate PR) pair alongside the qualitative rationale. -The maximum possible score is 130, but scores above 100 should be capped at 100. Record the numeric score for each (job, candidate PR) pair alongside the qualitative rationale. +**Sub-component resolution**: For repos with 3+ PRs in the originating payload, examine each PR's changed file paths to identify sub-components. Score component exclusivity based on sub-component overlap rather than repo-level overlap. For example, in the `hypershift` repo, PRs modifying `control-plane-operator/` vs `hypershift-operator/` vs `test/` are distinct sub-components. #### 6.2: Propose Revert Candidates