fix(ci): make revert_scoring_accuracy judge reward correct results

.claude-plugin/marketplace.json

@@ -105,7 +105,7 @@
       "name": "ci",
       "source": "./plugins/ci",
       "description": "A plugin to work with OpenShift CI and analyze Prow job results",
-      "version": "0.0.89",
+      "version": "0.0.90",
       "category": "ci",
       "keywords": [
         "prow",

docs/index.html

@@ -541,7 +541,7 @@ <h1>ai-helpers</h1>
     {
       "name": "ci",
       "description": "Tools for working with OpenShift CI and analyzing Prow job results",
-      "version": "0.0.89",
+      "version": "0.0.90",
       "has_readme": true,
       "commands": [
         {

plugins/ci/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "ci",
   "description": "Tools for working with OpenShift CI and analyzing Prow job results",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "author": {
     "name": "github.com/openshift-eng"
   }

plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml

@@ -7,16 +7,11 @@ payload_completed_at: "2026-03-27T04:56:37Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/cluster-authentication-operator/pull/839"
     component: "cluster-authentication-operator"
-    min_confidence: 85
-    expected_confidence: 92
     description: "Ginkgo v2 migration broke test binary stdout output, corrupting JSON test listing in openshift-tests"
     expected_failing_jobs:
       - "metal-ipi-ovn-ipv4"
       - "metal-ipi-ovn-ipv6"
-
-expected_ci_config_issues:
   - pr_url: "https://github.com/openshift/release/pull/76852"
-    repo: "openshift/release"
     description: "Changed vSphere VCM default lease to multi-tenant, breaking UPI workflows that require single-tenant leases"
     expected_failing_jobs:
       - "install-analysis-all"

plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-05-30T12:55:22Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/origin/pull/31006"
     component: "tests"
-    min_confidence: 85
-    expected_confidence: 95
     description: "add oc-mirror to extensionBinaries — caused widespread test failures across serial, techpreview, metal, and upgrade jobs"
     expected_failing_jobs:
       - "aws-ovn-serial"

plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-05-07T20:17:14Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/cluster-network-operator/pull/2959"
     component: "cluster-network-operator"
-    min_confidence: 80
-    expected_confidence: 90
     description: "Added remaining CNO NetworkPolicies that blocked egress from cloud-network-config-controller, causing CrashLoopBackOff due to API server timeout"
     expected_failing_jobs:
       - "aws-ovn-upgrade"

plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml

@@ -7,16 +7,12 @@ payload_completed_at: "2026-03-20T12:21:33Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/cluster-ingress-operator/pull/1354"
     component: "cluster-ingress-operator"
-    min_confidence: 85
-    expected_confidence: 95
     description: "Replaced OLM-based Istio install with Sail Library, causing GatewayAPIController failures"
     expected_failing_jobs:
       - "aws-ovn-techpreview"
       - "aws-ovn-techpreview-serial"
   - pr_url: "https://github.com/openshift/oc/pull/2232"
     component: "oc"
-    min_confidence: 85
-    expected_confidence: 90
     description: "Changed manifest reading to in-memory, causing nil pointer dereference in oc adm release mirror"
     expected_failing_jobs:
       - "metal-ipi-ovn-ipv6"

plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-03-31T10:56:34Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/cloud-credential-operator/pull/978"
     component: "cloud-credential-operator"
-    min_confidence: 85
-    expected_confidence: 95
     description: "Set operator condition to Progressing when pod identity webhook pods are updating, causing CCO Progressing=True during MCO rolling phase"
     expected_failing_jobs:
       - "aws-ovn-upgrade"

plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml

@@ -7,23 +7,12 @@ payload_completed_at: "2026-03-31T22:55:24Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/hypershift/pull/7790"
     component: "hypershift"
-    min_confidence: 85
-    expected_confidence: 95
     description: "Added guest cluster metrics forwarder feature and EnsureMetricsForwarderWorking test; HCCO silently skips deployment creation when prerequisites missing"
     expected_failing_jobs:
       - "e2e-aks"
       - "e2e-aws"
-  - pr_url: "https://github.com/openshift/hypershift/pull/8138"
-    component: "hypershift"
-    min_confidence: 40
-    expected_confidence: 50
-    description: "Reverted AWS guest resource cleanup timeout extension, causing intermittent teardown timeouts"
-    expected_failing_jobs:
-      - "e2e-aws"
 
 notes: >
   Payload 4.22.0-0.ci-2026-03-31-170515 was rejected with 2 failed
-  hypershift blocking jobs. Primary revert candidate: hypershift#7790
-  (score 95) introduced the metrics forwarder feature and its test.
-  Secondary candidate: hypershift#8138 (score 50) reduced cleanup
-  timeout causing intermittent teardown failures.
+  hypershift blocking jobs. Revert candidate hypershift#7790 introduced
+  the metrics forwarder feature and its test.

plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-03-19T01:17:31Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/operator-framework-olm/pull/1256"
     component: "operator-lifecycle-manager"
-    min_confidence: 85
-    expected_confidence: 95
     description: "OLM PR #1256 added e2e.Logf calls in the olmv0-tests-ext client-setup path (getAdminContextNameCached in tests-extension/test/qe/util/client.go); the klog INFO lines (starting with 'I') hit stdout during OTE test listing and corrupt the JSON stream"
     expected_failing_jobs:
       - "metal-ipi-ovn-ipv4"

plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-04-28T03:07:53Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/cluster-monitoring-operator/pull/2814"
     component: "cluster-monitoring-operator"
-    min_confidence: 85
-    expected_confidence: 95
     description: "Minimal and telemetry CP monitors change caused monitoring test regressions"
     expected_failing_jobs:
       - "aws-ovn-techpreview-serial"

plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-06-17T17:06:42Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/release/pull/78140"
     component: "openshift/release"
-    min_confidence: 85
-    expected_confidence: 95
     description: "step-registry mirror migration to Cloudflare R2 broke gather-must-gather curl in bare-metal proxy environments"
     expected_failing_jobs:
       - "metal-ipi-ovn-ipv4"

plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-07-23T07:44:48Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/oc/pull/2279"
     component: "cli"
-    min_confidence: 85
-    expected_confidence: 100
     description: "oc adm upgrade recommend output changed while the TechPreview serial tests still required the removed conditions preamble"
     expected_failing_jobs:
       - "aws-ovn-techpreview-serial"

plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml

@@ -7,8 +7,6 @@ payload_completed_at: "2026-07-25T16:05:26Z"
 expected_candidates:
   - pr_url: "https://github.com/openshift/cluster-version-operator/pull/1427"
     component: "cluster-version-operator"
-    min_confidence: 85
-    expected_confidence: 85
     description: "OTA-1997 changed the CVO's own Deployment manifest to reference a Go template field (agentic-skills image) that only the 5.0 CVO understands; during a 4.22->5.0 major upgrade the 4.22 CVO cannot render it and silently skips the CVO Deployment manifest, stranding a 4.22 CVO that re-applies 4.22 operator Deployments"
     expected_failing_jobs:
       - "aws-ovn-upgrade"

plugins/ci/evals/eval-payload-analysis.yaml

@@ -87,14 +87,10 @@ dataset:
         judges; used by the point_in_time_integrity judge to detect
         post-hoc leakage. Must always match the input.yaml value.
       - 'expected_candidates': list of expected revert candidates with pr_url,
-        component, min_confidence, expected_confidence, description,
+        component, description,
         expected_failing_jobs. An explicit empty list means the correct
         analysis recommends NO revert — any candidate at or above the
         revert threshold is a false positive.
-      - 'expected_ci_config_issues' (optional): openshift/release CI-config
-        culprits (pr_url, repo, description, expected_failing_jobs). These
-        never appear in payload changelogs, so identifying them requires
-        the step-registry lookup path.
       - 'notes': free text context
 
     Snapshots are stored as tar.gz archives in
@@ -449,14 +445,15 @@ judges:
     if: "annotations.get('expected_candidates') is not None or annotations.get('has_revert_candidates') is False"
     description: |
       Deterministic check that every expected revert candidate appears in the
-      results YAML at or above its min_confidence. A no-revert case is a valid
+      results YAML at or above the revert threshold (85). A no-revert case is a valid
       expectation and passes vacuously (nothing to miss); the false-positive
       direction is enforced separately by no_unexpected_reverts. A case declares
       no-revert either explicitly (expected_candidates: []) or implicitly
       (has_revert_candidates: false with the key omitted) — both are scored.
       Skipped only when a case defines neither expectation.
     check: |
       import yaml, os
+      REVERT_THRESHOLD = 85
       all_f = {**outputs.get("files", {}), **outputs.get("modified_files", {})}
       yf = [v for k, v in all_f.items() if os.path.basename(k).startswith("payload-results-") and k.endswith(".yaml")]
       if not yf:
@@ -479,8 +476,8 @@ judges:
               score = float(got.get("confidence_score", 0))
           except (TypeError, ValueError):
               score = 0.0
-          if score < float(exp.get("min_confidence", 0)):
-              low.append(f"{url} scored {score:g} < min {exp.get('min_confidence')}")
+          if score < REVERT_THRESHOLD:
+              low.append(f"{url} scored {score:g} < revert threshold {REVERT_THRESHOLD}")
           # Bidirectional substring match: the model may legitimately report
           # either the short job stem (e2e-aws-ovn) or the full periodic name
           # (periodic-ci-...-e2e-aws-ovn), and annotations may use either form.
@@ -506,8 +503,8 @@ judges:
   - name: no_unexpected_reverts
     description: |
       Deterministic false-revert gate: no candidate may reach the revert
-      threshold (>= 85) unless its pr_url is listed in expected_candidates
-      or expected_ci_config_issues. RHCOS RPM candidates are exempt — the
+      threshold (>= 85) unless its pr_url is listed in expected_candidates.
+      RHCOS RPM candidates are exempt — the
       skill scores them but never proposes reverting them. This is the hard
       check behind the adversarial false-revert cases.
     check: |
@@ -519,9 +516,8 @@ judges:
       data = yaml.safe_load(yf[0]) or {}
       ann = outputs.get("annotations", {})
       allowed = set()
-      for key in ("expected_candidates", "expected_ci_config_issues"):
-          for c in ann.get(key, []) or []:
-              allowed.add(str(c.get("pr_url", "")).rstrip("/"))
+      for c in ann.get("expected_candidates", []) or []:
+          allowed.add(str(c.get("pr_url", "")).rstrip("/"))
       REVERT_THRESHOLD = 85
       rogue = []
       for c in data.get("candidates", []) or []:
@@ -695,37 +691,33 @@ judges:
 
       {{ annotations }}
 
-      Two failure directions matter equally:
-      - Missed reverts: every entry in expected_candidates must be identified
-        at or above its min_confidence and linked to its expected_failing_jobs.
-      - False reverts: when expected_candidates is empty, the correct output
-        has NO candidate at or above the revert threshold. Recommending a
-        revert in that situation is the worst outcome, not a near miss.
+      Score the revert decision: which candidate(s) to revert, the confidence
+      assigned to each, and their failing-job linkage.
+
+      A candidate scored 85 or above is a proposed revert; below 85 is a
+      low-confidence maybe that triggers no action and is ignored.
+
+      - Every expected revert must score 85 or above; below that is a mistake.
+      - An extra candidate counts as a false revert only at 85 or above.
 
       Evaluate on a 1-5 scale:
 
-      Score 1: Wrong verdict in either direction — a candidate at/above the
-               revert threshold that is not in expected_candidates (including
-               any revert recommendation when expected_candidates is empty),
-               OR all expected candidates missed, OR no scoring applied at all.
-      Score 2: Some expected candidates identified but confidence scores are way
-               off (>20 points from expected), key failing-job linkages are
-               missing, or an unexpected candidate is scored just below
-               threshold on weak evidence.
-      Score 3: Correct revert/no-revert verdict, but scores are off by 10-20
-               points, rubric signals are not itemized per candidate, or
-               dismissals of non-candidates are asserted without cited evidence.
-      Score 4: Correct verdict with scores within 10 points of
-               expected_confidence. Rubric correctly applied with itemized
-               breakdown. Most expected failing jobs correctly linked.
-               Infrastructure failures correctly distinguished from product
-               failures.
-      Score 5: Correct verdict with scores within 5 points of
-               expected_confidence. Rubric meticulously applied — each signal
-               justified with specific evidence per the attached SKILL.md.
-               All expected failing jobs correctly linked. For no-revert cases,
-               each dismissed candidate's exoneration is backed by concrete
-               evidence rather than assertion.
+      Score 5: A 100% correct result. Every expected revert is identified and
+               scored 85 or above with the correct failing-job links, and no
+               unexpected revert is proposed — or no revert is proposed when
+               none is expected.
+      Score 4: Every expected revert is identified with the correct failing-job
+               links and no unexpected reverts, but with one small mistake —
+               one expected revert scored just below 85, or one factual error
+               in the failing-job linkage.
+      Score 3: Every expected revert is identified with the correct failing-job
+               links and no unexpected reverts, but with larger scoring errors
+               such as multiple confidence scores being wrong.
+      Score 2: At least one expected revert is missed, or a false revert is
+               proposed, but the result is still supported by some investigation
+               and evidence.
+      Score 1: Expected reverts are missed or fabricated with little or no
+               supporting investigation or evidence — the result is wrong.
 
   - name: point_in_time_integrity
     score_range: [1, 5]