feat(jira): add agent-eval-harness eval for jira:create skill

plugins/jira/evals/cases/create/case-001-story-cntrlplane-basic/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Story
+expected_project_key: CNTRLPLANE
+expected_conventions: [cntrlplane]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: customfield_10855
+expected_version_format: array
+expected_affects_version: null
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: story
+notes: >
+  Basic CNTRLPLANE story with --version flag. Tests version normalization
+  (4.22 → openshift-4.22) and array format for customfield_10855
+  [{"id": "PLACEHOLDER_VERSION_ID"}]. Description should follow user story
+  template (As a / I want / so that) with acceptance criteria.

plugins/jira/evals/cases/create/case-001-story-cntrlplane-basic/input.yaml

@@ -0,0 +1,8 @@
+type: story
+project_key: CNTRLPLANE
+summary: "Add user dashboard for cluster metrics"
+flags: "--version 4.22"
+interactive_responses: |
+  Who: cluster administrators
+  What: view real-time cluster metrics in a unified dashboard
+  Why: reduce time spent switching between monitoring tools

plugins/jira/evals/cases/create/case-002-bug-ocpbugs-basic/annotations.yaml

@@ -0,0 +1,20 @@
+expected_issue_type: Bug
+expected_project_key: OCPBUGS
+expected_conventions: [ocpbugs]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: customfield_10855
+expected_version_format: string
+expected_affects_version: [{"name": "4.21"}]
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: bug
+notes: >
+  OCPBUGS bug with both version fields. Tests that Affects Version uses
+  versions field as [{"name": "4.21"}] and Target Version uses
+  customfield_10855 as plain string "openshift-4.21" (NOT array format).
+  Description should follow bug template with problem description, version,
+  reproducibility, steps, actual/expected results.

plugins/jira/evals/cases/create/case-002-bug-ocpbugs-basic/input.yaml

@@ -0,0 +1,15 @@
+type: bug
+project_key: OCPBUGS
+summary: "API server returns 500 when listing pods with label selector"
+flags: ""
+interactive_responses: |
+  Affects version: 4.21
+  Target version: openshift-4.21
+  Reproducibility: Always
+  Steps to reproduce:
+    1. Create a pod with label app=test
+    2. Run oc get pods -l app=test
+    3. Observe 500 error in response
+  Actual results: API server returns HTTP 500 Internal Server Error
+  Expected results: API server returns the list of matching pods
+  Additional info: Occurs on clusters with more than 100 namespaces

plugins/jira/evals/cases/create/case-003-epic-cntrlplane/annotations.yaml

@@ -0,0 +1,21 @@
+expected_issue_type: Epic
+expected_project_key: CNTRLPLANE
+expected_conventions: [cntrlplane]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: null
+expected_version_format: null
+expected_affects_version: null
+expected_custom_fields:
+  customfield_10011: "Multi-cluster metrics aggregation"
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: epic
+notes: >
+  CNTRLPLANE epic with parent linking. Tests Epic Name custom field
+  (customfield_10011) which must match the summary exactly. Also tests
+  parent linking via additional_fields.parent.key = "CNTRLPLANE-500".
+  Description should follow epic template with objective, scope, and
+  acceptance criteria.

plugins/jira/evals/cases/create/case-003-epic-cntrlplane/input.yaml

@@ -0,0 +1,12 @@
+type: epic
+project_key: CNTRLPLANE
+summary: "Multi-cluster metrics aggregation"
+flags: "--parent CNTRLPLANE-500"
+interactive_responses: |
+  Objective: Enable aggregation of metrics across multiple managed clusters into a unified view
+  Scope: Prometheus federation, custom metrics pipeline, dashboard integration
+  Acceptance criteria:
+    1. Metrics from up to 50 managed clusters aggregated within 5 minutes
+    2. Custom dashboards show cross-cluster resource utilization
+    3. Alert rules can reference metrics from any managed cluster
+  Timeframe: 2-3 sprints

plugins/jira/evals/cases/create/case-004-story-hypershift-aro/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Story
+expected_project_key: CNTRLPLANE
+expected_conventions: [cntrlplane, hypershift]
+expected_labels: ["ai-generated-jira", "aro-hcp"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: "HyperShift / ARO"
+expected_version_field: customfield_10855
+expected_version_format: array
+expected_affects_version: null
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: story
+notes: >
+  CNTRLPLANE story with HyperShift ARO conventions layered. Tests that
+  "ARO HCP" in the summary triggers auto-detection of component
+  "HyperShift / ARO" and label "aro-hcp". Both cntrlplane and hypershift
+  conventions should be loaded. Version uses CNTRLPLANE array format.

plugins/jira/evals/cases/create/case-004-story-hypershift-aro/input.yaml

@@ -0,0 +1,8 @@
+type: story
+project_key: CNTRLPLANE
+summary: "Enable private API server for ARO HCP clusters"
+flags: "--version 4.22"
+interactive_responses: |
+  Who: ARO HCP cluster administrators
+  What: configure private API server endpoints for hosted control planes on Azure
+  Why: meet enterprise network isolation requirements for regulated workloads

plugins/jira/evals/cases/create/case-005-bug-ocpbugs-hypershift-rosa/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Bug
+expected_project_key: OCPBUGS
+expected_conventions: [ocpbugs, hypershift]
+expected_labels: ["ai-generated-jira", "rosa-hcp"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: "HyperShift / ROSA"
+expected_version_field: customfield_10855
+expected_version_format: string
+expected_affects_version: [{"name": "4.21"}]
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: bug
+notes: >
+  OCPBUGS bug with both ocpbugs and hypershift conventions layered. Tests
+  that "ROSA HCP" in summary triggers component "HyperShift / ROSA" and
+  label "rosa-hcp". Version fields use OCPBUGS format: versions as
+  [{"name": "4.21"}] and customfield_10855 as string "openshift-4.22".

plugins/jira/evals/cases/create/case-005-bug-ocpbugs-hypershift-rosa/input.yaml

@@ -0,0 +1,15 @@
+type: bug
+project_key: OCPBUGS
+summary: "ROSA HCP cluster upgrade fails with OLM operator timeout"
+flags: ""
+interactive_responses: |
+  Affects version: 4.21
+  Target version: openshift-4.22
+  Reproducibility: Sometimes
+  Steps to reproduce:
+    1. Create ROSA HCP cluster on version 4.21
+    2. Initiate upgrade to 4.22
+    3. Wait for upgrade to complete
+  Actual results: Upgrade stalls at 60% with OLM operator timeout after 30 minutes
+  Expected results: Upgrade completes successfully within 45 minutes
+  Additional info: Affects approximately 30% of upgrade attempts on clusters with 5+ installed operators

plugins/jira/evals/cases/create/case-006-task-cntrlplane-plain/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Task
+expected_project_key: CNTRLPLANE
+expected_conventions: [cntrlplane]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: null
+expected_version_format: null
+expected_affects_version: null
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: task
+notes: >
+  Plain CNTRLPLANE task with no team conventions beyond project level.
+  No version or component specified. Description should follow task
+  template with what/why and acceptance criteria. Tests that the skill
+  correctly identifies this as non-user-facing technical work.

plugins/jira/evals/cases/create/case-006-task-cntrlplane-plain/input.yaml

@@ -0,0 +1,11 @@
+type: task
+project_key: CNTRLPLANE
+summary: "Update API documentation for cluster provisioning endpoints"
+flags: ""
+interactive_responses: |
+  What: Update the API documentation to reflect recent changes to the cluster provisioning endpoints
+  Why: Current documentation is outdated after the v2 API migration
+  Acceptance criteria:
+    1. All provisioning endpoints documented with request/response examples
+    2. Deprecation notices added for v1 endpoints
+    3. OpenAPI spec updated and validated

plugins/jira/evals/cases/create/case-007-feature-cntrlplane/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Feature
+expected_project_key: CNTRLPLANE
+expected_conventions: [cntrlplane]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: null
+expected_version_format: null
+expected_affects_version: null
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: feature
+notes: >
+  CNTRLPLANE feature with strategic-level content. No version or component.
+  Description should follow feature template with market problem, strategic
+  value, and success criteria. Tests that the skill applies the feature
+  reference guide correctly.

plugins/jira/evals/cases/create/case-007-feature-cntrlplane/input.yaml

@@ -0,0 +1,12 @@
+type: feature
+project_key: CNTRLPLANE
+summary: "Advanced search capabilities for cluster fleet management"
+flags: ""
+interactive_responses: |
+  Market problem: Platform engineers managing 100+ clusters cannot efficiently find and filter clusters by custom attributes, leading to operational delays
+  Strategic value: Reduces mean time to cluster identification from 15 minutes to under 30 seconds, directly impacting incident response time
+  Success criteria:
+    - 80% of fleet management users adopt search within 3 months
+    - Average search latency under 500ms for fleets of 500+ clusters
+    - Support for at least 20 filterable attributes including custom labels
+  Timeline: Spans releases 4.22 through 4.23

plugins/jira/evals/cases/create/case-008-feature-request-rfe/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Feature Request
+expected_project_key: RFE
+expected_conventions: []
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: null
+expected_version_format: null
+expected_affects_version: null
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: feature-request
+notes: >
+  RFE project feature request. Tests default project key (RFE) and
+  issue type name ("Feature Request"). No project-specific conventions
+  apply. Description should follow the 4-question feature request
+  workflow (title, nature, business requirements, affected components).

plugins/jira/evals/cases/create/case-008-feature-request-rfe/input.yaml

@@ -0,0 +1,9 @@
+type: feature-request
+project_key: RFE
+summary: "Support custom SSL certificates for ROSA HCP ingress"
+flags: ""
+interactive_responses: |
+  Title: Support custom SSL certificates for ROSA HCP ingress
+  Nature: Customers need to use their own CA-signed certificates for ingress controllers on ROSA HCP clusters instead of the default Let's Encrypt certificates
+  Business requirements: Enterprise customers in regulated industries (finance, healthcare) require custom certificates to meet compliance requirements. Currently they must use workarounds that break during cluster upgrades.
+  Affected packages/components: Ingress, Networking

plugins/jira/evals/cases/create/case-009-story-gcp-hcp/annotations.yaml

@@ -0,0 +1,22 @@
+expected_issue_type: Story
+expected_project_key: GCP
+expected_conventions: [gcp-hcp]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: "hypershift-operator-gcp"
+expected_version_field: null
+expected_version_format: null
+expected_affects_version: null
+expected_custom_fields:
+  customfield_10028: 2.0
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: gcp-story
+notes: >
+  GCP project story with GCP-HCP team conventions. Tests GCP-specific
+  template (user story + context + requirements + technical approach +
+  dependencies + acceptance criteria checklist). Should auto-estimate
+  Story Points (customfield_10028) using the GCP sizing guide — adding
+  a Prometheus metric is a 2-point task per the GCP examples. Component
+  provided via flag.

plugins/jira/evals/cases/create/case-009-story-gcp-hcp/input.yaml

@@ -0,0 +1,16 @@
+type: story
+project_key: GCP
+summary: "Add Prometheus metric for hosted control plane CPU utilization"
+flags: "--component hypershift-operator-gcp"
+interactive_responses: |
+  Who: GCP HCP platform operations team
+  What: expose a Prometheus metric tracking CPU utilization per hosted control plane
+  Why: enable proactive scaling decisions and capacity planning for the management cluster
+  Context: Currently CPU utilization is only visible through GCP Cloud Monitoring, requiring context switching
+  Requirements: Metric must be available within 30 seconds of collection, labeled by cluster ID
+  Technical approach: Add a custom collector to the hypershift-operator-gcp that scrapes cgroup metrics
+  Dependencies: Requires prometheus-operator CRD access on the management cluster
+  Acceptance criteria:
+    - Metric hcp_cpu_utilization_ratio exposed on /metrics endpoint
+    - Grafana dashboard updated with CPU utilization panel
+    - Alert rule fires when utilization exceeds 80% for 5 minutes

plugins/jira/evals/cases/create/case-010-bug-ocpbugs-version-flag/annotations.yaml

@@ -0,0 +1,19 @@
+expected_issue_type: Bug
+expected_project_key: OCPBUGS
+expected_conventions: [ocpbugs]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: customfield_10855
+expected_version_format: string
+expected_affects_version: [{"name": "4.21"}]
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: bug
+notes: >
+  OCPBUGS bug with --version flag. Tests that the version flag is applied
+  correctly with OCPBUGS conventions: customfield_10855 as plain string
+  "openshift-4.21" and versions as [{"name": "4.21"}]. No component
+  specified. Description should follow bug template.

plugins/jira/evals/cases/create/case-010-bug-ocpbugs-version-flag/input.yaml

@@ -0,0 +1,15 @@
+type: bug
+project_key: OCPBUGS
+summary: "NodePort service intermittently unreachable on OVN-Kubernetes clusters"
+flags: "--version 4.21"
+interactive_responses: |
+  Affects version: 4.21
+  Target version: openshift-4.21
+  Reproducibility: Sometimes
+  Steps to reproduce:
+    1. Create a NodePort service on an OVN-Kubernetes cluster
+    2. Access the service via node IP and allocated port
+    3. Repeat access every 5 seconds for 10 minutes
+  Actual results: Approximately 5% of requests fail with connection refused
+  Expected results: All requests succeed consistently
+  Additional info: Issue is specific to OVN-Kubernetes; does not reproduce with OpenShift SDN

plugins/jira/evals/cases/create/case-011-story-hypershift-rosa/annotations.yaml

@@ -0,0 +1,18 @@
+expected_issue_type: Story
+expected_project_key: CNTRLPLANE
+expected_conventions: [cntrlplane, hypershift]
+expected_labels: ["ai-generated-jira", "rosa-hcp"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: "HyperShift / ROSA"
+expected_version_field: customfield_10855
+expected_version_format: array
+expected_affects_version: null
+expected_custom_fields: {}
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: story
+notes: >
+  CNTRLPLANE story with ROSA HCP keywords triggering HyperShift conventions.
+  Tests auto-detection of "ROSA HCP" → component "HyperShift / ROSA" and
+  label "rosa-hcp". Version uses CNTRLPLANE array format.

plugins/jira/evals/cases/create/case-011-story-hypershift-rosa/input.yaml

@@ -0,0 +1,8 @@
+type: story
+project_key: CNTRLPLANE
+summary: "Implement node auto-repair for ROSA HCP worker nodes"
+flags: "--version 4.22"
+interactive_responses: |
+  Who: ROSA HCP cluster operators
+  What: automatically detect and replace unhealthy worker nodes in hosted clusters on AWS
+  Why: minimize manual intervention and reduce cluster downtime caused by node failures

plugins/jira/evals/cases/create/case-012-epic-gcp-hcp/annotations.yaml

@@ -0,0 +1,20 @@
+expected_issue_type: Epic
+expected_project_key: GCP
+expected_conventions: [gcp-hcp]
+expected_labels: ["ai-generated-jira"]
+expected_security: "Red Hat Employee"
+expected_content_format: markdown
+expected_component: null
+expected_version_field: null
+expected_version_format: null
+expected_affects_version: null
+expected_custom_fields:
+  customfield_10011: "Implement CMEK support for GCP hosted control plane storage"
+expected_validation_warnings: []
+expected_creation_blocked: false
+description_template: gcp-epic
+notes: >
+  GCP project epic with GCP-HCP team conventions. Tests Epic Name custom
+  field (customfield_10011) matching summary, and GCP epic template (use
+  case/context, current/desired state, scope, story breakdown, acceptance
+  criteria). No story points on epics.