Add agent-eval step registry and ai-helpers eval presubmit

ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml

@@ -17,6 +17,16 @@ resources:
     requests:
       cpu: 100m
       memory: 200Mi
+tests:
+- always_run: false
+  as: eval-payload-analysis
+  optional: true
+  steps:
+    env:
+      EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml
+      EVAL_MODEL: claude-opus-4-6
+      EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh
+    workflow: openshift-claude-agent-eval
 zz_generated_metadata:
   branch: main
   org: openshift-eng

ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml

@@ -1,5 +1,80 @@
 presubmits:
   openshift-eng/ai-helpers:
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build13
+    context: ci/prow/eval-payload-analysis
+    decorate: true
+    decoration_config:
+      sparse_checkout_files:
+      - images/Dockerfile
+    labels:
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis
+    optional: true
+    rerun_command: /test eval-payload-analysis
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --target=eval-payload-analysis
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )eval-payload-analysis,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:

ci-operator/step-registry/openshift/claude/agent-eval/OWNERS

@@ -0,0 +1,14 @@
+approvers:
+- bentito
+- bryan-cox
+- cblecker
+- enxebre
+- prashanth684
+- stbenjam
+reviewers:
+- bentito
+- bryan-cox
+- cblecker
+- enxebre
+- prashanth684
+- stbenjam

ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh

@@ -0,0 +1,215 @@
+#!/bin/bash
+#
+# Run agent-eval-harness against Claude Code skills.
+#
+# Required env:
+#   EVAL_CONFIG  -- path to eval.yaml (relative to repo root)
+#
+# Optional env:
+#   EVAL_MODEL        -- model for the skill under test (default: claude-sonnet-4-6)
+#   EVAL_BASELINE     -- run-id of a previous run to compare against
+#   EVAL_EXTRA_ARGS   -- additional args passed to /eval-run
+#   EVAL_SETUP_SCRIPT -- script to run before eval (e.g. snapshot extraction)
+#   CLAUDE_MODEL      -- model for the eval harness orchestrator (default: claude-sonnet-4-6)
+#   MLFLOW_PORT       -- port for local MLflow server (default: 5000)
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+echo "Starting claude-agent-eval"
+
+# The repo is at /opt/ai-helpers; WORKDIR is /workspace
+cd /opt/ai-helpers
+
+echo "Config: ${EVAL_CONFIG}"
+echo "Skill model: ${EVAL_MODEL}"
+
+# -----------------------------------------------------------------------
+# Install dependencies
+# -----------------------------------------------------------------------
+echo "Installing mlflow..."
+export PATH="$HOME/.local/bin:$PATH"
+# mlflow 3.x requires sqlite >= 3.36.0; RHEL 9 ships 3.34.1
+python3 -m pip install --quiet 'mlflow==2.20.2'
+echo "mlflow installed."
+
+# Start local MLflow server in background
+echo "Starting local MLflow server on port ${MLFLOW_PORT}..."
+export MLFLOW_TRACKING_URI="http://127.0.0.1:${MLFLOW_PORT}"
+mlflow server --port "${MLFLOW_PORT}" --host 127.0.0.1 &
+MLFLOW_PID=$!
+
+for i in $(seq 1 30); do
+    if curl -sf "http://127.0.0.1:${MLFLOW_PORT}/health" >/dev/null 2>&1; then
+        echo "MLflow server ready (PID ${MLFLOW_PID})."
+        break
+    fi
+    if ! kill -0 "${MLFLOW_PID}" 2>/dev/null; then
+        echo "ERROR: MLflow server crashed."
+        exit 1
+    fi
+    if [[ $i -eq 30 ]]; then
+        echo "ERROR: MLflow server did not become ready in 30s."
+        exit 1
+    fi
+    sleep 1
+done
+
+# -----------------------------------------------------------------------
+# Verify eval config exists
+# -----------------------------------------------------------------------
+if [[ ! -f "${EVAL_CONFIG}" ]]; then
+    echo "ERROR: EVAL_CONFIG not found at ${EVAL_CONFIG}"
+    exit 1
+fi
+
+# -----------------------------------------------------------------------
+# Run optional setup script (e.g. extract snapshots, populate fixtures)
+# -----------------------------------------------------------------------
+if [[ -n "${EVAL_SETUP_SCRIPT}" ]]; then
+    if [[ ! -f "${EVAL_SETUP_SCRIPT}" ]]; then
+        echo "ERROR: EVAL_SETUP_SCRIPT not found: ${EVAL_SETUP_SCRIPT}"
+        exit 1
+    fi
+    echo ""
+    echo "=== Running setup script: ${EVAL_SETUP_SCRIPT} ==="
+    EVAL_SNAPSHOT_DIR=$(bash "${EVAL_SETUP_SCRIPT}")
+    export EVAL_SNAPSHOT_DIR
+    echo "Snapshot dir: ${EVAL_SNAPSHOT_DIR}"
+fi
+
+# -----------------------------------------------------------------------
+# Install plugins
+# -----------------------------------------------------------------------
+echo ""
+echo "=== Installing plugins ==="
+EVAL_HARNESS_DIR="/tmp/agent-eval-harness"
+git clone --depth 1 https://github.com/opendatahub-io/agent-eval-harness.git "${EVAL_HARNESS_DIR}"
+echo "agent-eval-harness cloned."
+
+# -----------------------------------------------------------------------
+# Artifact copy trap
+# -----------------------------------------------------------------------
+copy_artifacts() {
+    echo "Copying eval artifacts..."
+    if [[ -d "${AGENT_EVAL_RUNS_DIR:-eval/runs}" ]]; then
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "report.html" -exec cp {} "${ARTIFACT_DIR}/eval-report-summary.html" \; 2>/dev/null || true
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "summary.yaml" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "run_result.json" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+    fi
+    find . -name "*-summary.html" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+
+    # Copy MLflow data
+    if [[ -d "mlruns" ]]; then
+        tar -czf "${ARTIFACT_DIR}/mlflow-data.tar.gz" mlruns/ 2>/dev/null || true
+    fi
+
+    # Archive Claude session for continue-session support
+    CLAUDE_HOME="/home/claude/.claude"
+    if [[ -d "${CLAUDE_HOME}/projects" ]]; then
+        echo "Archiving Claude session logs..."
+        tar -czf "${ARTIFACT_DIR}/claude-sessions-$(date +%Y%m%d-%H%M%S).tar.gz" \
+            -C "${CLAUDE_HOME}" projects/ 2>/dev/null && \
+            touch "${SHARED_DIR}/claude-session-available" || true
+    fi
+
+    # Stop MLflow server and all child processes (gunicorn workers)
+    if [[ -n "${MLFLOW_PID:-}" ]]; then
+        pkill -P "${MLFLOW_PID}" 2>/dev/null || true
+        kill "${MLFLOW_PID}" 2>/dev/null || true
+    fi
+}
+trap copy_artifacts EXIT TERM INT
+
+# -----------------------------------------------------------------------
+# Workaround: --continue + -p is broken (anthropics/claude-code#42376).
+# -----------------------------------------------------------------------
+export CLAUDE_CODE_ENTRYPOINT=sdk-cli
+
+# -----------------------------------------------------------------------
+# Build arguments
+# -----------------------------------------------------------------------
+RUN_ID="ci-$(date +%Y%m%d-%H%M%S)-${EVAL_MODEL}"
+ALLOWED_TOOLS="Bash Read Write Edit Grep Glob Agent Skill"
+
+EVAL_RUN_ARGS="--config ${EVAL_CONFIG} --model ${EVAL_MODEL} --run-id ${RUN_ID}"
+if [[ -n "${EVAL_BASELINE}" ]]; then
+    EVAL_RUN_ARGS="${EVAL_RUN_ARGS} --baseline ${EVAL_BASELINE}"
+fi
+if [[ -n "${EVAL_EXTRA_ARGS}" ]]; then
+    EVAL_RUN_ARGS="${EVAL_RUN_ARGS} ${EVAL_EXTRA_ARGS}"
+fi
+
+# -----------------------------------------------------------------------
+# Run evaluation
+# -----------------------------------------------------------------------
+echo ""
+echo "=== Running eval ==="
+echo "Run ID: ${RUN_ID}"
+echo "Args: ${EVAL_RUN_ARGS}"
+
+EVAL_START=$(date +%s)
+EVAL_EXIT=0
+timeout 7200 claude \
+    --model "${CLAUDE_MODEL}" \
+    --plugin-dir "${EVAL_HARNESS_DIR}" \
+    --allowedTools "${ALLOWED_TOOLS}" \
+    --output-format stream-json \
+    --max-turns 100 \
+    -p "/eval-run ${EVAL_RUN_ARGS}" \
+    --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval.log" || EVAL_EXIT=$?
+EVAL_DURATION=$(( $(date +%s) - EVAL_START ))
+
+echo "eval-run completed in ${EVAL_DURATION}s (exit ${EVAL_EXIT})"
+
+# -----------------------------------------------------------------------
+# Run eval-mlflow to upload results
+# -----------------------------------------------------------------------
+echo ""
+echo "=== Running eval-mlflow ==="
+
+MLFLOW_EXIT=0
+timeout 600 claude \
+    --model "${CLAUDE_MODEL}" \
+    --plugin-dir "${EVAL_HARNESS_DIR}" \
+    --continue \
+    --allowedTools "${ALLOWED_TOOLS}" \
+    --output-format stream-json \
+    --max-turns 20 \
+    -p "/eval-mlflow --action all --run-id ${RUN_ID} --config ${EVAL_CONFIG}" \
+    --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval-mlflow.log" || MLFLOW_EXIT=$?
+
+echo "eval-mlflow completed with exit code ${MLFLOW_EXIT}"
+
+# -----------------------------------------------------------------------
+# Generate JUnit XML
+# -----------------------------------------------------------------------
+JUNIT_FILE="${ARTIFACT_DIR}/junit_claude-eval.xml"
+FAILURE_COUNT=0
+TESTCASE="[sig-claude] Skill evaluation should pass"
+
+if [[ "${EVAL_EXIT}" -ne 0 ]]; then
+    FAILURE_COUNT=1
+    TESTCASES="  <testcase name=\"${TESTCASE}\" time=\"${EVAL_DURATION}\">
+    <failure message=\"eval-run failed (exit ${EVAL_EXIT})\">eval-run exited with code ${EVAL_EXIT}.</failure>
+  </testcase>"
+else
+    TESTCASES="  <testcase name=\"${TESTCASE}\" time=\"${EVAL_DURATION}\"/>"
+fi
+
+cat > "${JUNIT_FILE}" <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="claude-eval" tests="1" failures="${FAILURE_COUNT}" time="${EVAL_DURATION}">
+${TESTCASES}
+</testsuite>
+EOF
+
+echo "JUnit XML written to ${JUNIT_FILE}"
+
+if [[ "${EVAL_EXIT}" -ne 0 ]]; then
+    echo "Evaluation failed."
+    exit 1
+fi
+
+echo "Evaluation complete."

ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.metadata.json

@@ -0,0 +1,21 @@
+{
+	"path": "openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml",
+	"owners": {
+		"approvers": [
+			"bentito",
+			"bryan-cox",
+			"cblecker",
+			"enxebre",
+			"prashanth684",
+			"stbenjam"
+		],
+		"reviewers": [
+			"bentito",
+			"bryan-cox",
+			"cblecker",
+			"enxebre",
+			"prashanth684",
+			"stbenjam"
+		]
+	}
+}

ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml

@@ -0,0 +1,45 @@
+ref:
+  as: openshift-claude-agent-eval
+  from_image:
+    namespace: ci
+    name: claude-ai-helpers
+    tag: latest
+  commands: openshift-claude-agent-eval-commands.sh
+  credentials:
+  - namespace: test-credentials
+    name: hypershift-team-claude-prow
+    mount_path: /var/run/claude-code-service-account
+  env:
+  - name: EVAL_CONFIG
+    default: "eval.yaml"
+  - name: EVAL_MODEL
+    default: "claude-sonnet-4-6"
+  - name: EVAL_BASELINE
+    default: ""
+  - name: EVAL_EXTRA_ARGS
+    default: ""
+  - name: EVAL_SETUP_SCRIPT
+    default: ""
+  - name: MLFLOW_PORT
+    default: "5000"
+  - name: CLAUDE_MODEL
+    default: "claude-sonnet-4-6"
+  - name: CLAUDE_CODE_USE_VERTEX
+    default: "1"
+  - name: CLOUD_ML_REGION
+    default: "global"
+  - name: ANTHROPIC_VERTEX_PROJECT_ID
+    default: "itpc-gcp-hybrid-pe-eng-claude"
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    default: "/var/run/claude-code-service-account/claude-prow"
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 2Gi
+  timeout: 3h0m0s
+  grace_period: 1m0s
+  documentation: |-
+    Runs skill evaluations using the agent-eval-harness. Installs the
+    eval harness plugin, starts a local MLflow server, runs /eval-run
+    and /eval-mlflow with the specified config and model, and produces
+    JUnit XML + artifacts. MLflow data is archived for download.