From 94b833b787a5621db0440c58dac77f972b337c77 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:02:05 -0400
Subject: [PATCH 01/17] Add agent-eval step registry and ai-helpers eval
 presubmit

Add a reusable eval step (openshift-claude-agent-eval) that runs
the agent-eval-harness against any skill, and an optional presubmit
for ai-helpers payload-analysis eval using Opus.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-eng-ai-helpers-main.yaml        |  10 +
 ...nshift-eng-ai-helpers-main-presubmits.yaml |  75 +++++++
 .../openshift/claude/agent-eval/OWNERS        |  14 ++
 .../openshift-claude-agent-eval-commands.sh   | 188 ++++++++++++++++++
 ...nshift-claude-agent-eval-ref.metadata.json |  21 ++
 .../openshift-claude-agent-eval-ref.yaml      |  47 +++++
 ...t-claude-agent-eval-workflow.metadata.json |  21 ++
 .../openshift-claude-agent-eval-workflow.yaml |  10 +
 8 files changed, 386 insertions(+)
 create mode 100644 ci-operator/step-registry/openshift/claude/agent-eval/OWNERS
 create mode 100755 ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
 create mode 100644 ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.metadata.json
 create mode 100644 ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml
 create mode 100644 ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.metadata.json
 create mode 100644 ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.yaml

diff --git a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
index 9e1b1eb55c7f5..d337f46250aad 100644
--- a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
+++ b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
@@ -17,6 +17,16 @@ resources:
     requests:
       cpu: 100m
       memory: 200Mi
+tests:
+- as: eval-payload-analysis
+  always_run: false
+  optional: true
+  steps:
+    env:
+      EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml
+      EVAL_MODEL: claude-opus-4-6
+      EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh
+    workflow: openshift-claude-agent-eval
 zz_generated_metadata:
   branch: main
   org: openshift-eng
diff --git a/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml b/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml
index c95ecaa06dd1f..1841042666a40 100644
--- a/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml
+++ b/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml
@@ -1,5 +1,80 @@
 presubmits:
   openshift-eng/ai-helpers:
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build13
+    context: ci/prow/eval-payload-analysis
+    decorate: true
+    decoration_config:
+      sparse_checkout_files:
+      - images/Dockerfile
+    labels:
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis
+    optional: true
+    rerun_command: /test eval-payload-analysis
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --target=eval-payload-analysis
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )eval-payload-analysis,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/OWNERS b/ci-operator/step-registry/openshift/claude/agent-eval/OWNERS
new file mode 100644
index 0000000000000..3259ba3e017ba
--- /dev/null
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/OWNERS
@@ -0,0 +1,14 @@
+approvers:
+- bentito
+- bryan-cox
+- cblecker
+- enxebre
+- prashanth684
+- stbenjam
+reviewers:
+- bentito
+- bryan-cox
+- cblecker
+- enxebre
+- prashanth684
+- stbenjam
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
new file mode 100755
index 0000000000000..7731d6e4f3b01
--- /dev/null
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -0,0 +1,188 @@
+#!/bin/bash
+#
+# Run agent-eval-harness against Claude Code skills.
+#
+# Required env:
+#   EVAL_CONFIG  -- path to eval.yaml (relative to repo root)
+#
+# Optional env:
+#   EVAL_MODEL        -- model for the skill under test (default: claude-sonnet-4-6)
+#   EVAL_JUDGE_MODEL  -- model for LLM judges (default: claude-sonnet-4-6)
+#   EVAL_BASELINE     -- run-id of a previous run to compare against
+#   EVAL_EXTRA_ARGS   -- additional args passed to /eval-run
+#   EVAL_SETUP_SCRIPT -- script to run before eval (e.g. snapshot extraction)
+#   CLAUDE_MODEL      -- model for the eval harness orchestrator (default: claude-sonnet-4-6)
+#   MLFLOW_PORT       -- port for local MLflow server (default: 5000)
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+echo "Starting claude-agent-eval"
+echo "Config: ${EVAL_CONFIG}"
+echo "Skill model: ${EVAL_MODEL}"
+echo "Judge model: ${EVAL_JUDGE_MODEL}"
+
+# -----------------------------------------------------------------------
+# Install dependencies
+# -----------------------------------------------------------------------
+echo "Installing mlflow..."
+pip install --quiet mlflow 2>&1 | tail -1
+echo "mlflow installed."
+
+# Start local MLflow server in background
+echo "Starting local MLflow server on port ${MLFLOW_PORT}..."
+export MLFLOW_TRACKING_URI="http://127.0.0.1:${MLFLOW_PORT}"
+mlflow server --port "${MLFLOW_PORT}" --host 127.0.0.1 &
+MLFLOW_PID=$!
+
+for i in $(seq 1 30); do
+    if curl -sf "http://127.0.0.1:${MLFLOW_PORT}/health" >/dev/null 2>&1; then
+        echo "MLflow server ready (PID ${MLFLOW_PID})."
+        break
+    fi
+    if [[ $i -eq 30 ]]; then
+        echo "Warning: MLflow server did not become ready in 30s. Continuing anyway."
+    fi
+    sleep 1
+done
+
+# -----------------------------------------------------------------------
+# Run optional setup script (e.g. extract snapshots, populate fixtures)
+# -----------------------------------------------------------------------
+if [[ -n "${EVAL_SETUP_SCRIPT}" ]] && [[ -f "${EVAL_SETUP_SCRIPT}" ]]; then
+    echo ""
+    echo "=== Running setup script: ${EVAL_SETUP_SCRIPT} ==="
+    bash "${EVAL_SETUP_SCRIPT}"
+fi
+
+# -----------------------------------------------------------------------
+# Install plugins
+# -----------------------------------------------------------------------
+echo ""
+echo "=== Installing plugins ==="
+claude plugin install agent-eval-harness@opendatahub-skills
+echo "agent-eval-harness plugin installed."
+
+# -----------------------------------------------------------------------
+# Artifact copy trap
+# -----------------------------------------------------------------------
+copy_artifacts() {
+    echo "Copying eval artifacts..."
+    if [[ -d "${AGENT_EVAL_RUNS_DIR:-eval/runs}" ]]; then
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "report.html" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "summary.yaml" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "run_result.json" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+    fi
+    find . -name "eval-summary-*.html" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+
+    # Copy MLflow data
+    if [[ -d "mlruns" ]]; then
+        tar -czf "${ARTIFACT_DIR}/mlflow-data.tar.gz" mlruns/ 2>/dev/null || true
+    fi
+
+    # Archive Claude session for continue-session support
+    CLAUDE_HOME="/home/claude/.claude"
+    if [[ -d "${CLAUDE_HOME}/projects" ]]; then
+        echo "Archiving Claude session logs..."
+        tar -czf "${ARTIFACT_DIR}/claude-sessions-$(date +%Y%m%d-%H%M%S).tar.gz" \
+            -C "${CLAUDE_HOME}" projects/ 2>/dev/null && \
+            touch "${SHARED_DIR}/claude-session-available" || true
+    fi
+
+    # Stop MLflow server
+    if [[ -n "${MLFLOW_PID:-}" ]]; then
+        kill "${MLFLOW_PID}" 2>/dev/null || true
+    fi
+}
+trap copy_artifacts EXIT TERM INT
+
+# -----------------------------------------------------------------------
+# Workaround: --continue + -p is broken (anthropics/claude-code#42376).
+# -----------------------------------------------------------------------
+export CLAUDE_CODE_ENTRYPOINT=sdk-cli
+
+# -----------------------------------------------------------------------
+# Build arguments
+# -----------------------------------------------------------------------
+RUN_ID="ci-$(date +%Y%m%d-%H%M%S)-${EVAL_MODEL}"
+ALLOWED_TOOLS="Bash Read Write Edit Grep Glob Agent Skill"
+
+EVAL_RUN_ARGS="--config ${EVAL_CONFIG} --model ${EVAL_MODEL} --run-id ${RUN_ID}"
+if [[ -n "${EVAL_BASELINE}" ]]; then
+    EVAL_RUN_ARGS="${EVAL_RUN_ARGS} --baseline ${EVAL_BASELINE}"
+fi
+if [[ -n "${EVAL_EXTRA_ARGS}" ]]; then
+    EVAL_RUN_ARGS="${EVAL_RUN_ARGS} ${EVAL_EXTRA_ARGS}"
+fi
+
+# -----------------------------------------------------------------------
+# Run evaluation
+# -----------------------------------------------------------------------
+echo ""
+echo "=== Running eval ==="
+echo "Run ID: ${RUN_ID}"
+echo "Args: ${EVAL_RUN_ARGS}"
+
+EVAL_START=$(date +%s)
+EVAL_EXIT=0
+timeout 7200 claude \
+    --model "${CLAUDE_MODEL}" \
+    --allowedTools "${ALLOWED_TOOLS}" \
+    --output-format stream-json \
+    --max-turns 100 \
+    -p "/eval-run ${EVAL_RUN_ARGS}" \
+    --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval.log" || EVAL_EXIT=$?
+EVAL_DURATION=$(( $(date +%s) - EVAL_START ))
+
+echo "eval-run completed in ${EVAL_DURATION}s (exit ${EVAL_EXIT})"
+
+# -----------------------------------------------------------------------
+# Run eval-mlflow to upload results
+# -----------------------------------------------------------------------
+echo ""
+echo "=== Running eval-mlflow ==="
+
+MLFLOW_EXIT=0
+timeout 600 claude \
+    --model "${CLAUDE_MODEL}" \
+    --continue \
+    --allowedTools "${ALLOWED_TOOLS}" \
+    --output-format stream-json \
+    --max-turns 20 \
+    -p "/eval-mlflow --action all --run-id ${RUN_ID} --config ${EVAL_CONFIG}" \
+    --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval-mlflow.log" || MLFLOW_EXIT=$?
+
+echo "eval-mlflow completed with exit code ${MLFLOW_EXIT}"
+
+# -----------------------------------------------------------------------
+# Generate JUnit XML
+# -----------------------------------------------------------------------
+JUNIT_FILE="${ARTIFACT_DIR}/junit_claude-eval.xml"
+FAILURE_COUNT=0
+TESTCASE="[sig-claude] Skill evaluation should pass"
+
+if [[ "${EVAL_EXIT}" -ne 0 ]]; then
+    FAILURE_COUNT=1
+    TESTCASES="  <testcase name=\"${TESTCASE}\" time=\"${EVAL_DURATION}\">
+    <failure message=\"eval-run failed (exit ${EVAL_EXIT})\">eval-run exited with code ${EVAL_EXIT}.</failure>
+  </testcase>"
+else
+    TESTCASES="  <testcase name=\"${TESTCASE}\" time=\"${EVAL_DURATION}\"/>"
+fi
+
+cat > "${JUNIT_FILE}" <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="claude-eval" tests="1" failures="${FAILURE_COUNT}" time="${EVAL_DURATION}">
+${TESTCASES}
+</testsuite>
+EOF
+
+echo "JUnit XML written to ${JUNIT_FILE}"
+
+if [[ "${EVAL_EXIT}" -ne 0 ]]; then
+    echo "Evaluation failed."
+    exit 1
+fi
+
+echo "Evaluation complete."
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.metadata.json b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.metadata.json
new file mode 100644
index 0000000000000..631d08f83823f
--- /dev/null
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.metadata.json
@@ -0,0 +1,21 @@
+{
+	"path": "openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml",
+	"owners": {
+		"approvers": [
+			"bentito",
+			"bryan-cox",
+			"cblecker",
+			"enxebre",
+			"prashanth684",
+			"stbenjam"
+		],
+		"reviewers": [
+			"bentito",
+			"bryan-cox",
+			"cblecker",
+			"enxebre",
+			"prashanth684",
+			"stbenjam"
+		]
+	}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml
new file mode 100644
index 0000000000000..e74ca34c62b3d
--- /dev/null
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml
@@ -0,0 +1,47 @@
+ref:
+  as: openshift-claude-agent-eval
+  from_image:
+    namespace: ci
+    name: claude-ai-helpers
+    tag: latest
+  commands: openshift-claude-agent-eval-commands.sh
+  credentials:
+  - namespace: test-credentials
+    name: hypershift-team-claude-prow
+    mount_path: /var/run/claude-code-service-account
+  env:
+  - name: EVAL_CONFIG
+    default: "eval.yaml"
+  - name: EVAL_MODEL
+    default: "claude-sonnet-4-6"
+  - name: EVAL_JUDGE_MODEL
+    default: "claude-sonnet-4-6"
+  - name: EVAL_BASELINE
+    default: ""
+  - name: EVAL_EXTRA_ARGS
+    default: ""
+  - name: EVAL_SETUP_SCRIPT
+    default: ""
+  - name: MLFLOW_PORT
+    default: "5000"
+  - name: CLAUDE_MODEL
+    default: "claude-sonnet-4-6"
+  - name: CLAUDE_CODE_USE_VERTEX
+    default: "1"
+  - name: CLOUD_ML_REGION
+    default: "global"
+  - name: ANTHROPIC_VERTEX_PROJECT_ID
+    default: "itpc-gcp-hybrid-pe-eng-claude"
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    default: "/var/run/claude-code-service-account/claude-prow"
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 2Gi
+  timeout: 3h0m0s
+  grace_period: 1m0s
+  documentation: |-
+    Runs skill evaluations using the agent-eval-harness. Installs the
+    eval harness plugin, starts a local MLflow server, runs /eval-run
+    and /eval-mlflow with the specified config and model, and produces
+    JUnit XML + artifacts. MLflow data is archived for download.
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.metadata.json b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.metadata.json
new file mode 100644
index 0000000000000..10beb8f7d974e
--- /dev/null
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.metadata.json
@@ -0,0 +1,21 @@
+{
+	"path": "openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"bentito",
+			"bryan-cox",
+			"cblecker",
+			"enxebre",
+			"prashanth684",
+			"stbenjam"
+		],
+		"reviewers": [
+			"bentito",
+			"bryan-cox",
+			"cblecker",
+			"enxebre",
+			"prashanth684",
+			"stbenjam"
+		]
+	}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.yaml
new file mode 100644
index 0000000000000..f0542d59b2c0f
--- /dev/null
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-workflow.yaml
@@ -0,0 +1,10 @@
+workflow:
+  as: openshift-claude-agent-eval
+  steps:
+    test:
+    - ref: openshift-claude-agent-eval
+    post:
+    - ref: openshift-claude-post
+  documentation: |-
+    Runs skill evaluations using the agent-eval-harness and generates
+    a continue-session page in artifacts for resuming locally.

From 4a3fab5503f768a55c96a255a66ef2eda87391d5 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:17:42 -0400
Subject: [PATCH 02/17] Pin mlflow, error on missing setup script, update
 owners

- Pin mlflow==3.12.0 for supply chain protection
- Error exit if EVAL_SETUP_SCRIPT is set but file not found
- Add MLflow server startup, /eval-mlflow step, and mlruns archiving
- Remove presubmit changed-skills auto-detection
- Update OWNERS to ai-helpers team

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../agent-eval/openshift-claude-agent-eval-commands.sh    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 7731d6e4f3b01..d61b40c5bc0cb 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -27,7 +27,7 @@ echo "Judge model: ${EVAL_JUDGE_MODEL}"
 # Install dependencies
 # -----------------------------------------------------------------------
 echo "Installing mlflow..."
-pip install --quiet mlflow 2>&1 | tail -1
+pip install --quiet 'mlflow==3.12.0' 2>&1 | tail -1
 echo "mlflow installed."
 
 # Start local MLflow server in background
@@ -50,7 +50,11 @@ done
 # -----------------------------------------------------------------------
 # Run optional setup script (e.g. extract snapshots, populate fixtures)
 # -----------------------------------------------------------------------
-if [[ -n "${EVAL_SETUP_SCRIPT}" ]] && [[ -f "${EVAL_SETUP_SCRIPT}" ]]; then
+if [[ -n "${EVAL_SETUP_SCRIPT}" ]]; then
+    if [[ ! -f "${EVAL_SETUP_SCRIPT}" ]]; then
+        echo "ERROR: EVAL_SETUP_SCRIPT not found: ${EVAL_SETUP_SCRIPT}"
+        exit 1
+    fi
     echo ""
     echo "=== Running setup script: ${EVAL_SETUP_SCRIPT} ==="
     bash "${EVAL_SETUP_SCRIPT}"

From bbef4aa9e452cfb6ed2e8c0c17185bd8640275c9 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:29:40 -0400
Subject: [PATCH 03/17] Add marketplace registration before plugin install

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index d61b40c5bc0cb..2d65266414883 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -65,6 +65,7 @@ fi
 # -----------------------------------------------------------------------
 echo ""
 echo "=== Installing plugins ==="
+claude plugin marketplace add opendatahub-io/skills-registry
 claude plugin install agent-eval-harness@opendatahub-skills
 echo "agent-eval-harness plugin installed."
 

From c0043045759327c13225adf34158459794c71d76 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:30:47 -0400
Subject: [PATCH 04/17] Add hello-world smoke test fallback when eval config
 missing

When EVAL_CONFIG doesn't exist in the repo checkout, generates a
minimal hello-world eval inline so the infrastructure can be tested
without any eval config checked in.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-claude-agent-eval-commands.sh   | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 2d65266414883..5defa47b8d1bb 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -47,6 +47,69 @@ for i in $(seq 1 30); do
     sleep 1
 done
 
+# -----------------------------------------------------------------------
+# Generate hello-world smoke test if no eval config exists
+# -----------------------------------------------------------------------
+if [[ ! -f "${EVAL_CONFIG}" ]]; then
+    echo "EVAL_CONFIG not found at ${EVAL_CONFIG}, generating hello-world smoke test..."
+    mkdir -p "$(dirname "${EVAL_CONFIG}")"
+    cat > "${EVAL_CONFIG}" <<'EVALEOF'
+name: hello-world-smoke-test
+description: Smoke test for eval infrastructure using hello-world plugin
+skill: hello-world:echo
+
+execution:
+  mode: case
+  arguments: "{name}"
+  timeout: 60
+
+runner:
+  type: claude-code
+  plugin_dirs:
+    - plugins/hello-world
+
+dataset:
+  path: eval/cases/hello-world
+
+outputs:
+  files:
+    - "*.txt"
+
+traces:
+  stdout: true
+
+judges:
+  greeting_correct:
+    type: check
+    check: |
+      stdout = record.get("outputs", {}).get("stdout", "")
+      name = record.get("inputs", {}).get("name", "world")
+      expected = f"Hello {name}"
+      if expected.lower() in stdout.lower():
+          score = 1.0
+          rationale = f"Found expected greeting: {expected}"
+      else:
+          score = 0.0
+          rationale = f"Expected '{expected}' in output, got: {stdout[:200]}"
+
+thresholds:
+  greeting_correct:
+    min_pass_rate: 1.0
+EVALEOF
+
+    mkdir -p eval/cases/hello-world/case-001-default
+    cat > eval/cases/hello-world/case-001-default/input.yaml <<'CASEEOF'
+name: "world"
+CASEEOF
+
+    mkdir -p eval/cases/hello-world/case-002-named
+    cat > eval/cases/hello-world/case-002-named/input.yaml <<'CASEEOF'
+name: "Alice"
+CASEEOF
+
+    echo "Generated smoke test config at ${EVAL_CONFIG}"
+fi
+
 # -----------------------------------------------------------------------
 # Run optional setup script (e.g. extract snapshots, populate fixtures)
 # -----------------------------------------------------------------------

From 869b467805d4716e7d0282a073232446a968f79a Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:31:43 -0400
Subject: [PATCH 05/17] Clear EVAL_SETUP_SCRIPT when falling back to smoke test

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 5defa47b8d1bb..258532435085f 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -52,6 +52,7 @@ done
 # -----------------------------------------------------------------------
 if [[ ! -f "${EVAL_CONFIG}" ]]; then
     echo "EVAL_CONFIG not found at ${EVAL_CONFIG}, generating hello-world smoke test..."
+    EVAL_SETUP_SCRIPT=""
     mkdir -p "$(dirname "${EVAL_CONFIG}")"
     cat > "${EVAL_CONFIG}" <<'EVALEOF'
 name: hello-world-smoke-test

From 05e21a9f8f6b7fe261f746d2623870f80a6b69b6 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:32:38 -0400
Subject: [PATCH 06/17] Use private artifact bucket for eval job

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
index d337f46250aad..bdc415eb05821 100644
--- a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
+++ b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
@@ -20,6 +20,7 @@ resources:
 tests:
 - as: eval-payload-analysis
   always_run: false
+  hidden: true
   optional: true
   steps:
     env:

From a0b92644513b04430b97149fb61fd0b223407989 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 12:39:45 -0400
Subject: [PATCH 07/17] Remove hidden setting from eval job

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
index bdc415eb05821..d337f46250aad 100644
--- a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
+++ b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
@@ -20,7 +20,6 @@ resources:
 tests:
 - as: eval-payload-analysis
   always_run: false
-  hidden: true
   optional: true
   steps:
     env:

From c774acd59567c2f3b9b006604f72f5c52cc37889 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 13:00:35 -0400
Subject: [PATCH 08/17] Use python3 -m pip instead of bare pip

The ai-helpers image has python3 and python3-pip but no bare pip
command on PATH.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 258532435085f..08bf569795553 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -27,7 +27,7 @@ echo "Judge model: ${EVAL_JUDGE_MODEL}"
 # Install dependencies
 # -----------------------------------------------------------------------
 echo "Installing mlflow..."
-pip install --quiet 'mlflow==3.12.0' 2>&1 | tail -1
+python3 -m pip install --quiet 'mlflow==3.12.0' 2>&1 | tail -1
 echo "mlflow installed."
 
 # Start local MLflow server in background

From c43ebdee426c9d44f466cdeee7b1e47c574aebc5 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 13:10:23 -0400
Subject: [PATCH 09/17] make ci-operator-config

---
 .../ai-helpers/openshift-eng-ai-helpers-main.yaml             | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
index d337f46250aad..e81f60d1bdea6 100644
--- a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
+++ b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml
@@ -18,8 +18,8 @@ resources:
       cpu: 100m
       memory: 200Mi
 tests:
-- as: eval-payload-analysis
-  always_run: false
+- always_run: false
+  as: eval-payload-analysis
   optional: true
   steps:
     env:

From 378e598138815b81dba7ed88e4511dd9ad67ce20 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 13:21:30 -0400
Subject: [PATCH 10/17] Run mlflow server via python3 -m mlflow

The mlflow binary isn't on PATH after pip install --user.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 08bf569795553..407bc88985f15 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -33,7 +33,7 @@ echo "mlflow installed."
 # Start local MLflow server in background
 echo "Starting local MLflow server on port ${MLFLOW_PORT}..."
 export MLFLOW_TRACKING_URI="http://127.0.0.1:${MLFLOW_PORT}"
-mlflow server --port "${MLFLOW_PORT}" --host 127.0.0.1 &
+python3 -m mlflow server --port "${MLFLOW_PORT}" --host 127.0.0.1 &
 MLFLOW_PID=$!
 
 for i in $(seq 1 30); do

From 8a5ade6e180823afa25afe87594192f3769b9363 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 13:29:43 -0400
Subject: [PATCH 11/17] Fix mlflow PATH and SSH host key for plugin install

Add ~/.local/bin to PATH after pip install so the mlflow binary is
available. Add ssh-keyscan for github.com before plugin install to
avoid host key verification failure.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../agent-eval/openshift-claude-agent-eval-commands.sh       | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 407bc88985f15..07c92e9764292 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -28,12 +28,13 @@ echo "Judge model: ${EVAL_JUDGE_MODEL}"
 # -----------------------------------------------------------------------
 echo "Installing mlflow..."
 python3 -m pip install --quiet 'mlflow==3.12.0' 2>&1 | tail -1
+export PATH="$HOME/.local/bin:$PATH"
 echo "mlflow installed."
 
 # Start local MLflow server in background
 echo "Starting local MLflow server on port ${MLFLOW_PORT}..."
 export MLFLOW_TRACKING_URI="http://127.0.0.1:${MLFLOW_PORT}"
-python3 -m mlflow server --port "${MLFLOW_PORT}" --host 127.0.0.1 &
+mlflow server --port "${MLFLOW_PORT}" --host 127.0.0.1 &
 MLFLOW_PID=$!
 
 for i in $(seq 1 30); do
@@ -129,6 +130,8 @@ fi
 # -----------------------------------------------------------------------
 echo ""
 echo "=== Installing plugins ==="
+mkdir -p ~/.ssh
+ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
 claude plugin marketplace add opendatahub-io/skills-registry
 claude plugin install agent-eval-harness@opendatahub-skills
 echo "agent-eval-harness plugin installed."

From 90b002667409f4d86344a8196dcaf6bfae205ad8 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 13:57:42 -0400
Subject: [PATCH 12/17] Fix mlflow, SSH, and PATH issues in eval step

- Downgrade mlflow to 2.20.2: mlflow 3.x requires sqlite3_deserialize
  which is missing from RHEL 9's sqlite 3.34.1 (added in 3.36.0)
- Add ~/.local/bin to PATH after pip install
- Force HTTPS for git clones (SSH port 22 blocked in CI)
- Fail hard if MLflow server doesn't start

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-claude-agent-eval-commands.sh          | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 07c92e9764292..d9a266c75316d 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -27,7 +27,7 @@ echo "Judge model: ${EVAL_JUDGE_MODEL}"
 # Install dependencies
 # -----------------------------------------------------------------------
 echo "Installing mlflow..."
-python3 -m pip install --quiet 'mlflow==3.12.0' 2>&1 | tail -1
+python3 -m pip install --quiet 'mlflow==2.20.2' 2>&1 | tail -1
 export PATH="$HOME/.local/bin:$PATH"
 echo "mlflow installed."
 
@@ -42,8 +42,13 @@ for i in $(seq 1 30); do
         echo "MLflow server ready (PID ${MLFLOW_PID})."
         break
     fi
+    if ! kill -0 "${MLFLOW_PID}" 2>/dev/null; then
+        echo "ERROR: MLflow server crashed."
+        exit 1
+    fi
     if [[ $i -eq 30 ]]; then
-        echo "Warning: MLflow server did not become ready in 30s. Continuing anyway."
+        echo "ERROR: MLflow server did not become ready in 30s."
+        exit 1
     fi
     sleep 1
 done
@@ -130,8 +135,7 @@ fi
 # -----------------------------------------------------------------------
 echo ""
 echo "=== Installing plugins ==="
-mkdir -p ~/.ssh
-ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
+git config --global url."https://github.com/".insteadOf "git@github.com:"
 claude plugin marketplace add opendatahub-io/skills-registry
 claude plugin install agent-eval-harness@opendatahub-skills
 echo "agent-eval-harness plugin installed."

From 1f14e8209b35f824a6cde9329bfbe14a735257a5 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 14:07:03 -0400
Subject: [PATCH 13/17] Add comment for mlflow version pin, remove pipe on pip
 install

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index d9a266c75316d..17f5681db412d 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -27,7 +27,8 @@ echo "Judge model: ${EVAL_JUDGE_MODEL}"
 # Install dependencies
 # -----------------------------------------------------------------------
 echo "Installing mlflow..."
-python3 -m pip install --quiet 'mlflow==2.20.2' 2>&1 | tail -1
+# mlflow 3.x requires sqlite >= 3.36.0; RHEL 9 ships 3.34.1
+python3 -m pip install --quiet 'mlflow==2.20.2'
 export PATH="$HOME/.local/bin:$PATH"
 echo "mlflow installed."
 

From c4adbdaeab73668d28817e9a4600e84229ece744 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 14:35:53 -0400
Subject: [PATCH 14/17] Clone eval harness directly, use --plugin-dir

The opendatahub-io/skills-registry marketplace pins a commit SHA
that can't be cloned with git --branch. Clone the repo directly
and pass --plugin-dir to claude instead.

Also move PATH export before pip install to suppress warnings.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-claude-agent-eval-commands.sh           | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 17f5681db412d..98831d720e7d3 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -27,9 +27,9 @@ echo "Judge model: ${EVAL_JUDGE_MODEL}"
 # Install dependencies
 # -----------------------------------------------------------------------
 echo "Installing mlflow..."
+export PATH="$HOME/.local/bin:$PATH"
 # mlflow 3.x requires sqlite >= 3.36.0; RHEL 9 ships 3.34.1
 python3 -m pip install --quiet 'mlflow==2.20.2'
-export PATH="$HOME/.local/bin:$PATH"
 echo "mlflow installed."
 
 # Start local MLflow server in background
@@ -136,10 +136,9 @@ fi
 # -----------------------------------------------------------------------
 echo ""
 echo "=== Installing plugins ==="
-git config --global url."https://github.com/".insteadOf "git@github.com:"
-claude plugin marketplace add opendatahub-io/skills-registry
-claude plugin install agent-eval-harness@opendatahub-skills
-echo "agent-eval-harness plugin installed."
+EVAL_HARNESS_DIR="/tmp/agent-eval-harness"
+git clone --depth 1 https://github.com/opendatahub-io/agent-eval-harness.git "${EVAL_HARNESS_DIR}"
+echo "agent-eval-harness cloned."
 
 # -----------------------------------------------------------------------
 # Artifact copy trap
@@ -205,6 +204,7 @@ EVAL_START=$(date +%s)
 EVAL_EXIT=0
 timeout 7200 claude \
     --model "${CLAUDE_MODEL}" \
+    --plugin-dir "${EVAL_HARNESS_DIR}" \
     --allowedTools "${ALLOWED_TOOLS}" \
     --output-format stream-json \
     --max-turns 100 \
@@ -223,6 +223,7 @@ echo "=== Running eval-mlflow ==="
 MLFLOW_EXIT=0
 timeout 600 claude \
     --model "${CLAUDE_MODEL}" \
+    --plugin-dir "${EVAL_HARNESS_DIR}" \
     --continue \
     --allowedTools "${ALLOWED_TOOLS}" \
     --output-format stream-json \

From 6458632e83194ce7e910dac06ad6b0383f833a6f Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 15:33:30 -0400
Subject: [PATCH 15/17] ci: strip hello-world smoke test, fix integration gaps

- Remove inline hello-world eval fallback (now require EVAL_CONFIG)
- cd /opt/ai-helpers where the repo lives (WORKDIR is /workspace)
- Capture and export EVAL_SNAPSHOT_DIR from setup script stdout
- Copy report.html as eval-report-summary.html for prow lens
- Kill gunicorn workers with pkill -P before killing mlflow parent

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../openshift-claude-agent-eval-commands.sh   | 76 ++++---------------
 1 file changed, 13 insertions(+), 63 deletions(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 98831d720e7d3..7c0e99eb4651d 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -19,6 +19,10 @@ set -o errexit
 set -o pipefail
 
 echo "Starting claude-agent-eval"
+
+# The repo is at /opt/ai-helpers; WORKDIR is /workspace
+cd /opt/ai-helpers
+
 echo "Config: ${EVAL_CONFIG}"
 echo "Skill model: ${EVAL_MODEL}"
 echo "Judge model: ${EVAL_JUDGE_MODEL}"
@@ -55,67 +59,11 @@ for i in $(seq 1 30); do
 done
 
 # -----------------------------------------------------------------------
-# Generate hello-world smoke test if no eval config exists
+# Verify eval config exists
 # -----------------------------------------------------------------------
 if [[ ! -f "${EVAL_CONFIG}" ]]; then
-    echo "EVAL_CONFIG not found at ${EVAL_CONFIG}, generating hello-world smoke test..."
-    EVAL_SETUP_SCRIPT=""
-    mkdir -p "$(dirname "${EVAL_CONFIG}")"
-    cat > "${EVAL_CONFIG}" <<'EVALEOF'
-name: hello-world-smoke-test
-description: Smoke test for eval infrastructure using hello-world plugin
-skill: hello-world:echo
-
-execution:
-  mode: case
-  arguments: "{name}"
-  timeout: 60
-
-runner:
-  type: claude-code
-  plugin_dirs:
-    - plugins/hello-world
-
-dataset:
-  path: eval/cases/hello-world
-
-outputs:
-  files:
-    - "*.txt"
-
-traces:
-  stdout: true
-
-judges:
-  greeting_correct:
-    type: check
-    check: |
-      stdout = record.get("outputs", {}).get("stdout", "")
-      name = record.get("inputs", {}).get("name", "world")
-      expected = f"Hello {name}"
-      if expected.lower() in stdout.lower():
-          score = 1.0
-          rationale = f"Found expected greeting: {expected}"
-      else:
-          score = 0.0
-          rationale = f"Expected '{expected}' in output, got: {stdout[:200]}"
-
-thresholds:
-  greeting_correct:
-    min_pass_rate: 1.0
-EVALEOF
-
-    mkdir -p eval/cases/hello-world/case-001-default
-    cat > eval/cases/hello-world/case-001-default/input.yaml <<'CASEEOF'
-name: "world"
-CASEEOF
-
-    mkdir -p eval/cases/hello-world/case-002-named
-    cat > eval/cases/hello-world/case-002-named/input.yaml <<'CASEEOF'
-name: "Alice"
-CASEEOF
-
-    echo "Generated smoke test config at ${EVAL_CONFIG}"
+    echo "ERROR: EVAL_CONFIG not found at ${EVAL_CONFIG}"
+    exit 1
 fi
 
 # -----------------------------------------------------------------------
@@ -128,7 +76,8 @@ if [[ -n "${EVAL_SETUP_SCRIPT}" ]]; then
     fi
     echo ""
     echo "=== Running setup script: ${EVAL_SETUP_SCRIPT} ==="
-    bash "${EVAL_SETUP_SCRIPT}"
+    export EVAL_SNAPSHOT_DIR=$(bash "${EVAL_SETUP_SCRIPT}")
+    echo "Snapshot dir: ${EVAL_SNAPSHOT_DIR}"
 fi
 
 # -----------------------------------------------------------------------
@@ -146,11 +95,11 @@ echo "agent-eval-harness cloned."
 copy_artifacts() {
     echo "Copying eval artifacts..."
     if [[ -d "${AGENT_EVAL_RUNS_DIR:-eval/runs}" ]]; then
-        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "report.html" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+        find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "report.html" -exec cp {} "${ARTIFACT_DIR}/eval-report-summary.html" \; 2>/dev/null || true
         find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "summary.yaml" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
         find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" -name "run_result.json" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
     fi
-    find . -name "eval-summary-*.html" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
+    find . -name "*-summary.html" -exec cp {} "${ARTIFACT_DIR}/" \; 2>/dev/null || true
 
     # Copy MLflow data
     if [[ -d "mlruns" ]]; then
@@ -166,8 +115,9 @@ copy_artifacts() {
             touch "${SHARED_DIR}/claude-session-available" || true
     fi
 
-    # Stop MLflow server
+    # Stop MLflow server and all child processes (gunicorn workers)
     if [[ -n "${MLFLOW_PID:-}" ]]; then
+        pkill -P "${MLFLOW_PID}" 2>/dev/null || true
         kill "${MLFLOW_PID}" 2>/dev/null || true
     fi
 }

From f1bfab950cd1850cf02fe39da66bff9bb6a624f6 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 15:36:45 -0400
Subject: [PATCH 16/17] ci: remove unused EVAL_JUDGE_MODEL env var

Judge model is configured in the eval config's models.judge field,
not passed via CLI.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh   | 2 --
 .../claude/agent-eval/openshift-claude-agent-eval-ref.yaml      | 2 --
 2 files changed, 4 deletions(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index 7c0e99eb4651d..e550e2c1d285c 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -7,7 +7,6 @@
 #
 # Optional env:
 #   EVAL_MODEL        -- model for the skill under test (default: claude-sonnet-4-6)
-#   EVAL_JUDGE_MODEL  -- model for LLM judges (default: claude-sonnet-4-6)
 #   EVAL_BASELINE     -- run-id of a previous run to compare against
 #   EVAL_EXTRA_ARGS   -- additional args passed to /eval-run
 #   EVAL_SETUP_SCRIPT -- script to run before eval (e.g. snapshot extraction)
@@ -25,7 +24,6 @@ cd /opt/ai-helpers
 
 echo "Config: ${EVAL_CONFIG}"
 echo "Skill model: ${EVAL_MODEL}"
-echo "Judge model: ${EVAL_JUDGE_MODEL}"
 
 # -----------------------------------------------------------------------
 # Install dependencies
diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml
index e74ca34c62b3d..872e7ba5a425c 100644
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml
@@ -14,8 +14,6 @@ ref:
     default: "eval.yaml"
   - name: EVAL_MODEL
     default: "claude-sonnet-4-6"
-  - name: EVAL_JUDGE_MODEL
-    default: "claude-sonnet-4-6"
   - name: EVAL_BASELINE
     default: ""
   - name: EVAL_EXTRA_ARGS

From d984b0d0c21df6944321f111f459e7bf7798ac90 Mon Sep 17 00:00:00 2001
From: Stephen Benjamin <stbenjam+ai@redhat.com>
Date: Mon, 1 Jun 2026 16:06:59 -0400
Subject: [PATCH 17/17] =?UTF-8?q?ci:=20fix=20shellcheck=20SC2155=20?=
 =?UTF-8?q?=E2=80=94=20declare=20and=20assign=20separately?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../claude/agent-eval/openshift-claude-agent-eval-commands.sh  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
index e550e2c1d285c..7723a216ef307 100755
--- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
+++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh
@@ -74,7 +74,8 @@ if [[ -n "${EVAL_SETUP_SCRIPT}" ]]; then
     fi
     echo ""
     echo "=== Running setup script: ${EVAL_SETUP_SCRIPT} ==="
-    export EVAL_SNAPSHOT_DIR=$(bash "${EVAL_SETUP_SCRIPT}")
+    EVAL_SNAPSHOT_DIR=$(bash "${EVAL_SETUP_SCRIPT}")
+    export EVAL_SNAPSHOT_DIR
     echo "Snapshot dir: ${EVAL_SNAPSHOT_DIR}"
 fi