From ffe9d4a791c1fe81e8acc77dce727e9464b9d0dc Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 17:21:57 +0530 Subject: [PATCH 1/4] Add eval discovery mode to agent-eval workflow Co-Authored-By: Claude Opus 4.6 --- .../openshift-eng-edge-tooling-main.yaml | 40 +++ ...hift-eng-edge-tooling-main-presubmits.yaml | 306 ++++++++++++++++++ 2 files changed, 346 insertions(+) diff --git a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml index c572084c2ac20..72ec63cc03e18 100644 --- a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml +++ b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml @@ -55,6 +55,46 @@ tests: clone: true from: root run_if_changed: (SKILL\.md|^scripts/lint-skills\.py|^Makefile|^plugins/.*/skills/) +- always_run: false + as: eval-cluster-diagnostic + optional: true + steps: + env: + EVAL_CONFIG: plugins/two-node/evals/cluster-diagnostic.yaml + EVAL_MODEL: claude-opus-4-6 + EVAL_PARALLELISM: "3" + workflow: openshift-claude-agent-eval +- as: eval-cluster-diagnostic-changed + optional: true + run_if_changed: ^plugins/two-node/evals/.*cluster-diagnostic + steps: + env: + EVAL_CASES_DIR: plugins/two-node/evals/cluster-diagnostic/cases + EVAL_CHANGED_ONLY: "true" + EVAL_CONFIG: plugins/two-node/evals/cluster-diagnostic.yaml + EVAL_MODEL: claude-opus-4-6 + EVAL_PARALLELISM: "3" + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-threat-model-tnf + optional: true + steps: + env: + EVAL_CONFIG: plugins/two-node/evals/threat-model-tnf.yaml + EVAL_MODEL: claude-opus-4-6 + EVAL_PARALLELISM: "3" + workflow: openshift-claude-agent-eval +- as: eval-threat-model-tnf-changed + optional: true + run_if_changed: ^plugins/two-node/evals/.*threat-model-tnf + steps: + env: + EVAL_CASES_DIR: plugins/two-node/evals/threat-model-tnf/cases + EVAL_CHANGED_ONLY: "true" + EVAL_CONFIG: plugins/two-node/evals/threat-model-tnf.yaml + EVAL_MODEL: claude-opus-4-6 + EVAL_PARALLELISM: "3" + workflow: openshift-claude-agent-eval - as: ocp-ci-monitor cron: 0 7 * * 1-5 reporter_config: diff --git a/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml b/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml index 458e2d110f7b6..d6d7ded3b3140 100644 --- a/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml @@ -1,5 +1,311 @@ presubmits: openshift-eng/edge-tooling: + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build12 + context: ci/prow/eval-cluster-diagnostic + decorate: true + decoration_config: + sparse_checkout_files: + - images/Containerfile.ci + - images/Containerfile.markdownlint + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-edge-tooling-main-eval-cluster-diagnostic + optional: true + rerun_command: /test eval-cluster-diagnostic + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-cluster-diagnostic + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-cluster-diagnostic,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build12 + context: ci/prow/eval-cluster-diagnostic-changed + decorate: true + decoration_config: + sparse_checkout_files: + - images/Containerfile.ci + - images/Containerfile.markdownlint + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-edge-tooling-main-eval-cluster-diagnostic-changed + optional: true + rerun_command: /test eval-cluster-diagnostic-changed + run_if_changed: ^plugins/two-node/evals/.*cluster-diagnostic + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-cluster-diagnostic-changed + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-cluster-diagnostic-changed,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build12 + context: ci/prow/eval-threat-model-tnf + decorate: true + decoration_config: + sparse_checkout_files: + - images/Containerfile.ci + - images/Containerfile.markdownlint + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-edge-tooling-main-eval-threat-model-tnf + optional: true + rerun_command: /test eval-threat-model-tnf + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-threat-model-tnf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-threat-model-tnf,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build12 + context: ci/prow/eval-threat-model-tnf-changed + decorate: true + decoration_config: + sparse_checkout_files: + - images/Containerfile.ci + - images/Containerfile.markdownlint + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-edge-tooling-main-eval-threat-model-tnf-changed + optional: true + rerun_command: /test eval-threat-model-tnf-changed + run_if_changed: ^plugins/two-node/evals/.*threat-model-tnf + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-threat-model-tnf-changed + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-threat-model-tnf-changed,?($|\s.*) - agent: kubernetes always_run: true branches: From 2ce7768ba7bc6e689d8cfe09f591c7879d9e4426 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:18:11 +0530 Subject: [PATCH 2/4] Add EVAL_DISCOVER support to agent-eval workflow When EVAL_DISCOVER is set ("true" or a glob pattern), the workflow auto-discovers eval configs, diffs against PULL_BASE_SHA to run only affected evals, and produces per-eval JUnit test cases. Single EVAL_CONFIG mode is preserved for backward compatibility. Co-Authored-By: Claude Opus 4.6 --- .../openshift-eng-edge-tooling-main.yaml | 31 +-- ...hift-eng-edge-tooling-main-presubmits.yaml | 175 +-------------- .../openshift-claude-agent-eval-commands.sh | 205 ++++++++++++------ .../openshift-claude-agent-eval-ref.yaml | 2 + 4 files changed, 153 insertions(+), 260 deletions(-) diff --git a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml index 72ec63cc03e18..f6db83f5fc7b6 100644 --- a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml +++ b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml @@ -56,42 +56,21 @@ tests: from: root run_if_changed: (SKILL\.md|^scripts/lint-skills\.py|^Makefile|^plugins/.*/skills/) - always_run: false - as: eval-cluster-diagnostic + as: eval-all optional: true steps: env: - EVAL_CONFIG: plugins/two-node/evals/cluster-diagnostic.yaml + EVAL_DISCOVER: "true" EVAL_MODEL: claude-opus-4-6 EVAL_PARALLELISM: "3" workflow: openshift-claude-agent-eval -- as: eval-cluster-diagnostic-changed +- as: eval-changed optional: true - run_if_changed: ^plugins/two-node/evals/.*cluster-diagnostic + run_if_changed: ^plugins/.*/evals/|^plugins/.*/skills/ steps: env: - EVAL_CASES_DIR: plugins/two-node/evals/cluster-diagnostic/cases EVAL_CHANGED_ONLY: "true" - EVAL_CONFIG: plugins/two-node/evals/cluster-diagnostic.yaml - EVAL_MODEL: claude-opus-4-6 - EVAL_PARALLELISM: "3" - workflow: openshift-claude-agent-eval -- always_run: false - as: eval-threat-model-tnf - optional: true - steps: - env: - EVAL_CONFIG: plugins/two-node/evals/threat-model-tnf.yaml - EVAL_MODEL: claude-opus-4-6 - EVAL_PARALLELISM: "3" - workflow: openshift-claude-agent-eval -- as: eval-threat-model-tnf-changed - optional: true - run_if_changed: ^plugins/two-node/evals/.*threat-model-tnf - steps: - env: - EVAL_CASES_DIR: plugins/two-node/evals/threat-model-tnf/cases - EVAL_CHANGED_ONLY: "true" - EVAL_CONFIG: plugins/two-node/evals/threat-model-tnf.yaml + EVAL_DISCOVER: "true" EVAL_MODEL: claude-opus-4-6 EVAL_PARALLELISM: "3" workflow: openshift-claude-agent-eval diff --git a/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml b/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml index d6d7ded3b3140..15fc381ae1859 100644 --- a/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml @@ -6,7 +6,7 @@ presubmits: - ^main$ - ^main- cluster: build12 - context: ci/prow/eval-cluster-diagnostic + context: ci/prow/eval-all decorate: true decoration_config: sparse_checkout_files: @@ -15,9 +15,9 @@ presubmits: labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-eng-edge-tooling-main-eval-cluster-diagnostic + name: pull-ci-openshift-eng-edge-tooling-main-eval-all optional: true - rerun_command: /test eval-cluster-diagnostic + rerun_command: /test eval-all spec: containers: - args: @@ -25,7 +25,7 @@ presubmits: - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - - --target=eval-cluster-diagnostic + - --target=eval-all command: - ci-operator env: @@ -75,14 +75,14 @@ presubmits: - name: result-aggregator secret: secretName: result-aggregator - trigger: (?m)^/test( | .* )eval-cluster-diagnostic,?($|\s.*) + trigger: (?m)^/test( | .* )eval-all,?($|\s.*) - agent: kubernetes always_run: false branches: - ^main$ - ^main- cluster: build12 - context: ci/prow/eval-cluster-diagnostic-changed + context: ci/prow/eval-changed decorate: true decoration_config: sparse_checkout_files: @@ -91,10 +91,10 @@ presubmits: labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-eng-edge-tooling-main-eval-cluster-diagnostic-changed + name: pull-ci-openshift-eng-edge-tooling-main-eval-changed optional: true - rerun_command: /test eval-cluster-diagnostic-changed - run_if_changed: ^plugins/two-node/evals/.*cluster-diagnostic + rerun_command: /test eval-changed + run_if_changed: ^plugins/.*/evals/|^plugins/.*/skills/ spec: containers: - args: @@ -102,7 +102,7 @@ presubmits: - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - - --target=eval-cluster-diagnostic-changed + - --target=eval-changed command: - ci-operator env: @@ -152,160 +152,7 @@ presubmits: - name: result-aggregator secret: secretName: result-aggregator - trigger: (?m)^/test( | .* )eval-cluster-diagnostic-changed,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - ^main$ - - ^main- - cluster: build12 - context: ci/prow/eval-threat-model-tnf - decorate: true - decoration_config: - sparse_checkout_files: - - images/Containerfile.ci - - images/Containerfile.markdownlint - labels: - ci.openshift.io/generator: prowgen - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-eng-edge-tooling-main-eval-threat-model-tnf - optional: true - rerun_command: /test eval-threat-model-tnf - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --target=eval-threat-model-tnf - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - trigger: (?m)^/test( | .* )eval-threat-model-tnf,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - ^main$ - - ^main- - cluster: build12 - context: ci/prow/eval-threat-model-tnf-changed - decorate: true - decoration_config: - sparse_checkout_files: - - images/Containerfile.ci - - images/Containerfile.markdownlint - labels: - ci.openshift.io/generator: prowgen - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-eng-edge-tooling-main-eval-threat-model-tnf-changed - optional: true - rerun_command: /test eval-threat-model-tnf-changed - run_if_changed: ^plugins/two-node/evals/.*threat-model-tnf - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --target=eval-threat-model-tnf-changed - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - trigger: (?m)^/test( | .* )eval-threat-model-tnf-changed,?($|\s.*) + trigger: (?m)^/test( | .* )eval-changed,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh index 288bbd6ac6913..2c663b18273fd 100755 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh @@ -9,6 +9,7 @@ # EVAL_MODEL -- model for the skill under test (default: claude-sonnet-4-6) # EVAL_PARALLELISM -- number of test cases to run concurrently (default: 1) # EVAL_CASES -- comma-separated list of case IDs to run (default: all) +# EVAL_DISCOVER -- "true" or glob pattern to auto-discover eval configs # EVAL_BASELINE -- run-id of a previous run to compare against # EVAL_EXTRA_ARGS -- additional args passed to /eval-run # EVAL_SETUP_SCRIPT -- script to run before eval (e.g. snapshot extraction) @@ -34,15 +35,67 @@ fi # The repo is at /opt/ai-helpers; WORKDIR is /workspace cd /opt/ai-helpers -echo "Config: ${EVAL_CONFIG}" echo "Skill model: ${EVAL_MODEL}" # ----------------------------------------------------------------------- -# Verify eval config exists +# Build list of eval configs to run (single or discovery mode) # ----------------------------------------------------------------------- -if [[ ! -f "${EVAL_CONFIG}" ]]; then - echo "ERROR: EVAL_CONFIG not found at ${EVAL_CONFIG}" - exit 1 +CONFIGS_TO_RUN=() +if [[ -n "${EVAL_DISCOVER}" ]]; then + if [[ -n "${EVAL_CONFIG}" ]] && [[ "${EVAL_CONFIG}" != "eval.yaml" ]]; then + echo "ERROR: EVAL_DISCOVER and EVAL_CONFIG are mutually exclusive" + exit 1 + fi + + # Default discovery pattern: all YAML files directly under */evals/ directories + DISCOVER_PATTERN="${EVAL_DISCOVER}" + if [[ "${EVAL_DISCOVER}" == "true" ]]; then + DISCOVER_PATTERN="plugins/*/evals/*.yaml" + fi + + echo "=== Discovering eval configs: ${DISCOVER_PATTERN} ===" + while IFS= read -r config; do + [[ -n "${config}" ]] && CONFIGS_TO_RUN+=("${config}") + done < <(find . -path "./${DISCOVER_PATTERN}" -name '*.yaml' ! -path '*/cases/*' | sed 's|^\./||' | sort) + + echo "Found ${#CONFIGS_TO_RUN[@]} eval config(s):" + printf ' %s\n' "${CONFIGS_TO_RUN[@]}" + + if [[ ${#CONFIGS_TO_RUN[@]} -eq 0 ]]; then + echo "No eval configs found matching ${DISCOVER_PATTERN}" + exit 0 + fi + + # Filter to only changed evals when EVAL_CHANGED_ONLY is set + if [[ "${EVAL_CHANGED_ONLY}" == "true" ]] && [[ -n "${PULL_BASE_SHA:-}" ]]; then + echo "" + echo "=== Filtering to changed evals ===" + CHANGED_FILES=$(git diff --name-only "${PULL_BASE_SHA}...HEAD" || true) + + FILTERED=() + for config in "${CONFIGS_TO_RUN[@]}"; do + config_name=$(basename "${config}" .yaml) + config_dir=$(dirname "${config}") + + if echo "${CHANGED_FILES}" | grep -qE "(${config}|${config_dir}/${config_name}/|skills/${config_name}/)"; then + echo " MATCH: ${config}" + FILTERED+=("${config}") + fi + done + + if [[ ${#FILTERED[@]} -eq 0 ]]; then + echo "No eval configs affected by changes, skipping." + exit 0 + fi + CONFIGS_TO_RUN=("${FILTERED[@]}") + fi +else + echo "Config: ${EVAL_CONFIG}" + if [[ ! -f "${EVAL_CONFIG}" ]]; then + echo "ERROR: EVAL_CONFIG not found at ${EVAL_CONFIG}" + exit 1 + fi + CONFIGS_TO_RUN=("${EVAL_CONFIG}") fi # ----------------------------------------------------------------------- @@ -96,92 +149,104 @@ trap copy_artifacts EXIT TERM INT export CLAUDE_CODE_ENTRYPOINT=sdk-cli # ----------------------------------------------------------------------- -# Auto-detect changed eval cases from PR diff +# Build common arguments and run evals # ----------------------------------------------------------------------- -if [[ "${EVAL_CHANGED_ONLY}" == "true" ]] && [[ -n "${EVAL_CASES_DIR}" ]] && [[ -z "${EVAL_CASES}" ]]; then +ALLOWED_TOOLS="Bash Read Write Edit Grep Glob Agent Skill" +OVERALL_EXIT=0 +JUNIT_TESTCASES="" +TOTAL_DURATION=0 +FAILURE_COUNT=0 + +for config in "${CONFIGS_TO_RUN[@]}"; do + config_name=$(basename "${config}" .yaml) echo "" - echo "=== Detecting changed eval cases ===" - if [[ -z "${PULL_BASE_SHA:-}" ]]; then - echo "PULL_BASE_SHA not set, running all cases." - else - if ! CHANGED_FILES=$(git diff --name-only "${PULL_BASE_SHA}...HEAD" -- "${EVAL_CASES_DIR}"); then - echo "Failed to diff against PULL_BASE_SHA (${PULL_BASE_SHA}); running all cases." - elif [[ -n "${CHANGED_FILES}" ]]; then - DETECTED_CASES=$(echo "${CHANGED_FILES}" | sed "s|^${EVAL_CASES_DIR}/||" | cut -d'/' -f1 | sort -u | paste -sd, -) - echo "Changed cases: ${DETECTED_CASES}" - EVAL_CASES="${DETECTED_CASES}" + echo "========================================" + echo "=== Running eval: ${config_name} ===" + echo "========================================" + + RUN_ID="ci-$(date +%Y%m%d-%H%M%S)-${config_name}-${EVAL_MODEL}" + + # Per-config changed-case detection + CASE_ARGS="" + if [[ "${EVAL_CHANGED_ONLY}" == "true" ]] && [[ -n "${PULL_BASE_SHA:-}" ]]; then + # In discovery mode, derive cases dir from config path convention + if [[ -n "${EVAL_DISCOVER}" ]]; then + CASES_DIR="$(dirname "${config}")/${config_name}/cases" + elif [[ -n "${EVAL_CASES_DIR}" ]]; then + CASES_DIR="${EVAL_CASES_DIR}" else - echo "No changed cases detected in ${EVAL_CASES_DIR}, skipping eval." - exit 0 + CASES_DIR="" + fi + + if [[ -n "${CASES_DIR}" ]] && [[ -d "${CASES_DIR}" ]]; then + if CASE_CHANGES=$(git diff --name-only "${PULL_BASE_SHA}...HEAD" -- "${CASES_DIR}"); then + if [[ -n "${CASE_CHANGES}" ]]; then + DETECTED=$(echo "${CASE_CHANGES}" | sed "s|^${CASES_DIR}/||" | cut -d'/' -f1 | sort -u | paste -sd, -) + echo "Changed cases: ${DETECTED}" + CASE_ARGS="--cases ${DETECTED//,/ }" + fi + fi fi fi -fi -# ----------------------------------------------------------------------- -# Build arguments -# ----------------------------------------------------------------------- -RUN_ID="ci-$(date +%Y%m%d-%H%M%S)-${EVAL_MODEL}" -ALLOWED_TOOLS="Bash Read Write Edit Grep Glob Agent Skill" + # Include explicit EVAL_CASES if set (single-config mode) + if [[ -z "${CASE_ARGS}" ]] && [[ -n "${EVAL_CASES}" ]]; then + CASE_ARGS="--cases ${EVAL_CASES//,/ }" + fi -EVAL_RUN_ARGS="--config ${EVAL_CONFIG} --model ${EVAL_MODEL} --run-id ${RUN_ID} --parallelism ${EVAL_PARALLELISM}" -if [[ -n "${EVAL_CASES}" ]]; then - EVAL_RUN_ARGS="${EVAL_RUN_ARGS} --cases ${EVAL_CASES//,/ }" -fi -if [[ -n "${EVAL_BASELINE}" ]]; then - EVAL_RUN_ARGS="${EVAL_RUN_ARGS} --baseline ${EVAL_BASELINE}" -fi -if [[ -n "${EVAL_EXTRA_ARGS}" ]]; then - EVAL_RUN_ARGS="${EVAL_RUN_ARGS} ${EVAL_EXTRA_ARGS}" -fi + EVAL_RUN_ARGS="--config ${config} --model ${EVAL_MODEL} --run-id ${RUN_ID} --parallelism ${EVAL_PARALLELISM}" + [[ -n "${CASE_ARGS}" ]] && EVAL_RUN_ARGS="${EVAL_RUN_ARGS} ${CASE_ARGS}" + [[ -n "${EVAL_BASELINE}" ]] && EVAL_RUN_ARGS="${EVAL_RUN_ARGS} --baseline ${EVAL_BASELINE}" + [[ -n "${EVAL_EXTRA_ARGS}" ]] && EVAL_RUN_ARGS="${EVAL_RUN_ARGS} ${EVAL_EXTRA_ARGS}" + + echo "Run ID: ${RUN_ID}" + echo "Args: ${EVAL_RUN_ARGS}" + + EVAL_START=$(date +%s) + THIS_EXIT=0 + timeout 7200 claude \ + --model "${CLAUDE_MODEL}" \ + --plugin-dir "${EVAL_HARNESS_DIR}" \ + --allowedTools "${ALLOWED_TOOLS}" \ + --output-format stream-json \ + --max-turns "${EVAL_MAX_TURNS}" \ + -p "/eval-run ${EVAL_RUN_ARGS}" \ + --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval-${config_name}.log" || THIS_EXIT=$? + THIS_DURATION=$(( $(date +%s) - EVAL_START )) + TOTAL_DURATION=$(( TOTAL_DURATION + THIS_DURATION )) + + TESTCASE="[sig-claude] ${config_name} evaluation" + if [[ "${THIS_EXIT}" -ne 0 ]]; then + OVERALL_EXIT=1 + FAILURE_COUNT=$(( FAILURE_COUNT + 1 )) + JUNIT_TESTCASES="${JUNIT_TESTCASES} + + eval-run exited with code ${THIS_EXIT}. + " + else + JUNIT_TESTCASES="${JUNIT_TESTCASES} + " + fi -# ----------------------------------------------------------------------- -# Run evaluation -# ----------------------------------------------------------------------- -echo "" -echo "=== Running eval ===" -echo "Run ID: ${RUN_ID}" -echo "Args: ${EVAL_RUN_ARGS}" - -EVAL_START=$(date +%s) -EVAL_EXIT=0 -timeout 7200 claude \ - --model "${CLAUDE_MODEL}" \ - --plugin-dir "${EVAL_HARNESS_DIR}" \ - --allowedTools "${ALLOWED_TOOLS}" \ - --output-format stream-json \ - --max-turns "${EVAL_MAX_TURNS}" \ - -p "/eval-run ${EVAL_RUN_ARGS}" \ - --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval.log" || EVAL_EXIT=$? -EVAL_DURATION=$(( $(date +%s) - EVAL_START )) - -echo "eval-run completed in ${EVAL_DURATION}s (exit ${EVAL_EXIT})" + echo "=== ${config_name}: completed in ${THIS_DURATION}s (exit ${THIS_EXIT}) ===" +done # ----------------------------------------------------------------------- # Generate JUnit XML # ----------------------------------------------------------------------- JUNIT_FILE="${ARTIFACT_DIR}/junit_claude-eval.xml" -FAILURE_COUNT=0 -TESTCASE="[sig-claude] Skill evaluation should pass" - -if [[ "${EVAL_EXIT}" -ne 0 ]]; then - FAILURE_COUNT=1 - TESTCASES=" - eval-run exited with code ${EVAL_EXIT}. - " -else - TESTCASES=" " -fi +TEST_COUNT=${#CONFIGS_TO_RUN[@]} cat > "${JUNIT_FILE}" < - -${TESTCASES} + +${JUNIT_TESTCASES} EOF echo "JUnit XML written to ${JUNIT_FILE}" -if [[ "${EVAL_EXIT}" -ne 0 ]]; then +if [[ "${OVERALL_EXIT}" -ne 0 ]]; then echo "Evaluation failed." exit 1 fi diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml index 00ccd7cec3ac8..30a0bbc54fd64 100644 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml @@ -28,6 +28,8 @@ ref: default: "" - name: EVAL_CHANGED_ONLY default: "" + - name: EVAL_DISCOVER + default: "" - name: EVAL_MAX_TURNS default: "100" - name: CLAUDE_MODEL From 88a7f89e28b3d3b5ef724aa1c44df3988357a3ef Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:57:31 +0530 Subject: [PATCH 3/4] Add incremental JUnit writes and step timeout guard Write JUnit XML after each eval config completes so results are preserved if the step gets killed mid-loop. Add a step-level time guard (2h50m) to skip remaining configs before hitting the 3h limit. Co-Authored-By: Claude Opus 4.6 --- .../openshift-claude-agent-eval-commands.sh | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh index 2c663b18273fd..0d9b3b0217aae 100755 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh @@ -156,8 +156,26 @@ OVERALL_EXIT=0 JUNIT_TESTCASES="" TOTAL_DURATION=0 FAILURE_COUNT=0 +CONFIGS_RUN=0 +JUNIT_FILE="${ARTIFACT_DIR}/junit_claude-eval.xml" +STEP_START=${SECONDS} +STEP_TIMEOUT=10200 # 2h50m — leave margin within the 3h step limit + +write_junit() { + cat > "${JUNIT_FILE}" < + +${JUNIT_TESTCASES} + +JEOF +} for config in "${CONFIGS_TO_RUN[@]}"; do + ELAPSED=$(( SECONDS - STEP_START )) + if [[ ${ELAPSED} -ge ${STEP_TIMEOUT} ]]; then + echo "WARNING: approaching step timeout (${ELAPSED}s elapsed), skipping remaining configs." + break + fi config_name=$(basename "${config}" .yaml) echo "" echo "========================================" @@ -228,21 +246,11 @@ for config in "${CONFIGS_TO_RUN[@]}"; do " fi + CONFIGS_RUN=$(( CONFIGS_RUN + 1 )) echo "=== ${config_name}: completed in ${THIS_DURATION}s (exit ${THIS_EXIT}) ===" -done -# ----------------------------------------------------------------------- -# Generate JUnit XML -# ----------------------------------------------------------------------- -JUNIT_FILE="${ARTIFACT_DIR}/junit_claude-eval.xml" -TEST_COUNT=${#CONFIGS_TO_RUN[@]} - -cat > "${JUNIT_FILE}" < - -${JUNIT_TESTCASES} - -EOF + write_junit +done echo "JUnit XML written to ${JUNIT_FILE}" From 188e695e54f922e6aee6095c27a6018925106d8d Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Wed, 8 Jul 2026 23:09:01 +0530 Subject: [PATCH 4/4] Fix config_name collision and EVAL_CHANGED_ONLY skip logic Derive config_name from full path (tr '/' '-') instead of basename to prevent log/JUnit/RUN_ID collisions when multiple configs share the same filename. Also add early continue when EVAL_CHANGED_ONLY is set but no changed cases are detected, preventing unintended full eval runs. Use consistent ${EVAL_CHANGED_ONLY:-} across all references. Co-Authored-By: Claude Opus 4.6 --- .../openshift-claude-agent-eval-commands.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh index 0d9b3b0217aae..adce666bd7750 100755 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh @@ -67,7 +67,7 @@ if [[ -n "${EVAL_DISCOVER}" ]]; then fi # Filter to only changed evals when EVAL_CHANGED_ONLY is set - if [[ "${EVAL_CHANGED_ONLY}" == "true" ]] && [[ -n "${PULL_BASE_SHA:-}" ]]; then + if [[ "${EVAL_CHANGED_ONLY:-}" == "true" ]] && [[ -n "${PULL_BASE_SHA:-}" ]]; then echo "" echo "=== Filtering to changed evals ===" CHANGED_FILES=$(git diff --name-only "${PULL_BASE_SHA}...HEAD" || true) @@ -176,7 +176,8 @@ for config in "${CONFIGS_TO_RUN[@]}"; do echo "WARNING: approaching step timeout (${ELAPSED}s elapsed), skipping remaining configs." break fi - config_name=$(basename "${config}" .yaml) + config_name=$(echo "${config}" | sed 's|\.yaml$||' | tr '/' '-') + config_basename=$(basename "${config}" .yaml) echo "" echo "========================================" echo "=== Running eval: ${config_name} ===" @@ -186,10 +187,10 @@ for config in "${CONFIGS_TO_RUN[@]}"; do # Per-config changed-case detection CASE_ARGS="" - if [[ "${EVAL_CHANGED_ONLY}" == "true" ]] && [[ -n "${PULL_BASE_SHA:-}" ]]; then + if [[ "${EVAL_CHANGED_ONLY:-}" == "true" ]] && [[ -n "${PULL_BASE_SHA:-}" ]]; then # In discovery mode, derive cases dir from config path convention if [[ -n "${EVAL_DISCOVER}" ]]; then - CASES_DIR="$(dirname "${config}")/${config_name}/cases" + CASES_DIR="$(dirname "${config}")/${config_basename}/cases" elif [[ -n "${EVAL_CASES_DIR}" ]]; then CASES_DIR="${EVAL_CASES_DIR}" else @@ -207,6 +208,12 @@ for config in "${CONFIGS_TO_RUN[@]}"; do fi fi + # Skip configs with no changed cases in changed-only mode + if [[ "${EVAL_CHANGED_ONLY:-}" == "true" ]] && [[ -z "${CASE_ARGS}" ]] && [[ -z "${EVAL_CASES}" ]]; then + echo "No changed cases for ${config_name}, skipping." + continue + fi + # Include explicit EVAL_CASES if set (single-config mode) if [[ -z "${CASE_ARGS}" ]] && [[ -n "${EVAL_CASES}" ]]; then CASE_ARGS="--cases ${EVAL_CASES//,/ }"