diff --git a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml index 9b08d5a64cc56..8c80f3b8a2b6c 100644 --- a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml +++ b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml @@ -27,7 +27,62 @@ tests: env: EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml EVAL_MAX_TURNS: "2500" - EVAL_MODEL: claude-opus-4-6 + EVAL_MODEL: claude-opus-4-8 + EVAL_PARALLELISM: "5" + EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-payload-analysis-opus47 + optional: true + steps: + env: + EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml + EVAL_MAX_TURNS: "2500" + EVAL_MODEL: claude-opus-4-7 + EVAL_PARALLELISM: "5" + EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-payload-analysis-opus48 + optional: true + steps: + env: + EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml + EVAL_MAX_TURNS: "2500" + EVAL_MODEL: claude-opus-4-8 + EVAL_PARALLELISM: "5" + EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-payload-analysis-sonnet46 + optional: true + steps: + env: + EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml + EVAL_MAX_TURNS: "2500" + EVAL_MODEL: claude-sonnet-4-6 + EVAL_PARALLELISM: "5" + EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-payload-analysis-sonnet46-1m + optional: true + steps: + env: + EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml + EVAL_MAX_TURNS: "2500" + EVAL_MODEL: claude-sonnet-4-6[1m] + EVAL_PARALLELISM: "5" + EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-payload-analysis-haiku45 + optional: true + steps: + env: + EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml + EVAL_MAX_TURNS: "2500" + EVAL_MODEL: claude-haiku-4-5 EVAL_PARALLELISM: "5" EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh workflow: openshift-claude-agent-eval @@ -39,7 +94,7 @@ tests: EVAL_CASES: case-001 EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml EVAL_MAX_TURNS: "250" - EVAL_MODEL: claude-opus-4-6 + EVAL_MODEL: claude-opus-4-8 EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh workflow: openshift-claude-agent-eval zz_generated_metadata: diff --git a/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml b/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml index 9ef9152d37772..ba3041f0af8f3 100644 --- a/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main-presubmits.yaml @@ -75,6 +75,81 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )eval-payload-analysis,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/eval-payload-analysis-haiku45 + decorate: true + decoration_config: + sparse_checkout_files: + - images/Dockerfile + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis-haiku45 + optional: true + rerun_command: /test eval-payload-analysis-haiku45 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-payload-analysis-haiku45 + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-payload-analysis-haiku45,?($|\s.*) - agent: kubernetes always_run: false branches: @@ -150,6 +225,306 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )eval-payload-analysis-minimal,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/eval-payload-analysis-opus47 + decorate: true + decoration_config: + sparse_checkout_files: + - images/Dockerfile + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis-opus47 + optional: true + rerun_command: /test eval-payload-analysis-opus47 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-payload-analysis-opus47 + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-payload-analysis-opus47,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/eval-payload-analysis-opus48 + decorate: true + decoration_config: + sparse_checkout_files: + - images/Dockerfile + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis-opus48 + optional: true + rerun_command: /test eval-payload-analysis-opus48 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-payload-analysis-opus48 + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-payload-analysis-opus48,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/eval-payload-analysis-sonnet46 + decorate: true + decoration_config: + sparse_checkout_files: + - images/Dockerfile + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis-sonnet46 + optional: true + rerun_command: /test eval-payload-analysis-sonnet46 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-payload-analysis-sonnet46 + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-payload-analysis-sonnet46,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/eval-payload-analysis-sonnet46-1m + decorate: true + decoration_config: + sparse_checkout_files: + - images/Dockerfile + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ai-helpers-main-eval-payload-analysis-sonnet46-1m + optional: true + rerun_command: /test eval-payload-analysis-sonnet46-1m + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-payload-analysis-sonnet46-1m + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-payload-analysis-sonnet46-1m,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml index 97540e3342e6d..12466a231a04d 100644 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml @@ -4,13 +4,13 @@ ref: commands: openshift-claude-agent-eval-commands.sh credentials: - namespace: test-credentials - name: hypershift-team-claude-prow + name: sa-claude-model-eval mount_path: /var/run/claude-code-service-account env: - name: EVAL_CONFIG default: "eval.yaml" - name: EVAL_MODEL - default: "claude-opus-4-6" + default: "claude-opus-4-8" - name: EVAL_BASELINE default: "" - name: EVAL_PARALLELISM @@ -24,15 +24,15 @@ ref: - name: EVAL_MAX_TURNS default: "100" - name: CLAUDE_MODEL - default: "claude-opus-4-6" + default: "claude-opus-4-8" - name: CLAUDE_CODE_USE_VERTEX default: "1" - name: CLOUD_ML_REGION default: "global" - name: ANTHROPIC_VERTEX_PROJECT_ID - default: "itpc-gcp-hybrid-pe-eng-claude" + default: "gcp-jboyer-san-gemini" - name: GOOGLE_APPLICATION_CREDENTIALS - default: "/var/run/claude-code-service-account/claude-prow" + default: "/var/run/claude-code-service-account/token" resources: requests: cpu: 1000m