diff --git a/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.22.yaml b/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.22.yaml index 7ebab5acd23e5..db3e8f5bfd526 100644 --- a/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.22.yaml +++ b/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.22.yaml @@ -3,6 +3,10 @@ base_images: name: ubi namespace: ocp tag: "9" + claude-ai-helpers: + name: claude-ai-helpers + namespace: ci + tag: latest operator-sdk: name: "4.17" namespace: origin @@ -51,6 +55,9 @@ tests: BASE_DOMAIN: ocp-ci.medik8s-ci.devcluster.openshift.com INSTALL_NAMESPACE: nhc-install OPERATOR_RELEASED_VERSION: 0.12.0 + TEST_STEPS: e2e-test + post: + - ref: medik8s-analyze-e2e-failure test: - as: e2e-install cli: latest diff --git a/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.23.yaml b/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.23.yaml index e32bd44702d46..83797e9e4877d 100644 --- a/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.23.yaml +++ b/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.23.yaml @@ -3,6 +3,10 @@ base_images: name: ubi namespace: ocp tag: "9" + claude-ai-helpers: + name: claude-ai-helpers + namespace: ci + tag: latest operator-sdk: name: "4.17" namespace: origin @@ -51,6 +55,9 @@ tests: BASE_DOMAIN: ocp-ci.medik8s-ci.devcluster.openshift.com INSTALL_NAMESPACE: nhc-install OPERATOR_RELEASED_VERSION: 0.12.0 + TEST_STEPS: e2e-test + post: + - ref: medik8s-analyze-e2e-failure test: - as: e2e-install cli: latest diff --git a/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__5.0.yaml b/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__5.0.yaml index d591308acef98..2b0c33d04d99e 100644 --- a/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__5.0.yaml +++ b/ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__5.0.yaml @@ -3,6 +3,10 @@ base_images: name: ubi namespace: ocp tag: "9" + claude-ai-helpers: + name: claude-ai-helpers + namespace: ci + tag: latest operator-sdk: name: "4.17" namespace: origin @@ -51,6 +55,9 @@ tests: BASE_DOMAIN: ocp-ci.medik8s-ci.devcluster.openshift.com INSTALL_NAMESPACE: nhc-install OPERATOR_RELEASED_VERSION: 0.12.0 + TEST_STEPS: e2e-test + post: + - ref: medik8s-analyze-e2e-failure test: - as: e2e-install cli: latest diff --git a/ci-operator/step-registry/medik8s/analyze-e2e-failure/OWNERS b/ci-operator/step-registry/medik8s/analyze-e2e-failure/OWNERS new file mode 100644 index 0000000000000..eb734f4329eac --- /dev/null +++ b/ci-operator/step-registry/medik8s/analyze-e2e-failure/OWNERS @@ -0,0 +1,4 @@ +approvers: + - medik8s-admins +reviewers: + - medik8s-admins diff --git a/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-commands.sh b/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-commands.sh new file mode 100755 index 0000000000000..48b730578a4d9 --- /dev/null +++ b/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-commands.sh @@ -0,0 +1,143 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Medik8s E2E Failure Analyzer ===" + +# --------------------------------------------------------------------------- +# 1. Construct GCS base path and wait for test step artifacts +# --------------------------------------------------------------------------- +JOB_NAME="${JOB_NAME:-unknown}" +BUILD_ID="${BUILD_ID:-unknown}" +JOB_TYPE="${JOB_TYPE:-}" +PULL_NUMBER="${PULL_NUMBER:-}" +REPO_OWNER="${REPO_OWNER:-}" +REPO_NAME="${REPO_NAME:-}" + +if [[ "$JOB_TYPE" == "presubmit" ]] && [[ -n "$PULL_NUMBER" ]]; then + GCS_BUCKET_PATH="pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}" +else + GCS_BUCKET_PATH="logs/${JOB_NAME}/${BUILD_ID}" +fi + +GCSWEB_BASE="https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results" +PROW_JOB_URL="${GCSWEB_BASE}/${GCS_BUCKET_PATH}" +ARTIFACTS_BASE="${GCSWEB_BASE}/${GCS_BUCKET_PATH}/artifacts/${TEST_NAME}" + +echo "Waiting for test step artifacts in GCS..." +FAILURE_DETECTED=false +FAILED_STEP="" +MAX_WAIT=600 +POLL_INTERVAL=15 +WAITED=0 + +while [[ $WAITED -lt $MAX_WAIT ]]; do + for STEP_NAME in $TEST_STEPS; do + FINISHED_JSON=$(curl -sL "${ARTIFACTS_BASE}/${STEP_NAME}/finished.json" 2>/dev/null || true) + if echo "$FINISHED_JSON" | jq -e '.passed == false' &>/dev/null; then + echo "Detected test failure in ${STEP_NAME}/finished.json (waited ${WAITED}s)" + FAILURE_DETECTED=true + FAILED_STEP="$STEP_NAME" + break 2 + elif echo "$FINISHED_JSON" | jq -e '.passed == true' &>/dev/null; then + echo "Test step ${STEP_NAME} passed — skipping analysis." + exit 0 + fi + done + + echo " Waiting for artifacts... (${WAITED}s/${MAX_WAIT}s)" + sleep "$POLL_INTERVAL" + WAITED=$((WAITED + POLL_INTERVAL)) +done + +if [[ "$FAILURE_DETECTED" == "false" ]]; then + echo "Timed out waiting for test step artifacts after ${WAITED}s — skipping analysis." + exit 0 +fi + +# --------------------------------------------------------------------------- +# 2. Verify Claude Code CLI +# --------------------------------------------------------------------------- +if ! command -v claude &>/dev/null; then + echo "ERROR: Claude Code CLI not found — skipping analysis" + exit 0 +fi + +echo "Claude Code CLI: $(claude --version 2>/dev/null || echo 'unknown')" + +# --------------------------------------------------------------------------- +# 3. Run Claude with the pre-installed ai-helpers skill +# --------------------------------------------------------------------------- +echo "Prow job URL: $PROW_JOB_URL" +echo "Failed step: $FAILED_STEP" + +SYSTEM_PROMPT="IMPORTANT CI CONTEXT: +- You are running inside the CI job itself as a post-step. +- This step's artifact directory is: ${ARTIFACT_DIR} +- Other steps' artifacts (build-log, JUnit, intervals) are available via GCS at: ${PROW_JOB_URL} +- You have network access to download artifacts from GCS using curl. +- Write the final analysis report to: ${ARTIFACT_DIR}/failure-analysis.md +- Use --fast mode (do NOT use AskUserQuestion). +- Do NOT prompt for JIRA export — just write the markdown analysis. + +MEDIK8S CONTEXT: +- Medik8s operators handle automated node remediation in OpenShift/Kubernetes clusters. +- Operators: self-node-remediation (SNR), node-healthcheck-operator (NHC), fence-agents-remediation (FAR), machine-deletion-remediation (MDR), node-maintenance-operator (NMO), storage-based-remediation (SBR). +- E2E tests use Ginkgo v2 + Gomega framework. +- Tests run on ephemeral OCP clusters provisioned via IPI-AWS." + +echo "" +echo "Running Claude with /ci:prow-job-analysis skill..." +echo "" + +set +e +timeout 1200 claude -p "/ci:prow-job-analysis ${PROW_JOB_URL} --fast" \ + --append-system-prompt "$SYSTEM_PROMPT" \ + --allowedTools "Bash Read Write Edit Grep Glob WebFetch Skill" \ + --max-turns 100 \ + --model "$CLAUDE_MODEL" \ + --verbose \ + --output-format stream-json \ + 2> "${ARTIFACT_DIR}/claude-failure-analysis.log" \ + | tee "${ARTIFACT_DIR}/claude-failure-analysis.json" +CLAUDE_EXIT=$? +set -e + +if [[ "$CLAUDE_EXIT" -eq 124 ]]; then + echo "Claude timed out — report may be incomplete" +fi + +# --------------------------------------------------------------------------- +# 4. Extract token usage +# --------------------------------------------------------------------------- + +TOKENS_JSON=$(grep '"type":"result"' "${ARTIFACT_DIR}/claude-failure-analysis.json" 2>/dev/null \ + | head -1 \ + | jq '{ + total_cost_usd: (.total_cost_usd // 0), + duration_ms: (.duration_ms // 0), + num_turns: (.num_turns // 0), + input_tokens: (.usage.input_tokens // 0), + output_tokens: (.usage.output_tokens // 0), + cache_read_input_tokens: (.usage.cache_read_input_tokens // 0), + cache_creation_input_tokens: (.usage.cache_creation_input_tokens // 0) + }' 2>/dev/null \ + || echo '{"total_cost_usd":0,"duration_ms":0,"num_turns":0,"input_tokens":0,"output_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}') + +DURATION_MS=$(echo "$TOKENS_JSON" | jq -r '.duration_ms // 0') +DURATION_S=$(printf '%.0f' "$(echo "${DURATION_MS} / 1000" | bc -l 2>/dev/null)" 2>/dev/null || echo 0) +NUM_TURNS=$(echo "$TOKENS_JSON" | jq -r '.num_turns // 0') + +jq -j 'select(.type == "assistant") | .message.content[]? | select(.type == "text") | .text // empty' \ + "${ARTIFACT_DIR}/claude-failure-analysis.json" \ + > "${ARTIFACT_DIR}/claude-failure-analysis-text.txt" 2>/dev/null || true + +echo "$TOKENS_JSON" > "${SHARED_DIR}/claude-failure-analysis-tokens.json" 2>/dev/null || true + +echo "" +echo "=== Failure Analysis Complete ===" +echo "Claude exit code: $CLAUDE_EXIT" +echo "Duration: ${DURATION_S}s" +echo "Turns: ${NUM_TURNS}" +echo "Analysis: ${ARTIFACT_DIR}/failure-analysis.md" + +exit 0 diff --git a/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.metadata.json b/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.metadata.json new file mode 100644 index 0000000000000..2675d88313b5b --- /dev/null +++ b/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.yaml", + "owners": { + "approvers": [ + "medik8s-admins" + ], + "reviewers": [ + "medik8s-admins" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.yaml b/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.yaml new file mode 100644 index 0000000000000..99d2b026842e1 --- /dev/null +++ b/ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.yaml @@ -0,0 +1,51 @@ +ref: + as: medik8s-analyze-e2e-failure + from: claude-ai-helpers + best_effort: true + commands: medik8s-analyze-e2e-failure-commands.sh + timeout: 30m0s + grace_period: 30s + env: + - name: CLAUDE_CODE_USE_VERTEX + default: "1" + documentation: |- + Enable Vertex AI for Claude Code. + - name: CLOUD_ML_REGION + default: "global" + documentation: |- + Google Cloud region for Vertex AI. + - name: ANTHROPIC_VERTEX_PROJECT_ID + default: "itpc-gcp-hybrid-pe-eng-claude" + documentation: |- + Google Cloud project ID for Vertex AI authentication. + - name: GOOGLE_APPLICATION_CREDENTIALS + default: "/var/run/claude-code-service-account/token" + documentation: |- + Path to the Google Cloud service account JSON key file for Vertex AI authentication. + - name: CLAUDE_MODEL + default: "claude-opus-4-6" + documentation: |- + Claude model to use for test failure analysis. + - name: TEST_NAME + default: "openshift-e2e" + documentation: |- + The test name (as: field) in the ci-operator config. Used to construct + the GCS artifact path. Override if your test uses a different name. + - name: TEST_STEPS + default: "test-command" + documentation: |- + Space-separated list of test step names to check for finished.json. + These are the inner step names (as: fields) within the test definition. + resources: + requests: + cpu: 100m + memory: 256Mi + credentials: + - namespace: test-credentials + name: sa-claude-openshift-ci + mount_path: /var/run/claude-code-service-account + documentation: |- + Post-step that uses Claude to analyze e2e test failures for medik8s operators. + Only runs when test failures are detected (polls GCS for finished.json). + On success (no test failures), exits early with no cost. + Produces a markdown analysis report in ARTIFACT_DIR.