diff --git a/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-commands.sh b/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-commands.sh index f6694a58d9b01..4bb470dc44134 100644 --- a/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-commands.sh +++ b/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-commands.sh @@ -28,22 +28,6 @@ else echo "Warning: Slack webhook not found at ${SLACK_WEBHOOK_URL}. Notifications will be skipped." fi -JIRA_API_TOKEN="" -if [ -f "${JIRA_API_TOKEN_PATH}" ]; then - JIRA_API_TOKEN=$(cat "${JIRA_API_TOKEN_PATH}") - echo "Jira API token loaded." -else - echo "Warning: Jira API token not found at ${JIRA_API_TOKEN_PATH}. Jira operations will not be available." -fi - -JIRA_USERNAME="" -if [ -f "${JIRA_USERNAME_PATH}" ]; then - JIRA_USERNAME=$(cat "${JIRA_USERNAME_PATH}") - echo "Jira username loaded." -else - echo "Warning: Jira username not found at ${JIRA_USERNAME_PATH}. Jira operations will not be available." -fi - # Install gcloud CLI for GCS artifact access (no root required) echo "Installing gcloud CLI..." curl -sSL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz | tar -xz -C /tmp @@ -63,7 +47,6 @@ PAYLOAD_URL="${RELEASE_CONTROLLER_URL}/releasestream/${STREAM_NAME}/release/${PA echo "Version: ${VERSION}, Stream: ${STREAM}" echo "Release API: ${API_URL}" -echo "Automatic reverts enabled: ${ENABLE_PAYLOAD_REVERT}" # Poll until blocking jobs finish OR the payload reaches a terminal state. # The release controller can report jobs as Pending even after they complete @@ -93,7 +76,13 @@ echo "" while true; do POLL_COUNT=$((POLL_COUNT + 1)) - RELEASE_JSON=$(curl -sf "${API_URL}") + RELEASE_JSON=$(curl -sf "${API_URL}" || true) + if [[ -z "${RELEASE_JSON}" ]]; then + echo " Warning: Failed to fetch release API. Retrying next poll..." + sleep ${POLL_INTERVAL} + ELAPSED=$((ELAPSED + POLL_INTERVAL)) + continue + fi PENDING=$(echo "${RELEASE_JSON}" | jq '[.results.blockingJobs // {} | to_entries[] | select(.value.state == "Pending")] | length') FAILED=$(echo "${RELEASE_JSON}" | jq '[.results.blockingJobs // {} | to_entries[] | select(.value.state == "Failed")] | length') SUCCEEDED=$(echo "${RELEASE_JSON}" | jq '[.results.blockingJobs // {} | to_entries[] | select(.value.state == "Succeeded")] | length') @@ -125,7 +114,7 @@ while true; do SLACK_TEXT=":green-check: *Payload Accepted for <${PAYLOAD_URL}|${PAYLOAD_TAG}>* All ${TOTAL} blocking jobs succeeded.${RETRY_INFO} -_Agent: ${CLAUDE_MODEL}_" +_Model: ${CLAUDE_MODEL}_" set +x jq -n --arg text "$SLACK_TEXT" '{text: $text}' | \ @@ -200,6 +189,21 @@ copy_reports() { } trap copy_reports EXIT TERM INT +# Create payload snapshot deterministically (no tokens spent) +echo "" +echo "=== Creating payload snapshot ===" +SNAPSHOT_SCRIPT="/opt/ai-helpers/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py" +SNAPSHOT_DIR="${WORKDIR}/snapshot" +PHASE_SNAPSHOT_START=$(date +%s) +python3 "${SNAPSHOT_SCRIPT}" "${PAYLOAD_TAG}" --output-dir "${SNAPSHOT_DIR}" +PHASE_SNAPSHOT_DURATION=$(( $(date +%s) - PHASE_SNAPSHOT_START )) +echo "Snapshot created in ${PHASE_SNAPSHOT_DURATION}s" +echo "Archiving payload snapshot to artifacts..." +tar -czf "${ARTIFACT_DIR}/snapshot-${PAYLOAD_TAG}.tar.gz" -C "${SNAPSHOT_DIR}" . + +SNAPSHOT_DATA_DIR=$(dirname "$(find "${SNAPSHOT_DIR}" -name summary.json -print -quit)") +echo "Snapshot data dir: ${SNAPSHOT_DATA_DIR}" + # Install the must-gather plugin for analyzing must-gather archives echo "Installing must-gather plugin..." claude plugin install must-gather@ai-helpers @@ -209,7 +213,9 @@ ALLOWED_TOOLS="Bash Read Write Edit Grep Glob WebFetch WebSearch Task Skill" SYSTEM_PROMPT="You are a diligent senior OpenShift release engineer triaging failures. -**CRITICAL**: You have many ci, must-gather, and jira skills at your disposal. You MUST load the relevant skills using the Skill tool BEFORE you begin any work. Do NOT improvise or guess. This applies equally to subagents: instruct every subagent to review its available skills and load the appropriate ones before beginning its investigation. A subagent that does not load a skill will produce shallow, unreliable analysis." +**CRITICAL**: You have many ci and must-gather skills at your disposal. You MUST load the relevant skills using the Skill tool BEFORE you begin any work. Do NOT improvise or guess. This applies equally to subagents: instruct every subagent to review its available skills and load the appropriate ones before beginning its investigation. A subagent that does not load a skill will produce shallow, unreliable analysis. + +After completing your analysis, you MUST use the Skill tool to invoke ci:payload-results-yaml and ci:payload-autodl-json to generate the structured output files. NEVER write these files directly — the skills enforce the canonical schema." PHASE_ANALYSIS_START=$(date +%s) CLAUDE_EXIT=0 @@ -219,11 +225,9 @@ timeout 3600 claude \ --output-format stream-json \ --max-turns 100 \ --append-system-prompt "${SYSTEM_PROMPT}" \ - -p "/ci:analyze-payload ${PAYLOAD_TAG}" \ + -p "/ci:payload-analysis ${PAYLOAD_TAG} --snapshot-dir ${SNAPSHOT_DATA_DIR}" \ --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-output.log" || CLAUDE_EXIT=$? -PHASE_ANALYSIS_DURATION=$(( $(date +%s) - PHASE_ANALYSIS_START )) - # If Claude timed out (exit 124), nudge it to wrap up with a shorter timeout PHASE_NUDGE_START=$(date +%s) NUDGE_EXIT=0 @@ -241,74 +245,64 @@ if [[ "${CLAUDE_EXIT}" -eq 124 ]]; then fi PHASE_NUDGE_DURATION=$(( $(date +%s) - PHASE_NUDGE_START )) -# Optionally stage reverts for high-confidence candidates -PHASE_REVERT_START=$(date +%s) -REVERT_EXIT=0 -if [[ "${ENABLE_PAYLOAD_REVERT}" == "true" ]]; then - if [[ -z "${GITHUB_TOKEN:-}" ]]; then - echo "Warning: Automatic reverts enabled but no GitHub token is available. Skipping reverts." - elif ls "${WORKDIR}"/payload-results-*.yaml 1>/dev/null 2>&1; then - echo "" - echo "=== Staging reverts for high-confidence candidates ===" - - # Configure Jira MCP server for creating TRT issues - REVERT_ALLOWED_TOOLS="${ALLOWED_TOOLS}" - if [[ -n "${JIRA_API_TOKEN}" ]] && [[ -n "${JIRA_USERNAME}" ]]; then - echo "Configuring Jira MCP server..." - set +x - claude mcp add \ - -e JIRA_URL="${JIRA_URL}" \ - -e JIRA_API_TOKEN="${JIRA_API_TOKEN}" \ - -e JIRA_USERNAME="${JIRA_USERNAME}" \ - --transport stdio \ - jira -- uvx mcp-atlassian@0.21.0 - echo "Jira MCP server configured." - REVERT_ALLOWED_TOOLS="${REVERT_ALLOWED_TOOLS} mcp__jira__*" - else - echo "Warning: Jira API token or username not available. TRT issues will not be created." - fi +# Validate structured output files, retry up to 3 times if missing/invalid +VALIDATE_YAML="/opt/ai-helpers/plugins/ci/skills/payload-results-yaml/scripts/validate.py" +VALIDATE_JSON="/opt/ai-helpers/plugins/ci/skills/payload-autodl-json/scripts/validate.py" - timeout 3600 claude \ - --model "${CLAUDE_MODEL}" \ - --continue \ - --allowedTools "${REVERT_ALLOWED_TOOLS}" \ - --output-format stream-json \ - --max-turns 50 \ - -p "/ci:payload-revert ${PAYLOAD_TAG}" \ - --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-revert.log" || REVERT_EXIT=$? +for attempt in 1 2 3; do + YAML_OK=false + JSON_OK=false - else - echo "Warning: No payload results YAML found. Skipping reverts." + RESULTS_YAML=$(find . -maxdepth 1 -name "payload-results-*.yaml" -print -quit) + RESULTS_JSON=$(find . -maxdepth 1 -name "payload-analysis-*-autodl.json" -print -quit) + + if [[ -n "${RESULTS_YAML}" ]] && python3 "${VALIDATE_YAML}" "${RESULTS_YAML}"; then + YAML_OK=true + fi + if [[ -n "${RESULTS_JSON}" ]] && python3 "${VALIDATE_JSON}" "${RESULTS_JSON}"; then + JSON_OK=true fi -else - echo "Automatic reverts not enabled. Skipping revert stage." -fi -PHASE_REVERT_DURATION=$(( $(date +%s) - PHASE_REVERT_START )) + + if $YAML_OK && $JSON_OK; then + echo "Structured outputs validated (attempt ${attempt})." + break + fi + + if [[ "${attempt}" -eq 3 ]]; then + echo "ERROR: Structured outputs still invalid after 3 attempts." + PHASE_ANALYSIS_DURATION=$(( $(date +%s) - PHASE_ANALYSIS_START )) + exit 1 + fi + + MISSING="" + if ! $YAML_OK; then MISSING="ci:payload-results-yaml"; fi + if ! $JSON_OK; then MISSING="${MISSING:+${MISSING} and }ci:payload-autodl-json"; fi + echo "Attempt ${attempt}: Missing/invalid outputs (${MISSING}). Re-invoking Claude..." + + timeout 600 claude \ + --model "${CLAUDE_MODEL}" \ + --continue \ + --allowedTools "${ALLOWED_TOOLS}" \ + --output-format stream-json \ + --max-turns 10 \ + -p "Your structured output files are missing or invalid. Use the Skill tool to invoke ${MISSING} to regenerate them now." \ + --verbose 2>&1 | tee -a "${ARTIFACT_DIR}/claude-output.log" || true +done + +PHASE_ANALYSIS_DURATION=$(( $(date +%s) - PHASE_ANALYSIS_START )) # Generate JUnit XML for timeout and phase duration tracking JUNIT_FILE="${ARTIFACT_DIR}/junit_claude-ci.xml" PHASE_PREFIX="[sig-claude]" TIMEOUT_TESTCASE="${PHASE_PREFIX} Claude should complete in a reasonable time" -TOTAL_DURATION=$(( PHASE_WAIT_DURATION + PHASE_ANALYSIS_DURATION + PHASE_NUDGE_DURATION + PHASE_REVERT_DURATION )) +TOTAL_DURATION=$(( PHASE_WAIT_DURATION + PHASE_SNAPSHOT_DURATION + PHASE_ANALYSIS_DURATION + PHASE_NUDGE_DURATION )) PHASE_CASES=" + " TIMEOUT_CASES="" FAILURE_COUNT=0 - -if [[ "${ENABLE_PAYLOAD_REVERT}" == "true" ]]; then - if [[ "${REVERT_EXIT}" -ne 0 ]]; then - FAILURE_COUNT=$(( FAILURE_COUNT + 1 )) - PHASE_CASES="${PHASE_CASES} - - Claude payload revert exited with code ${REVERT_EXIT}. - " - else - PHASE_CASES="${PHASE_CASES} - " - fi -fi TIMEOUT_TEST_COUNT=0 if [[ "${CLAUDE_EXIT}" -eq 124 ]]; then @@ -343,9 +337,9 @@ else TIMEOUT_CASES=" " fi -PHASE_COUNT=2 -if [[ "${ENABLE_PAYLOAD_REVERT}" == "true" ]]; then - PHASE_COUNT=3 +PHASE_COUNT=3 +if [[ "${CLAUDE_EXIT}" -eq 124 ]]; then + PHASE_COUNT=$((PHASE_COUNT + 1)) fi TEST_COUNT=$(( PHASE_COUNT + TIMEOUT_TEST_COUNT )) cat > "${JUNIT_FILE}" </dev/null) || SUMMARY="" if [[ -n "${SLACK_WEBHOOK}" ]]; then @@ -388,7 +382,7 @@ if [[ -n "${SLACK_WEBHOOK}" ]]; then ${SUMMARY:-No summary available.} <${PROW_JOB_URL}|:point_right: View Full Analysis Report> -_Agent: ${CLAUDE_MODEL}_" +_Model: ${CLAUDE_MODEL}_" set +x jq -n --arg text "$SLACK_TEXT" '{text: $text}' | \ diff --git a/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-ref.yaml b/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-ref.yaml index d8f0a95eec2ff..94aeb9b6b6dce 100644 --- a/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-ref.yaml +++ b/ci-operator/step-registry/openshift/claude/payload/agent/openshift-claude-payload-agent-ref.yaml @@ -7,7 +7,7 @@ ref: commands: openshift-claude-payload-agent-commands.sh credentials: - namespace: test-credentials - name: hypershift-team-claude-prow + name: sa-claude-openshift-ci mount_path: /var/run/claude-code-service-account - namespace: test-credentials name: trt-alert-slack-webhook-url @@ -15,9 +15,6 @@ ref: - namespace: test-credentials name: claude-payload-agent-github-token mount_path: /var/run/github-token - - namespace: test-credentials - name: claude-payload-agent-jira-token - mount_path: /var/run/jira-token env: - name: PAYLOAD_TAG - name: CLAUDE_CODE_USE_VERTEX @@ -27,21 +24,13 @@ ref: - name: ANTHROPIC_VERTEX_PROJECT_ID default: "itpc-gcp-hybrid-pe-eng-claude" - name: GOOGLE_APPLICATION_CREDENTIALS - default: "/var/run/claude-code-service-account/claude-prow" + default: "/var/run/claude-code-service-account/token" - name: CLAUDE_MODEL default: "claude-opus-4-6" - name: SLACK_WEBHOOK_URL default: "/var/run/slack-webhook/url" - name: GITHUB_TOKEN_PATH default: "/var/run/github-token/token" - - name: JIRA_API_TOKEN_PATH - default: "/var/run/jira-token/token" - - name: JIRA_USERNAME_PATH - default: "/var/run/jira-token/username" - - name: JIRA_URL - default: "https://redhat.atlassian.net" - - name: ENABLE_PAYLOAD_REVERT - default: "false" - name: ENABLE_SLACK_NOTIFICATIONS default: "true" resources: @@ -51,9 +40,7 @@ ref: timeout: 12h0m0s grace_period: 1m0s documentation: |- - Analyzes rejected release payloads using Claude AI and optionally - stages reverts for high-confidence causes. Receives a PAYLOAD_TAG - from the release controller, polls until blocking jobs complete, - runs the analyze-payload skill to identify root causes, generates - an HTML report, and when ENABLE_PAYLOAD_REVERT is true, stages - reverts for high-confidence candidates via payload-revert. + Analyzes rejected release payloads using Claude AI. Creates a + deterministic snapshot of payload data, then runs the + payload-analysis skill to identify root causes and generate an + HTML report with revert recommendations.