diff --git a/ci-operator/step-registry/hypershift/review-agent/process/hypershift-review-agent-process-commands.sh b/ci-operator/step-registry/hypershift/review-agent/process/hypershift-review-agent-process-commands.sh index 3acfd179a7537..16a29b5929c57 100644 --- a/ci-operator/step-registry/hypershift/review-agent/process/hypershift-review-agent-process-commands.sh +++ b/ci-operator/step-registry/hypershift/review-agent/process/hypershift-review-agent-process-commands.sh @@ -45,6 +45,7 @@ echo "Installing Claude Code plugins..." claude plugin marketplace add openshift-eng/ai-helpers claude plugin marketplace add RedHatProductSecurity/prodsec-skills claude plugin install openshift-developer@ai-helpers +claude plugin install prow-agent@ai-helpers cd /tmp/hypershift @@ -142,6 +143,113 @@ DISALLOWED_TOOLS=( "Bash(cat*claude-code-service-account*)" ) + +# OTEL / BigQuery telemetry support +EXTRACT_METRICS="/opt/ai-helpers/plugins/prow-agent/scripts/extract_metrics.py" +OTEL_LOG="${ARTIFACT_DIR}/claude-otel.jsonl" + +# Wrapper: run claude via agentic-ci for native OTEL collection +run_claude() { + local phase=$1; shift + local pr_number=$1; shift + local prompt="$1"; shift + + local phase_otel="/tmp/claude-${pr_number}-${phase}-otel.jsonl" + + agentic-ci run \ + --backend local \ + --harness claude-code \ + --model "${CLAUDE_MODEL}" \ + --workdir /tmp/hypershift \ + --no-streaming \ + "${prompt}" \ + -- \ + --permission-mode default \ + --verbose \ + --output-format stream-json \ + "$@" \ + | grep '^{' + local rc=${PIPESTATUS[0]} + + for f in /tmp/agentic-ci-run.*/claude-otel.jsonl; do + if [ -f "$f" ]; then + cat "$f" >> "${phase_otel}" + cat "$f" >> "${OTEL_LOG}" + fi + done + rm -rf /tmp/agentic-ci-run.* + return $rc +} + +extract_session_metrics() { + local pr_number=$1 + local phase=$2 + + if [ ! -f "${EXTRACT_METRICS}" ]; then + echo "Warning: extract_metrics.py not found, skipping session metrics" + return 0 + fi + + local phase_otel="/tmp/claude-${pr_number}-${phase}-otel.jsonl" + if [ ! -f "$phase_otel" ] || [ ! -s "$phase_otel" ]; then + echo "Warning: No OTEL data for ${phase}, skipping session metrics" + return 0 + fi + + python3 "${EXTRACT_METRICS}" "$phase_otel" \ + "${ARTIFACT_DIR}/claude-${pr_number}-${phase}-session-metrics-autodl.json" \ + 2>&1 || echo "Warning: Failed to extract session metrics for ${phase}" +} + +get_session_id() { + local json_file=$1 + grep '"type":"result"' "$json_file" 2>/dev/null | head -1 | jq -r '.session_id // ""' 2>/dev/null || echo "" +} + +generate_autodl() { + local pr_number=$1 + local result=$2 + local session_id=${3:-} + local analyzed_at + analyzed_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + local autodl_file="${ARTIFACT_DIR}/review-agent-pr-${pr_number}-autodl.json" + + jq -n \ + --arg pr_number "$pr_number" \ + --arg result "$result" \ + --arg session_id "$session_id" \ + --arg analyzed_at "$analyzed_at" \ + --arg job_name "${JOB_NAME:-}" \ + --arg build_id "${BUILD_ID:-}" \ + '{ + table_name: "address_review_agent", + schema: { + session_id: "string", + agent: "string", + pr_number: "string", + result: "string", + analyzed_at: "string", + job_name: "string", + build_id: "string" + }, + schema_mapping: null, + rows: [{ + session_id: $session_id, + agent: "address-review", + pr_number: $pr_number, + result: $result, + analyzed_at: $analyzed_at, + job_name: $job_name, + build_id: $build_id + }], + chunk_size: 0, + expiration_days: 0, + partition_column: "" + }' > "$autodl_file" + echo "Generated autodl: ${autodl_file}" +} + # Helper: extract token usage from stream-json output and save to SHARED_DIR extract_tokens() { local JSON_FILE=$1 @@ -206,15 +314,12 @@ echo "==========================================" PHASE1_START=$(date +%s) set +e -claude -p "/openshift-developer:address-review-pr $PR_NUMBER" \ +run_claude "address-review" "$PR_NUMBER" "/openshift-developer:address-review-pr $PR_NUMBER" \ --append-system-prompt "You are addressing review comments on PR #$PR_NUMBER in openshift/hypershift. The PR was created from the hypershift-community fork." \ --allowedTools "Bash Read Write Edit Grep Glob WebFetch Agent Skill Task" \ --disallowedTools "${DISALLOWED_TOOLS[@]}" \ --max-turns 200 \ --effort max \ - --model "$CLAUDE_MODEL" \ - --verbose \ - --output-format stream-json \ 2> "/tmp/claude-pr-${PR_NUMBER}-output.log" \ | tee "/tmp/claude-pr-${PR_NUMBER}-output.json" EXIT_CODE=$? @@ -225,10 +330,13 @@ if [ -f "/tmp/claude-pr-${PR_NUMBER}-output.json" ]; then cp "/tmp/claude-pr-${PR_NUMBER}-output.json" "${ARTIFACT_DIR}/claude-pr-${PR_NUMBER}-output.json" 2>/dev/null || true fi -# Extract artifacts for the report step (same helper functions as jira-agent) +# Extract artifacts for the report step extract_artifacts "/tmp/claude-pr-${PR_NUMBER}-output.json" "claude-pr-${PR_NUMBER}-review" +extract_session_metrics "$PR_NUMBER" "address-review" extract_tokens "/tmp/claude-pr-${PR_NUMBER}-output.json" "${SHARED_DIR}/claude-pr-${PR_NUMBER}-review-tokens.json" echo "Token usage: $(cat "${SHARED_DIR}/claude-pr-${PR_NUMBER}-review-tokens.json")" +REVIEW_SESSION_ID=$(get_session_id "/tmp/claude-pr-${PR_NUMBER}-output.json") +generate_autodl "$PR_NUMBER" "$([ $EXIT_CODE -eq 0 ] && echo success || echo failed)" "$REVIEW_SESSION_ID" PHASE_END=$(date +%s) PHASE_DURATION=$((PHASE_END - PHASE1_START)) @@ -245,6 +353,132 @@ else echo "$PR_NUMBER $TIMESTAMP FAILED" >> "$STATE_FILE" fi +# Generate conversation transcript HTML from stream-json output +echo "Generating conversation transcript..." +python3 - "$STATE_FILE" "$ARTIFACT_DIR" << 'TRANSCRIPT_PY' +import json, html, sys, os + +state_file = sys.argv[1] +artifact_dir = sys.argv[2] +output_file = os.path.join(artifact_dir, "review-agent-transcript.html") + +def parse_stream_json(path): + blocks = [] + cost = turns = inp = outp = duration = 0 + model = session = "unknown" + for line in open(path): + line = line.strip() + if not line: + continue + try: + m = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + t = m.get("type", "") + if t == "system" and m.get("subtype") == "init": + model = m.get("model", "unknown") + session = m.get("session_id", "unknown") + if t == "result": + cost = m.get("total_cost_usd", 0) + turns = m.get("num_turns", 0) + u = m.get("usage", {}) + inp = u.get("input_tokens", 0) + outp = u.get("output_tokens", 0) + duration = m.get("duration_ms", 0) + if t == "assistant": + for c in m.get("message", {}).get("content", []): + ct = c.get("type", "") + if ct == "text": + blocks.append(("assistant", c.get("text", ""))) + elif ct == "tool_use": + name = c.get("name", "") + inp_str = json.dumps(c.get("input", {})) + blocks.append(("tool_use", f"{name}: {inp_str[:300]}")) + if t == "user": + r = m.get("tool_use_result", "") + if isinstance(r, str) and r: + blocks.append(("tool_result", r[:1000])) + elif isinstance(r, list): + for item in r: + if isinstance(item, dict) and item.get("type") == "text": + blocks.append(("tool_result", item.get("text", "")[:1000])) + return blocks, {"cost": cost, "turns": turns, "input": inp, "output": outp, + "duration": duration, "model": model, "session": session} + +def render_blocks(blocks): + out = [] + for kind, text in blocks: + escaped = html.escape(text) + if kind == "assistant": + out.append(f'
Assistant
{escaped}
') + elif kind == "tool_use": + out.append(f'
Tool Call
{escaped}
') + elif kind == "tool_result": + cls = "e" if text.startswith("Error:") else "r" + label = "Error" if cls == "e" else "Result" + out.append(f'
{label}
{escaped}
') + return "\n".join(out) + +if not os.path.exists(state_file): + print("No state file, skipping transcript") + sys.exit(0) + +prs = [l.strip().split()[0] for l in open(state_file) if l.strip()] +sections = [] +total_cost = 0 + +for pr_number in prs: + stream_file = f"/tmp/claude-pr-{pr_number}-output.json" + if not os.path.exists(stream_file): + continue + blocks, stats = parse_stream_json(stream_file) + if not blocks: + continue + total_cost += stats["cost"] + dur_s = stats["duration"] // 1000 + dur_str = f"{dur_s // 60}m {dur_s % 60}s" if dur_s >= 60 else f"{dur_s}s" + sections.append(f''' +

PR #{pr_number}

+
+Address Review — {stats["turns"]} turns, ${stats["cost"]:.4f}, {dur_str} +
{render_blocks(blocks)}
+
+
''') + +with open(output_file, "w") as f: + f.write(f''' + + + +Review Agent Transcript + + + +

Back to summary report

+

Review Agent Transcript

+

Total cost: ${total_cost:.4f}

+{"".join(sections) if sections else "

No transcript data available.

"} + +''') +print(f"Transcript written: {len(sections)} PR(s)") +TRANSCRIPT_PY + echo "" echo "=== Processing Summary ===" echo "PR: #$PR_NUMBER" diff --git a/ci-operator/step-registry/hypershift/review-agent/report/hypershift-review-agent-report-commands.sh b/ci-operator/step-registry/hypershift/review-agent/report/hypershift-review-agent-report-commands.sh index 202f1d4c6a109..e642043a5d38b 100644 --- a/ci-operator/step-registry/hypershift/review-agent/report/hypershift-review-agent-report-commands.sh +++ b/ci-operator/step-registry/hypershift/review-agent/report/hypershift-review-agent-report-commands.sh @@ -239,6 +239,8 @@ ${SUMMARY_ROWS} +

View full conversation transcript

+

Details

${DETAIL_SECTIONS}