diff --git a/.changeset/add-graders-run-command.md b/.changeset/add-graders-run-command.md new file mode 100644 index 00000000000..cece5d0d7ae --- /dev/null +++ b/.changeset/add-graders-run-command.md @@ -0,0 +1,5 @@ +--- +"gh-aw": minor +--- + +Add `gh aw graders run` to execute one workflow grader with a saved run payload or JSON from standard input. diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index 3ee3d26b528..5730bc929e3 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -1137,6 +1137,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1158,6 +1159,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/ace-editor.lock.yml b/.github/workflows/ace-editor.lock.yml index 04ac3455390..479cd5b710c 100644 --- a/.github/workflows/ace-editor.lock.yml +++ b/.github/workflows/ace-editor.lock.yml @@ -1142,6 +1142,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/agent-job-health.lock.yml b/.github/workflows/agent-job-health.lock.yml index 05197efcbfa..3fb10edc21a 100644 --- a/.github/workflows/agent-job-health.lock.yml +++ b/.github/workflows/agent-job-health.lock.yml @@ -1356,6 +1356,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1375,6 +1376,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/agent-performance-analyzer.lock.yml b/.github/workflows/agent-performance-analyzer.lock.yml index e9dbb02ef5d..b4663aeb9f9 100644 --- a/.github/workflows/agent-performance-analyzer.lock.yml +++ b/.github/workflows/agent-performance-analyzer.lock.yml @@ -1361,6 +1361,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1382,6 +1383,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/agent-persona-explorer.lock.yml b/.github/workflows/agent-persona-explorer.lock.yml index eaa39b2bcd4..488fdc04ca7 100644 --- a/.github/workflows/agent-persona-explorer.lock.yml +++ b/.github/workflows/agent-persona-explorer.lock.yml @@ -1232,6 +1232,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1253,6 +1254,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/agentic-token-trend-audit.lock.yml b/.github/workflows/agentic-token-trend-audit.lock.yml index eff5f174aa5..65adc0d6c9e 100644 --- a/.github/workflows/agentic-token-trend-audit.lock.yml +++ b/.github/workflows/agentic-token-trend-audit.lock.yml @@ -1299,6 +1299,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1320,6 +1321,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/ai-moderator.lock.yml b/.github/workflows/ai-moderator.lock.yml index 535c22af272..a937478df5f 100644 --- a/.github/workflows/ai-moderator.lock.yml +++ b/.github/workflows/ai-moderator.lock.yml @@ -1261,6 +1261,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1284,6 +1285,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/api-consumption-report.lock.yml b/.github/workflows/api-consumption-report.lock.yml index 9eb450094ed..7f48f4afb54 100644 --- a/.github/workflows/api-consumption-report.lock.yml +++ b/.github/workflows/api-consumption-report.lock.yml @@ -1335,6 +1335,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1356,6 +1357,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/approach-validator.lock.yml b/.github/workflows/approach-validator.lock.yml index a35a2dc3d12..b3f11b435f3 100644 --- a/.github/workflows/approach-validator.lock.yml +++ b/.github/workflows/approach-validator.lock.yml @@ -1344,6 +1344,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1363,6 +1364,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/archie.lock.yml b/.github/workflows/archie.lock.yml index 59735e0029d..c00c14eafc0 100644 --- a/.github/workflows/archie.lock.yml +++ b/.github/workflows/archie.lock.yml @@ -1240,6 +1240,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1261,6 +1262,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index 17a6176d01a..85f2b5b7eb8 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1170,6 +1170,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1191,6 +1192,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/archivx-agentic-workflows-analyzer.lock.yml b/.github/workflows/archivx-agentic-workflows-analyzer.lock.yml index d6628b89913..161b10215ba 100644 --- a/.github/workflows/archivx-agentic-workflows-analyzer.lock.yml +++ b/.github/workflows/archivx-agentic-workflows-analyzer.lock.yml @@ -1395,6 +1395,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1414,6 +1415,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/artifacts-summary.lock.yml b/.github/workflows/artifacts-summary.lock.yml index 4ea0084ed2c..fd82fde3429 100644 --- a/.github/workflows/artifacts-summary.lock.yml +++ b/.github/workflows/artifacts-summary.lock.yml @@ -1070,6 +1070,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1091,6 +1092,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index 009b7e03e3a..bcf7f72a6cc 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -1433,6 +1433,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1454,6 +1455,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index d07ed42258b..d43419feee7 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1164,6 +1164,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1187,6 +1188,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/avenger.lock.yml b/.github/workflows/avenger.lock.yml index 167811703d5..fa3b891e149 100644 --- a/.github/workflows/avenger.lock.yml +++ b/.github/workflows/avenger.lock.yml @@ -1249,6 +1249,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1270,6 +1271,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/aw-failure-investigator.lock.yml b/.github/workflows/aw-failure-investigator.lock.yml index d816f33411b..e6a075a87b0 100644 --- a/.github/workflows/aw-failure-investigator.lock.yml +++ b/.github/workflows/aw-failure-investigator.lock.yml @@ -1384,6 +1384,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1403,6 +1404,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 4a849a41411..5e4283dba4c 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -1269,6 +1269,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1288,6 +1289,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/bot-detection.lock.yml b/.github/workflows/bot-detection.lock.yml index ca0049862b2..5c95a660847 100644 --- a/.github/workflows/bot-detection.lock.yml +++ b/.github/workflows/bot-detection.lock.yml @@ -1253,6 +1253,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1274,6 +1275,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 0c8094df930..f2082be6446 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1210,6 +1210,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1231,6 +1232,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/changeset.lock.yml b/.github/workflows/changeset.lock.yml index 3c110e41399..7fc757ad22a 100644 --- a/.github/workflows/changeset.lock.yml +++ b/.github/workflows/changeset.lock.yml @@ -1170,6 +1170,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1191,6 +1192,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml index 56247a3bbb9..d21cfe885fd 100644 --- a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml +++ b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml @@ -1144,6 +1144,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1165,6 +1166,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index 27406b7220a..670f4d7aed8 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -1296,6 +1296,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1317,6 +1318,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/claude-code-user-docs-review.lock.yml b/.github/workflows/claude-code-user-docs-review.lock.yml index 786c60ac290..b184be9e960 100644 --- a/.github/workflows/claude-code-user-docs-review.lock.yml +++ b/.github/workflows/claude-code-user-docs-review.lock.yml @@ -1228,6 +1228,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1247,6 +1248,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index dca25911a90..14bd3d42a13 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1142,6 +1142,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1163,6 +1164,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/cli-version-checker.lock.yml b/.github/workflows/cli-version-checker.lock.yml index 6061ad45046..146790f1024 100644 --- a/.github/workflows/cli-version-checker.lock.yml +++ b/.github/workflows/cli-version-checker.lock.yml @@ -1225,6 +1225,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1246,6 +1247,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/cloclo.lock.yml b/.github/workflows/cloclo.lock.yml index 8ac32679332..9f23603b8c8 100644 --- a/.github/workflows/cloclo.lock.yml +++ b/.github/workflows/cloclo.lock.yml @@ -1570,6 +1570,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1591,6 +1592,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/code-scanning-fixer.lock.yml b/.github/workflows/code-scanning-fixer.lock.yml index a47fc4cf073..1b39cdb3323 100644 --- a/.github/workflows/code-scanning-fixer.lock.yml +++ b/.github/workflows/code-scanning-fixer.lock.yml @@ -1316,6 +1316,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1337,6 +1338,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/code-simplifier.lock.yml b/.github/workflows/code-simplifier.lock.yml index de8eedf08fe..b10f1021631 100644 --- a/.github/workflows/code-simplifier.lock.yml +++ b/.github/workflows/code-simplifier.lock.yml @@ -1196,6 +1196,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1217,6 +1218,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/codex-github-remote-mcp-test.lock.yml b/.github/workflows/codex-github-remote-mcp-test.lock.yml index d2cf7e7ddee..3e9b17a5c31 100644 --- a/.github/workflows/codex-github-remote-mcp-test.lock.yml +++ b/.github/workflows/codex-github-remote-mcp-test.lock.yml @@ -1098,6 +1098,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/daily-file-diet.lock.yml b/.github/workflows/daily-file-diet.lock.yml index 39ab12d953c..64e6121f806 100644 --- a/.github/workflows/daily-file-diet.lock.yml +++ b/.github/workflows/daily-file-diet.lock.yml @@ -1254,6 +1254,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/agent/graders/operational_value_evaluator.sh if-no-files-found: ignore @@ -1276,6 +1277,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/agent/graders/operational_value_evaluator.sh /tmp/gh-aw/safeoutputs.jsonl diff --git a/.github/workflows/daily-grader-audit.lock.yml b/.github/workflows/daily-grader-audit.lock.yml index 99c28a9884a..b48791078bb 100644 --- a/.github/workflows/daily-grader-audit.lock.yml +++ b/.github/workflows/daily-grader-audit.lock.yml @@ -1166,6 +1166,24 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require(path.join(actionsDir, 'trace_graders.cjs')); await main('eyJ2ZXJzaW9uIjoxLCJncmFkZXJzIjpbeyJpZCI6InRvb2wtc3VjY2Vzcy1yYXRlIiwibmFtZSI6IlRvb2wgU3VjY2VzcyBSYXRlIiwiZGVzY3JpcHRpb24iOiJGcmFjdGlvbiBvZiB0b29sIGNhbGxzIHRoYXQgc3VjY2VlZGVkIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJyYXRpbyIsImRpcmVjdGlvbiI6ImhpZ2hlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjAuOCwibWF4IjoxLCJtaW4iOjB9LHsiaWQiOiJ0b29sLWZhaWx1cmUtY291bnQiLCJuYW1lIjoiVG9vbCBGYWlsdXJlIENvdW50IiwiZGVzY3JpcHRpb24iOiJOdW1iZXIgb2YgdG9vbCBjYWxscyB0aGF0IGZhaWxlZCIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjV9LHsiaWQiOiJyZXRyaWVzIiwibmFtZSI6IlJldHJpZXMiLCJkZXNjcmlwdGlvbiI6Ik51bWJlciBvZiByZXRyeSBldmVudHMgZGV0ZWN0ZWQgaW4gZ2F0ZXdheSBsb2dzIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJjb3VudCIsImRpcmVjdGlvbiI6Imxvd2VyX2lzX2JldHRlciIsInRocmVzaG9sZCI6MTB9LHsiaWQiOiJsb29wcyIsIm5hbWUiOiJMb29wcyIsImRlc2NyaXB0aW9uIjoiQ29uc2VjdXRpdmUgaWRlbnRpY2FsIHRvb2wgY2FsbHMgKHNhbWUgbmFtZSBhbmQgYXJndW1lbnRzKSIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjN9LHsiaWQiOiJ0cmFqZWN0b3J5LWVmZmljaWVuY3kiLCJuYW1lIjoiVHJhamVjdG9yeSBFZmZpY2llbmN5IiwiZGVzY3JpcHRpb24iOiJSYXRpbyBvZiB1bmlxdWUgdG9vbCBuYW1lcyB0byB0b3RhbCB0b29sIGNhbGxzIChoaWdoZXIgPSBtb3JlIGRpdmVyc2UgdXNhZ2UpIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJyYXRpbyIsImRpcmVjdGlvbiI6ImhpZ2hlcl9pc19iZXR0ZXIiLCJtYXgiOjEsIm1pbiI6MH0seyJpZCI6ImV4ZWN1dGlvbi1zdGVwLWNvdW50IiwibmFtZSI6IkV4ZWN1dGlvbiBTdGVwIENvdW50IiwiZGVzY3JpcHRpb24iOiJUb3RhbCBMTE0gcmVxdWVzdCBjb3VudCIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIifSx7ImlkIjoiZXhlY3V0aW9uLWR1cmF0aW9uIiwibmFtZSI6IkV4ZWN1dGlvbiBEdXJhdGlvbiIsImRlc2NyaXB0aW9uIjoiVG90YWwgZXhlY3V0aW9uIGR1cmF0aW9uIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJtcyIsImRpcmVjdGlvbiI6Imxvd2VyX2lzX2JldHRlciJ9LHsiaWQiOiJ3b3JraW5nLXNldC1yZWJ1aWxkLWZhY3RvciIsIm5hbWUiOiJXb3JraW5nLVNldCBSZWJ1aWxkIEZhY3RvciIsImRlc2NyaXB0aW9uIjoiQ3VtdWxhdGl2ZSBpbnB1dCB0b2tlbnMgZGl2aWRlZCBieSBwZWFrIGludm9jYXRpb24gaW5wdXQgdG9rZW5zIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJmYWN0b3IiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJtaW4iOjF9LHsiaWQiOiJjb250ZXh0LWdyb3d0aCIsIm5hbWUiOiJDb250ZXh0IEdyb3d0aCIsImRlc2NyaXB0aW9uIjoiUmF0aW8gb2YgdG90YWwgdG9rZW5zIHRvIGZpcnN0LXJlcXVlc3QgdG9rZW5zIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJmYWN0b3IiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIifSx7ImlkIjoiYXJ0aWZhY3QtcHJvZHVjdGlvbiIsIm5hbWUiOiJBcnRpZmFjdCBQcm9kdWN0aW9uIiwiZGVzY3JpcHRpb24iOiJDb3VudCBvZiBvdXRwdXRzL2FydGlmYWN0cyBwcm9kdWNlZCBieSB0aGUgYWdlbnQiLCJzb3VyY2UiOiJidWlsdGluIiwiZW5hYmxlZCI6dHJ1ZSwidW5pdCI6ImNvdW50IiwiZGlyZWN0aW9uIjoiaGlnaGVyX2lzX2JldHRlciJ9XX0=', 'bnVsbA=='); + - name: Redact grader outputs + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { redactFilesInDir } = require(path.join(actionsDir, 'redact_secrets.cjs')); + await redactFilesInDir('/tmp/gh-aw/agent/graders'); + env: + GH_AW_SECRET_NAMES: 'ANTHROPIC_API_KEY,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Write agent output placeholder if missing if: always() run: | @@ -1193,6 +1211,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1212,6 +1231,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/daily-rendering-scripts-verifier.lock.yml b/.github/workflows/daily-rendering-scripts-verifier.lock.yml index 3e274a07d2a..6bd09d140c5 100644 --- a/.github/workflows/daily-rendering-scripts-verifier.lock.yml +++ b/.github/workflows/daily-rendering-scripts-verifier.lock.yml @@ -1395,6 +1395,24 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require(path.join(actionsDir, 'trace_graders.cjs')); await main('eyJ2ZXJzaW9uIjoxLCJncmFkZXJzIjpbeyJpZCI6InRvb2wtc3VjY2Vzcy1yYXRlIiwibmFtZSI6IlRvb2wgU3VjY2VzcyBSYXRlIiwiZGVzY3JpcHRpb24iOiJGcmFjdGlvbiBvZiB0b29sIGNhbGxzIHRoYXQgc3VjY2VlZGVkIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJyYXRpbyIsImRpcmVjdGlvbiI6ImhpZ2hlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjAuOCwibWF4IjoxLCJtaW4iOjB9LHsiaWQiOiJ0b29sLWZhaWx1cmUtY291bnQiLCJuYW1lIjoiVG9vbCBGYWlsdXJlIENvdW50IiwiZGVzY3JpcHRpb24iOiJOdW1iZXIgb2YgdG9vbCBjYWxscyB0aGF0IGZhaWxlZCIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjV9LHsiaWQiOiJyZXRyaWVzIiwibmFtZSI6IlJldHJpZXMiLCJkZXNjcmlwdGlvbiI6Ik51bWJlciBvZiByZXRyeSBldmVudHMgZGV0ZWN0ZWQgaW4gZ2F0ZXdheSBsb2dzIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJjb3VudCIsImRpcmVjdGlvbiI6Imxvd2VyX2lzX2JldHRlciIsInRocmVzaG9sZCI6MTB9LHsiaWQiOiJsb29wcyIsIm5hbWUiOiJMb29wcyIsImRlc2NyaXB0aW9uIjoiQ29uc2VjdXRpdmUgaWRlbnRpY2FsIHRvb2wgY2FsbHMgKHNhbWUgbmFtZSBhbmQgYXJndW1lbnRzKSIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjN9LHsiaWQiOiJ0cmFqZWN0b3J5LWVmZmljaWVuY3kiLCJuYW1lIjoiVHJhamVjdG9yeSBFZmZpY2llbmN5IiwiZGVzY3JpcHRpb24iOiJSYXRpbyBvZiB1bmlxdWUgdG9vbCBuYW1lcyB0byB0b3RhbCB0b29sIGNhbGxzIChoaWdoZXIgPSBtb3JlIGRpdmVyc2UgdXNhZ2UpIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJyYXRpbyIsImRpcmVjdGlvbiI6ImhpZ2hlcl9pc19iZXR0ZXIiLCJtYXgiOjEsIm1pbiI6MH0seyJpZCI6ImV4ZWN1dGlvbi1zdGVwLWNvdW50IiwibmFtZSI6IkV4ZWN1dGlvbiBTdGVwIENvdW50IiwiZGVzY3JpcHRpb24iOiJUb3RhbCBMTE0gcmVxdWVzdCBjb3VudCIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIifSx7ImlkIjoiZXhlY3V0aW9uLWR1cmF0aW9uIiwibmFtZSI6IkV4ZWN1dGlvbiBEdXJhdGlvbiIsImRlc2NyaXB0aW9uIjoiVG90YWwgZXhlY3V0aW9uIGR1cmF0aW9uIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJtcyIsImRpcmVjdGlvbiI6Imxvd2VyX2lzX2JldHRlciJ9LHsiaWQiOiJ3b3JraW5nLXNldC1yZWJ1aWxkLWZhY3RvciIsIm5hbWUiOiJXb3JraW5nLVNldCBSZWJ1aWxkIEZhY3RvciIsImRlc2NyaXB0aW9uIjoiQ3VtdWxhdGl2ZSBpbnB1dCB0b2tlbnMgZGl2aWRlZCBieSBwZWFrIGludm9jYXRpb24gaW5wdXQgdG9rZW5zIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJmYWN0b3IiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJtaW4iOjF9LHsiaWQiOiJjb250ZXh0LWdyb3d0aCIsIm5hbWUiOiJDb250ZXh0IEdyb3d0aCIsImRlc2NyaXB0aW9uIjoiUmF0aW8gb2YgdG90YWwgdG9rZW5zIHRvIGZpcnN0LXJlcXVlc3QgdG9rZW5zIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJmYWN0b3IiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIifSx7ImlkIjoiYXJ0aWZhY3QtcHJvZHVjdGlvbiIsIm5hbWUiOiJBcnRpZmFjdCBQcm9kdWN0aW9uIiwiZGVzY3JpcHRpb24iOiJDb3VudCBvZiBvdXRwdXRzL2FydGlmYWN0cyBwcm9kdWNlZCBieSB0aGUgYWdlbnQiLCJzb3VyY2UiOiJidWlsdGluIiwiZW5hYmxlZCI6dHJ1ZSwidW5pdCI6ImNvdW50IiwiZGlyZWN0aW9uIjoiaGlnaGVyX2lzX2JldHRlciJ9XX0=', 'bnVsbA=='); + - name: Redact grader outputs + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { redactFilesInDir } = require(path.join(actionsDir, 'redact_secrets.cjs')); + await redactFilesInDir('/tmp/gh-aw/agent/graders'); + env: + GH_AW_SECRET_NAMES: 'ANTHROPIC_API_KEY,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Write agent output placeholder if missing if: always() run: | @@ -1431,6 +1449,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1450,6 +1469,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/daily-trajectory-grader-implementer.lock.yml b/.github/workflows/daily-trajectory-grader-implementer.lock.yml index 3b01ee72559..ff0f009c6e5 100644 --- a/.github/workflows/daily-trajectory-grader-implementer.lock.yml +++ b/.github/workflows/daily-trajectory-grader-implementer.lock.yml @@ -1193,6 +1193,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1214,6 +1215,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/.github/workflows/pr-sous-chef.lock.yml b/.github/workflows/pr-sous-chef.lock.yml index 3ebfedefa36..3e482869cf1 100644 --- a/.github/workflows/pr-sous-chef.lock.yml +++ b/.github/workflows/pr-sous-chef.lock.yml @@ -1645,6 +1645,7 @@ jobs: /tmp/gh-aw/agent_output.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json if-no-files-found: ignore - name: Upload agent artifacts @@ -1668,6 +1669,7 @@ jobs: /tmp/gh-aw/otel.jsonl /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_payload.json /tmp/gh-aw/agent/graders/grader_results.json /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index ab6418f7f78..d94305632f0 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -14,6 +14,7 @@ const { executeOperationalValueEvaluator } = require("./operational_value_grader const TMP_GH_AW = "/tmp/gh-aw"; const GRADERS_DIR = path.join(TMP_GH_AW, "agent", "graders"); const MANIFEST_PATH = path.join(GRADERS_DIR, "grader_manifest.json"); +const PAYLOAD_PATH = path.join(GRADERS_DIR, "grader_payload.json"); const RESULTS_PATH = path.join(GRADERS_DIR, "grader_results.json"); const OPERATIONAL_VALUE_EVALUATOR_PATH = path.join(GRADERS_DIR, "operational_value_evaluator.sh"); @@ -764,6 +765,11 @@ async function main(manifestB64, execSpecB64) { // Single preprocessing pass core.info(`Graders: preprocessing trace files for ${enabledGraders.length} grader(s)...`); const trace = preprocessTrace(); + try { + fs.writeFileSync(PAYLOAD_PATH, JSON.stringify(trace)); + } catch (err) { + core.warning(`Graders: failed to write payload: ${getErrorMessage(err)}`); + } const runCreatedAt = process.env.GH_AW_RUN_CREATED_AT; const operationalValueRunMetadata = runCreatedAt ? { createdAt: runCreatedAt } : undefined; @@ -869,6 +875,7 @@ module.exports = { IMPLEMENTATION_ID, GRADERS_DIR, MANIFEST_PATH, + PAYLOAD_PATH, RESULTS_PATH, OPERATIONAL_VALUE_EVALUATOR_PATH, archiveOperationalValueEvaluator, diff --git a/docs/adr/58634-add-payload-based-grader-execution.md b/docs/adr/58634-add-payload-based-grader-execution.md new file mode 100644 index 00000000000..3f34996383b --- /dev/null +++ b/docs/adr/58634-add-payload-based-grader-execution.md @@ -0,0 +1,48 @@ +# ADR-58634: Add Payload-Based Grader Execution + +**Date**: 2026-09-05 +**Status**: Draft +**Deciders**: Unknown, adr-writer agent + +--- + +### Context + +This pull request adds a new `gh aw graders run [run-id]` CLI subcommand, updates grader tracing to persist `grader_payload.json`, and adjusts workflow artifact uploads so saved grader payloads are available for later replay. The implementation also adds redaction before archived grader outputs are uploaded and documents stdin-based and artifact-based usage in the CLI docs. Together, these changes introduce a new supported execution path for replaying one grader independently of the original workflow run, which changes how grader data is stored, transported, and re-executed. Because this establishes a reusable debugging and inspection mechanism across the CLI and workflow runtime, it should be recorded as an explicit architectural decision. + +### Decision + +We will support single-grader replay by persisting a preprocessed grader payload during workflow execution and exposing a `gh aw graders run` command that can execute a declared grader against either that archived payload or JSON from standard input. We will keep the replay contract centered on the preprocessed trace payload rather than rehydrating raw workflow logs, and we will redact archived grader outputs before upload to reduce secret exposure risk. This favors a simple, reproducible replay model that works for built-in graders, inline JavaScript graders, and the operational-value evaluator. + +### Alternatives Considered + +#### Alternative 1: Keep grader execution tied to full workflow runs only + +This would preserve the existing model where graders are observed only as part of a completed workflow run and inspected through final results rather than replayed directly. It was considered because it avoids introducing a new persisted payload artifact and another CLI execution mode. It was not chosen because the PR evidence shows a need to rerun one grader from either a saved run payload or stdin without rerunning the full workflow, which is better for debugging and iterative evaluation. + +#### Alternative 2: Reconstruct replay inputs from raw logs or artifacts on demand + +This would avoid storing a dedicated `grader_payload.json` by rebuilding the grader input each time from archived traces, logs, or other run artifacts. It was considered because it could reduce artifact surface area and avoid introducing a new persisted contract. It was not chosen because the implementation instead standardizes on one preprocessed payload written by `trace_graders.cjs`, which is simpler to validate, cheaper to consume from the CLI, and more deterministic across grader types. + +### Consequences + +#### Positive +- Developers can rerun one grader quickly from a prior run artifact or local JSON without rerunning the full workflow. +- The replay interface is consistent across built-in graders, inline graders, and operational-value grading because all consume the same preprocessed payload shape. +- Persisting the preprocessed payload makes grader investigations more reproducible and less dependent on log-mining or ad hoc reconstruction. +- Redacting grader files before artifact upload lowers the chance that replayable payloads expose sensitive values. + +#### Negative +- The project now has to maintain `grader_payload.json` as a durable artifact contract between workflow execution and CLI replay. +- Artifact handling becomes more complex because workflows must archive the payload and, in some cases, explicitly redact grader outputs before upload. +- Persisting replay payloads increases artifact volume and may preserve sensitive contextual data if redaction misses a case. +- The CLI replay path adds more validation and runtime surface area, including JSON validation, payload size limits, sandboxed JavaScript execution, and artifact download logic. + +#### Neutral +- Integration and unit tests now cover help text, invalid run IDs, stdin payload execution, inline script grading, and script-file operational-value grading. +- The CLI documentation now treats `graders run` and `graders operational-value` as separate replay-oriented entry points under the `graders` command. +- Future changes to trace preprocessing will need to preserve compatibility with the stored payload format or update both the runtime producer and replay consumer together. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* diff --git a/docs/src/content/docs/setup/cli.md b/docs/src/content/docs/setup/cli.md index fac7cfc0267..f40b084474a 100644 --- a/docs/src/content/docs/setup/cli.md +++ b/docs/src/content/docs/setup/cli.md @@ -617,9 +617,15 @@ The diff output shows: new or removed network domains, status changes (allowed #### `graders` -Inspect and replay workflow graders. `graders operational-value` regrades the operational-value observation from a completed workflow run at an explicit evidence cutoff. It verifies and executes the evaluator archived by the run without modifying the original artifact. +Inspect and replay workflow graders. `graders run` runs one grader declared by a +local workflow against a saved run payload or JSON from standard input. +`graders operational-value` regrades the operational-value observation from a +completed workflow run at an explicit evidence cutoff. It verifies and executes +the evaluator archived by the run without modifying the original artifact. ```bash wrap +gh aw graders run weekly-research loops 123456789 +cat payload.json | gh aw graders run weekly-research loops gh aw graders operational-value 123456789 \ --evidence-at 2026-08-30T12:00:00.000Z --json ``` diff --git a/pkg/cli/graders_command.go b/pkg/cli/graders_command.go index fe915d551d3..ea243d8fadf 100644 --- a/pkg/cli/graders_command.go +++ b/pkg/cli/graders_command.go @@ -2,6 +2,7 @@ package cli import ( "errors" + "os" "strconv" "github.com/github/gh-aw/pkg/constants" @@ -14,10 +15,47 @@ func NewGradersCommand() *cobra.Command { Use: "graders", Short: "Inspect and replay workflow graders", } + cmd.AddCommand(newGradersRunCommand()) cmd.AddCommand(newGradersOperationalValueCommand()) return cmd } +func newGradersRunCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "run [run-id]", + Short: "Run one workflow grader with a JSON payload", + Long: `Run one grader declared by a local workflow. When run-id is provided, the +preprocessed payload is downloaded from the run's agent artifact. Otherwise, +the JSON payload is read from standard input.`, + Example: ` ` + string(constants.CLIExtensionPrefix) + ` graders run weekly-research loops 123456789 + cat payload.json | ` + string(constants.CLIExtensionPrefix) + ` graders run weekly-research loops`, + Args: cobra.RangeArgs(2, 3), + RunE: func(cmd *cobra.Command, args []string) error { + var runID int64 + var err error + if len(args) == 3 { + runID, err = parseGraderRunID(args[2]) + if err != nil { + return err + } + } + repoOverride, _ := cmd.Flags().GetString("repo") + return runGrader(cmd.Context(), graderRunConfig{ + Workflow: args[0], + GraderID: args[1], + RunID: runID, + Repo: repoOverride, + Input: cmd.InOrStdin(), + Output: cmd.OutOrStdout(), + }) + }, + } + addRepoFlag(cmd) + cmd.SetIn(os.Stdin) + cmd.SetOut(os.Stdout) + return cmd +} + func newGradersOperationalValueCommand() *cobra.Command { cmd := &cobra.Command{ Use: "operational-value ", diff --git a/pkg/cli/graders_integration_test.go b/pkg/cli/graders_integration_test.go index 10b803a3d16..752ba67963b 100644 --- a/pkg/cli/graders_integration_test.go +++ b/pkg/cli/graders_integration_test.go @@ -23,7 +23,18 @@ func TestGradersCommandIntegration(t *testing.T) { { name: "graders help", args: []string{"graders", "--help"}, - wantOutput: []string{"Inspect and replay workflow graders", "operational-value"}, + wantOutput: []string{"Inspect and replay workflow graders", "operational-value", "run"}, + }, + { + name: "run grader help", + args: []string{"graders", "run", "--help"}, + wantOutput: []string{"Run one grader", " [run-id]", "--repo"}, + }, + { + name: "run grader rejects invalid run ID", + args: []string{"graders", "run", "workflow", "loops", "0"}, + wantErr: true, + wantOutput: []string{"run ID must be a positive integer"}, }, { name: "operational value help", diff --git a/pkg/cli/graders_run.cjs b/pkg/cli/graders_run.cjs new file mode 100644 index 00000000000..344346b6869 --- /dev/null +++ b/pkg/cli/graders_run.cjs @@ -0,0 +1,141 @@ +"use strict"; + +const fs = require("fs"); +const vm = require("vm"); + +const builtins = { + "tool-success-rate": trace => { + if (trace.toolCalls.length === 0) return 1; + const failures = trace.toolCalls.filter(call => call.success === false || call.status === "error" || call.status === "failure" || call.error !== undefined).length; + return (trace.toolCalls.length - failures) / trace.toolCalls.length; + }, + "tool-failure-count": trace => trace.toolCalls.filter(call => call.success === false || call.status === "error" || call.status === "failure" || call.error !== undefined).length, + retries: trace => trace.retryEvents.length, + loops: trace => { + let count = 0; + let previous = ""; + for (const call of trace.toolCalls) { + const key = `${String(call.name || call.tool)}:${JSON.stringify(call.arguments || call.params || "")}`; + if (key === previous) count++; + previous = key; + } + return count; + }, + "trajectory-efficiency": trace => { + if (trace.toolCalls.length === 0) return 1; + return Math.min(1, new Set(trace.toolCalls.map(call => String(call.name || call.tool || ""))).size / trace.toolCalls.length); + }, + "execution-step-count": trace => trace.totalRequests, + "execution-duration": trace => trace.totalDurationMs, + "working-set-rebuild-factor": trace => { + const entries = trace.tokenUsageEntries || []; + const valid = entries.map(entry => entry?.input_tokens).filter(value => typeof value === "number" && Number.isSafeInteger(value) && value >= 0); + if (valid.length === 0) return null; + const total = valid.reduce((sum, value) => sum + value, 0); + const peak = Math.max(...valid); + return peak === 0 ? null : Math.max(1, total / peak); + }, + "context-growth": trace => { + const entries = trace.tokenUsageEntries || []; + if (entries.length < 2) return 1; + const first = (Number(entries[0].input_tokens) || 0) + (Number(entries[0].output_tokens) || 0); + return first === 0 ? 1 : ((Number(trace.totalInputTokens) || 0) + (Number(trace.totalOutputTokens) || 0)) / first; + }, + "artifact-production": trace => trace.artifacts.length, +}; + +function deepFreeze(value) { + if (value === null || typeof value !== "object") return value; + Object.freeze(value); + for (const key of Object.getOwnPropertyNames(value)) { + if (value[key] !== null && typeof value[key] === "object" && !Object.isFrozen(value[key])) deepFreeze(value[key]); + } + return value; +} + +function runInline(grader, trace) { + const sandbox = { + trace: deepFreeze(structuredClone(trace)), + run: deepFreeze({ graderCount: 1 }), + workflow: deepFreeze({}), + config: deepFreeze(structuredClone(grader.config || {})), + Date: undefined, + fetch: undefined, + require: undefined, + process: undefined, + global: undefined, + globalThis: undefined, + Function: undefined, + eval: undefined, + }; + const context = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } }); + const safeMath = vm.runInContext( + `(() => { + const value = {}; + Object.defineProperties(value, Object.getOwnPropertyDescriptors(Math)); + Object.defineProperty(value, "random", { value: undefined }); + return Object.freeze(value); + })()`, + context, + { timeout: 1000 } + ); + const helpers = deepFreeze({ + clamp: (value, low, high) => Math.max(low, Math.min(high, value)), + ratio: (numerator, denominator) => (denominator === 0 ? 0 : numerator / denominator), + sum: values => values.reduce((sum, value) => sum + value, 0), + }); + const fn = vm.compileFunction(`"use strict";\n${grader.script}`, ["trace", "run", "workflow", "config", "helpers", "Math"], { + parsingContext: context, + filename: `grader:${grader.id}`, + }); + context.__grader = fn; + context.__helpers = helpers; + context.__math = safeMath; + return vm.runInContext("__grader(trace, run, workflow, config, __helpers, __math)", context, { + timeout: 5000, + filename: `grader:${grader.id}:invoke`, + }); +} + +function normalize(grader, raw) { + const object = raw !== null && typeof raw === "object" && !Array.isArray(raw) ? raw : {}; + const value = Object.hasOwn(object, "value") ? object.value : raw; + let passed = typeof object.passed === "boolean" ? object.passed : null; + const implementation = { id: "gh-aw/graders", version: 1, ...(grader.digest ? { digest: grader.digest } : {}) }; + if (value !== null && value !== undefined && (typeof value !== "number" || !Number.isFinite(value))) { + return { + id: grader.id, + name: grader.name || grader.id, + value: null, + unit: grader.unit || "", + passed: null, + status: "error", + source: grader.source, + error: `grader ${grader.id} returned non-finite value: ${String(value)}`, + implementation, + }; + } + if (passed === null && value !== null && value !== undefined && grader.threshold !== null && grader.threshold !== undefined) { + passed = grader.direction === "higher_is_better" ? value >= grader.threshold : value <= grader.threshold; + } + const status = value === null || value === undefined ? "unavailable" : passed === false ? "fail" : "pass"; + return { + id: grader.id, + name: grader.name || grader.id, + value: value ?? null, + unit: grader.unit || "", + passed, + status, + source: grader.source, + implementation, + ...(typeof object.severity === "string" && ["error", "warning", "info", "note"].includes(object.severity) ? { severity: object.severity } : {}), + ...(object.details ? { details: String(object.details) } : {}), + ...(object.message ? { message: String(object.message) } : {}), + }; +} + +const input = JSON.parse(fs.readFileSync(0, "utf8")); +const grader = input.grader; +const trace = deepFreeze(input.payload); +const raw = grader.source === "builtin" ? builtins[grader.id](trace) : runInline(grader, trace); +process.stdout.write(`${JSON.stringify(normalize(grader, raw), null, 2)}\n`); diff --git a/pkg/cli/graders_run.go b/pkg/cli/graders_run.go new file mode 100644 index 00000000000..2233b8874b9 --- /dev/null +++ b/pkg/cli/graders_run.go @@ -0,0 +1,256 @@ +package cli + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "slices" + "strconv" + "time" + + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/parser" + "github.com/github/gh-aw/pkg/repoutil" + "github.com/github/gh-aw/pkg/stringutil" + "github.com/github/gh-aw/pkg/workflow" +) + +const ( + maxGraderPayloadBytes = 50 * 1024 * 1024 + graderJSTimeout = 7 * time.Second +) + +//go:embed graders_run.cjs +var gradersRunScript []byte + +type graderRunConfig struct { + Workflow string + GraderID string + RunID int64 + Repo string + Input io.Reader + Output io.Writer +} + +type graderRunDefinition struct { + ID string `json:"id"` + Name string `json:"name"` + Source string `json:"source"` + Unit string `json:"unit"` + Direction string `json:"direction"` + Threshold *float64 `json:"threshold"` + Config map[string]any `json:"config,omitempty"` + Script string `json:"script,omitempty"` + Digest string `json:"digest,omitempty"` +} + +func runGrader(ctx context.Context, config graderRunConfig) error { + grader, err := loadGraderRunDefinition(config.Workflow, config.GraderID) + if err != nil { + return err + } + payload, err := loadGraderRunPayload(ctx, config) + if err != nil { + return err + } + if grader.Source == "operational-value" { + evaluatorHost := getGitHubHostForRepo("") + if config.Repo != "" { + ownerRepo, host := repoutil.NormalizeRepoForAPI(config.Repo) + evaluatorHost = getGitHubHostForRepo(ownerRepo) + if host != "" { + evaluatorHost = stringutil.NormalizeGitHubHostURL(host) + } + } + return runOperationalValuePayload(ctx, config.Workflow, payload, config.Output, evaluatorHost) + } + return runJavaScriptGrader(ctx, grader, payload, config.Output) +} + +func loadGraderRunDefinition(workflowArg, graderID string) (graderRunDefinition, error) { + workflowPath, err := ResolveWorkflowPath(workflowArg) + if err != nil { + return graderRunDefinition{}, err + } + content, err := os.ReadFile(workflowPath) + if err != nil { + return graderRunDefinition{}, fmt.Errorf("cannot read workflow %s: %w", workflowPath, err) + } + parsed, err := parser.ExtractFrontmatterFromContent(string(content)) + if err != nil { + return graderRunDefinition{}, fmt.Errorf("cannot parse workflow %s: %w", workflowPath, err) + } + graders, err := workflow.ParseGradersFromFrontmatter(parsed.Frontmatter) + if err != nil { + return graderRunDefinition{}, fmt.Errorf("cannot parse graders in %s: %w", workflowPath, err) + } + if graders == nil || graders.Graders[graderID] == nil { + return graderRunDefinition{}, fmt.Errorf("workflow %s does not declare grader %q", workflowPath, graderID) + } + definition := graders.Graders[graderID] + if definition.Enabled != nil && !*definition.Enabled { + return graderRunDefinition{}, fmt.Errorf("grader %q is disabled in workflow %s", graderID, workflowPath) + } + source := "inline" + if slices.Contains(workflow.BuiltinGraderIDs, graderID) { + source = "builtin" + } + if graderID == "operational-value" { + source = "operational-value" + } + return graderRunDefinition{ + ID: graderID, + Name: definition.Name, + Source: source, + Unit: definition.Unit, + Direction: definition.Direction, + Threshold: definition.Threshold, + Config: definition.Config, + Script: definition.Script, + Digest: definition.ScriptDigest(), + }, nil +} + +func loadGraderRunPayload(ctx context.Context, config graderRunConfig) (json.RawMessage, error) { + if config.RunID == 0 { + return readGraderPayload(config.Input, "standard input") + } + tempDir, err := os.MkdirTemp("", "gh-aw-grader-run-*") + if err != nil { + return nil, fmt.Errorf("failed to create grader run directory: %w", err) + } + defer os.RemoveAll(tempDir) + + params := buildConcurrentDownloadParams(tempDir, false, config.Repo, nil, false, nil) + names, err := listRunArtifactNames(ctx, config.RunID, params.dlOwner, params.dlRepo, params.dlHost, false) + if err != nil { + return nil, err + } + artifactNames := make([]string, 0, 2) + for _, name := range names { + if name == constants.AgentArtifactName.String() || name == constants.AgentOutputFallbackArtifactName.String() { + artifactNames = append(artifactNames, name) + } + } + if len(artifactNames) == 0 { + return nil, fmt.Errorf("run %d has no agent artifact", config.RunID) + } + if err := downloadArtifactsByName(ctx, downloadArtifactsOptions{ + runID: config.RunID, outputDir: tempDir, owner: params.dlOwner, repo: params.dlRepo, hostname: params.dlHost, + }, artifactNames); err != nil { + return nil, err + } + if err := flattenUnifiedArtifact(tempDir, false); err != nil { + return nil, fmt.Errorf("failed to unpack run %d agent artifact: %w", config.RunID, err) + } + payloadPath := findGraderFile(tempDir, constants.GraderPayloadFilename.String()) + if payloadPath == "" { + return nil, fmt.Errorf("run %d agent artifact does not contain %s", config.RunID, constants.GraderPayloadFilename) + } + file, err := os.Open(payloadPath) + if err != nil { + return nil, fmt.Errorf("cannot read grader payload for run %d: %w", config.RunID, err) + } + defer file.Close() + return readGraderPayload(file, "run artifact") +} + +func readGraderPayload(reader io.Reader, source string) (json.RawMessage, error) { + if reader == nil { + return nil, fmt.Errorf("cannot read grader payload from %s", source) + } + data, err := io.ReadAll(io.LimitReader(reader, maxGraderPayloadBytes+1)) + if err != nil { + return nil, fmt.Errorf("cannot read grader payload from %s: %w", source, err) + } + if len(data) > maxGraderPayloadBytes { + return nil, fmt.Errorf("grader payload from %s exceeds the %d-byte limit", source, maxGraderPayloadBytes) + } + if len(bytes.TrimSpace(data)) == 0 { + return nil, fmt.Errorf("grader payload from %s is empty", source) + } + if !json.Valid(data) { + return nil, fmt.Errorf("grader payload from %s is not valid JSON", source) + } + return data, nil +} + +func runJavaScriptGrader(ctx context.Context, grader graderRunDefinition, payload json.RawMessage, output io.Writer) error { + nodePath, err := exec.LookPath("node") + if err != nil { + return errors.New("node is required to run graders") + } + script, err := os.CreateTemp("", "gh-aw-grader-run-*.cjs") + if err != nil { + return fmt.Errorf("failed to stage grader runtime: %w", err) + } + scriptPath := script.Name() + defer os.Remove(scriptPath) + if err := script.Chmod(constants.FilePermSensitive); err != nil { + _ = script.Close() + return fmt.Errorf("failed to secure grader runtime: %w", err) + } + if _, err := script.Write(gradersRunScript); err != nil { + _ = script.Close() + return fmt.Errorf("failed to stage grader runtime: %w", err) + } + if err := script.Close(); err != nil { + return fmt.Errorf("failed to stage grader runtime: %w", err) + } + input, err := json.Marshal(struct { + Grader graderRunDefinition `json:"grader"` + Payload json.RawMessage `json:"payload"` + }{Grader: grader, Payload: payload}) + if err != nil { + return fmt.Errorf("failed to encode grader input: %w", err) + } + commandCtx, cancel := context.WithTimeout(ctx, graderJSTimeout) + defer cancel() + cmd := exec.CommandContext(commandCtx, nodePath, scriptPath) + cmd.Stdin = bytes.NewReader(input) + cmd.Stdout = output + stderr := &boundedCommandBuffer{limit: maxOperationalValueRegradeOutputBytes} + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + if errors.Is(commandCtx.Err(), context.DeadlineExceeded) { + return fmt.Errorf("grader timed out after %s", graderJSTimeout) + } + if message := bytes.TrimSpace(stderr.Bytes()); len(message) > 0 { + return errors.New(string(message)) + } + return fmt.Errorf("grader failed: %w", err) + } + return nil +} + +func runOperationalValuePayload(ctx context.Context, workflowArg string, payload json.RawMessage, output io.Writer, evaluatorHost string) error { + evaluator, err := loadOperationalValueReportEvaluator(ctx, workflowArg, evaluatorHost) + if err != nil { + return err + } + defer evaluator.cleanup() + result, err := runOperationalValueEvaluatorBash(ctx, "/bin/bash", evaluator.EvaluatorPath, + []string{evaluator.EvaluatorPath, "--grade-run"}, payload, operationalValueEvaluatorTimeout, evaluatorHost) + if err != nil { + return fmt.Errorf("operational-value evaluator --grade-run failed: %w", err) + } + if !json.Valid(result) { + return errors.New("operational-value evaluator returned invalid JSON") + } + _, err = fmt.Fprintln(output, string(result)) + return err +} + +func parseGraderRunID(value string) (int64, error) { + runID, err := strconv.ParseInt(value, 10, 64) + if err != nil || runID <= 0 { + return 0, errors.New("run ID must be a positive integer") + } + return runID, nil +} diff --git a/pkg/cli/graders_run_test.go b/pkg/cli/graders_run_test.go new file mode 100644 index 00000000000..32fcd34cda8 --- /dev/null +++ b/pkg/cli/graders_run_test.go @@ -0,0 +1,136 @@ +package cli + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeGraderRunWorkflow(t *testing.T, content string) string { + t.Helper() + root := t.TempDir() + workflowDir := filepath.Join(root, ".github", "workflows") + require.NoError(t, os.MkdirAll(workflowDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "test.md"), []byte(content), 0o600)) + previous, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Chdir(root)) + t.Cleanup(func() { require.NoError(t, os.Chdir(previous)) }) + return "test" +} + +func TestRunGraderFromStdin(t *testing.T) { + workflowID := writeGraderRunWorkflow(t, "---\ngraders: {}\n---\n") + var output bytes.Buffer + err := runGrader(context.Background(), graderRunConfig{ + Workflow: workflowID, + GraderID: "loops", + Input: bytes.NewBufferString(`{ + "toolCalls":[ + {"name":"view","arguments":{"path":"a"}}, + {"name":"view","arguments":{"path":"a"}} + ], + "tokenUsageEntries":[], + "retryEvents":[], + "artifacts":[] + }`), + Output: &output, + }) + require.NoError(t, err) + assert.JSONEq(t, `{ + "id":"loops", + "name":"Loops", + "value":1, + "unit":"count", + "passed":true, + "status":"pass", + "source":"builtin", + "implementation":{"id":"gh-aw/graders","version":1} + }`, output.String()) +} + +func TestRunInlineScriptGraderFromStdin(t *testing.T) { + workflowID := writeGraderRunWorkflow(t, `--- +graders: + custom-score: + script: | + return { value: trace.score, message: "computed" } + unit: ratio + direction: higher_is_better + threshold: 0.5 +--- +`) + var output bytes.Buffer + err := runGrader(context.Background(), graderRunConfig{ + Workflow: workflowID, + GraderID: "custom-score", + Input: bytes.NewBufferString(`{"score":0.75}`), + Output: &output, + }) + require.NoError(t, err) + assert.JSONEq(t, `{ + "id":"custom-score", + "name":"custom-score", + "value":0.75, + "unit":"ratio", + "passed":true, + "status":"pass", + "source":"inline", + "implementation":{ + "id":"gh-aw/graders", + "version":1, + "digest":"518c37ee83a83874d2added478398c3b391bdbaa7b92f6ea9567a016dd888640" + }, + "message":"computed" + }`, output.String()) +} + +func TestRunScriptFileGraderFromStdin(t *testing.T) { + workflowID := writeGraderRunWorkflow(t, `--- +graders: + operational-value: + run: .github/graders/test-operational-value.sh +--- +`) + require.NoError(t, os.Mkdir(".git", 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(".github", "graders"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(".github", "graders", "test-operational-value.sh"), []byte(`#!/usr/bin/env bash +set -euo pipefail +case "${1:-}" in +--definition) + printf '%s\n' '{"schemaVersion":4,"grader":"operational-value","repository":"example/repo","workflowName":"Test","sourcePath":".github/workflows/test.md","adoption":{"commit":"abc","adoptedAt":"2026-01-01T00:00:00Z"},"operationalValue":"Test direct script execution.","evidence":{"opportunity":"test","assignment":"payload","accepted":"stdin","repositories":["example/repo"],"collection":"test","maturation":"immediate","zeroRule":"none","missingRule":"null"},"primaryMetric":{"id":"score","formula":"payload score","direction":"higher_is_better"},"baseline":{"mode":"attainment-only","value":null,"evidenceCutoff":null,"provenance":[]},"validationExamples":{"sample":{"valid":true}}}' + ;; +--grade-run) + [[ "${GH_HOST:-}" == "ghe.example" ]] + payload=$(cat) + [[ "$payload" == '{"score":0.8}' ]] + printf '%s\n' '{"value":0.8,"source":"script-file"}' + ;; +*) exit 1 ;; +esac +`), 0o700)) + + var output bytes.Buffer + err := runGrader(context.Background(), graderRunConfig{ + Workflow: workflowID, + GraderID: "operational-value", + Repo: "ghe.example/example/repo", + Input: bytes.NewBufferString(`{"score":0.8}`), + Output: &output, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"value":0.8,"source":"script-file"}`, output.String()) +} + +func TestReadGraderPayloadValidation(t *testing.T) { + _, err := readGraderPayload(bytes.NewBufferString("not-json"), "standard input") + require.ErrorContains(t, err, "not valid JSON") + + _, err = parseGraderRunID("0") + require.ErrorContains(t, err, "positive integer") +} diff --git a/pkg/constants/job_constants.go b/pkg/constants/job_constants.go index be490e1d95b..fa37dd97faa 100644 --- a/pkg/constants/job_constants.go +++ b/pkg/constants/job_constants.go @@ -145,6 +145,9 @@ const GraderManifestFilename Filename = "grader_manifest.json" // by trace_graders.cjs. Contains deterministic metric values computed from trace files. const GraderResultsFilename Filename = "grader_results.json" +// GraderPayloadFilename is the filename of the preprocessed trace payload consumed by graders. +const GraderPayloadFilename Filename = "grader_payload.json" + // OperationalValueEvaluatorFilename is the filename of the frozen operational-value evaluator archived for replay. const OperationalValueEvaluatorFilename Filename = "operational_value_evaluator.sh" diff --git a/pkg/workflow/compiler_yaml_graders.go b/pkg/workflow/compiler_yaml_graders.go index 37e2058c119..729616efae7 100644 --- a/pkg/workflow/compiler_yaml_graders.go +++ b/pkg/workflow/compiler_yaml_graders.go @@ -210,9 +210,6 @@ func (c *Compiler) generateGraderRedactionStep(yaml *strings.Builder, yamlConten if data.Graders == nil || !data.Graders.HasGraders() { return } - if !data.Graders.HasCustomScripts() { - return - } secretReferences := CollectSecretReferences(yamlContent) c.stepOrderTracker.RecordSecretRedaction("Redact grader outputs") @@ -246,6 +243,7 @@ func (c *Compiler) generateGraderRedactionStep(yaml *strings.Builder, yamlConten func collectGraderArtifactPaths(graders *GradersConfig) []string { paths := []string{ constants.GradersDirSlash + constants.GraderManifestFilename.String(), + constants.GradersDirSlash + constants.GraderPayloadFilename.String(), constants.GradersDirSlash + constants.GraderResultsFilename.String(), } if graders != nil { diff --git a/pkg/workflow/graders_config_test.go b/pkg/workflow/graders_config_test.go index c5ace8fcd55..6f66e4c54fa 100644 --- a/pkg/workflow/graders_config_test.go +++ b/pkg/workflow/graders_config_test.go @@ -721,16 +721,19 @@ func TestCollectGraderArtifactPaths(t *testing.T) { paths := collectGraderArtifactPaths(&GradersConfig{ Graders: map[string]*GraderDefinition{"operational-value": grader}, }) - if len(paths) != 3 { - t.Fatalf("expected 3 paths, got %d", len(paths)) + if len(paths) != 4 { + t.Fatalf("expected 4 paths, got %d", len(paths)) } if !strings.Contains(paths[0], "grader_manifest.json") { t.Fatal("expected grader_manifest.json in paths") } - if !strings.Contains(paths[1], "grader_results.json") { + if !strings.Contains(paths[1], "grader_payload.json") { + t.Fatal("expected grader_payload.json in paths") + } + if !strings.Contains(paths[2], "grader_results.json") { t.Fatal("expected grader_results.json in paths") } - if !strings.Contains(paths[2], "operational_value_evaluator.sh") { + if !strings.Contains(paths[3], "operational_value_evaluator.sh") { t.Fatal("expected operational_value_evaluator.sh in paths") } } @@ -739,8 +742,8 @@ func TestCollectGraderArtifactPathsWithoutOperationalValue(t *testing.T) { paths := collectGraderArtifactPaths(&GradersConfig{ Graders: map[string]*GraderDefinition{"retries": {ID: "retries"}}, }) - if len(paths) != 2 { - t.Fatalf("expected manifest and results paths, got %v", paths) + if len(paths) != 3 { + t.Fatalf("expected manifest, payload, and results paths, got %v", paths) } } @@ -760,13 +763,12 @@ func TestCollectGraderArtifactPaths_AgentGradersDir(t *testing.T) { } } -// TestGenerateGraderRedactionStep_CustomOnly verifies that the redaction step is only emitted -// when custom (non-builtin) grader scripts are present. -func TestGenerateGraderRedactionStep_CustomOnly(t *testing.T) { +// TestGenerateGraderRedactionStep verifies every grader payload is redacted before upload. +func TestGenerateGraderRedactionStep(t *testing.T) { c := &Compiler{stepOrderTracker: NewStepOrderTracker()} var yaml strings.Builder - // Built-in only — no redaction step + // Built-in payloads also contain trace data and must be redacted. data := &WorkflowData{ Graders: &GradersConfig{ Graders: map[string]*GraderDefinition{ @@ -775,11 +777,12 @@ func TestGenerateGraderRedactionStep_CustomOnly(t *testing.T) { }, } c.generateGraderRedactionStep(&yaml, "", data) - if yaml.Len() > 0 { - t.Error("expected no redaction step for built-in-only graders") + if !strings.Contains(yaml.String(), "Redact grader outputs") { + t.Error("expected redaction step for built-in grader payload") } // Custom script — should emit redaction step + yaml.Reset() data.Graders.Graders["my-custom"] = &GraderDefinition{ ID: "my-custom", Script: "return {value: 1}", diff --git a/pkg/workflow/schemas/github-workflow.json b/pkg/workflow/schemas/github-workflow.json index 23083e8951d..e65d2332ab1 100644 --- a/pkg/workflow/schemas/github-workflow.json +++ b/pkg/workflow/schemas/github-workflow.json @@ -1944,110 +1944,117 @@ }, "workflow_call": { "$comment": "https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#workflow_call", - "type": "object", "description": "Allows workflows to be reused by other workflows.", - "properties": { - "inputs": { - "$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_callinputs", - "description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.", + "oneOf": [ + { + "type": "null" + }, + { "type": "object", - "patternProperties": { - "^[_a-zA-Z][a-zA-Z0-9_-]*$": { - "$comment": "https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputsinput_id", - "description": "A string identifier to associate with the input. The value of is a map of the input's metadata. The must be a unique identifier within the inputs object. The must start with a letter or _ and contain only alphanumeric characters, -, or _.", + "properties": { + "inputs": { + "$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_callinputs", + "description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.", "type": "object", - "properties": { - "description": { - "$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddescription", - "description": "A string description of the input parameter.", - "type": "string" - }, - "required": { - "$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_idrequired", - "description": "A boolean to indicate whether the action requires the input parameter. Set to true when the parameter is required.", - "type": "boolean" - }, - "type": { - "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinput_idtype", - "description": "Required if input is defined for the on.workflow_call keyword. The value of this parameter is a string specifying the data type of the input. This must be one of: boolean, number, or string.", - "type": "string", - "enum": ["boolean", "number", "string"] - }, - "default": { - "$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddefault", - "description": "The default value is used when an input parameter isn't specified in a workflow file.", - "anyOf": [ - { + "patternProperties": { + "^[_a-zA-Z][a-zA-Z0-9_-]*$": { + "$comment": "https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputsinput_id", + "description": "A string identifier to associate with the input. The value of is a map of the input's metadata. The must be a unique identifier within the inputs object. The must start with a letter or _ and contain only alphanumeric characters, -, or _.", + "type": "object", + "properties": { + "description": { + "$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddescription", + "description": "A string description of the input parameter.", + "type": "string" + }, + "required": { + "$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_idrequired", + "description": "A boolean to indicate whether the action requires the input parameter. Set to true when the parameter is required.", "type": "boolean" }, - { - "type": "number" + "type": { + "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinput_idtype", + "description": "Required if input is defined for the on.workflow_call keyword. The value of this parameter is a string specifying the data type of the input. This must be one of: boolean, number, or string.", + "type": "string", + "enum": ["boolean", "number", "string"] }, - { - "type": "string" + "default": { + "$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddefault", + "description": "The default value is used when an input parameter isn't specified in a workflow file.", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + } + ] } - ] + }, + "required": ["type"], + "additionalProperties": false } }, - "required": ["type"], "additionalProperties": false - } - }, - "additionalProperties": false - }, - "outputs": { - "$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_calloutputs", - "description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.", - "type": "object", - "patternProperties": { - "^[_a-zA-Z][a-zA-Z0-9_-]*$": { - "$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_id", - "description": "A string identifier to associate with the output. The value of is a map of the output's metadata. The must be a unique identifier within the outputs object. The must start with a letter or _ and contain only alphanumeric characters, -, or _.", + }, + "outputs": { + "$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_calloutputs", + "description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.", "type": "object", - "properties": { - "description": { - "$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_iddescription", - "description": "A string description of the output parameter.", - "type": "string" - }, - "value": { - "$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_idvalue", - "description": "The value that the output parameter will be mapped to. You can set this to a string or an expression with context. For example, you can use the steps context to set the value of an output to the output value of a step.", - "type": "string" + "patternProperties": { + "^[_a-zA-Z][a-zA-Z0-9_-]*$": { + "$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_id", + "description": "A string identifier to associate with the output. The value of is a map of the output's metadata. The must be a unique identifier within the outputs object. The must start with a letter or _ and contain only alphanumeric characters, -, or _.", + "type": "object", + "properties": { + "description": { + "$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_iddescription", + "description": "A string description of the output parameter.", + "type": "string" + }, + "value": { + "$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_idvalue", + "description": "The value that the output parameter will be mapped to. You can set this to a string or an expression with context. For example, you can use the steps context to set the value of an output to the output value of a step.", + "type": "string" + } + }, + "required": ["value"], + "additionalProperties": false } }, - "required": ["value"], "additionalProperties": false - } - }, - "additionalProperties": false - }, - "secrets": { - "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecrets", - "type": "object", - "description": "A map of the secrets that can be used in the called workflow. Within the called workflow, you can use the secrets context to refer to a secret.", - "patternProperties": { - "^[_a-zA-Z][a-zA-Z0-9_-]*$": { - "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_id", + }, + "secrets": { + "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecrets", "type": "object", - "description": "A string identifier to associate with the secret.", - "properties": { - "description": { - "description": "A string description of the secret parameter.", - "type": "string" - }, - "required": { - "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_idrequired", - "description": "A boolean specifying whether the secret must be supplied.", - "type": "boolean" + "description": "A map of the secrets that can be used in the called workflow. Within the called workflow, you can use the secrets context to refer to a secret.", + "patternProperties": { + "^[_a-zA-Z][a-zA-Z0-9_-]*$": { + "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_id", + "type": "object", + "description": "A string identifier to associate with the secret.", + "properties": { + "description": { + "description": "A string description of the secret parameter.", + "type": "string" + }, + "required": { + "$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_idrequired", + "description": "A boolean specifying whether the secret must be supplied.", + "type": "boolean" + } + }, + "additionalProperties": false } }, "additionalProperties": false } - }, - "additionalProperties": false + } } - } + ] }, "workflow_dispatch": { "$comment": "https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/",