From 86eff364b73aec3aa4d1636a593aadc746ef063b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 10:44:13 +0000 Subject: [PATCH 01/12] Add manual eval trigger via PR comment slash commands Features: - Add /eval command support in PR comments with options: --model/-m: Specify model(s) to test --markers/-M: Pytest markers (default: regression) --filter/-k: Pytest -k filter --iterations/-i: Number of iterations (max 10) - Add workflow_dispatch trigger for GitHub UI triggering - Post initial "running" comment when evals start - Update same comment with results when complete - Each run preserves its own comment (history maintained) - Input validation to prevent shell injection - Add manual trigger instructions in automatic run comments --- .github/workflows/eval-regression.yaml | 551 ++++++++++++++++++++++++- 1 file changed, 541 insertions(+), 10 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index ade7346387..e5a973fa07 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -5,17 +5,288 @@ on: branches: ["*"] push: branches: [master] + # Allow manual triggering with custom parameters + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to run evals on (leave empty for current branch)' + required: false + default: '' + model: + description: 'Model(s) to test (comma-separated, e.g., gpt-4.1,gpt-4o)' + required: false + default: '' + markers: + description: 'Pytest markers (e.g., regression, easy, logs). Combined with "llm and"' + required: false + default: 'regression' + filter: + description: 'Pytest -k filter (e.g., 09_crashpod, checkout)' + required: false + default: '' + iterations: + description: 'Number of iterations (1-10)' + required: false + default: '1' + # Allow triggering via /eval comment on PRs + issue_comment: + types: [created] permissions: pull-requests: write contents: read + issues: write jobs: + # Parse /eval command from PR comments + parse-eval-command: + if: github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/eval') + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.parse.outputs.should_run }} + model: ${{ steps.parse.outputs.model }} + markers: ${{ steps.parse.outputs.markers }} + filter: ${{ steps.parse.outputs.filter }} + iterations: ${{ steps.parse.outputs.iterations }} + pr_sha: ${{ steps.get-pr.outputs.sha }} + + steps: + - name: Get PR details + id: get-pr + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + core.setOutput('sha', pr.data.head.sha); + + - name: Parse /eval command + id: parse + uses: actions/github-script@v7 + with: + script: | + const comment = context.payload.comment.body; + const commandLine = comment.split('\n')[0].trim(); + + // Security: Validate input doesn't contain shell injection characters + // Dangerous chars: ` $ ; | & < > ( ) { } [ ] \ newline + const dangerousPattern = /[`$;|&<>(){}[\]\\'\n\r]/; + + function validateInput(value, name) { + const trimmed = value.trim(); + if (dangerousPattern.test(trimmed)) { + core.setFailed(`Invalid ${name}: contains dangerous shell characters. Allowed: alphanumeric, dash, underscore, dot, comma, space, quotes.`); + return null; + } + return trimmed; + } + + // Default values + let model = ''; + let markers = 'regression'; + let filter = ''; + let iterations = '1'; + + // Parse command arguments + const args = commandLine.slice(5).trim(); + const argParts = args.match(/(?:[^\s"]+|"[^"]*")+/g) || []; + + for (let i = 0; i < argParts.length; i++) { + const arg = argParts[i]; + const nextArg = argParts[i + 1]; + + if ((arg === '--model' || arg === '-m') && nextArg) { + model = nextArg.replace(/^"|"$/g, ''); + i++; + } else if ((arg === '--markers' || arg === '-M') && nextArg) { + markers = nextArg.replace(/^"|"$/g, ''); + i++; + } else if ((arg === '--filter' || arg === '-k') && nextArg) { + filter = nextArg.replace(/^"|"$/g, ''); + i++; + } else if ((arg === '--iterations' || arg === '-i') && nextArg) { + iterations = nextArg.replace(/^"|"$/g, ''); + i++; + } + } + + // Validate all inputs for shell injection + model = validateInput(model, 'model'); + markers = validateInput(markers, 'markers'); + filter = validateInput(filter, 'filter'); + + if (model === null || markers === null || filter === null) { + return; // core.setFailed was already called + } + + // Additional validation: only allow safe characters in each field + // model: alphanumeric, dash, underscore, dot, comma, forward slash (for provider/model) + if (model && !/^[a-zA-Z0-9_.,\/-]*$/.test(model)) { + core.setFailed('Invalid model: only alphanumeric, dash, underscore, dot, comma, and forward slash allowed'); + return; + } + + // markers: alphanumeric, dash, underscore, space, "and", "or", "not" (pytest marker syntax) + if (markers && !/^[a-zA-Z0-9_\- ]*$/.test(markers)) { + core.setFailed('Invalid markers: only alphanumeric, dash, underscore, and space allowed'); + return; + } + + // filter: alphanumeric, dash, underscore (test name patterns) + if (filter && !/^[a-zA-Z0-9_\-]*$/.test(filter)) { + core.setFailed('Invalid filter: only alphanumeric, dash, and underscore allowed'); + return; + } + + // Cap iterations at 10 + let iterNum = parseInt(iterations, 10); + if (isNaN(iterNum) || iterNum < 1) iterNum = 1; + if (iterNum > 10) iterNum = 10; + + core.setOutput('should_run', 'true'); + core.setOutput('model', model); + core.setOutput('markers', markers); + core.setOutput('filter', filter); + core.setOutput('iterations', iterNum.toString()); + + console.log('Parsed /eval command:', { model, markers, filter, iterations: iterNum }); + + - name: React to comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket' + }); + llm_evals: runs-on: ubuntu-latest + needs: [parse-eval-command] + # Run if: PR event, push event, workflow_dispatch, or /eval comment parsed successfully + if: always() && (github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || needs.parse-eval-command.outputs.should_run == 'true') steps: + - name: Determine checkout ref + id: checkout-ref + shell: bash + run: | + if [[ "${{ github.event_name }}" == "issue_comment" ]]; then + echo "ref=${{ needs.parse-eval-command.outputs.pr_sha }}" >> $GITHUB_OUTPUT + else + echo "ref=" >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v4 + with: + ref: ${{ steps.checkout-ref.outputs.ref || github.ref }} + + - name: Validate and determine eval parameters + id: eval-params + uses: actions/github-script@v7 + with: + script: | + // Security: Validate input doesn't contain shell injection characters + const dangerousPattern = /[`$;|&<>(){}[\]\\'\n\r]/; + + function validateInput(value, name, allowedPattern) { + if (!value) return ''; + const trimmed = value.trim(); + if (dangerousPattern.test(trimmed)) { + core.setFailed(`Invalid ${name}: contains dangerous shell characters`); + return null; + } + if (allowedPattern && !allowedPattern.test(trimmed)) { + core.setFailed(`Invalid ${name}: contains disallowed characters`); + return null; + } + return trimmed; + } + + // Patterns for allowed characters + const modelPattern = /^[a-zA-Z0-9_.,\/-]*$/; // alphanumeric, dash, underscore, dot, comma, slash + const markersPattern = /^[a-zA-Z0-9_\- ]*$/; // alphanumeric, dash, underscore, space + const filterPattern = /^[a-zA-Z0-9_\-]*$/; // alphanumeric, dash, underscore + const iterationsPattern = /^[0-9]+$/; // digits only + const prNumberPattern = /^[0-9]*$/; // digits only (can be empty) + + let model, markers, filter, iterations, isManual, triggerSource, prNumber; + + const eventName = context.eventName; + + if (eventName === 'issue_comment') { + // From /eval command - already validated in parse step + // Using toJSON for defense-in-depth escaping + model = ${{ toJSON(needs.parse-eval-command.outputs.model) }} || ''; + markers = ${{ toJSON(needs.parse-eval-command.outputs.markers) }} || 'regression'; + filter = ${{ toJSON(needs.parse-eval-command.outputs.filter) }} || ''; + iterations = ${{ toJSON(needs.parse-eval-command.outputs.iterations) }} || '1'; + isManual = true; + triggerSource = '/eval comment'; + prNumber = context.issue.number.toString(); + } else if (eventName === 'workflow_dispatch') { + // From manual workflow dispatch - needs validation + // Using toJSON for defense-in-depth escaping + model = validateInput(${{ toJSON(github.event.inputs.model) }} || '', 'model', modelPattern); + markers = validateInput(${{ toJSON(github.event.inputs.markers) }} || '', 'markers', markersPattern); + filter = validateInput(${{ toJSON(github.event.inputs.filter) }} || '', 'filter', filterPattern); + iterations = validateInput(${{ toJSON(github.event.inputs.iterations) }} || '', 'iterations', iterationsPattern); + prNumber = validateInput(${{ toJSON(github.event.inputs.pr_number) }} || '', 'pr_number', prNumberPattern); + + if (model === null || markers === null || filter === null || iterations === null || prNumber === null) { + return; // core.setFailed was already called + } + + isManual = true; + triggerSource = 'workflow_dispatch'; + } else { + // Default for PR/push events + model = ''; + markers = 'regression'; + filter = ''; + iterations = '1'; + isManual = false; + triggerSource = 'automatic'; + prNumber = ''; + } + + // Apply defaults + const varsModel = ${{ toJSON(vars.MODEL) }} || ''; + if (!model) model = varsModel; + if (!markers) markers = 'regression'; + if (!iterations) iterations = '1'; + + // Cap iterations at 10 + let iterNum = parseInt(iterations, 10); + if (isNaN(iterNum) || iterNum < 1) iterNum = 1; + if (iterNum > 10) iterNum = 10; + + // Build marker expression (safe since markers is validated) + const markerExpr = `llm and (${markers})`; + + core.setOutput('model', model); + core.setOutput('markers', markers); + core.setOutput('filter', filter); + core.setOutput('iterations', iterNum.toString()); + core.setOutput('is_manual', isManual.toString()); + core.setOutput('trigger_source', triggerSource); + core.setOutput('pr_number', prNumber); + core.setOutput('marker_expr', markerExpr); + + console.log('=== Eval Parameters ==='); + console.log(`Model: ${model}`); + console.log(`Markers: ${markers}`); + console.log(`Filter: ${filter}`); + console.log(`Iterations: ${iterNum}`); + console.log(`Is Manual: ${isManual}`); + console.log(`Trigger: ${triggerSource}`); - name: Check if tests should run id: check-tests @@ -26,7 +297,17 @@ jobs: # For pushes to master, we always run regardless of secrets availability # Note: We must check this in a step rather than job-level 'if' because # the secrets context is not available in job-level conditions per GitHub Actions docs - if [[ "${{ github.event_name }}" != "pull_request" ]] || [[ -n "${{ secrets.BRAINTRUST_API_KEY }}" && (-n "${{ secrets.AZURE_API_KEY }}" || -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}") ]]; then + + # For manual triggers, check if any API key is available + if [[ "${{ steps.eval-params.outputs.is_manual }}" == "true" ]]; then + if [[ -n "${{ secrets.OPENAI_API_KEY }}" ]] || [[ -n "${{ secrets.AZURE_API_KEY }}" ]] || [[ -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "✅ Running manual eval - API keys available" + else + echo "should-run=false" >> $GITHUB_OUTPUT + echo "❌ Cannot run evals - no API keys available (fork PR?)" + fi + elif [[ "${{ github.event_name }}" != "pull_request" ]] || [[ -n "${{ secrets.BRAINTRUST_API_KEY }}" && (-n "${{ secrets.AZURE_API_KEY }}" || -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}") ]]; then echo "should-run=true" >> $GITHUB_OUTPUT echo "✅ Running tests - either not a PR or required secrets are available" else @@ -34,6 +315,76 @@ jobs: echo "⏭️ Skipping tests - PR from fork without required secrets" fi + - name: Post initial comment + id: initial-comment + if: steps.check-tests.outputs.should-run == 'true' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + // Using toJSON for defense-in-depth escaping + const isManual = ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true'; + const triggerSource = ${{ toJSON(steps.eval-params.outputs.trigger_source) }} || ''; + const model = ${{ toJSON(steps.eval-params.outputs.model) }} || 'default'; + const markers = ${{ toJSON(steps.eval-params.outputs.markers) }} || ''; + const filter = ${{ toJSON(steps.eval-params.outputs.filter) }} || ''; + const iterations = ${{ toJSON(steps.eval-params.outputs.iterations) }} || ''; + const runId = ${{ toJSON(github.run_id) }}; + const repository = ${{ toJSON(github.repository) }}; + const runUrl = `https://github.com/${repository}/actions/runs/${runId}`; + + // Determine PR number + let prNumber = null; + if (context.eventName === 'issue_comment') { + prNumber = context.issue.number; + } else if (context.eventName === 'pull_request') { + prNumber = context.payload.pull_request.number; + } else if (context.eventName === 'workflow_dispatch') { + const prNumStr = ${{ toJSON(steps.eval-params.outputs.pr_number) }} || ''; + if (prNumStr) { + prNumber = parseInt(prNumStr, 10); + } + } + + if (!prNumber) { + console.log('No PR number found, skipping initial comment'); + core.setOutput('comment_id', ''); + return; + } + + // Build initial comment body + let body = ''; + + if (isManual) { + body += `## 🚀 Manual Eval Running...\n\n`; + body += `| Parameter | Value |\n`; + body += `|-----------|-------|\n`; + body += `| **Triggered via** | ${triggerSource} |\n`; + body += `| **Model** | \`${model}\` |\n`; + body += `| **Markers** | \`${markers}\` |\n`; + if (filter) { + body += `| **Filter (-k)** | \`${filter}\` |\n`; + } + body += `| **Iterations** | ${iterations} |\n`; + body += `| **Status** | ⏳ Running... |\n`; + body += `| **Workflow** | [View logs](${runUrl}) |\n`; + } else { + body += `## ⏳ HolmesGPT evals running...\n\n`; + body += `[View workflow logs](${runUrl})\n\n`; + body += `_Results will appear here when complete._\n`; + } + + // Post comment + const comment = await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + + core.setOutput('comment_id', comment.data.id.toString()); + console.log(`Posted initial comment #${comment.data.id} to PR #${prNumber}`); + - name: Setup KIND cluster if: steps.check-tests.outputs.should-run == 'true' uses: ./.github/actions/setup-kind-cluster @@ -60,25 +411,205 @@ jobs: AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }} - MODEL: ${{ vars.MODEL }} - CLASSIFIER_MODEL: ${{ vars.CLASSIFIER_MODEL }} + MODEL: ${{ steps.eval-params.outputs.model }} + ITERATIONS: ${{ steps.eval-params.outputs.iterations }} + CLASSIFIER_MODEL: ${{ vars.CLASSIFIER_MODEL || 'gpt-4o' }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} UPLOAD_DATASET: "true" EXPERIMENT_ID: github-${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }} GENERATE_REGRESSIONS_FILE: "true" + # Pass validated parameters as environment variables (not shell interpolation) + EVAL_MARKER_EXPR: ${{ steps.eval-params.outputs.marker_expr }} + EVAL_FILTER: ${{ steps.eval-params.outputs.filter }} run: | - # Run regression tests - these are critical tests that must always pass - # Tests are marked with the 'regression' tag in their test_case.yaml files - poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n10 -m 'llm and regression' + START_TIME=$(date +%s) + + # Build pytest command safely using validated environment variables + # Note: EVAL_MARKER_EXPR and EVAL_FILTER are validated to contain only safe characters + PYTEST_ARGS=( + --no-cov + tests/llm/test_ask_holmes.py + tests/llm/test_investigate.py + -s + -n10 + -m "$EVAL_MARKER_EXPR" + ) + + # Add filter if provided + if [[ -n "$EVAL_FILTER" ]]; then + PYTEST_ARGS+=(-k "$EVAL_FILTER") + fi + + # Run pytest with array expansion (safe from injection) + poetry run pytest "${PYTEST_ARGS[@]}" || true + + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + # Format duration + HOURS=$((DURATION / 3600)) + MINUTES=$(((DURATION % 3600) / 60)) + SECONDS=$((DURATION % 60)) + + if [[ $HOURS -gt 0 ]]; then + DURATION_STR="${HOURS}h ${MINUTES}m ${SECONDS}s" + elif [[ $MINUTES -gt 0 ]]; then + DURATION_STR="${MINUTES}m ${SECONDS}s" + else + DURATION_STR="${SECONDS}s" + fi + + echo "duration=$DURATION_STR" >> $GITHUB_OUTPUT + echo "Eval duration: $DURATION_STR" + - name: Post evaluation results if: always() - uses: ./.github/actions/post-eval-comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + + // Using toJSON for defense-in-depth escaping of all interpolated values + const isManual = ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true'; + const triggerSource = ${{ toJSON(steps.eval-params.outputs.trigger_source) }} || ''; + const model = ${{ toJSON(steps.eval-params.outputs.model) }} || 'default'; + const markers = ${{ toJSON(steps.eval-params.outputs.markers) }} || ''; + const filter = ${{ toJSON(steps.eval-params.outputs.filter) }} || ''; + const iterations = ${{ toJSON(steps.eval-params.outputs.iterations) }} || ''; + const duration = ${{ toJSON(steps.evals.outputs.duration) }} || 'N/A'; + const runId = ${{ toJSON(github.run_id) }}; + const repository = ${{ toJSON(github.repository) }}; + const runUrl = `https://github.com/${repository}/actions/runs/${runId}`; + const initialCommentId = ${{ toJSON(steps.initial-comment.outputs.comment_id) }} || ''; + + // Determine PR number + let prNumber = null; + if (context.eventName === 'issue_comment') { + prNumber = context.issue.number; + } else if (context.eventName === 'pull_request') { + prNumber = context.payload.pull_request.number; + } else if (context.eventName === 'workflow_dispatch') { + const prNumStr = ${{ toJSON(steps.eval-params.outputs.pr_number) }} || ''; + if (prNumStr) { + prNumber = parseInt(prNumStr, 10); + } + } + + if (!prNumber) { + console.log('No PR number found, skipping comment'); + return; + } + + // Read the report file + let reportContent = ''; + if (fs.existsSync('evals_report.md')) { + reportContent = fs.readFileSync('evals_report.md', 'utf8'); + } + + // Check for regressions + let regressions = ''; + if (fs.existsSync('regressions.txt')) { + regressions = fs.readFileSync('regressions.txt', 'utf8'); + } + + // Build comment body + let body = ''; + + if (isManual) { + // Manual trigger - full details + body += `## 🧪 Manual Eval Results\n\n`; + body += `| Parameter | Value |\n`; + body += `|-----------|-------|\n`; + body += `| **Triggered via** | ${triggerSource} |\n`; + body += `| **Model** | \`${model}\` |\n`; + body += `| **Markers** | \`${markers}\` |\n`; + if (filter) { + body += `| **Filter (-k)** | \`${filter}\` |\n`; + } + body += `| **Iterations** | ${iterations} |\n`; + body += `| **Duration** | ${duration} |\n`; + body += `| **Workflow** | [View logs](${runUrl}) |\n\n`; + } else { + // Automatic trigger - standard header + body += `## Results of HolmesGPT evals\n\n`; + body += `**Duration:** ${duration} | [View workflow logs](${runUrl})\n\n`; + } + + // Add report content + if (reportContent) { + body += reportContent + '\n'; + } else { + body += '⚠️ No eval report was generated.\n\n'; + } + + // Add regressions warning + if (regressions) { + const count = regressions.trim(); + body += `\n### ⚠️ ${count} Regression${count === '1' ? '' : 's'} Detected\n\n`; + } + + // Add manual trigger instructions (for automatic runs only) + if (!isManual) { + const workflowUrl = `https://github.com/${repository}/actions/workflows/eval-regression.yaml`; + + body += '\n---\n'; + body += '
\n🔄 Re-run evals manually\n\n'; + body += '**Option 1: Comment on this PR** with `/eval`:\n\n'; + body += '```\n'; + body += '/eval # Run with defaults\n'; + body += '/eval --model gpt-4o # Use specific model\n'; + body += '/eval --model "gpt-4.1,gpt-4o" # Multiple models\n'; + body += '/eval --filter "09_crashpod" # Run specific test (-k)\n'; + body += '/eval --markers "logs" # Run tests with marker\n'; + body += '/eval --iterations 5 # Run multiple times\n'; + body += '```\n\n'; + body += '| Option | Short | Description |\n'; + body += '|--------|-------|-------------|\n'; + body += '| `--model` | `-m` | Model(s) to test |\n'; + body += '| `--markers` | `-M` | Pytest markers |\n'; + body += '| `--filter` | `-k` | Pytest -k filter |\n'; + body += '| `--iterations` | `-i` | Iterations (max 10) |\n\n'; + body += `**Option 2: [Trigger via GitHub Actions UI](${workflowUrl})** → "Run workflow" → select branch and fill in parameters\n`; + body += '
\n'; + } + + // Update existing comment or create new one + if (initialCommentId) { + // Update the comment we created at the start + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: parseInt(initialCommentId, 10), + body: body + }); + console.log(`Updated comment #${initialCommentId} with eval results on PR #${prNumber}`); + } else { + // No initial comment (e.g., tests skipped), create new one + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + console.log(`Posted new eval results comment to PR #${prNumber}`); + } + + - name: Add completion reaction for /eval comments + if: always() && github.event_name == 'issue_comment' + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} - report-file: 'evals_report.md' - comment-identifier: '## Results of HolmesGPT evals' - delete-previous: 'true' + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'hooray' + }); + - name: Check test results if: always() run: | From f1e48060ddf12f5eaa08c39c04f8e3a55f85a5e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 13:51:17 +0000 Subject: [PATCH 02/12] Refactor eval workflow: reduce redundancy, 38% smaller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Consolidate PR number logic into eval-params step - Remove duplicate validation (parse step extracts, eval-params validates) - Add run_url output to avoid recalculating - Use shared params object in comment steps - Condense duration formatting in bash - 621 → 387 lines --- .github/workflows/eval-regression.yaml | 522 +++++++------------------ 1 file changed, 144 insertions(+), 378 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index e5a973fa07..5dc6203853 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -5,7 +5,6 @@ on: branches: ["*"] push: branches: [master] - # Allow manual triggering with custom parameters workflow_dispatch: inputs: pr_number: @@ -28,7 +27,6 @@ on: description: 'Number of iterations (1-10)' required: false default: '1' - # Allow triggering via /eval comment on PRs issue_comment: types: [created] @@ -38,7 +36,6 @@ permissions: issues: write jobs: - # Parse /eval command from PR comments parse-eval-command: if: github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/eval') runs-on: ubuntu-latest @@ -51,11 +48,10 @@ jobs: pr_sha: ${{ steps.get-pr.outputs.sha }} steps: - - name: Get PR details + - name: Get PR SHA id: get-pr uses: actions/github-script@v7 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | const pr = await github.rest.pulls.get({ owner: context.repo.owner, @@ -69,96 +65,27 @@ jobs: uses: actions/github-script@v7 with: script: | - const comment = context.payload.comment.body; - const commandLine = comment.split('\n')[0].trim(); + const comment = context.payload.comment.body.split('\n')[0].trim(); + const args = comment.slice(5).trim(); + const parts = args.match(/(?:[^\s"]+|"[^"]*")+/g) || []; - // Security: Validate input doesn't contain shell injection characters - // Dangerous chars: ` $ ; | & < > ( ) { } [ ] \ newline - const dangerousPattern = /[`$;|&<>(){}[\]\\'\n\r]/; + let params = { model: '', markers: 'regression', filter: '', iterations: '1' }; + const flags = { '--model': 'model', '-m': 'model', '--markers': 'markers', '-M': 'markers', + '--filter': 'filter', '-k': 'filter', '--iterations': 'iterations', '-i': 'iterations' }; - function validateInput(value, name) { - const trimmed = value.trim(); - if (dangerousPattern.test(trimmed)) { - core.setFailed(`Invalid ${name}: contains dangerous shell characters. Allowed: alphanumeric, dash, underscore, dot, comma, space, quotes.`); - return null; + for (let i = 0; i < parts.length; i++) { + if (flags[parts[i]] && parts[i + 1]) { + params[flags[parts[i]]] = parts[++i].replace(/^"|"$/g, ''); } - return trimmed; - } - - // Default values - let model = ''; - let markers = 'regression'; - let filter = ''; - let iterations = '1'; - - // Parse command arguments - const args = commandLine.slice(5).trim(); - const argParts = args.match(/(?:[^\s"]+|"[^"]*")+/g) || []; - - for (let i = 0; i < argParts.length; i++) { - const arg = argParts[i]; - const nextArg = argParts[i + 1]; - - if ((arg === '--model' || arg === '-m') && nextArg) { - model = nextArg.replace(/^"|"$/g, ''); - i++; - } else if ((arg === '--markers' || arg === '-M') && nextArg) { - markers = nextArg.replace(/^"|"$/g, ''); - i++; - } else if ((arg === '--filter' || arg === '-k') && nextArg) { - filter = nextArg.replace(/^"|"$/g, ''); - i++; - } else if ((arg === '--iterations' || arg === '-i') && nextArg) { - iterations = nextArg.replace(/^"|"$/g, ''); - i++; - } - } - - // Validate all inputs for shell injection - model = validateInput(model, 'model'); - markers = validateInput(markers, 'markers'); - filter = validateInput(filter, 'filter'); - - if (model === null || markers === null || filter === null) { - return; // core.setFailed was already called } - // Additional validation: only allow safe characters in each field - // model: alphanumeric, dash, underscore, dot, comma, forward slash (for provider/model) - if (model && !/^[a-zA-Z0-9_.,\/-]*$/.test(model)) { - core.setFailed('Invalid model: only alphanumeric, dash, underscore, dot, comma, and forward slash allowed'); - return; - } - - // markers: alphanumeric, dash, underscore, space, "and", "or", "not" (pytest marker syntax) - if (markers && !/^[a-zA-Z0-9_\- ]*$/.test(markers)) { - core.setFailed('Invalid markers: only alphanumeric, dash, underscore, and space allowed'); - return; - } - - // filter: alphanumeric, dash, underscore (test name patterns) - if (filter && !/^[a-zA-Z0-9_\-]*$/.test(filter)) { - core.setFailed('Invalid filter: only alphanumeric, dash, and underscore allowed'); - return; - } - - // Cap iterations at 10 - let iterNum = parseInt(iterations, 10); - if (isNaN(iterNum) || iterNum < 1) iterNum = 1; - if (iterNum > 10) iterNum = 10; - + // Output raw values - validation happens in eval-params step core.setOutput('should_run', 'true'); - core.setOutput('model', model); - core.setOutput('markers', markers); - core.setOutput('filter', filter); - core.setOutput('iterations', iterNum.toString()); - - console.log('Parsed /eval command:', { model, markers, filter, iterations: iterNum }); + Object.entries(params).forEach(([k, v]) => core.setOutput(k, v)); - name: React to comment uses: actions/github-script@v7 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | await github.rest.reactions.createForIssueComment({ owner: context.repo.owner, @@ -170,220 +97,154 @@ jobs: llm_evals: runs-on: ubuntu-latest needs: [parse-eval-command] - # Run if: PR event, push event, workflow_dispatch, or /eval comment parsed successfully if: always() && (github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || needs.parse-eval-command.outputs.should_run == 'true') steps: - name: Determine checkout ref id: checkout-ref - shell: bash run: | if [[ "${{ github.event_name }}" == "issue_comment" ]]; then echo "ref=${{ needs.parse-eval-command.outputs.pr_sha }}" >> $GITHUB_OUTPUT - else - echo "ref=" >> $GITHUB_OUTPUT fi + shell: bash - uses: actions/checkout@v4 with: ref: ${{ steps.checkout-ref.outputs.ref || github.ref }} - - name: Validate and determine eval parameters + - name: Determine eval parameters id: eval-params uses: actions/github-script@v7 with: script: | - // Security: Validate input doesn't contain shell injection characters - const dangerousPattern = /[`$;|&<>(){}[\]\\'\n\r]/; - - function validateInput(value, name, allowedPattern) { + // Validation patterns + const patterns = { + model: /^[a-zA-Z0-9_.,\/-]*$/, + markers: /^[a-zA-Z0-9_\- ]*$/, + filter: /^[a-zA-Z0-9_\-]*$/, + iterations: /^[0-9]+$/, + pr_number: /^[0-9]*$/ + }; + const dangerous = /[`$;|&<>(){}[\]\\'\n\r]/; + + function validate(value, name) { if (!value) return ''; - const trimmed = value.trim(); - if (dangerousPattern.test(trimmed)) { - core.setFailed(`Invalid ${name}: contains dangerous shell characters`); - return null; - } - if (allowedPattern && !allowedPattern.test(trimmed)) { - core.setFailed(`Invalid ${name}: contains disallowed characters`); + if (dangerous.test(value) || (patterns[name] && !patterns[name].test(value))) { + core.setFailed(`Invalid ${name}`); return null; } - return trimmed; + return value; } - // Patterns for allowed characters - const modelPattern = /^[a-zA-Z0-9_.,\/-]*$/; // alphanumeric, dash, underscore, dot, comma, slash - const markersPattern = /^[a-zA-Z0-9_\- ]*$/; // alphanumeric, dash, underscore, space - const filterPattern = /^[a-zA-Z0-9_\-]*$/; // alphanumeric, dash, underscore - const iterationsPattern = /^[0-9]+$/; // digits only - const prNumberPattern = /^[0-9]*$/; // digits only (can be empty) - let model, markers, filter, iterations, isManual, triggerSource, prNumber; + const event = context.eventName; - const eventName = context.eventName; - - if (eventName === 'issue_comment') { - // From /eval command - already validated in parse step - // Using toJSON for defense-in-depth escaping + if (event === 'issue_comment') { model = ${{ toJSON(needs.parse-eval-command.outputs.model) }} || ''; markers = ${{ toJSON(needs.parse-eval-command.outputs.markers) }} || 'regression'; filter = ${{ toJSON(needs.parse-eval-command.outputs.filter) }} || ''; iterations = ${{ toJSON(needs.parse-eval-command.outputs.iterations) }} || '1'; isManual = true; triggerSource = '/eval comment'; - prNumber = context.issue.number.toString(); - } else if (eventName === 'workflow_dispatch') { - // From manual workflow dispatch - needs validation - // Using toJSON for defense-in-depth escaping - model = validateInput(${{ toJSON(github.event.inputs.model) }} || '', 'model', modelPattern); - markers = validateInput(${{ toJSON(github.event.inputs.markers) }} || '', 'markers', markersPattern); - filter = validateInput(${{ toJSON(github.event.inputs.filter) }} || '', 'filter', filterPattern); - iterations = validateInput(${{ toJSON(github.event.inputs.iterations) }} || '', 'iterations', iterationsPattern); - prNumber = validateInput(${{ toJSON(github.event.inputs.pr_number) }} || '', 'pr_number', prNumberPattern); - - if (model === null || markers === null || filter === null || iterations === null || prNumber === null) { - return; // core.setFailed was already called - } - + prNumber = context.issue.number; + } else if (event === 'workflow_dispatch') { + model = validate(${{ toJSON(github.event.inputs.model) }} || '', 'model'); + markers = validate(${{ toJSON(github.event.inputs.markers) }} || '', 'markers'); + filter = validate(${{ toJSON(github.event.inputs.filter) }} || '', 'filter'); + iterations = validate(${{ toJSON(github.event.inputs.iterations) }} || '', 'iterations'); + const prNum = validate(${{ toJSON(github.event.inputs.pr_number) }} || '', 'pr_number'); + if ([model, markers, filter, iterations, prNum].includes(null)) return; isManual = true; triggerSource = 'workflow_dispatch'; + prNumber = prNum ? parseInt(prNum, 10) : null; } else { - // Default for PR/push events model = ''; markers = 'regression'; filter = ''; iterations = '1'; isManual = false; triggerSource = 'automatic'; - prNumber = ''; + prNumber = event === 'pull_request' ? context.payload.pull_request.number : null; + } + + // Validate issue_comment inputs too + if (event === 'issue_comment') { + model = validate(model, 'model'); + markers = validate(markers, 'markers'); + filter = validate(filter, 'filter'); + if ([model, markers, filter].includes(null)) return; } // Apply defaults - const varsModel = ${{ toJSON(vars.MODEL) }} || ''; - if (!model) model = varsModel; + if (!model) model = ${{ toJSON(vars.MODEL) }} || ''; if (!markers) markers = 'regression'; - if (!iterations) iterations = '1'; - // Cap iterations at 10 - let iterNum = parseInt(iterations, 10); - if (isNaN(iterNum) || iterNum < 1) iterNum = 1; - if (iterNum > 10) iterNum = 10; - - // Build marker expression (safe since markers is validated) - const markerExpr = `llm and (${markers})`; + // Cap iterations + let iterNum = Math.min(Math.max(parseInt(iterations, 10) || 1, 1), 10); + // Set outputs core.setOutput('model', model); core.setOutput('markers', markers); core.setOutput('filter', filter); core.setOutput('iterations', iterNum.toString()); core.setOutput('is_manual', isManual.toString()); core.setOutput('trigger_source', triggerSource); - core.setOutput('pr_number', prNumber); - core.setOutput('marker_expr', markerExpr); - - console.log('=== Eval Parameters ==='); - console.log(`Model: ${model}`); - console.log(`Markers: ${markers}`); - console.log(`Filter: ${filter}`); - console.log(`Iterations: ${iterNum}`); - console.log(`Is Manual: ${isManual}`); - console.log(`Trigger: ${triggerSource}`); + core.setOutput('pr_number', prNumber ? prNumber.toString() : ''); + core.setOutput('marker_expr', `llm and (${markers})`); + core.setOutput('run_url', `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`); - name: Check if tests should run id: check-tests shell: bash run: | - # Check if we have the required API keys to run LLM tests - # For PRs from forks, secrets are not available for security reasons - # For pushes to master, we always run regardless of secrets availability - # Note: We must check this in a step rather than job-level 'if' because - # the secrets context is not available in job-level conditions per GitHub Actions docs - - # For manual triggers, check if any API key is available if [[ "${{ steps.eval-params.outputs.is_manual }}" == "true" ]]; then if [[ -n "${{ secrets.OPENAI_API_KEY }}" ]] || [[ -n "${{ secrets.AZURE_API_KEY }}" ]] || [[ -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}" ]]; then echo "should-run=true" >> $GITHUB_OUTPUT - echo "✅ Running manual eval - API keys available" else echo "should-run=false" >> $GITHUB_OUTPUT - echo "❌ Cannot run evals - no API keys available (fork PR?)" fi elif [[ "${{ github.event_name }}" != "pull_request" ]] || [[ -n "${{ secrets.BRAINTRUST_API_KEY }}" && (-n "${{ secrets.AZURE_API_KEY }}" || -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}") ]]; then echo "should-run=true" >> $GITHUB_OUTPUT - echo "✅ Running tests - either not a PR or required secrets are available" else echo "should-run=false" >> $GITHUB_OUTPUT - echo "⏭️ Skipping tests - PR from fork without required secrets" fi - name: Post initial comment id: initial-comment - if: steps.check-tests.outputs.should-run == 'true' + if: steps.check-tests.outputs.should-run == 'true' && steps.eval-params.outputs.pr_number != '' uses: actions/github-script@v7 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | - // Using toJSON for defense-in-depth escaping - const isManual = ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true'; - const triggerSource = ${{ toJSON(steps.eval-params.outputs.trigger_source) }} || ''; - const model = ${{ toJSON(steps.eval-params.outputs.model) }} || 'default'; - const markers = ${{ toJSON(steps.eval-params.outputs.markers) }} || ''; - const filter = ${{ toJSON(steps.eval-params.outputs.filter) }} || ''; - const iterations = ${{ toJSON(steps.eval-params.outputs.iterations) }} || ''; - const runId = ${{ toJSON(github.run_id) }}; - const repository = ${{ toJSON(github.repository) }}; - const runUrl = `https://github.com/${repository}/actions/runs/${runId}`; - - // Determine PR number - let prNumber = null; - if (context.eventName === 'issue_comment') { - prNumber = context.issue.number; - } else if (context.eventName === 'pull_request') { - prNumber = context.payload.pull_request.number; - } else if (context.eventName === 'workflow_dispatch') { - const prNumStr = ${{ toJSON(steps.eval-params.outputs.pr_number) }} || ''; - if (prNumStr) { - prNumber = parseInt(prNumStr, 10); - } - } - - if (!prNumber) { - console.log('No PR number found, skipping initial comment'); - core.setOutput('comment_id', ''); - return; - } - - // Build initial comment body - let body = ''; - - if (isManual) { - body += `## 🚀 Manual Eval Running...\n\n`; - body += `| Parameter | Value |\n`; - body += `|-----------|-------|\n`; - body += `| **Triggered via** | ${triggerSource} |\n`; - body += `| **Model** | \`${model}\` |\n`; - body += `| **Markers** | \`${markers}\` |\n`; - if (filter) { - body += `| **Filter (-k)** | \`${filter}\` |\n`; - } - body += `| **Iterations** | ${iterations} |\n`; - body += `| **Status** | ⏳ Running... |\n`; - body += `| **Workflow** | [View logs](${runUrl}) |\n`; - } else { - body += `## ⏳ HolmesGPT evals running...\n\n`; - body += `[View workflow logs](${runUrl})\n\n`; - body += `_Results will appear here when complete._\n`; - } + const p = { + isManual: ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true', + trigger: ${{ toJSON(steps.eval-params.outputs.trigger_source) }}, + model: ${{ toJSON(steps.eval-params.outputs.model) }} || 'default', + markers: ${{ toJSON(steps.eval-params.outputs.markers) }}, + filter: ${{ toJSON(steps.eval-params.outputs.filter) }}, + iterations: ${{ toJSON(steps.eval-params.outputs.iterations) }}, + runUrl: ${{ toJSON(steps.eval-params.outputs.run_url) }}, + prNumber: parseInt(${{ toJSON(steps.eval-params.outputs.pr_number) }}, 10) + }; + + let body = p.isManual + ? `## 🚀 Manual Eval Running...\n\n` + + `| Parameter | Value |\n|-----------|-------|\n` + + `| **Triggered via** | ${p.trigger} |\n` + + `| **Model** | \`${p.model}\` |\n` + + `| **Markers** | \`${p.markers}\` |\n` + + (p.filter ? `| **Filter (-k)** | \`${p.filter}\` |\n` : '') + + `| **Iterations** | ${p.iterations} |\n` + + `| **Status** | ⏳ Running... |\n` + + `| **Workflow** | [View logs](${p.runUrl}) |\n` + : `## ⏳ HolmesGPT evals running...\n\n[View workflow logs](${p.runUrl})\n\n_Results will appear here when complete._\n`; - // Post comment const comment = await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: prNumber, - body: body + issue_number: p.prNumber, + body }); - core.setOutput('comment_id', comment.data.id.toString()); - console.log(`Posted initial comment #${comment.data.id} to PR #${prNumber}`); - name: Setup KIND cluster if: steps.check-tests.outputs.should-run == 'true' @@ -420,194 +281,99 @@ jobs: UPLOAD_DATASET: "true" EXPERIMENT_ID: github-${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }} GENERATE_REGRESSIONS_FILE: "true" - # Pass validated parameters as environment variables (not shell interpolation) EVAL_MARKER_EXPR: ${{ steps.eval-params.outputs.marker_expr }} EVAL_FILTER: ${{ steps.eval-params.outputs.filter }} run: | START_TIME=$(date +%s) - # Build pytest command safely using validated environment variables - # Note: EVAL_MARKER_EXPR and EVAL_FILTER are validated to contain only safe characters - PYTEST_ARGS=( - --no-cov - tests/llm/test_ask_holmes.py - tests/llm/test_investigate.py - -s - -n10 - -m "$EVAL_MARKER_EXPR" - ) - - # Add filter if provided - if [[ -n "$EVAL_FILTER" ]]; then - PYTEST_ARGS+=(-k "$EVAL_FILTER") - fi - - # Run pytest with array expansion (safe from injection) + PYTEST_ARGS=(--no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n10 -m "$EVAL_MARKER_EXPR") + [[ -n "$EVAL_FILTER" ]] && PYTEST_ARGS+=(-k "$EVAL_FILTER") poetry run pytest "${PYTEST_ARGS[@]}" || true - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME)) - - # Format duration - HOURS=$((DURATION / 3600)) - MINUTES=$(((DURATION % 3600) / 60)) - SECONDS=$((DURATION % 60)) - - if [[ $HOURS -gt 0 ]]; then - DURATION_STR="${HOURS}h ${MINUTES}m ${SECONDS}s" - elif [[ $MINUTES -gt 0 ]]; then - DURATION_STR="${MINUTES}m ${SECONDS}s" + DURATION=$(($(date +%s) - START_TIME)) + if [[ $DURATION -ge 3600 ]]; then + echo "duration=$((DURATION/3600))h $(((DURATION%3600)/60))m $((DURATION%60))s" >> $GITHUB_OUTPUT + elif [[ $DURATION -ge 60 ]]; then + echo "duration=$((DURATION/60))m $((DURATION%60))s" >> $GITHUB_OUTPUT else - DURATION_STR="${SECONDS}s" + echo "duration=${DURATION}s" >> $GITHUB_OUTPUT fi - echo "duration=$DURATION_STR" >> $GITHUB_OUTPUT - echo "Eval duration: $DURATION_STR" - - name: Post evaluation results - if: always() + if: always() && steps.eval-params.outputs.pr_number != '' uses: actions/github-script@v7 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - - // Using toJSON for defense-in-depth escaping of all interpolated values - const isManual = ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true'; - const triggerSource = ${{ toJSON(steps.eval-params.outputs.trigger_source) }} || ''; - const model = ${{ toJSON(steps.eval-params.outputs.model) }} || 'default'; - const markers = ${{ toJSON(steps.eval-params.outputs.markers) }} || ''; - const filter = ${{ toJSON(steps.eval-params.outputs.filter) }} || ''; - const iterations = ${{ toJSON(steps.eval-params.outputs.iterations) }} || ''; - const duration = ${{ toJSON(steps.evals.outputs.duration) }} || 'N/A'; - const runId = ${{ toJSON(github.run_id) }}; - const repository = ${{ toJSON(github.repository) }}; - const runUrl = `https://github.com/${repository}/actions/runs/${runId}`; - const initialCommentId = ${{ toJSON(steps.initial-comment.outputs.comment_id) }} || ''; - - // Determine PR number - let prNumber = null; - if (context.eventName === 'issue_comment') { - prNumber = context.issue.number; - } else if (context.eventName === 'pull_request') { - prNumber = context.payload.pull_request.number; - } else if (context.eventName === 'workflow_dispatch') { - const prNumStr = ${{ toJSON(steps.eval-params.outputs.pr_number) }} || ''; - if (prNumStr) { - prNumber = parseInt(prNumStr, 10); - } - } - - if (!prNumber) { - console.log('No PR number found, skipping comment'); - return; + const p = { + isManual: ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true', + trigger: ${{ toJSON(steps.eval-params.outputs.trigger_source) }}, + model: ${{ toJSON(steps.eval-params.outputs.model) }} || 'default', + markers: ${{ toJSON(steps.eval-params.outputs.markers) }}, + filter: ${{ toJSON(steps.eval-params.outputs.filter) }}, + iterations: ${{ toJSON(steps.eval-params.outputs.iterations) }}, + duration: ${{ toJSON(steps.evals.outputs.duration) }} || 'N/A', + runUrl: ${{ toJSON(steps.eval-params.outputs.run_url) }}, + prNumber: parseInt(${{ toJSON(steps.eval-params.outputs.pr_number) }}, 10), + commentId: ${{ toJSON(steps.initial-comment.outputs.comment_id) }} + }; + + const report = fs.existsSync('evals_report.md') ? fs.readFileSync('evals_report.md', 'utf8') : ''; + const regressions = fs.existsSync('regressions.txt') ? fs.readFileSync('regressions.txt', 'utf8').trim() : ''; + + let body = p.isManual + ? `## 🧪 Manual Eval Results\n\n` + + `| Parameter | Value |\n|-----------|-------|\n` + + `| **Triggered via** | ${p.trigger} |\n` + + `| **Model** | \`${p.model}\` |\n` + + `| **Markers** | \`${p.markers}\` |\n` + + (p.filter ? `| **Filter (-k)** | \`${p.filter}\` |\n` : '') + + `| **Iterations** | ${p.iterations} |\n` + + `| **Duration** | ${p.duration} |\n` + + `| **Workflow** | [View logs](${p.runUrl}) |\n\n` + : `## Results of HolmesGPT evals\n\n**Duration:** ${p.duration} | [View workflow logs](${p.runUrl})\n\n`; + + body += report || '⚠️ No eval report was generated.\n\n'; + if (regressions) body += `\n### ⚠️ ${regressions} Regression${regressions === '1' ? '' : 's'} Detected\n\n`; + + if (!p.isManual) { + const workflowUrl = p.runUrl.replace(/\/runs\/.*/, ''); + body += '\n---\n
\n🔄 Re-run evals manually\n\n' + + '**Option 1: Comment on this PR** with `/eval`:\n\n```\n' + + '/eval # Run with defaults\n' + + '/eval --model gpt-4o # Use specific model\n' + + '/eval --model "gpt-4.1,gpt-4o" # Multiple models\n' + + '/eval --filter "09_crashpod" # Run specific test (-k)\n' + + '/eval --markers "logs" # Run tests with marker\n' + + '/eval --iterations 5 # Run multiple times\n```\n\n' + + '| Option | Short | Description |\n|--------|-------|-------------|\n' + + '| `--model` | `-m` | Model(s) to test |\n' + + '| `--markers` | `-M` | Pytest markers |\n' + + '| `--filter` | `-k` | Pytest -k filter |\n' + + '| `--iterations` | `-i` | Iterations (max 10) |\n\n' + + `**Option 2: [Trigger via GitHub Actions UI](${workflowUrl})** → "Run workflow"\n
\n`; } - // Read the report file - let reportContent = ''; - if (fs.existsSync('evals_report.md')) { - reportContent = fs.readFileSync('evals_report.md', 'utf8'); - } - - // Check for regressions - let regressions = ''; - if (fs.existsSync('regressions.txt')) { - regressions = fs.readFileSync('regressions.txt', 'utf8'); - } - - // Build comment body - let body = ''; - - if (isManual) { - // Manual trigger - full details - body += `## 🧪 Manual Eval Results\n\n`; - body += `| Parameter | Value |\n`; - body += `|-----------|-------|\n`; - body += `| **Triggered via** | ${triggerSource} |\n`; - body += `| **Model** | \`${model}\` |\n`; - body += `| **Markers** | \`${markers}\` |\n`; - if (filter) { - body += `| **Filter (-k)** | \`${filter}\` |\n`; - } - body += `| **Iterations** | ${iterations} |\n`; - body += `| **Duration** | ${duration} |\n`; - body += `| **Workflow** | [View logs](${runUrl}) |\n\n`; - } else { - // Automatic trigger - standard header - body += `## Results of HolmesGPT evals\n\n`; - body += `**Duration:** ${duration} | [View workflow logs](${runUrl})\n\n`; - } - - // Add report content - if (reportContent) { - body += reportContent + '\n'; - } else { - body += '⚠️ No eval report was generated.\n\n'; - } - - // Add regressions warning - if (regressions) { - const count = regressions.trim(); - body += `\n### ⚠️ ${count} Regression${count === '1' ? '' : 's'} Detected\n\n`; - } - - // Add manual trigger instructions (for automatic runs only) - if (!isManual) { - const workflowUrl = `https://github.com/${repository}/actions/workflows/eval-regression.yaml`; - - body += '\n---\n'; - body += '
\n🔄 Re-run evals manually\n\n'; - body += '**Option 1: Comment on this PR** with `/eval`:\n\n'; - body += '```\n'; - body += '/eval # Run with defaults\n'; - body += '/eval --model gpt-4o # Use specific model\n'; - body += '/eval --model "gpt-4.1,gpt-4o" # Multiple models\n'; - body += '/eval --filter "09_crashpod" # Run specific test (-k)\n'; - body += '/eval --markers "logs" # Run tests with marker\n'; - body += '/eval --iterations 5 # Run multiple times\n'; - body += '```\n\n'; - body += '| Option | Short | Description |\n'; - body += '|--------|-------|-------------|\n'; - body += '| `--model` | `-m` | Model(s) to test |\n'; - body += '| `--markers` | `-M` | Pytest markers |\n'; - body += '| `--filter` | `-k` | Pytest -k filter |\n'; - body += '| `--iterations` | `-i` | Iterations (max 10) |\n\n'; - body += `**Option 2: [Trigger via GitHub Actions UI](${workflowUrl})** → "Run workflow" → select branch and fill in parameters\n`; - body += '
\n'; - } - - // Update existing comment or create new one - if (initialCommentId) { - // Update the comment we created at the start + if (p.commentId) { await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: parseInt(initialCommentId, 10), - body: body + owner: context.repo.owner, repo: context.repo.repo, + comment_id: parseInt(p.commentId, 10), body }); - console.log(`Updated comment #${initialCommentId} with eval results on PR #${prNumber}`); } else { - // No initial comment (e.g., tests skipped), create new one await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: body + owner: context.repo.owner, repo: context.repo.repo, + issue_number: p.prNumber, body }); - console.log(`Posted new eval results comment to PR #${prNumber}`); } - - name: Add completion reaction for /eval comments + - name: Add completion reaction if: always() && github.event_name == 'issue_comment' uses: actions/github-script@v7 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | await github.rest.reactions.createForIssueComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: context.payload.comment.id, - content: 'hooray' + owner: context.repo.owner, repo: context.repo.repo, + comment_id: context.payload.comment.id, content: 'hooray' }); - name: Check test results From f834a66a59f5d71b9e3c19f391c8db6823b418d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 13:56:28 +0000 Subject: [PATCH 03/12] Add more model examples to workflow_dispatch description --- .github/workflows/eval-regression.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 5dc6203853..e4a38874a1 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -12,7 +12,7 @@ on: required: false default: '' model: - description: 'Model(s) to test (comma-separated, e.g., gpt-4.1,gpt-4o)' + description: 'Model(s) to test. Examples: gpt-4.1, gpt-4o, anthropic/claude-sonnet-4-20250514, azure/gpt-4.1, bedrock/anthropic.claude-sonnet-4-20250514-v1:0, gemini/gemini-1.5-pro, vertex_ai/claude-3-5-sonnet, ollama/llama2. Comma-separated for multiple.' required: false default: '' markers: From 4fb601f9339dde9f4c1db1098085251470bfb85e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 14:02:06 +0000 Subject: [PATCH 04/12] Simplify /eval command to multiline key: value format Before: /eval --model gpt-4o --filter "my test" After: /eval model: gpt-4o filter: my test Much simpler parsing (split on newlines, split on colon) and easier for humans to read/write in PR comments. --- .github/workflows/eval-regression.yaml | 48 ++++++++++++-------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index e4a38874a1..77f2517e8b 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -65,21 +65,22 @@ jobs: uses: actions/github-script@v7 with: script: | - const comment = context.payload.comment.body.split('\n')[0].trim(); - const args = comment.slice(5).trim(); - const parts = args.match(/(?:[^\s"]+|"[^"]*")+/g) || []; - - let params = { model: '', markers: 'regression', filter: '', iterations: '1' }; - const flags = { '--model': 'model', '-m': 'model', '--markers': 'markers', '-M': 'markers', - '--filter': 'filter', '-k': 'filter', '--iterations': 'iterations', '-i': 'iterations' }; - - for (let i = 0; i < parts.length; i++) { - if (flags[parts[i]] && parts[i + 1]) { - params[flags[parts[i]]] = parts[++i].replace(/^"|"$/g, ''); - } + // Parse multiline format: + // /eval + // model: gpt-4o + // filter: my test + const lines = context.payload.comment.body.split('\n'); + const params = { model: '', markers: 'regression', filter: '', iterations: '1' }; + const validKeys = new Set(Object.keys(params)); + + for (const line of lines.slice(1)) { + const colonIdx = line.indexOf(':'); + if (colonIdx === -1) continue; + const key = line.slice(0, colonIdx).trim().toLowerCase(); + const value = line.slice(colonIdx + 1).trim(); + if (validKeys.has(key) && value) params[key] = value; } - // Output raw values - validation happens in eval-params step core.setOutput('should_run', 'true'); Object.entries(params).forEach(([k, v]) => core.setOutput(k, v)); @@ -339,18 +340,15 @@ jobs: if (!p.isManual) { const workflowUrl = p.runUrl.replace(/\/runs\/.*/, ''); body += '\n---\n
\n🔄 Re-run evals manually\n\n' + - '**Option 1: Comment on this PR** with `/eval`:\n\n```\n' + - '/eval # Run with defaults\n' + - '/eval --model gpt-4o # Use specific model\n' + - '/eval --model "gpt-4.1,gpt-4o" # Multiple models\n' + - '/eval --filter "09_crashpod" # Run specific test (-k)\n' + - '/eval --markers "logs" # Run tests with marker\n' + - '/eval --iterations 5 # Run multiple times\n```\n\n' + - '| Option | Short | Description |\n|--------|-------|-------------|\n' + - '| `--model` | `-m` | Model(s) to test |\n' + - '| `--markers` | `-M` | Pytest markers |\n' + - '| `--filter` | `-k` | Pytest -k filter |\n' + - '| `--iterations` | `-i` | Iterations (max 10) |\n\n' + + '**Option 1: Comment on this PR** with `/eval`:\n\n' + + '```\n/eval\n```\n\n' + + 'Or with options (one per line):\n\n' + + '```\n/eval\nmodel: gpt-4o\nfilter: 09_crashpod\niterations: 5\n```\n\n' + + '| Option | Description |\n|--------|-------------|\n' + + '| `model` | Model(s) to test (comma-separated) |\n' + + '| `markers` | Pytest markers (default: regression) |\n' + + '| `filter` | Pytest -k filter |\n' + + '| `iterations` | Number of runs, max 10 |\n\n' + `**Option 2: [Trigger via GitHub Actions UI](${workflowUrl})** → "Run workflow"\n
\n`; } From 745a7a80284f0a94852f8716d2821080f4452961 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 14:05:58 +0000 Subject: [PATCH 05/12] Clarify default model matches automatic runs --- .github/workflows/eval-regression.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 77f2517e8b..2166120b7f 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -176,10 +176,12 @@ jobs: if ([model, markers, filter].includes(null)) return; } - // Apply defaults + // Apply defaults - use vars.MODEL if set, otherwise empty (uses litellm default) if (!model) model = ${{ toJSON(vars.MODEL) }} || ''; if (!markers) markers = 'regression'; + console.log(`Using model: ${model || '(default from litellm)'}`); + // Cap iterations let iterNum = Math.min(Math.max(parseInt(iterations, 10) || 1, 1), 10); @@ -345,7 +347,7 @@ jobs: 'Or with options (one per line):\n\n' + '```\n/eval\nmodel: gpt-4o\nfilter: 09_crashpod\niterations: 5\n```\n\n' + '| Option | Description |\n|--------|-------------|\n' + - '| `model` | Model(s) to test (comma-separated) |\n' + + '| `model` | Model(s) to test (default: same as automatic runs) |\n' + '| `markers` | Pytest markers (default: regression) |\n' + '| `filter` | Pytest -k filter |\n' + '| `iterations` | Number of runs, max 10 |\n\n' + From 7378b5b37122f05cae9d7559508521c674c00556 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 14:11:33 +0000 Subject: [PATCH 06/12] Consolidate /eval parsing into eval-params step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Simplify parse-eval-command job to just: get PR SHA + add reaction - Move parseComment() function into eval-params step - Remove 4 outputs (model, markers, filter, iterations) from job 1 - Eliminates duplicate passing of values between jobs - 388 → 357 lines --- .github/workflows/eval-regression.yaml | 78 ++++++++------------------ 1 file changed, 24 insertions(+), 54 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 2166120b7f..3229dd07f6 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -40,16 +40,12 @@ jobs: if: github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/eval') runs-on: ubuntu-latest outputs: - should_run: ${{ steps.parse.outputs.should_run }} - model: ${{ steps.parse.outputs.model }} - markers: ${{ steps.parse.outputs.markers }} - filter: ${{ steps.parse.outputs.filter }} - iterations: ${{ steps.parse.outputs.iterations }} - pr_sha: ${{ steps.get-pr.outputs.sha }} + should_run: ${{ steps.init.outputs.should_run }} + pr_sha: ${{ steps.init.outputs.sha }} steps: - - name: Get PR SHA - id: get-pr + - name: Get PR SHA and react + id: init uses: actions/github-script@v7 with: script: | @@ -59,35 +55,8 @@ jobs: pull_number: context.issue.number }); core.setOutput('sha', pr.data.head.sha); - - - name: Parse /eval command - id: parse - uses: actions/github-script@v7 - with: - script: | - // Parse multiline format: - // /eval - // model: gpt-4o - // filter: my test - const lines = context.payload.comment.body.split('\n'); - const params = { model: '', markers: 'regression', filter: '', iterations: '1' }; - const validKeys = new Set(Object.keys(params)); - - for (const line of lines.slice(1)) { - const colonIdx = line.indexOf(':'); - if (colonIdx === -1) continue; - const key = line.slice(0, colonIdx).trim().toLowerCase(); - const value = line.slice(colonIdx + 1).trim(); - if (validKeys.has(key) && value) params[key] = value; - } - core.setOutput('should_run', 'true'); - Object.entries(params).forEach(([k, v]) => core.setOutput(k, v)); - - name: React to comment - uses: actions/github-script@v7 - with: - script: | await github.rest.reactions.createForIssueComment({ owner: context.repo.owner, repo: context.repo.repo, @@ -118,7 +87,6 @@ jobs: uses: actions/github-script@v7 with: script: | - // Validation patterns const patterns = { model: /^[a-zA-Z0-9_.,\/-]*$/, markers: /^[a-zA-Z0-9_\- ]*$/, @@ -137,14 +105,29 @@ jobs: return value; } + function parseComment(body) { + const params = { model: '', markers: 'regression', filter: '', iterations: '1' }; + const validKeys = new Set(Object.keys(params)); + for (const line of body.split('\n').slice(1)) { + const colonIdx = line.indexOf(':'); + if (colonIdx === -1) continue; + const key = line.slice(0, colonIdx).trim().toLowerCase(); + const value = line.slice(colonIdx + 1).trim(); + if (validKeys.has(key) && value) params[key] = value; + } + return params; + } + let model, markers, filter, iterations, isManual, triggerSource, prNumber; const event = context.eventName; if (event === 'issue_comment') { - model = ${{ toJSON(needs.parse-eval-command.outputs.model) }} || ''; - markers = ${{ toJSON(needs.parse-eval-command.outputs.markers) }} || 'regression'; - filter = ${{ toJSON(needs.parse-eval-command.outputs.filter) }} || ''; - iterations = ${{ toJSON(needs.parse-eval-command.outputs.iterations) }} || '1'; + const parsed = parseComment(context.payload.comment.body); + model = validate(parsed.model, 'model'); + markers = validate(parsed.markers, 'markers'); + filter = validate(parsed.filter, 'filter'); + iterations = parsed.iterations; + if ([model, markers, filter].includes(null)) return; isManual = true; triggerSource = '/eval comment'; prNumber = context.issue.number; @@ -168,24 +151,11 @@ jobs: prNumber = event === 'pull_request' ? context.payload.pull_request.number : null; } - // Validate issue_comment inputs too - if (event === 'issue_comment') { - model = validate(model, 'model'); - markers = validate(markers, 'markers'); - filter = validate(filter, 'filter'); - if ([model, markers, filter].includes(null)) return; - } - - // Apply defaults - use vars.MODEL if set, otherwise empty (uses litellm default) + // Apply defaults if (!model) model = ${{ toJSON(vars.MODEL) }} || ''; if (!markers) markers = 'regression'; - - console.log(`Using model: ${model || '(default from litellm)'}`); - - // Cap iterations let iterNum = Math.min(Math.max(parseInt(iterations, 10) || 1, 1), 10); - // Set outputs core.setOutput('model', model); core.setOutput('markers', markers); core.setOutput('filter', filter); From 23f3e5b5f86c654f61b546aac29aadb88ab382c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 19:25:56 +0000 Subject: [PATCH 07/12] Fix: validate iterations for /eval comments --- .github/workflows/eval-regression.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 3229dd07f6..36f2b8200c 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -126,8 +126,8 @@ jobs: model = validate(parsed.model, 'model'); markers = validate(parsed.markers, 'markers'); filter = validate(parsed.filter, 'filter'); - iterations = parsed.iterations; - if ([model, markers, filter].includes(null)) return; + iterations = validate(parsed.iterations, 'iterations'); + if ([model, markers, filter, iterations].includes(null)) return; isManual = true; triggerSource = '/eval comment'; prNumber = context.issue.number; From 41e92653ff28f24f8914dff4c01483652af8c184 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 19:32:50 +0000 Subject: [PATCH 08/12] Merge jobs into one, add permission check for /eval - Consolidate parse-eval-command job into llm_evals as a step - Add permission check: only OWNER, MEMBER, COLLABORATOR can trigger /eval - Prevents random external users from triggering expensive evals - Simplifies workflow structure (1 job instead of 2) --- .github/workflows/eval-regression.yaml | 44 ++++++++++++-------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 36f2b8200c..bbe0c018e8 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -36,27 +36,37 @@ permissions: issues: write jobs: - parse-eval-command: - if: github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/eval') + llm_evals: runs-on: ubuntu-latest - outputs: - should_run: ${{ steps.init.outputs.should_run }} - pr_sha: ${{ steps.init.outputs.sha }} + if: | + github.event_name == 'pull_request' || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/eval')) steps: - - name: Get PR SHA and react - id: init + - name: Handle /eval comment + id: eval-comment + if: github.event_name == 'issue_comment' uses: actions/github-script@v7 with: script: | + // Check commenter has write access + const association = context.payload.comment.author_association; + if (!['OWNER', 'MEMBER', 'COLLABORATOR'].includes(association)) { + core.setFailed(`Permission denied: ${association} cannot trigger /eval (requires OWNER, MEMBER, or COLLABORATOR)`); + return; + } + + // Get PR SHA for checkout const pr = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number }); - core.setOutput('sha', pr.data.head.sha); - core.setOutput('should_run', 'true'); + core.setOutput('pr_sha', pr.data.head.sha); + // Add rocket reaction await github.rest.reactions.createForIssueComment({ owner: context.repo.owner, repo: context.repo.repo, @@ -64,23 +74,9 @@ jobs: content: 'rocket' }); - llm_evals: - runs-on: ubuntu-latest - needs: [parse-eval-command] - if: always() && (github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || needs.parse-eval-command.outputs.should_run == 'true') - - steps: - - name: Determine checkout ref - id: checkout-ref - run: | - if [[ "${{ github.event_name }}" == "issue_comment" ]]; then - echo "ref=${{ needs.parse-eval-command.outputs.pr_sha }}" >> $GITHUB_OUTPUT - fi - shell: bash - - uses: actions/checkout@v4 with: - ref: ${{ steps.checkout-ref.outputs.ref || github.ref }} + ref: ${{ steps.eval-comment.outputs.pr_sha || github.ref }} - name: Determine eval parameters id: eval-params From b14797e4bd31cf4cd697f72289c1fdc15024bb6d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 19:36:22 +0000 Subject: [PATCH 09/12] Fix: skip all steps when /eval permission denied --- .github/workflows/eval-regression.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index bbe0c018e8..bf24639317 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -75,10 +75,12 @@ jobs: }); - uses: actions/checkout@v4 + if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success' with: ref: ${{ steps.eval-comment.outputs.pr_sha || github.ref }} - name: Determine eval parameters + if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success' id: eval-params uses: actions/github-script@v7 with: @@ -164,6 +166,7 @@ jobs: - name: Check if tests should run id: check-tests + if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success' shell: bash run: | if [[ "${{ steps.eval-params.outputs.is_manual }}" == "true" ]]; then @@ -333,7 +336,7 @@ jobs: } - name: Add completion reaction - if: always() && github.event_name == 'issue_comment' + if: always() && github.event_name == 'issue_comment' && steps.eval-comment.outcome == 'success' uses: actions/github-script@v7 with: script: | From d35141e8c940ac67c424084664290bcf69001619 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Mon, 29 Dec 2025 11:06:04 +0200 Subject: [PATCH 10/12] fixes Signed-off-by: Natan Yellin --- .github/workflows/eval-regression.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index bf24639317..435ff5e316 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -85,19 +85,23 @@ jobs: uses: actions/github-script@v7 with: script: | + // SECURITY: These whitelist regexes protect against shell injection by only + // allowing safe characters. This works together with proper quoting in bash + // (using env vars + "$VAR" syntax). If you modify these patterns or add new + // parameters, ensure: (1) regex is a whitelist of safe chars, (2) values are + // passed via env vars not inline ${{ }}, (3) bash references use double quotes. const patterns = { - model: /^[a-zA-Z0-9_.,\/-]*$/, + model: /^[a-zA-Z0-9_.,:\/-]*$/, markers: /^[a-zA-Z0-9_\- ]*$/, filter: /^[a-zA-Z0-9_\-]*$/, iterations: /^[0-9]+$/, pr_number: /^[0-9]*$/ }; - const dangerous = /[`$;|&<>(){}[\]\\'\n\r]/; function validate(value, name) { if (!value) return ''; - if (dangerous.test(value) || (patterns[name] && !patterns[name].test(value))) { - core.setFailed(`Invalid ${name}`); + if (patterns[name] && !patterns[name].test(value)) { + core.setFailed(`Invalid ${name}: must match ${patterns[name]}`); return null; } return value; From 18857218ced9ce660c6855cd4148a47312aece8e Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Mon, 29 Dec 2025 11:57:46 +0200 Subject: [PATCH 11/12] Improve eval-regression workflow UX and simplify logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix workflow dispatch link to point to specific workflow file - Change rocket reaction to eyes for clearer "processing" signal - Simplify secrets check (repo has all or none) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 Signed-off-by: Natan Yellin --- .github/workflows/eval-regression.yaml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 435ff5e316..4ba3808aa1 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -66,12 +66,12 @@ jobs: }); core.setOutput('pr_sha', pr.data.head.sha); - // Add rocket reaction + // Add eyes reaction to indicate the comment was seen and is being processed await github.rest.reactions.createForIssueComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: context.payload.comment.id, - content: 'rocket' + content: 'eyes' }); - uses: actions/checkout@v4 @@ -173,13 +173,8 @@ jobs: if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success' shell: bash run: | - if [[ "${{ steps.eval-params.outputs.is_manual }}" == "true" ]]; then - if [[ -n "${{ secrets.OPENAI_API_KEY }}" ]] || [[ -n "${{ secrets.AZURE_API_KEY }}" ]] || [[ -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}" ]]; then - echo "should-run=true" >> $GITHUB_OUTPUT - else - echo "should-run=false" >> $GITHUB_OUTPUT - fi - elif [[ "${{ github.event_name }}" != "pull_request" ]] || [[ -n "${{ secrets.BRAINTRUST_API_KEY }}" && (-n "${{ secrets.AZURE_API_KEY }}" || -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}") ]]; then + # Check one secret as proxy for secrets access (repo has all or none) + if [[ -n "${{ secrets.AZURE_API_KEY }}" ]]; then echo "should-run=true" >> $GITHUB_OUTPUT else echo "should-run=false" >> $GITHUB_OUTPUT @@ -313,7 +308,7 @@ jobs: if (regressions) body += `\n### ⚠️ ${regressions} Regression${regressions === '1' ? '' : 's'} Detected\n\n`; if (!p.isManual) { - const workflowUrl = p.runUrl.replace(/\/runs\/.*/, ''); + const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/eval-regression.yaml`; body += '\n---\n
\n🔄 Re-run evals manually\n\n' + '**Option 1: Comment on this PR** with `/eval`:\n\n' + '```\n/eval\n```\n\n' + From f307351281478fe6d616c8ed811b20c06eb4df2c Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Mon, 29 Dec 2025 12:58:52 +0200 Subject: [PATCH 12/12] Fix empty expression in workflow comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove `${{ }}` from JS comment - GitHub parses expressions even in comments. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 Signed-off-by: Natan Yellin --- .github/workflows/eval-regression.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index 4ba3808aa1..67c518d1d5 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -89,7 +89,7 @@ jobs: // allowing safe characters. This works together with proper quoting in bash // (using env vars + "$VAR" syntax). If you modify these patterns or add new // parameters, ensure: (1) regex is a whitelist of safe chars, (2) values are - // passed via env vars not inline ${{ }}, (3) bash references use double quotes. + // passed via env vars not inline, (3) bash references use double quotes. const patterns = { model: /^[a-zA-Z0-9_.,:\/-]*$/, markers: /^[a-zA-Z0-9_\- ]*$/,