Allow manually triggering evals

.github/workflows/eval-regression.yaml

@@ -5,35 +5,218 @@ on:
     branches: ["*"]
   push:
     branches: [master]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number to run evals on (leave empty for current branch)'
+        required: false
+        default: ''
+      model:
+        description: 'Model(s) to test. Examples: gpt-4.1, gpt-4o, anthropic/claude-sonnet-4-20250514, azure/gpt-4.1, bedrock/anthropic.claude-sonnet-4-20250514-v1:0, gemini/gemini-1.5-pro, vertex_ai/claude-3-5-sonnet, ollama/llama2. Comma-separated for multiple.'
+        required: false
+        default: ''
+      markers:
+        description: 'Pytest markers (e.g., regression, easy, logs). Combined with "llm and"'
+        required: false
+        default: 'regression'
+      filter:
+        description: 'Pytest -k filter (e.g., 09_crashpod, checkout)'
+        required: false
+        default: ''
+      iterations:
+        description: 'Number of iterations (1-10)'
+        required: false
+        default: '1'
+  issue_comment:
+    types: [created]
 
 permissions:
   pull-requests: write
   contents: read
+  issues: write
 
 jobs:
   llm_evals:
     runs-on: ubuntu-latest
+    if: |
+      github.event_name == 'pull_request' ||
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/eval'))
 
     steps:
+      - name: Handle /eval comment
+        id: eval-comment
+        if: github.event_name == 'issue_comment'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            // Check commenter has write access
+            const association = context.payload.comment.author_association;
+            if (!['OWNER', 'MEMBER', 'COLLABORATOR'].includes(association)) {
+              core.setFailed(`Permission denied: ${association} cannot trigger /eval (requires OWNER, MEMBER, or COLLABORATOR)`);
+              return;
+            }
+
+            // Get PR SHA for checkout
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+            core.setOutput('pr_sha', pr.data.head.sha);
+
+            // Add eyes reaction to indicate the comment was seen and is being processed
+            await github.rest.reactions.createForIssueComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: context.payload.comment.id,
+              content: 'eyes'
+            });
+
       - uses: actions/checkout@v4
+        if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success'
+        with:
+          ref: ${{ steps.eval-comment.outputs.pr_sha || github.ref }}
+
+      - name: Determine eval parameters
+        if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success'
+        id: eval-params
+        uses: actions/github-script@v7
+        with:
+          script: |
+            // SECURITY: These whitelist regexes protect against shell injection by only
+            // allowing safe characters. This works together with proper quoting in bash
+            // (using env vars + "$VAR" syntax). If you modify these patterns or add new
+            // parameters, ensure: (1) regex is a whitelist of safe chars, (2) values are
+            // passed via env vars not inline, (3) bash references use double quotes.
+            const patterns = {
+              model: /^[a-zA-Z0-9_.,:\/-]*$/,
+              markers: /^[a-zA-Z0-9_\- ]*$/,
+              filter: /^[a-zA-Z0-9_\-]*$/,
+              iterations: /^[0-9]+$/,
+              pr_number: /^[0-9]*$/
+            };
+
+            function validate(value, name) {
+              if (!value) return '';
+              if (patterns[name] && !patterns[name].test(value)) {
+                core.setFailed(`Invalid ${name}: must match ${patterns[name]}`);
+                return null;
+              }
+              return value;
+            }
+
+            function parseComment(body) {
+              const params = { model: '', markers: 'regression', filter: '', iterations: '1' };
+              const validKeys = new Set(Object.keys(params));
+              for (const line of body.split('\n').slice(1)) {
+                const colonIdx = line.indexOf(':');
+                if (colonIdx === -1) continue;
+                const key = line.slice(0, colonIdx).trim().toLowerCase();
+                const value = line.slice(colonIdx + 1).trim();
+                if (validKeys.has(key) && value) params[key] = value;
+              }
+              return params;
+            }
+
+            let model, markers, filter, iterations, isManual, triggerSource, prNumber;
+            const event = context.eventName;
+
+            if (event === 'issue_comment') {
+              const parsed = parseComment(context.payload.comment.body);
+              model = validate(parsed.model, 'model');
+              markers = validate(parsed.markers, 'markers');
+              filter = validate(parsed.filter, 'filter');
+              iterations = validate(parsed.iterations, 'iterations');
+              if ([model, markers, filter, iterations].includes(null)) return;
+              isManual = true;
+              triggerSource = '/eval comment';
+              prNumber = context.issue.number;
+            } else if (event === 'workflow_dispatch') {
+              model = validate(${{ toJSON(github.event.inputs.model) }} || '', 'model');
+              markers = validate(${{ toJSON(github.event.inputs.markers) }} || '', 'markers');
+              filter = validate(${{ toJSON(github.event.inputs.filter) }} || '', 'filter');
+              iterations = validate(${{ toJSON(github.event.inputs.iterations) }} || '', 'iterations');
+              const prNum = validate(${{ toJSON(github.event.inputs.pr_number) }} || '', 'pr_number');
+              if ([model, markers, filter, iterations, prNum].includes(null)) return;
+              isManual = true;
+              triggerSource = 'workflow_dispatch';
+              prNumber = prNum ? parseInt(prNum, 10) : null;
+            } else {
+              model = '';
+              markers = 'regression';
+              filter = '';
+              iterations = '1';
+              isManual = false;
+              triggerSource = 'automatic';
+              prNumber = event === 'pull_request' ? context.payload.pull_request.number : null;
+            }
+
+            // Apply defaults
+            if (!model) model = ${{ toJSON(vars.MODEL) }} || '';
+            if (!markers) markers = 'regression';
+            let iterNum = Math.min(Math.max(parseInt(iterations, 10) || 1, 1), 10);
+
+            core.setOutput('model', model);
+            core.setOutput('markers', markers);
+            core.setOutput('filter', filter);
+            core.setOutput('iterations', iterNum.toString());
+            core.setOutput('is_manual', isManual.toString());
+            core.setOutput('trigger_source', triggerSource);
+            core.setOutput('pr_number', prNumber ? prNumber.toString() : '');
+            core.setOutput('marker_expr', `llm and (${markers})`);
+            core.setOutput('run_url', `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`);
 
       - name: Check if tests should run
         id: check-tests
+        if: github.event_name != 'issue_comment' || steps.eval-comment.outcome == 'success'
         shell: bash
         run: |
-          # Check if we have the required API keys to run LLM tests
-          # For PRs from forks, secrets are not available for security reasons
-          # For pushes to master, we always run regardless of secrets availability
-          # Note: We must check this in a step rather than job-level 'if' because
-          # the secrets context is not available in job-level conditions per GitHub Actions docs
-          if [[ "${{ github.event_name }}" != "pull_request" ]] || [[ -n "${{ secrets.BRAINTRUST_API_KEY }}" && (-n "${{ secrets.AZURE_API_KEY }}" || -n "${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}") ]]; then
+          # Check one secret as proxy for secrets access (repo has all or none)
+          if [[ -n "${{ secrets.AZURE_API_KEY }}" ]]; then
             echo "should-run=true" >> $GITHUB_OUTPUT
-            echo "✅ Running tests - either not a PR or required secrets are available"
           else
             echo "should-run=false" >> $GITHUB_OUTPUT
-            echo "⏭️ Skipping tests - PR from fork without required secrets"
           fi
 
+      - name: Post initial comment
+        id: initial-comment
+        if: steps.check-tests.outputs.should-run == 'true' && steps.eval-params.outputs.pr_number != ''
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const p = {
+              isManual: ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true',
+              trigger: ${{ toJSON(steps.eval-params.outputs.trigger_source) }},
+              model: ${{ toJSON(steps.eval-params.outputs.model) }} || 'default',
+              markers: ${{ toJSON(steps.eval-params.outputs.markers) }},
+              filter: ${{ toJSON(steps.eval-params.outputs.filter) }},
+              iterations: ${{ toJSON(steps.eval-params.outputs.iterations) }},
+              runUrl: ${{ toJSON(steps.eval-params.outputs.run_url) }},
+              prNumber: parseInt(${{ toJSON(steps.eval-params.outputs.pr_number) }}, 10)
+            };
+
+            let body = p.isManual
+              ? `## 🚀 Manual Eval Running...\n\n` +
+                `| Parameter | Value |\n|-----------|-------|\n` +
+                `| **Triggered via** | ${p.trigger} |\n` +
+                `| **Model** | \`${p.model}\` |\n` +
+                `| **Markers** | \`${p.markers}\` |\n` +
+                (p.filter ? `| **Filter (-k)** | \`${p.filter}\` |\n` : '') +
+                `| **Iterations** | ${p.iterations} |\n` +
+                `| **Status** | ⏳ Running... |\n` +
+                `| **Workflow** | [View logs](${p.runUrl}) |\n`
+              : `## ⏳ HolmesGPT evals running...\n\n[View workflow logs](${p.runUrl})\n\n_Results will appear here when complete._\n`;
+
+            const comment = await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: p.prNumber,
+              body
+            });
+            core.setOutput('comment_id', comment.data.id.toString());
+
       - name: Setup KIND cluster
         if: steps.check-tests.outputs.should-run == 'true'
         uses: ./.github/actions/setup-kind-cluster
@@ -60,25 +243,107 @@ jobs:
           AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
           AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
           AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }}
-          MODEL: ${{ vars.MODEL }}
-          CLASSIFIER_MODEL: ${{ vars.CLASSIFIER_MODEL }}
+          MODEL: ${{ steps.eval-params.outputs.model }}
+          ITERATIONS: ${{ steps.eval-params.outputs.iterations }}
+          CLASSIFIER_MODEL: ${{ vars.CLASSIFIER_MODEL || 'gpt-4o' }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           UPLOAD_DATASET: "true"
           EXPERIMENT_ID: github-${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }}
           GENERATE_REGRESSIONS_FILE: "true"
+          EVAL_MARKER_EXPR: ${{ steps.eval-params.outputs.marker_expr }}
+          EVAL_FILTER: ${{ steps.eval-params.outputs.filter }}
         run: |
-          # Run regression tests - these are critical tests that must always pass
-          # Tests are marked with the 'regression' tag in their test_case.yaml files
-          poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n10 -m 'llm and regression'
+          START_TIME=$(date +%s)
+
+          PYTEST_ARGS=(--no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n10 -m "$EVAL_MARKER_EXPR")
+          [[ -n "$EVAL_FILTER" ]] && PYTEST_ARGS+=(-k "$EVAL_FILTER")
+          poetry run pytest "${PYTEST_ARGS[@]}" || true
+
+          DURATION=$(($(date +%s) - START_TIME))
+          if [[ $DURATION -ge 3600 ]]; then
+            echo "duration=$((DURATION/3600))h $(((DURATION%3600)/60))m $((DURATION%60))s" >> $GITHUB_OUTPUT
+          elif [[ $DURATION -ge 60 ]]; then
+            echo "duration=$((DURATION/60))m $((DURATION%60))s" >> $GITHUB_OUTPUT
+          else
+            echo "duration=${DURATION}s" >> $GITHUB_OUTPUT
+          fi
+
       - name: Post evaluation results
-        if: always()
-        uses: ./.github/actions/post-eval-comment
+        if: always() && steps.eval-params.outputs.pr_number != ''
+        uses: actions/github-script@v7
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          report-file: 'evals_report.md'
-          comment-identifier: '## Results of HolmesGPT evals'
-          delete-previous: 'true'
+          script: |
+            const fs = require('fs');
+            const p = {
+              isManual: ${{ toJSON(steps.eval-params.outputs.is_manual) }} === 'true',
+              trigger: ${{ toJSON(steps.eval-params.outputs.trigger_source) }},
+              model: ${{ toJSON(steps.eval-params.outputs.model) }} || 'default',
+              markers: ${{ toJSON(steps.eval-params.outputs.markers) }},
+              filter: ${{ toJSON(steps.eval-params.outputs.filter) }},
+              iterations: ${{ toJSON(steps.eval-params.outputs.iterations) }},
+              duration: ${{ toJSON(steps.evals.outputs.duration) }} || 'N/A',
+              runUrl: ${{ toJSON(steps.eval-params.outputs.run_url) }},
+              prNumber: parseInt(${{ toJSON(steps.eval-params.outputs.pr_number) }}, 10),
+              commentId: ${{ toJSON(steps.initial-comment.outputs.comment_id) }}
+            };
+
+            const report = fs.existsSync('evals_report.md') ? fs.readFileSync('evals_report.md', 'utf8') : '';
+            const regressions = fs.existsSync('regressions.txt') ? fs.readFileSync('regressions.txt', 'utf8').trim() : '';
+
+            let body = p.isManual
+              ? `## 🧪 Manual Eval Results\n\n` +
+                `| Parameter | Value |\n|-----------|-------|\n` +
+                `| **Triggered via** | ${p.trigger} |\n` +
+                `| **Model** | \`${p.model}\` |\n` +
+                `| **Markers** | \`${p.markers}\` |\n` +
+                (p.filter ? `| **Filter (-k)** | \`${p.filter}\` |\n` : '') +
+                `| **Iterations** | ${p.iterations} |\n` +
+                `| **Duration** | ${p.duration} |\n` +
+                `| **Workflow** | [View logs](${p.runUrl}) |\n\n`
+              : `## Results of HolmesGPT evals\n\n**Duration:** ${p.duration} | [View workflow logs](${p.runUrl})\n\n`;
+
+            body += report || '⚠️ No eval report was generated.\n\n';
+            if (regressions) body += `\n### ⚠️ ${regressions} Regression${regressions === '1' ? '' : 's'} Detected\n\n`;
+
+            if (!p.isManual) {
+              const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/eval-regression.yaml`;
+              body += '\n---\n<details>\n<summary>🔄 <b>Re-run evals manually</b></summary>\n\n' +
+                '**Option 1: Comment on this PR** with `/eval`:\n\n' +
+                '```\n/eval\n```\n\n' +
+                'Or with options (one per line):\n\n' +
+                '```\n/eval\nmodel: gpt-4o\nfilter: 09_crashpod\niterations: 5\n```\n\n' +
+                '| Option | Description |\n|--------|-------------|\n' +
+                '| `model` | Model(s) to test (default: same as automatic runs) |\n' +
+                '| `markers` | Pytest markers (default: regression) |\n' +
+                '| `filter` | Pytest -k filter |\n' +
+                '| `iterations` | Number of runs, max 10 |\n\n' +
+                `**Option 2: [Trigger via GitHub Actions UI](${workflowUrl})** → "Run workflow"\n</details>\n`;
+            }
+
+            if (p.commentId) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                comment_id: parseInt(p.commentId, 10), body
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: p.prNumber, body
+              });
+            }
+
+      - name: Add completion reaction
+        if: always() && github.event_name == 'issue_comment' && steps.eval-comment.outcome == 'success'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.reactions.createForIssueComment({
+              owner: context.repo.owner, repo: context.repo.repo,
+              comment_id: context.payload.comment.id, content: 'hooray'
+            });
+
       - name: Check test results
         if: always()
         run: |