diff --git a/.github/workflows/evaluation.yml b/.github/workflows/evaluation.yml index 06e20b291b..79848eac6d 100644 --- a/.github/workflows/evaluation.yml +++ b/.github/workflows/evaluation.yml @@ -18,11 +18,11 @@ on: model: description: 'Copilot model to use' required: false - default: 'claude-opus-4.5' + default: 'claude-opus-4.6' judge-model: description: 'Judge model to use for evaluation' required: false - default: 'claude-opus-4.5' + default: 'claude-opus-4.6' runs: description: 'Number of runs per test' required: false @@ -51,9 +51,10 @@ concurrency: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOTNET_VERSION: '10.0.x' - DEFAULT_MODEL: 'claude-opus-4.5' - DEFAULT_JUDGE_MODEL: 'claude-opus-4.5' - DEFAULT_RUNS: '3' + DEFAULT_MODEL: 'claude-opus-4.6' + DEFAULT_JUDGE_MODEL: 'claude-opus-4.6' + # Use more runs for main branch to get more stable benchmark data, but fewer runs for PRs to reduce quota pressure. + DEFAULT_RUNS: ${{ github.ref == 'refs/heads/main' && '5' || '3' }} DEFAULT_PARALLEL_SKILLS: '3' DEFAULT_PARALLEL_RUNS: '3' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf1965c72f..3b3ab9cf71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -224,12 +224,15 @@ node dist/index.js --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/s node dist/index.js --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills/common-build-errors # Fewer runs for faster iteration (default is 5) -node dist/index.js --runs 1 --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills +node dist/index.js --runs 3 --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills # Use a specific model -node dist/index.js --model claude-sonnet-4.5 --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills +node dist/index.js --model claude-opus-4.6 --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills ``` +> [!WARNING] +> If you share the results in a Pull Request, make sure to have `--runs` configured to at least 3 but better 5 for reliable results. + ### CI evaluation Tests run automatically on pull requests that modify files under `src/`. The evaluation workflow discovers changed components and runs the skill-validator for each one. Results are posted as a PR comment and uploaded as build artifacts.