diff --git a/.github/workflows/evaluation.yml b/.github/workflows/evaluation.yml index 06e20b291b..ac391ffc00 100644 --- a/.github/workflows/evaluation.yml +++ b/.github/workflows/evaluation.yml @@ -33,6 +33,11 @@ on: required: false type: number default: 3 + parallel-scenarios: + description: 'Max concurrent scenarios per skill' + required: false + type: number + default: 3 parallel-runs: description: 'Max concurrent runs per scenario' required: false @@ -55,6 +60,7 @@ env: DEFAULT_JUDGE_MODEL: 'claude-opus-4.5' DEFAULT_RUNS: '3' DEFAULT_PARALLEL_SKILLS: '3' + DEFAULT_PARALLEL_SCENARIOS: '3' DEFAULT_PARALLEL_RUNS: '3' permissions: @@ -166,6 +172,10 @@ jobs: if [ -n "$PARALLEL_SKILLS" ]; then ARGS="$ARGS --parallel-skills $PARALLEL_SKILLS" fi + PARALLEL_SCENARIOS="${{ github.event.inputs.parallel-scenarios || env.DEFAULT_PARALLEL_SCENARIOS }}" + if [ -n "$PARALLEL_SCENARIOS" ]; then + ARGS="$ARGS --parallel-scenarios $PARALLEL_SCENARIOS" + fi PARALLEL_RUNS="${{ github.event.inputs.parallel-runs || env.DEFAULT_PARALLEL_RUNS }}" if [ -n "$PARALLEL_RUNS" ]; then ARGS="$ARGS --parallel-runs $PARALLEL_RUNS" diff --git a/eng/skill-validator/README.md b/eng/skill-validator/README.md index b41a962edc..4b6ca32bd7 100644 --- a/eng/skill-validator/README.md +++ b/eng/skill-validator/README.md @@ -222,6 +222,7 @@ The default of 5 runs provides sufficient precision for significance testing (va | `--min-improvement ` | `0.1` | Minimum improvement score (0–1) | | `--runs ` | `5` | Runs per scenario (averaged for stability) | | `--parallel-skills ` | `1` | Max concurrent skills to evaluate | +| `--parallel-scenarios ` | `1` | Max concurrent scenarios per skill | | `--parallel-runs ` | `1` | Max concurrent runs per scenario | | `--confidence-level ` | `0.95` | Confidence level for statistical intervals (0–1) | | `--judge-timeout ` | `300` | Judge LLM timeout in seconds | diff --git a/eng/skill-validator/src/cli.ts b/eng/skill-validator/src/cli.ts index 06bb3738b9..54465633ec 100644 --- a/eng/skill-validator/src/cli.ts +++ b/eng/skill-validator/src/cli.ts @@ -17,6 +17,7 @@ import type { RunResult, ScenarioComparison, PairwiseJudgeResult, + EvalScenario, } from "./types.js"; import type { ModelInfo } from "@github/copilot-sdk"; @@ -116,6 +117,7 @@ export function createProgram(): Command { .option("--judge-mode ", "Judge mode: pairwise, independent, or both", "pairwise") .option("--runs ", "Number of runs per scenario for averaging", "5") .option("--parallel-skills ", "Max concurrent skills to evaluate", "1") + .option("--parallel-scenarios ", "Max concurrent scenarios per skill", "1") .option("--parallel-runs ", "Max concurrent runs per scenario", "1") .option("--judge-timeout ", "Judge timeout in seconds", "300") .option("--confidence-level ", "Confidence level for statistical intervals (0-1)", "0.95") @@ -146,6 +148,7 @@ export function createProgram(): Command { judgeMode: opts.judgeMode || "pairwise", runs: Math.max(1, parseInt(opts.runs, 10) || 5), parallelSkills: Math.max(1, parseInt(opts.parallelSkills, 10) || 1), + parallelScenarios: Math.max(1, parseInt(opts.parallelScenarios, 10) || 1), parallelRuns: Math.max(1, parseInt(opts.parallelRuns, 10) || 1), judgeTimeout: parseInt(opts.judgeTimeout, 10) * 1000, confidenceLevel: parseFloat(opts.confidenceLevel || "0.95"), @@ -250,10 +253,11 @@ export async function run(config: ValidatorConfig): Promise { log(warning); } - const comparisons: ScenarioComparison[] = []; const singleScenario = skill.evalConfig.scenarios.length === 1; + const scenarioLimit = pLimit(Math.max(1, config.parallelScenarios)); - for (const scenario of skill.evalConfig.scenarios) { + // Execute all scenarios for this skill (parallel if parallelScenarios > 1) + const executeScenario = async (scenario: EvalScenario): Promise => { const tag = singleScenario ? `[${skill.name}]` : `[${skill.name}/${scenario.name}]`; const scenarioLog = (msg: string) => spinner.log(`${tag} ${msg}`); const runLimit = pLimit(Math.max(1, config.parallelRuns)); @@ -429,8 +433,13 @@ export async function run(config: ValidatorConfig): Promise { ); comparison.perRunScores = perRunScores; - comparisons.push(comparison); - } + return comparison; + }; + + const scenarioPromises = skill.evalConfig.scenarios.map((scenario) => + scenarioLimit(() => executeScenario(scenario)) + ); + const comparisons = await Promise.all(scenarioPromises); const verdict = computeVerdict( skill, diff --git a/eng/skill-validator/src/types.ts b/eng/skill-validator/src/types.ts index cad55024a2..953c2a1745 100644 --- a/eng/skill-validator/src/types.ts +++ b/eng/skill-validator/src/types.ts @@ -184,6 +184,7 @@ export interface ValidatorConfig { judgeMode: JudgeMode; runs: number; parallelSkills: number; + parallelScenarios: number; parallelRuns: number; judgeTimeout: number; confidenceLevel: number;