[Infra] feat: add support for parallel scenarios in skill evaluation

.github/workflows/evaluation.yml

@@ -33,6 +33,11 @@ on:
         required: false
         type: number
         default: 3
+      parallel-scenarios:
+        description: 'Max concurrent scenarios per skill'
+        required: false
+        type: number
+        default: 3
       parallel-runs:
         description: 'Max concurrent runs per scenario'
         required: false
@@ -55,6 +60,7 @@ env:
   DEFAULT_JUDGE_MODEL: 'claude-opus-4.5'
   DEFAULT_RUNS: '3'
   DEFAULT_PARALLEL_SKILLS: '3'
+  DEFAULT_PARALLEL_SCENARIOS: '3'
   DEFAULT_PARALLEL_RUNS: '3'
 
 permissions:
@@ -166,6 +172,10 @@ jobs:
           if [ -n "$PARALLEL_SKILLS" ]; then
             ARGS="$ARGS --parallel-skills $PARALLEL_SKILLS"
           fi
+          PARALLEL_SCENARIOS="${{ github.event.inputs.parallel-scenarios || env.DEFAULT_PARALLEL_SCENARIOS }}"
+          if [ -n "$PARALLEL_SCENARIOS" ]; then
+            ARGS="$ARGS --parallel-scenarios $PARALLEL_SCENARIOS"
+          fi
           PARALLEL_RUNS="${{ github.event.inputs.parallel-runs || env.DEFAULT_PARALLEL_RUNS }}"
           if [ -n "$PARALLEL_RUNS" ]; then
             ARGS="$ARGS --parallel-runs $PARALLEL_RUNS"

eng/skill-validator/README.md

@@ -222,6 +222,7 @@ The default of 5 runs provides sufficient precision for significance testing (va
 | `--min-improvement <n>` | `0.1` | Minimum improvement score (0–1) |
 | `--runs <n>` | `5` | Runs per scenario (averaged for stability) |
 | `--parallel-skills <n>` | `1` | Max concurrent skills to evaluate |
+| `--parallel-scenarios <n>` | `1` | Max concurrent scenarios per skill |
 | `--parallel-runs <n>` | `1` | Max concurrent runs per scenario |
 | `--confidence-level <n>` | `0.95` | Confidence level for statistical intervals (0–1) |
 | `--judge-timeout <n>` | `300` | Judge LLM timeout in seconds |

eng/skill-validator/src/cli.ts

@@ -17,6 +17,7 @@ import type {
   RunResult,
   ScenarioComparison,
   PairwiseJudgeResult,
+  EvalScenario,
 } from "./types.js";
 import type { ModelInfo } from "@github/copilot-sdk";
 
@@ -116,6 +117,7 @@ export function createProgram(): Command {
     .option("--judge-mode <mode>", "Judge mode: pairwise, independent, or both", "pairwise")
     .option("--runs <number>", "Number of runs per scenario for averaging", "5")
     .option("--parallel-skills <number>", "Max concurrent skills to evaluate", "1")
+    .option("--parallel-scenarios <number>", "Max concurrent scenarios per skill", "1")
     .option("--parallel-runs <number>", "Max concurrent runs per scenario", "1")
     .option("--judge-timeout <number>", "Judge timeout in seconds", "300")
     .option("--confidence-level <number>", "Confidence level for statistical intervals (0-1)", "0.95")
@@ -146,6 +148,7 @@ export function createProgram(): Command {
         judgeMode: opts.judgeMode || "pairwise",
         runs: Math.max(1, parseInt(opts.runs, 10) || 5),
         parallelSkills: Math.max(1, parseInt(opts.parallelSkills, 10) || 1),
+        parallelScenarios: Math.max(1, parseInt(opts.parallelScenarios, 10) || 1),
         parallelRuns: Math.max(1, parseInt(opts.parallelRuns, 10) || 1),
         judgeTimeout: parseInt(opts.judgeTimeout, 10) * 1000,
         confidenceLevel: parseFloat(opts.confidenceLevel || "0.95"),
@@ -250,10 +253,11 @@ export async function run(config: ValidatorConfig): Promise<number> {
       log(warning);
     }
 
-    const comparisons: ScenarioComparison[] = [];
     const singleScenario = skill.evalConfig.scenarios.length === 1;
+    const scenarioLimit = pLimit(Math.max(1, config.parallelScenarios));
 
-    for (const scenario of skill.evalConfig.scenarios) {
+    // Execute all scenarios for this skill (parallel if parallelScenarios > 1)
+    const executeScenario = async (scenario: EvalScenario): Promise<ScenarioComparison> => {
       const tag = singleScenario ? `[${skill.name}]` : `[${skill.name}/${scenario.name}]`;
       const scenarioLog = (msg: string) => spinner.log(`${tag} ${msg}`);
       const runLimit = pLimit(Math.max(1, config.parallelRuns));
@@ -429,8 +433,13 @@ export async function run(config: ValidatorConfig): Promise<number> {
       );
       comparison.perRunScores = perRunScores;
 
-      comparisons.push(comparison);
-    }
+      return comparison;
+    };
+
+    const scenarioPromises = skill.evalConfig.scenarios.map((scenario) =>
+      scenarioLimit(() => executeScenario(scenario))
+    );
+    const comparisons = await Promise.all(scenarioPromises);
 
     const verdict = computeVerdict(
       skill,

eng/skill-validator/src/types.ts

@@ -184,6 +184,7 @@ export interface ValidatorConfig {
   judgeMode: JudgeMode;
   runs: number;
   parallelSkills: number;
+  parallelScenarios: number;
   parallelRuns: number;
   judgeTimeout: number;
   confidenceLevel: number;