Fix 429 rate limiting in evaluation pipeline

.github/workflows/evaluation-run.yml

@@ -175,9 +175,15 @@ jobs:
             exit 1
           fi
 
-          # Pick a random token
-          IDX=$((RANDOM % ${#TOKENS[@]}))
-          echo "Selected ${NAMES[$IDX]} (1 of ${#TOKENS[@]} available tokens)"
+          # Assign token deterministically by matrix job index to avoid collisions.
+          # Falls back to RANDOM if strategy.job-index is unavailable.
+          JOB_INDEX="${{ strategy.job-index }}"
+          if [ -n "$JOB_INDEX" ]; then
+            IDX=$(( JOB_INDEX % ${#TOKENS[@]} ))
+          else
+            IDX=$((RANDOM % ${#TOKENS[@]}))
+          fi
+          echo "Selected ${NAMES[$IDX]} (1 of ${#TOKENS[@]} available tokens, job-index=${JOB_INDEX:-random})"
 
           # Mask the value so it won't appear in logs, then export
           echo "::add-mask::${TOKENS[$IDX]}"

.github/workflows/evaluation.yml

@@ -137,11 +137,11 @@ jobs:
     with:
       entries: ${{ needs.discover.outputs.entries }}
       runs: ${{ needs.discover.outputs.is_infra == 'true' && '1' || (github.ref == 'refs/heads/main' && '5' || '3') }}
-      # Infra changes evaluate all plugins in parallel — reduce per-job concurrency
-      # to avoid API rate limits and timeouts from contention.
-      parallel-skills: ${{ needs.discover.outputs.is_infra == 'true' && '2' || '5' }}
-      parallel-scenarios: ${{ needs.discover.outputs.is_infra == 'true' && '3' || '5' }}
-      parallel-runs: ${{ needs.discover.outputs.is_infra == 'true' && '3' || '5' }}
+      # Infra changes and scheduled runs evaluate all plugins in parallel — reduce
+      # per-job concurrency to avoid API rate limits and timeouts from contention.
+      parallel-skills: ${{ (needs.discover.outputs.is_infra == 'true' || github.event_name == 'schedule') && '2' || '5' }}
+      parallel-scenarios: ${{ (needs.discover.outputs.is_infra == 'true' || github.event_name == 'schedule') && '3' || '5' }}
+      parallel-runs: ${{ (needs.discover.outputs.is_infra == 'true' || github.event_name == 'schedule') && '3' || '5' }}
       pr-number: ${{ github.event.pull_request.number || '' }}
     secrets:
       COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}

eng/skill-validator/src/Commands/ValidateCommand.cs

@@ -122,7 +122,12 @@ public static async Task<int> Run(ValidatorConfig config)
         try
         {
             var client = await AgentRunner.GetSharedClient(config.Verbose);
-            var models = await client.ListModelsAsync();
+            var models = await RetryHelper.ExecuteWithRetry(
+                async _ => await client.ListModelsAsync(),
+                label: "ListModels",
+                maxRetries: 3,
+                baseDelayMs: 2_000,
+                totalTimeoutMs: 60_000);
             var modelIds = models.Select(m => m.Id).ToList();
             var modelsToValidate = new List<string> { config.Model };
             if (config.JudgeModel != config.Model) modelsToValidate.Add(config.JudgeModel);

eng/skill-validator/src/Services/AgentRunner.cs

@@ -3,6 +3,7 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using SkillValidator.Models;
+using SkillValidator.Utilities;
 using GitHub.Copilot.SDK;
 
 namespace SkillValidator.Services;
@@ -154,6 +155,16 @@ internal static SessionConfig BuildSessionConfig(
     }
 
     public static async Task<RunMetrics> RunAgent(RunOptions options)
+    {
+        return await RetryHelper.ExecuteWithRetry(
+            async ct => await RunAgentCore(options, ct),
+            label: $"RunAgent({options.Scenario.Name}, {(options.Skill is not null ? "skilled" : "baseline")})",
+            maxRetries: 2,
+            baseDelayMs: 5_000,
+            totalTimeoutMs: (options.Scenario.Timeout + 60) * 1000);
+    }
+
+    private static async Task<RunMetrics> RunAgentCore(RunOptions options, CancellationToken cancellationToken)
     {
         var workDir = await SetupWorkDir(options.Scenario, options.Skill?.Path, options.EvalPath);
         if (options.Verbose)
@@ -176,7 +187,8 @@ public static async Task<RunMetrics> RunAgent(RunOptions options)
 
             var done = new TaskCompletionSource();
             var effectiveTimeout = options.Scenario.Timeout;
-            using var cts = new CancellationTokenSource(effectiveTimeout * 1000);
+            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            cts.CancelAfter(effectiveTimeout * 1000);
             cts.Token.Register(() =>
                 done.TrySetException(new TimeoutException($"Scenario timed out after {effectiveTimeout}s")));
 
@@ -259,9 +271,21 @@ public static async Task<RunMetrics> RunAgent(RunOptions options)
                 DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
                 new Dictionary<string, JsonNode?> { ["message"] = JsonValue.Create(te.ToString()) }));
         }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw; // Budget exhausted — let RetryHelper handle it.
+        }
         catch (Exception error)
         {
             var msg = error.ToString();
+
+            // Re-throw rate-limit (429) errors so RetryHelper can retry them.
+            if (msg.Contains("429", StringComparison.Ordinal)
+                || msg.Contains("rate limit", StringComparison.OrdinalIgnoreCase))
+            {
+                throw;
+            }
+
             if (error is TimeoutException || error.InnerException is TimeoutException
                 || msg.Contains("timed out", StringComparison.OrdinalIgnoreCase))
             {