Add --keep-sessions flag and rejudge command to skill-validator

eng/skill-validator/src/Commands/ValidateCommand.cs

@@ -19,7 +19,8 @@ public static RootCommand Create()
         var verboseOpt = new Option<bool>("--verbose") { Description = "Show detailed per-scenario breakdowns" };
         var modelOpt = new Option<string>("--model") { Description = "Model to use for agent runs", DefaultValueFactory = _ => "claude-opus-4.6" };
         var judgeModelOpt = new Option<string?>("--judge-model") { Description = "Model to use for judging (defaults to --model)" };
-        var judgeModeOpt = new Option<string>("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" };
+        var judgeModeOpt = new Option<string>("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }
+            .AcceptOnlyFromAmong("pairwise", "independent", "both");
         var runsOpt = new Option<int>("--runs") { Description = "Number of runs per scenario for averaging", DefaultValueFactory = _ => 5 };
         var parallelSkillsOpt = new Option<int>("--parallel-skills") { Description = "Max concurrent skills to evaluate", DefaultValueFactory = _ => 1 };
         var parallelScenariosOpt = new Option<int>("--parallel-scenarios") { Description = "Max concurrent scenarios per skill", DefaultValueFactory = _ => 1 };
@@ -31,6 +32,7 @@ public static RootCommand Create()
         var reporterOpt = new Option<string[]>("--reporter") { Description = "Reporter (console, json, junit, markdown). Can be repeated.", AllowMultipleArgumentsPerToken = true };
         var noOverfittingCheckOpt = new Option<bool>("--no-overfitting-check") { Description = "Disable LLM-based overfitting analysis (on by default)" };
         var overfittingFixOpt = new Option<bool>("--overfitting-fix") { Description = "Generate a fixed eval.yaml with improved rubric items/assertions" };
+        var keepSessionsOpt = new Option<bool>("--keep-sessions") { Description = "Preserve agent session data in the results directory for later rejudging" };
         var noiseSkillsDirOpt = new Option<string?>("--noise-skills-dir") { Description = "Directory containing skills to load as noise. Enables the noise test: re-runs scenarios with all noise skills loaded and measures degradation." };
         var noiseMaxDegradationOpt = new Option<double>("--noise-max-degradation") { Description = "Maximum acceptable average quality degradation (0-1) in noise test (only positive degradations count)", DefaultValueFactory = _ => 0.2 };
         var noiseMaxScenarioDegradationOpt = new Option<double>("--noise-max-scenario-degradation") { Description = "Maximum acceptable quality degradation (0-1) for any single noise-test scenario", DefaultValueFactory = _ => 0.4 };
@@ -57,6 +59,7 @@ public static RootCommand Create()
             reporterOpt,
             noOverfittingCheckOpt,
             overfittingFixOpt,
+            keepSessionsOpt,
             noiseSkillsDirOpt,
             noiseMaxDegradationOpt,
             noiseMaxScenarioDegradationOpt,
@@ -105,6 +108,7 @@ public static RootCommand Create()
                 TestsDir = parseResult.GetValue(testsDirOpt),
                 OverfittingCheck = !parseResult.GetValue(noOverfittingCheckOpt),
                 OverfittingFix = parseResult.GetValue(overfittingFixOpt),
+                KeepSessions = parseResult.GetValue(keepSessionsOpt),
                 NoiseSkillsDir = parseResult.GetValue(noiseSkillsDirOpt),
                 NoiseDegradationLimit = parseResult.GetValue(noiseMaxDegradationOpt),
                 NoiseMaxScenarioDegradation = parseResult.GetValue(noiseMaxScenarioDegradationOpt),
@@ -320,14 +324,33 @@ public static async Task<int> Run(ValidatorConfig config)
             Console.WriteLine($"\x1b[33m⚠  Running with {config.Runs} run(s). For statistically significant results, use --runs 5 or higher.\x1b[0m");
 
         bool usePairwise = config.JudgeMode is JudgeMode.Pairwise or JudgeMode.Both;
+        bool effectiveKeepSessions = config.KeepSessions && config.ResultsDir is not null;
+
+        string? sessionsDir = null;
+        SessionDatabase? sessionDb = null;
+        string? timestampedResultsDir = null;
+        if (effectiveKeepSessions)
+        {
+            timestampedResultsDir = Path.Combine(config.ResultsDir!, Reporter.FormatTimestamp(DateTime.Now));
+            Directory.CreateDirectory(timestampedResultsDir);
+            sessionsDir = Path.Combine(timestampedResultsDir, "sessions");
+            Directory.CreateDirectory(sessionsDir);
+            sessionDb = new SessionDatabase(Path.Combine(timestampedResultsDir, "sessions.db"));
+            sessionDb.SetSchemaInfo("judge_model", config.JudgeModel);
+            Console.WriteLine($"Session persistence enabled: {timestampedResultsDir}");
+        }
+        else if (config.KeepSessions)
+        {
+            Console.WriteLine("\x1b[33m⚠  --keep-sessions was set without --results-dir; sessions will not be persisted.\x1b[0m");
+        }
 
         using var spinner = new Spinner();
         using var skillLimit = new ConcurrencyLimiter(config.ParallelSkills);
 
         // Evaluate skills
         spinner.Start($"Evaluating {allSkills.Count} skill(s)...");
         var skillTasks = allSkills.Select(skill =>
-            skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills)));
+            skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills, sessionsDir, sessionDb)));
         var settled = await Task.WhenAll(skillTasks.Select(async t =>
         {
             try { return (Result: await t, Error: (Exception?)null); }
@@ -350,7 +373,7 @@ public static async Task<int> Run(ValidatorConfig config)
         }
 
         await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose,
-            config.Model, config.JudgeModel, config.ResultsDir,
+            config.Model, config.JudgeModel, config.ResultsDir, timestampedResultsDir,
             rejectedCount: rejectionMessages.Count);
 
         if (rejectionMessages.Count > 0)
@@ -362,7 +385,8 @@ await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose,
         }
 
         await AgentRunner.StopAllClients();
-        await AgentRunner.CleanupWorkDirs();
+        await AgentRunner.CleanupWorkDirs(effectiveKeepSessions);
+        sessionDb?.Dispose();
 
         // Always fail on execution errors, even in --verdict-warn-only mode
         if (rejectionMessages.Count > 0) return 1;
@@ -432,7 +456,9 @@ internal static List<string> CheckAggregateDescriptionLimits(IReadOnlyList<Skill
         ValidatorConfig config,
         bool usePairwise,
         Spinner spinner,
-        IReadOnlyList<SkillInfo> noiseSkills)
+        IReadOnlyList<SkillInfo> noiseSkills,
+        string? sessionsDir,
+        SessionDatabase? sessionDb)
     {
         var prefix = $"[{skill.Name}]";
         var log = (string msg) => spinner.Log($"{prefix} {msg}");
@@ -501,11 +527,12 @@ internal static List<string> CheckAggregateDescriptionLimits(IReadOnlyList<Skill
                 config.JudgeModel, config.Verbose, config.JudgeTimeout, workDir));
         }
 
+        var skillSha = sessionDb is not null ? SessionDatabase.ComputeDirectorySha(skill.Path) : null;
         bool singleScenario = skill.EvalConfig!.Scenarios.Count == 1;
         using var scenarioLimit = new ConcurrencyLimiter(config.ParallelScenarios);
 
         var scenarioTasks = skill.EvalConfig.Scenarios.Select(scenario =>
-            scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner)));
+            scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb, skillSha)));
         var comparisons = (await Task.WhenAll(scenarioTasks)).ToList();
 
         // Await overfitting result (non-fatal — never blocks an otherwise-successful evaluation)
@@ -587,7 +614,10 @@ private static async Task<ScenarioComparison> ExecuteScenario(
         ValidatorConfig config,
         bool usePairwise,
         bool singleScenario,
-        Spinner spinner)
+        Spinner spinner,
+        string? sessionsDir,
+        SessionDatabase? sessionDb,
+        string? skillSha)
     {
         var tag = singleScenario ? $"[{skill.Name}]" : $"[{skill.Name}/{scenario.Name}]";
         var scenarioLog = (string msg) => spinner.Log($"{tag} {msg}");
@@ -597,7 +627,7 @@ private static async Task<ScenarioComparison> ExecuteScenario(
             scenarioLog("📋 Starting scenario");
 
         var runTasks = Enumerable.Range(0, config.Runs).Select(i =>
-            runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner)));
+            runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb, skillSha)));
         var runResults = await Task.WhenAll(runTasks);
 
         scenarioLog($"✓ All {config.Runs} run(s) complete");
@@ -717,7 +747,10 @@ private static async Task<RunExecutionResult> ExecuteRun(
         ValidatorConfig config,
         bool usePairwise,
         bool singleScenario,
-        Spinner spinner)
+        Spinner spinner,
+        string? sessionsDir,
+        SessionDatabase? sessionDb,
+        string? skillSha)
     {
         var runTag = config.Runs > 1
             ? (singleScenario ? $"[{skill.Name}/{runIndex + 1}]" : $"[{skill.Name}/{scenario.Name}/{runIndex + 1}]")
@@ -728,21 +761,46 @@ private static async Task<RunExecutionResult> ExecuteRun(
             runLog("running agents...");
 
         var pluginRoot = SkillDiscovery.FindPluginRoot(skill.Path);
+        var baselineSessionId = Guid.NewGuid().ToString("N");
+        var isolatedSessionId = Guid.NewGuid().ToString("N");
+        var pluginSessionId = Guid.NewGuid().ToString("N");
+
+        var baselineConfigDir = sessionsDir is not null ? Path.Combine("sessions", baselineSessionId) : null;
+        var isolatedConfigDir = sessionsDir is not null ? Path.Combine("sessions", isolatedSessionId) : null;
+        var pluginConfigDir = sessionsDir is not null ? Path.Combine("sessions", pluginSessionId) : null;
+        var rubricJson = JsonSerializer.Serialize(scenario.Rubric?.ToArray() ?? [], SkillValidatorJsonContext.Default.StringArray);
+
+        sessionDb?.RegisterSession(baselineSessionId, skill.Name, skill.Path, scenario.Name, runIndex,
+            "baseline", config.Model, baselineConfigDir, null, scenario.Prompt, skillSha, rubricJson);
+        sessionDb?.RegisterSession(isolatedSessionId, skill.Name, skill.Path, scenario.Name, runIndex,
+            "with-skill-isolated", config.Model, isolatedConfigDir, null, scenario.Prompt, skillSha, rubricJson);
+        sessionDb?.RegisterSession(pluginSessionId, skill.Name, skill.Path, scenario.Name, runIndex,
+            "with-skill-plugin", config.Model, pluginConfigDir, null, scenario.Prompt, skillSha, rubricJson);
 
         var agentTasks = await Task.WhenAll(
             // 1. Baseline: no plugin, no skills — vanilla agent
             AgentRunner.RunAgent(new RunOptions(scenario, null, skill.EvalPath, config.Model, config.Verbose,
-                PluginRoot: null, Log: runLog)),
+                PluginRoot: null, Log: runLog, SessionsDir: sessionsDir, SessionId: baselineSessionId)),
             // 2. Skilled-isolated: single skill only (current behavior)
             AgentRunner.RunAgent(new RunOptions(scenario, skill, skill.EvalPath, config.Model, config.Verbose,
-                PluginRoot: null, Log: runLog)),
+                PluginRoot: null, Log: runLog, SessionsDir: sessionsDir, SessionId: isolatedSessionId)),
             // 3. Skilled-plugin: load entire plugin from plugin root directory
             AgentRunner.RunAgent(new RunOptions(scenario, skill, skill.EvalPath, config.Model, config.Verbose,
-                PluginRoot: pluginRoot, Log: runLog)));
+                PluginRoot: pluginRoot, Log: runLog, SessionsDir: sessionsDir, SessionId: pluginSessionId)));
         var baselineMetrics = agentTasks[0];
         var isolatedMetrics = agentTasks[1];
         var pluginMetrics = agentTasks[2];
 
+        if (sessionDb is not null)
+        {
+            var baselineStatus = baselineMetrics.TimedOut ? "timed_out" : "completed";
+            var isolatedStatus = isolatedMetrics.TimedOut ? "timed_out" : "completed";
+            var pluginStatus = pluginMetrics.TimedOut ? "timed_out" : "completed";
+            sessionDb.CompleteSession(baselineSessionId, baselineStatus, JsonSerializer.Serialize(baselineMetrics, SkillValidatorJsonContext.Default.RunMetrics));
+            sessionDb.CompleteSession(isolatedSessionId, isolatedStatus, JsonSerializer.Serialize(isolatedMetrics, SkillValidatorJsonContext.Default.RunMetrics));
+            sessionDb.CompleteSession(pluginSessionId, pluginStatus, JsonSerializer.Serialize(pluginMetrics, SkillValidatorJsonContext.Default.RunMetrics));
+        }
+
         // Evaluate assertions on all three runs
         if (scenario.Assertions is { Count: > 0 })
         {
@@ -786,6 +844,13 @@ private static async Task<RunExecutionResult> ExecuteRun(
         var isolatedJudge = await SafeJudge(isolatedJudgeTask, "isolated", runLog);
         var pluginJudge = await SafeJudge(pluginJudgeTask, "plugin", runLog);
 
+        if (sessionDb is not null)
+        {
+            sessionDb.SaveJudgeResult(baselineSessionId, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult));
+            sessionDb.SaveJudgeResult(isolatedSessionId, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult));
+            sessionDb.SaveJudgeResult(pluginSessionId, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult));
+        }
+
         var baselineResult = new RunResult(baselineMetrics, baselineJudge);
         var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge);
         var pluginResult = new RunResult(pluginMetrics, pluginJudge);
@@ -805,6 +870,10 @@ private static async Task<RunExecutionResult> ExecuteRun(
                     scenario, baselineMetrics, worseSkilled,
                     new PairwiseJudgeOptions(config.JudgeModel, config.Verbose, config.JudgeTimeout, baselineMetrics.WorkDir, skill.Path, worseSkilled.WorkDir),
                     runLog);
+                if (sessionDb is not null && pairwise is not null)
+                {
+                    sessionDb.SavePairwiseResult(baselineSessionId, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult));
+                }
             }
             catch (Exception error)
             {

eng/skill-validator/src/Models/Models.cs

@@ -376,6 +376,7 @@ public sealed record ValidatorConfig
     public string? TestsDir { get; init; }
     public bool OverfittingCheck { get; init; } = true;
     public bool OverfittingFix { get; init; }
+    public bool KeepSessions { get; init; }
     public string? NoiseSkillsDir { get; init; }
     public double NoiseDegradationLimit { get; init; } = 0.2;
     public double NoiseMaxScenarioDegradation { get; init; } = 0.4;

eng/skill-validator/src/Program.cs

@@ -3,6 +3,7 @@
 
 var rootCommand = ValidateCommand.Create();
 rootCommand.Add(ConsolidateCommand.Create());
+rootCommand.Add(RejudgeCommand.Create());
 
 var parseResult = rootCommand.Parse(args);
 return await parseResult.InvokeAsync();

eng/skill-validator/src/Services/AgentRunner.cs

@@ -16,13 +16,16 @@ public sealed record RunOptions(
     bool Verbose,
     string? PluginRoot = null,
     Action<string>? Log = null,
-    IReadOnlyList<SkillInfo>? AdditionalSkills = null);
+    IReadOnlyList<SkillInfo>? AdditionalSkills = null,
+    string? SessionsDir = null,
+    string? SessionId = null);
 
 public static class AgentRunner
 {
     private static readonly ConcurrentDictionary<string, CopilotClient> _pluginClients = new(StringComparer.OrdinalIgnoreCase);
     private static readonly SemaphoreSlim _clientLock = new(1, 1);
     private static readonly ConcurrentBag<string> _workDirs = [];
+    private static readonly ConcurrentBag<string> _configDirs = [];
     private static string? _capturedGitHubToken;
     private static bool _tokenCaptured;
 
@@ -102,11 +105,16 @@ public static async Task StopAllClients()
     public static Task StopSharedClient() => StopAllClients();
 
     /// <summary>Remove all temporary working directories created during runs.</summary>
-    public static Task CleanupWorkDirs()
+    public static Task CleanupWorkDirs(bool keepSessions = false)
     {
         var dirs = _workDirs.ToArray();
         _workDirs.Clear();
-        return Task.WhenAll(dirs.Select(dir =>
+
+        var configDirsToClean = keepSessions ? [] : _configDirs.ToArray();
+        _configDirs.Clear();
+
+        var allDirs = dirs.Concat(configDirsToClean);
+        return Task.WhenAll(allDirs.Select(dir =>
         {
             try { Directory.Delete(dir, true); } catch { }
             return Task.CompletedTask;
@@ -206,16 +214,29 @@ internal static SessionConfig BuildSessionConfig(
         IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null,
         IReadOnlyList<SkillInfo>? additionalSkills = null,
         Action<string>? log = null,
-        bool verbose = false)
+        bool verbose = false,
+        string? sessionsDir = null,
+        string? sessionId = null)
     {
         // The SDK expects SkillDirectories entries to be parent directories that
         // it scans for child folders containing SKILL.md.
         var skillPath = skill is not null ? Path.GetDirectoryName(skill.Path) : null;
 
-        // Create a unique temporary config directory for this session to not share any data
-        var configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}");
-        Directory.CreateDirectory(configDir);
-        _workDirs.Add(configDir);
+        string configDir;
+        if (sessionsDir is not null)
+        {
+            // Persistent session dir — use sessionId as folder name for DB linkage
+            var dirName = sessionId ?? Guid.NewGuid().ToString("N");
+            configDir = Path.Combine(sessionsDir, dirName);
+            Directory.CreateDirectory(configDir);
+            _configDirs.Add(configDir);
+        }
+        else
+        {
+            configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}");
+            Directory.CreateDirectory(configDir);
+            _configDirs.Add(configDir);
+        }
         if (verbose)
             log?.Invoke($"      📂 Config dir: {configDir} ({(skill is not null ? "skilled" : "baseline")})");
 
@@ -406,7 +427,7 @@ private static async Task<RunMetrics> RunAgentCore(RunOptions options, Cancellat
             var client = await GetPluginClient(options.PluginRoot, options.Verbose);
 
             await using var session = await client.CreateSessionAsync(
-                BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills, options.Log, options.Verbose));
+                BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills, options.Log, options.Verbose, options.SessionsDir, options.SessionId));
 
             var done = new TaskCompletionSource();
             var effectiveTimeout = options.Scenario.Timeout;

eng/skill-validator/src/Services/Reporter.cs

@@ -14,13 +14,15 @@ public static async Task ReportResults(
         string? model = null,
         string? judgeModel = null,
         string? resultsDir = null,
+        string? timestampedResultsDir = null,
         int rejectedCount = 0)
     {
         bool needsResultsDir = reporters.Any(r =>
             r.Type is ReporterType.Json or ReporterType.Junit or ReporterType.Markdown);
-        string? effectiveResultsDir = resultsDir is not null && needsResultsDir
-            ? Path.Combine(resultsDir, FormatTimestamp(DateTime.Now))
-            : null;
+        string? effectiveResultsDir = timestampedResultsDir
+            ?? (resultsDir is not null && needsResultsDir
+                ? Path.Combine(resultsDir, FormatTimestamp(DateTime.Now))
+                : null);
 
         if (effectiveResultsDir is not null)
             Directory.CreateDirectory(effectiveResultsDir);