feat: noise-test evaluation option

eng/skill-validator/src/Commands/ValidateCommand.cs

@@ -30,6 +30,8 @@ public static RootCommand Create()
         var reporterOpt = new Option<string[]>("--reporter") { Description = "Reporter (console, json, junit, markdown). Can be repeated.", AllowMultipleArgumentsPerToken = true };
         var noOverfittingCheckOpt = new Option<bool>("--no-overfitting-check") { Description = "Disable LLM-based overfitting analysis (on by default)" };
         var overfittingFixOpt = new Option<bool>("--overfitting-fix") { Description = "Generate a fixed eval.yaml with improved rubric items/assertions" };
+        var noiseSkillsDirOpt = new Option<string?>("--noise-skills-dir") { Description = "Directory containing skills to load as noise. Enables the noise test: re-runs scenarios with all noise skills loaded and measures degradation." };
+        var noiseMaxDegradationOpt = new Option<double>("--noise-max-degradation") { Description = "Maximum acceptable quality degradation (0-1) in noise test", DefaultValueFactory = _ => 0.2 };
 
         var command = new RootCommand("Validate that agent skills meaningfully improve agent performance")
         {
@@ -53,6 +55,8 @@ public static RootCommand Create()
             reporterOpt,
             noOverfittingCheckOpt,
             overfittingFixOpt,
+            noiseSkillsDirOpt,
+            noiseMaxDegradationOpt,
         };
 
         command.SetAction(async (parseResult, _) =>
@@ -98,6 +102,8 @@ public static RootCommand Create()
                 TestsDir = parseResult.GetValue(testsDirOpt),
                 OverfittingCheck = !parseResult.GetValue(noOverfittingCheckOpt),
                 OverfittingFix = parseResult.GetValue(overfittingFixOpt),
+                NoiseSkillsDir = parseResult.GetValue(noiseSkillsDirOpt),
+                NoiseMaxDegradation = parseResult.GetValue(noiseMaxDegradationOpt),
             };
 
             return await Run(config);
@@ -164,6 +170,14 @@ public static async Task<int> Run(ValidatorConfig config)
 
         Console.WriteLine($"Found {allSkills.Count} skill(s)\n");
 
+        // Discover noise skills when --noise-skills-dir is provided
+        var noiseSkills = new List<SkillInfo>();
+        if (config.NoiseSkillsDir is not null)
+        {
+            noiseSkills.AddRange(await SkillDiscovery.DiscoverSkills(config.NoiseSkillsDir, config.TestsDir));
+            Console.WriteLine($"Noise test enabled: discovered {noiseSkills.Count} noise skill(s) from {config.NoiseSkillsDir}");
+        }
+
         // Check per-plugin aggregate description size
         var aggregateFailures = CheckAggregateDescriptionLimits(allSkills);
         if (aggregateFailures.Count > 0)
@@ -184,7 +198,7 @@ public static async Task<int> Run(ValidatorConfig config)
         // Evaluate skills
         spinner.Start($"Evaluating {allSkills.Count} skill(s)...");
         var skillTasks = allSkills.Select(skill =>
-            skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner)));
+            skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills)));
         var settled = await Task.WhenAll(skillTasks.Select(async t =>
         {
             try { return (Result: await t, Error: (Exception?)null); }
@@ -280,7 +294,8 @@ internal static List<string> CheckAggregateDescriptionLimits(IReadOnlyList<Skill
         SkillInfo skill,
         ValidatorConfig config,
         bool usePairwise,
-        Spinner spinner)
+        Spinner spinner,
+        IReadOnlyList<SkillInfo> noiseSkills)
     {
         var prefix = $"[{skill.Name}]";
         var log = (string msg) => spinner.Log($"{prefix} {msg}");
@@ -382,6 +397,28 @@ internal static List<string> CheckAggregateDescriptionLimits(IReadOnlyList<Skill
             }
         }
 
+        // --- Noise test: run scenarios with all skills loaded ---
+        if (config.NoiseSkillsDir is not null && noiseSkills.Count > 0)
+        {
+            try
+            {
+                var noiseResult = await ExecuteNoiseTest(skill, noiseSkills, config, usePairwise, spinner);
+                verdict.NoiseTestResult = noiseResult;
+                if (!noiseResult.Passed)
+                {
+                    log($"\x1b[33m\u26a0\ufe0f  Noise test: quality degraded by {noiseResult.OverallDegradation * 100:F1}% with {noiseResult.TotalSkillsLoaded} skills loaded\x1b[0m");
+                }
+                else
+                {
+                    log($"\u2705 Noise test passed ({noiseResult.TotalSkillsLoaded} skills loaded, degradation: {noiseResult.OverallDegradation * 100:F1}%)");
+                }
+            }
+            catch (Exception ex)
+            {
+                log($"\u26a0\ufe0f Noise test failed: {ex.Message}");
+            }
+        }
+
         var notActivated = comparisons.Where(c => c.SkillActivation is { Activated: false }).ToList();
         // Separate unexpected non-activations (expect_activation defaulting to true)
         // from expected ones (negative tests with expect_activation: false).
@@ -598,6 +635,122 @@ private static async Task<RunExecutionResult> ExecuteRun(
         return new RunExecutionResult(baseline, withSkillResult, pairwise, skillActivation);
     }
 
+    // --- Noise test: run scenarios with all discovered skills loaded ---
+
+    private static async Task<NoiseTestResult> ExecuteNoiseTest(
+        SkillInfo targetSkill,
+        IReadOnlyList<SkillInfo> allSkills,
+        ValidatorConfig config,
+        bool usePairwise,
+        Spinner spinner)
+    {
+        var prefix = $"[{targetSkill.Name}/noise]";
+        var log = (string msg) => spinner.Log($"{prefix} {msg}");
+
+        var otherSkills = allSkills.Where(s =>
+            !string.Equals(s.Path, targetSkill.Path, StringComparison.OrdinalIgnoreCase)).ToList();
+        int totalLoaded = otherSkills.Count + 1; // target + others
+
+        log($"🔊 Running noise test with {totalLoaded} skills loaded...");
+
+        var noiseScenarios = new List<NoiseScenarioResult>();
+        using var scenarioLimit = new ConcurrencyLimiter(config.ParallelScenarios);
+
+        var tasks = targetSkill.EvalConfig!.Scenarios
+            .Where(s => s.ExpectActivation) // only test positive scenarios
+            .Select(scenario => scenarioLimit.RunAsync(async () =>
+            {
+                var tag = $"[{targetSkill.Name}/noise/{scenario.Name}]";
+                var scenarioLog = (string msg) => spinner.Log($"{tag} {msg}");
+
+                scenarioLog("running skill-only vs all-skills...");
+
+                // Run with target skill only
+                var skillOnlyMetrics = await AgentRunner.RunAgent(new RunOptions(
+                    scenario, targetSkill, targetSkill.EvalPath, config.Model, config.Verbose, scenarioLog));
+
+                // Run with all skills loaded
+                var allSkillsMetrics = await AgentRunner.RunAgent(new RunOptions(
+                    scenario, targetSkill, targetSkill.EvalPath, config.Model, config.Verbose, scenarioLog, AdditionalSkills: otherSkills));
+
+                // Evaluate assertions on both
+                if (scenario.Assertions is { Count: > 0 })
+                {
+                    skillOnlyMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(
+                        scenario.Assertions, skillOnlyMetrics.AgentOutput, skillOnlyMetrics.WorkDir);
+                    allSkillsMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(
+                        scenario.Assertions, allSkillsMetrics.AgentOutput, allSkillsMetrics.WorkDir);
+                }
+                var soConstraints = AssertionEvaluator.EvaluateConstraints(scenario, skillOnlyMetrics);
+                var asConstraints = AssertionEvaluator.EvaluateConstraints(scenario, allSkillsMetrics);
+                skillOnlyMetrics.AssertionResults = [..skillOnlyMetrics.AssertionResults, ..soConstraints];
+                allSkillsMetrics.AssertionResults = [..allSkillsMetrics.AssertionResults, ..asConstraints];
+
+                skillOnlyMetrics.TaskCompleted = scenario.Assertions is { Count: > 0 } || soConstraints.Count > 0
+                    ? skillOnlyMetrics.AssertionResults.All(a => a.Passed)
+                    : skillOnlyMetrics.ErrorCount == 0;
+                allSkillsMetrics.TaskCompleted = scenario.Assertions is { Count: > 0 } || asConstraints.Count > 0
+                    ? allSkillsMetrics.AssertionResults.All(a => a.Passed)
+                    : allSkillsMetrics.ErrorCount == 0;
+
+                // Judge both runs
+                var judgeOpts = new JudgeOptions(config.JudgeModel, config.Verbose, config.JudgeTimeout, skillOnlyMetrics.WorkDir, targetSkill.Path);
+                JudgeResult skillOnlyJudge, allSkillsJudge;
+                try
+                {
+                    skillOnlyJudge = await Services.Judge.JudgeRun(scenario, skillOnlyMetrics, judgeOpts);
+                }
+                catch
+                {
+                    skillOnlyJudge = new JudgeResult([], 3, "Judge failed");
+                }
+                try
+                {
+                    allSkillsJudge = await Services.Judge.JudgeRun(scenario, allSkillsMetrics,
+                        judgeOpts with { WorkDir = allSkillsMetrics.WorkDir });
+                }
+                catch
+                {
+                    allSkillsJudge = new JudgeResult([], 3, "Judge failed");
+                }
+
+                var skillOnlyResult = new RunResult(skillOnlyMetrics, skillOnlyJudge);
+                var allSkillsResult = new RunResult(allSkillsMetrics, allSkillsJudge);
+
+                // Compare: skill-only is "baseline", all-skills is "with-skill"
+                // A positive score means all-skills is *better*, negative means degradation
+                var comparison = Comparator.CompareScenario(scenario.Name, skillOnlyResult, allSkillsResult);
+                var degradation = -comparison.ImprovementScore; // positive = degradation
+
+                var activation = MetricsCollector.ExtractSkillActivation(
+                    allSkillsMetrics.Events, skillOnlyMetrics.ToolCallBreakdown);
+
+                scenarioLog($"✓ degradation: {degradation * 100:F1}%, target skill activated: {activation.Activated}");
+
+                return new NoiseScenarioResult(
+                    scenario.Name,
+                    skillOnlyResult,
+                    allSkillsResult,
+                    degradation,
+                    comparison.Breakdown,
+                    activation,
+                    totalLoaded);
+            }));
+
+        noiseScenarios = (await Task.WhenAll(tasks)).ToList();
+
+        double overallDegradation = noiseScenarios.Count > 0
+            ? noiseScenarios.Average(s => s.DegradationScore)
+            : 0;
+        bool passed = overallDegradation <= config.NoiseMaxDegradation;
+
+        string reason = passed
+            ? $"Quality degradation {overallDegradation * 100:F1}% within threshold of {config.NoiseMaxDegradation * 100:F1}% ({totalLoaded} skills loaded)"
+            : $"Quality degradation {overallDegradation * 100:F1}% exceeds threshold of {config.NoiseMaxDegradation * 100:F1}% ({totalLoaded} skills loaded)";
+
+        return new NoiseTestResult(noiseScenarios, overallDegradation, passed, reason, totalLoaded);
+    }
+
     private static RunResult AverageResults(List<RunResult> runs)
     {
         if (runs.Count == 1) return runs[0];

eng/skill-validator/src/Models/Models.cs

@@ -227,6 +227,7 @@ public sealed class SkillVerdict
     public IReadOnlyList<string>? ProfileWarnings { get; set; }
     public bool SkillNotActivated { get; set; }
     public OverfittingResult? OverfittingResult { get; set; }
+    public NoiseTestResult? NoiseTestResult { get; set; }
 }
 
 // --- Overfitting assessment ---
@@ -274,6 +275,24 @@ public sealed record OverfittingJudgeOptions(
     int Timeout,
     string WorkDir);
 
+// --- Multi-skill noise test ---
+
+public sealed record NoiseScenarioResult(
+    string ScenarioName,
+    RunResult WithSkillOnly,
+    RunResult WithAllSkills,
+    double DegradationScore,
+    MetricBreakdown Breakdown,
+    SkillActivationInfo? SkillActivation,
+    int TotalSkillsLoaded);
+
+public sealed record NoiseTestResult(
+    IReadOnlyList<NoiseScenarioResult> Scenarios,
+    double OverallDegradation,
+    bool Passed,
+    string Reason,
+    int TotalSkillsLoaded);
+
 // --- Config ---
 
 public sealed record ReporterSpec(ReporterType Type);
@@ -308,6 +327,8 @@ public sealed record ValidatorConfig
     public string? TestsDir { get; init; }
     public bool OverfittingCheck { get; init; } = true;
     public bool OverfittingFix { get; init; }
+    public string? NoiseSkillsDir { get; init; }
+    public double NoiseMaxDegradation { get; init; } = 0.2;
 }
 
 public static class DefaultWeights

eng/skill-validator/src/Services/AgentRunner.cs

@@ -13,7 +13,8 @@ public sealed record RunOptions(
     string? EvalPath,
     string Model,
     bool Verbose,
-    Action<string>? Log = null);
+    Action<string>? Log = null,
+    IReadOnlyList<SkillInfo>? AdditionalSkills = null);
 
 public static class AgentRunner
 {
@@ -104,7 +105,8 @@ public static bool CheckPermission(PermissionRequest request, string workDir, st
 
     internal static SessionConfig BuildSessionConfig(
         SkillInfo? skill, string model, string workDir,
-        IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null)
+        IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null,
+        IReadOnlyList<SkillInfo>? additionalSkills = null)
     {
         var skillPath = skill is not null ? Path.GetDirectoryName(skill.Path) : null;
 
@@ -113,6 +115,19 @@ internal static SessionConfig BuildSessionConfig(
         Directory.CreateDirectory(configDir);
         _workDirs.Add(configDir);
 
+        // Build skill directories list: primary skill + any additional skills
+        var skillDirs = new List<string>();
+        if (skillPath is not null) skillDirs.Add(skillPath);
+        if (additionalSkills is { Count: > 0 })
+        {
+            foreach (var s in additionalSkills)
+            {
+                var dir = Path.GetDirectoryName(s.Path);
+                if (dir is not null && !skillDirs.Contains(dir, StringComparer.OrdinalIgnoreCase))
+                    skillDirs.Add(dir);
+            }
+        }
+
         // Convert MCPServerDef records to the SDK's Dictionary<string, object> shape
         Dictionary<string, object>? sdkMcp = null;
         if (mcpServers is { Count: > 0 })
@@ -138,7 +153,7 @@ internal static SessionConfig BuildSessionConfig(
             Model = model,
             Streaming = true,
             WorkingDirectory = workDir,
-            SkillDirectories = skill is not null ? [skillPath!] : [],
+            SkillDirectories = skillDirs,
             ConfigDir = configDir,
             McpServers = sdkMcp,
             InfiniteSessions = new InfiniteSessionConfig { Enabled = false },
@@ -172,7 +187,7 @@ public static async Task<RunMetrics> RunAgent(RunOptions options)
             var client = await GetSharedClient(options.Verbose);
 
             await using var session = await client.CreateSessionAsync(
-                BuildSessionConfig(options.Skill, options.Model, workDir, options.Skill?.McpServers));
+                BuildSessionConfig(options.Skill, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills));
 
             var done = new TaskCompletionSource();
             var effectiveTimeout = options.Scenario.Timeout;

eng/skill-validator/src/Services/Reporter.cs

@@ -132,6 +132,25 @@ private static void ReportConsole(IReadOnlyList<SkillVerdict> verdicts, bool ver
                         Console.WriteLine($"    \x1b[2m•\x1b[0m [{item.Classification}] \x1b[2m{item.AssertionSummary}\x1b[0m\n      \x1b[2m— {item.Reasoning}\x1b[0m");
                 }
             }
+
+            // Noise test results
+            if (verdict.NoiseTestResult is { } noiseResult)
+            {
+                Console.WriteLine();
+                var noiseIcon = noiseResult.Passed ? "✅" : "⚠️";
+                var noiseColor = noiseResult.Passed ? "\x1b[32m" : "\x1b[33m";
+                Console.WriteLine($"  🔊 Noise test ({noiseResult.TotalSkillsLoaded} skills loaded): {noiseColor}{noiseResult.OverallDegradation * 100:F1}% degradation\x1b[0m {noiseIcon}");
+                Console.WriteLine($"  \x1b[2m{noiseResult.Reason}\x1b[0m");
+
+                foreach (var ns in noiseResult.Scenarios)
+                {
+                    var nsIcon = ns.DegradationScore <= 0 ? "\x1b[32m↑\x1b[0m" : "\x1b[33m↓\x1b[0m";
+                    var activated = ns.SkillActivation?.Activated == true ? "✅" : "⚠️ not activated";
+                    Console.WriteLine($"    {nsIcon} {ns.ScenarioName}  degradation: {ns.DegradationScore * 100:F1}%  target skill: {activated}");
+                    Console.WriteLine($"      \x1b[2mskill-only: {ns.WithSkillOnly.JudgeResult.OverallScore:F1}/5 → all-skills: {ns.WithAllSkills.JudgeResult.OverallScore:F1}/5\x1b[0m");
+                }
+            }
+
             if (verdict.Scenarios.Count > 0)
             {
                 Console.WriteLine();
@@ -363,6 +382,41 @@ public static string GenerateMarkdownSummary(
         if (anyTimeout)
             sb.AppendLine("\n> ⏰ **timeout** — run hit the scenario timeout limit; scoring may be impacted by aborting model execution before it could produce its full output");
 
+        // Noise test results
+        var withNoise = verdicts.Where(v => v.NoiseTestResult is not null).ToList();
+        if (withNoise.Count > 0)
+        {
+            sb.AppendLine();
+            sb.AppendLine("### Noise Test (Multi-Skill Loading)");
+            sb.AppendLine();
+            sb.AppendLine("| Skill | Skills Loaded | Degradation | Verdict |");
+            sb.AppendLine("|-------|--------------|-------------|---------|");
+            foreach (var v in withNoise)
+            {
+                var nr = v.NoiseTestResult!;
+                var icon = nr.Passed ? "✅" : "⚠️";
+                sb.AppendLine($"| {v.SkillName} | {nr.TotalSkillsLoaded} | {nr.OverallDegradation * 100:F1}% | {icon} |");
+            }
+
+            foreach (var v in withNoise)
+            {
+                var nr = v.NoiseTestResult!;
+                if (nr.Scenarios.Count > 0)
+                {
+                    sb.AppendLine();
+                    sb.AppendLine($"**{v.SkillName}** noise scenarios:");
+                    sb.AppendLine();
+                    sb.AppendLine("| Scenario | Skill-Only | All-Skills | Degradation | Target Activated |");
+                    sb.AppendLine("|----------|-----------|------------|-------------|-----------------|");
+                    foreach (var ns in nr.Scenarios)
+                    {
+                        var activated = ns.SkillActivation?.Activated == true ? "✅" : "⚠️";
+                        sb.AppendLine($"| {ns.ScenarioName} | {ns.WithSkillOnly.JudgeResult.OverallScore:F1}/5 | {ns.WithAllSkills.JudgeResult.OverallScore:F1}/5 | {ns.DegradationScore * 100:F1}% | {activated} |");
+                    }
+                }
+            }
+        }
+
         sb.AppendLine($"\nModel: {model ?? "unknown"} | Judge: {judgeModel ?? "unknown"}");
         return sb.ToString();
     }