feat: noise-test evaluation option

eng/skill-validator/src/Models/Models.cs

@@ -259,6 +259,7 @@ public sealed class SkillVerdict
     public IReadOnlyList<string>? ProfileWarnings { get; set; }
     public bool SkillNotActivated { get; set; }
     public OverfittingResult? OverfittingResult { get; set; }
+    public NoiseTestResult? NoiseTestResult { get; set; }
 }
 
 // --- Overfitting assessment ---
@@ -306,6 +307,24 @@ public sealed record OverfittingJudgeOptions(
     int Timeout,
     string WorkDir);
 
+// --- Multi-skill noise test ---
+
+public sealed record NoiseScenarioResult(
+    string ScenarioName,
+    RunResult WithSkillOnly,
+    RunResult WithAllSkills,
+    double DegradationScore,
+    MetricBreakdown Breakdown,
+    SkillActivationInfo? SkillActivation,
+    int TotalSkillsLoaded);
+
+public sealed record NoiseTestResult(
+    IReadOnlyList<NoiseScenarioResult> Scenarios,
+    double OverallDegradation,
+    bool Passed,
+    string Reason,
+    int TotalSkillsLoaded);
+
 // --- Config ---
 
 public sealed record ReporterSpec(ReporterType Type);
@@ -340,6 +359,9 @@ public sealed record ValidatorConfig
     public string? TestsDir { get; init; }
     public bool OverfittingCheck { get; init; } = true;
     public bool OverfittingFix { get; init; }
+    public string? NoiseSkillsDir { get; init; }
+    public double NoiseDegradationLimit { get; init; } = 0.2;
+    public double NoiseMaxScenarioDegradation { get; init; } = 0.4;
 }
 
 public static class DefaultWeights

eng/skill-validator/src/Services/AgentRunner.cs

@@ -14,7 +14,8 @@ public sealed record RunOptions(
     string? EvalPath,
     string Model,
     bool Verbose,
-    Action<string>? Log = null);
+    Action<string>? Log = null,
+    IReadOnlyList<SkillInfo>? AdditionalSkills = null);
 
 public static class AgentRunner
 {
@@ -105,15 +106,44 @@ public static bool CheckPermission(PermissionRequest request, string workDir, st
 
     internal static SessionConfig BuildSessionConfig(
         SkillInfo? skill, string model, string workDir,
-        IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null)
+        IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null,
+        IReadOnlyList<SkillInfo>? additionalSkills = null)
     {
+        // The SDK expects SkillDirectories entries to be parent directories that
+        // it scans for child folders containing SKILL.md.
         var skillPath = skill is not null ? Path.GetDirectoryName(skill.Path) : null;
 
         // Create a unique temporary config directory for this session to not share any data
         var configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}");
         Directory.CreateDirectory(configDir);
         _workDirs.Add(configDir);
 
+        // Build skill directories list: primary skill + any additional skills.
+        // For additional skills we stage a temp directory with copies of each
+        // skill's SKILL.md so the SDK discovers exactly those skills — not
+        // every sibling that happens to share the same parent directory.
+        var skillDirs = new List<string>();
+        if (skillPath is not null) skillDirs.Add(skillPath);
+        if (additionalSkills is { Count: > 0 })
+        {
+            var stageDir = Path.Combine(Path.GetTempPath(), $"sv-noise-{Guid.NewGuid():N}");
+            Directory.CreateDirectory(stageDir);
+            _workDirs.Add(stageDir);
+
+            foreach (var s in additionalSkills)
+            {
+                var skillMdPath = Path.Combine(s.Path, "SKILL.md");
+                if (!File.Exists(skillMdPath))
+                    continue;
+
+                var stagedSkillDir = Path.Combine(stageDir, Path.GetFileName(s.Path));
+                Directory.CreateDirectory(stagedSkillDir);
+                File.Copy(skillMdPath, Path.Combine(stagedSkillDir, "SKILL.md"));
+            }
+
+            skillDirs.Add(stageDir);
+        }
+
         // Convert MCPServerDef records to the SDK's Dictionary<string, object> shape
         Dictionary<string, object>? sdkMcp = null;
         if (mcpServers is { Count: > 0 })
@@ -139,7 +169,7 @@ internal static SessionConfig BuildSessionConfig(
             Model = model,
             Streaming = true,
             WorkingDirectory = workDir,
-            SkillDirectories = skill is not null ? [skillPath!] : [],
+            SkillDirectories = skillDirs,
             ConfigDir = configDir,
             McpServers = sdkMcp,
             InfiniteSessions = new InfiniteSessionConfig { Enabled = false },
@@ -183,7 +213,7 @@ private static async Task<RunMetrics> RunAgentCore(RunOptions options, Cancellat
             var client = await GetSharedClient(options.Verbose);
 
             await using var session = await client.CreateSessionAsync(
-                BuildSessionConfig(options.Skill, options.Model, workDir, options.Skill?.McpServers));
+                BuildSessionConfig(options.Skill, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills));
 
             var done = new TaskCompletionSource();
             var effectiveTimeout = options.Scenario.Timeout;

eng/skill-validator/src/Services/Reporter.cs

@@ -133,6 +133,25 @@ private static void ReportConsole(IReadOnlyList<SkillVerdict> verdicts, bool ver
                         Console.WriteLine($"    \x1b[2m•\x1b[0m [{item.Classification}] \x1b[2m{item.AssertionSummary}\x1b[0m\n      \x1b[2m— {item.Reasoning}\x1b[0m");
                 }
             }
+
+            // Noise test results
+            if (verdict.NoiseTestResult is { } noiseResult)
+            {
+                Console.WriteLine();
+                var noiseIcon = noiseResult.Passed ? "✅" : "⚠️";
+                var noiseColor = noiseResult.Passed ? "\x1b[32m" : "\x1b[33m";
+                Console.WriteLine($"  🔊 Noise test ({noiseResult.TotalSkillsLoaded} skills loaded): {noiseColor}{noiseResult.OverallDegradation * 100:F1}% avg degradation\x1b[0m {noiseIcon}");
+                Console.WriteLine($"  \x1b[2m{noiseResult.Reason}\x1b[0m");
+
+                foreach (var ns in noiseResult.Scenarios)
+                {
+                    var nsIcon = ns.DegradationScore <= 0 ? "\x1b[32m↑\x1b[0m" : "\x1b[33m↓\x1b[0m";
+                    var activated = ns.SkillActivation?.Activated == true ? "✅" : "⚠️ not activated";
+                    Console.WriteLine($"    {nsIcon} {ns.ScenarioName}  degradation: {ns.DegradationScore * 100:F1}%  target skill: {activated}");
+                    Console.WriteLine($"      \x1b[2mskill-only: {ns.WithSkillOnly.JudgeResult.OverallScore:F1}/5 → all-skills: {ns.WithAllSkills.JudgeResult.OverallScore:F1}/5\x1b[0m");
+                }
+            }
+
             if (verdict.Scenarios.Count > 0)
             {
                 Console.WriteLine();
@@ -404,6 +423,41 @@ public static string GenerateMarkdownSummary(
         if (anyTimeout)
             sb.AppendLine("\n> ⏰ **timeout** — run hit the scenario timeout limit; scoring may be impacted by aborting model execution before it could produce its full output");
 
+        // Noise test results
+        var withNoise = verdicts.Where(v => v.NoiseTestResult is not null).ToList();
+        if (withNoise.Count > 0)
+        {
+            sb.AppendLine();
+            sb.AppendLine("### Noise Test (Multi-Skill Loading)");
+            sb.AppendLine();
+            sb.AppendLine("| Skill | Skills Loaded | Degradation | Verdict |");
+            sb.AppendLine("|-------|--------------|-------------|---------|");
+            foreach (var v in withNoise)
+            {
+                var nr = v.NoiseTestResult!;
+                var icon = nr.Passed ? "✅" : "⚠️";
+                sb.AppendLine($"| {v.SkillName} | {nr.TotalSkillsLoaded} | {nr.OverallDegradation * 100:F1}% | {icon} |");
+            }
+
+            foreach (var v in withNoise)
+            {
+                var nr = v.NoiseTestResult!;
+                if (nr.Scenarios.Count > 0)
+                {
+                    sb.AppendLine();
+                    sb.AppendLine($"**{v.SkillName}** noise scenarios:");
+                    sb.AppendLine();
+                    sb.AppendLine("| Scenario | Skill-Only | All-Skills | Degradation | Target Activated |");
+                    sb.AppendLine("|----------|-----------|------------|-------------|-----------------|");
+                    foreach (var ns in nr.Scenarios)
+                    {
+                        var activated = ns.SkillActivation?.Activated == true ? "✅" : "⚠️";
+                        sb.AppendLine($"| {ns.ScenarioName} | {ns.WithSkillOnly.JudgeResult.OverallScore:F1}/5 | {ns.WithAllSkills.JudgeResult.OverallScore:F1}/5 | {ns.DegradationScore * 100:F1}% | {activated} |");
+                    }
+                }
+            }
+        }
+
         sb.AppendLine($"\nModel: {model ?? "unknown"} | Judge: {judgeModel ?? "unknown"}");
 
         return sb.ToString();

eng/skill-validator/src/Services/SkillDiscovery.cs

@@ -33,6 +33,29 @@ public static async Task<IReadOnlyList<SkillInfo>> DiscoverSkills(string targetP
         return skills;
     }
 
+    /// <summary>
+    /// Recursively discover all skills under a directory tree by finding SKILL.md files.
+    /// </summary>
+    public static async Task<IReadOnlyList<SkillInfo>> DiscoverSkillsRecursive(string targetPath, string? testsDir = null)
+    {
+        if (!Directory.Exists(targetPath))
+            return [];
+
+        var skills = new List<SkillInfo>();
+        foreach (var skillMdPath in Directory.EnumerateFiles(targetPath, "SKILL.md", SearchOption.AllDirectories))
+        {
+            var dirPath = Path.GetDirectoryName(skillMdPath)!;
+            if (Path.GetFileName(dirPath).StartsWith('.'))
+                continue;
+
+            var skill = await DiscoverSkillAt(dirPath, testsDir);
+            if (skill is not null)
+                skills.Add(skill);
+        }
+
+        return skills;
+    }
+
     private static async Task<SkillInfo?> DiscoverSkillAt(string dirPath, string? testsDir)
     {
         var skillMdPath = Path.Combine(dirPath, "SKILL.md");
@@ -49,11 +72,9 @@ public static async Task<IReadOnlyList<SkillInfo>> DiscoverSkills(string targetP
         string? evalPath = null;
         EvalConfig? evalConfig = null;
 
-        var evalFilePath = testsDir is not null
-            ? Path.Combine(testsDir, Path.GetFileName(dirPath), "eval.yaml")
-            : Path.Combine(dirPath, "tests", "eval.yaml");
+        var evalFilePath = ResolveEvalPath(dirPath, testsDir);
 
-        if (File.Exists(evalFilePath))
+        if (evalFilePath is not null && File.Exists(evalFilePath))
         {
             evalPath = evalFilePath;
             var evalContent = await File.ReadAllTextAsync(evalFilePath);
@@ -119,6 +140,39 @@ await File.ReadAllTextAsync(candidate),
         return null;
     }
 
+    /// <summary>
+    /// Resolve the eval.yaml path for a skill. Tries flat layout first,
+    /// then searches one level of subdirectories under testsDir.
+    /// </summary>
+    private static string? ResolveEvalPath(string skillDirPath, string? testsDir)
+    {
+        var skillDirName = Path.GetFileName(skillDirPath);
+
+        if (testsDir is null)
+        {
+            var inTree = Path.Combine(skillDirPath, "tests", "eval.yaml");
+            return File.Exists(inTree) ? inTree : null;
+        }
+
+        // Flat: testsDir/<skill-name>/eval.yaml
+        var flat = Path.Combine(testsDir, skillDirName, "eval.yaml");
+        if (File.Exists(flat))
+            return flat;
+
+        // Nested: testsDir/<subdir>/<skill-name>/eval.yaml (e.g., tests/dotnet/csharp-scripts/eval.yaml)
+        if (Directory.Exists(testsDir))
+        {
+            foreach (var subDir in Directory.GetDirectories(testsDir))
+            {
+                var nested = Path.Combine(subDir, skillDirName, "eval.yaml");
+                if (File.Exists(nested))
+                    return nested;
+            }
+        }
+
+        return null;
+    }
+
     private static readonly IDeserializer FrontmatterDeserializer = new StaticDeserializerBuilder(new SkillValidatorYamlContext())
         .WithNamingConvention(UnderscoredNamingConvention.Instance)
         .IgnoreUnmatchedProperties()

eng/skill-validator/src/SkillValidator.csproj

@@ -18,7 +18,7 @@
     <PublishAot>true</PublishAot>
 
     <!-- dotnet run args for local invocation -->
-    <RunArguments>--results-dir &quot;$([MSBuild]::NormalizePath('$(ArtifactsPath)', 'TestResults', '$(AssemblyName)'))&quot; --parallel-skills 3 --parallel-scenarios 3 --parallel-runs 3</RunArguments>
+    <RunArguments>--results-dir "$([MSBuild]::NormalizePath('$(ArtifactsPath)', 'TestResults', '$(AssemblyName)'))" --parallel-skills 3 --parallel-scenarios 3 --parallel-runs 3</RunArguments>
   </PropertyGroup>
 
   <ItemGroup>

eng/skill-validator/src/SkillValidatorJsonContext.cs

@@ -30,6 +30,8 @@ namespace SkillValidator;
 [JsonSerializable(typeof(RubricOverfitAssessment))]
 [JsonSerializable(typeof(AssertionOverfitAssessment))]
 [JsonSerializable(typeof(OverfittingSeverity))]
+[JsonSerializable(typeof(NoiseScenarioResult))]
+[JsonSerializable(typeof(NoiseTestResult))]
 [JsonSerializable(typeof(PairwiseMagnitude))]
 [JsonSerializable(typeof(AssertionType))]
 [JsonSerializable(typeof(MCPServerDef))]

eng/skill-validator/tests/DiscoveryTests.cs

@@ -133,4 +133,93 @@ public async Task ReturnsNullWhenNoPluginJson()
             Directory.Delete(tmpDir, true);
         }
     }
+
+    [Fact]
+    public async Task DiscoverSkillsRecursiveFindsNestedSkills()
+    {
+        // Simulates plugins/<plugin>/skills/<skill>/SKILL.md layout
+        var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
+        var skill1Dir = Path.Combine(tmpDir, "plugin-a", "skills", "skill-one");
+        var skill2Dir = Path.Combine(tmpDir, "plugin-b", "skills", "skill-two");
+        Directory.CreateDirectory(skill1Dir);
+        Directory.CreateDirectory(skill2Dir);
+        try
+        {
+            await File.WriteAllTextAsync(Path.Combine(skill1Dir, "SKILL.md"), "---\nname: skill-one\ndescription: first\n---\nBody", TestContext.Current.CancellationToken);
+            await File.WriteAllTextAsync(Path.Combine(skill2Dir, "SKILL.md"), "---\nname: skill-two\ndescription: second\n---\nBody", TestContext.Current.CancellationToken);
+
+            var skills = await SkillDiscovery.DiscoverSkillsRecursive(tmpDir);
+            Assert.Equal(2, skills.Count);
+            var names = skills.Select(s => s.Name).OrderBy(n => n).ToList();
+            Assert.Equal("skill-one", names[0]);
+            Assert.Equal("skill-two", names[1]);
+        }
+        finally
+        {
+            Directory.Delete(tmpDir, true);
+        }
+    }
+
+    [Fact]
+    public async Task DiscoverSkillsRecursiveReturnsEmptyForMissingDir()
+    {
+        var skills = await SkillDiscovery.DiscoverSkillsRecursive(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")));
+        Assert.Empty(skills);
+    }
+
+    [Fact]
+    public async Task ResolveEvalPathFindsNestedTestDir()
+    {
+        // Layout: tests/<plugin-name>/<skill-name>/eval.yaml
+        var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
+        var skillDir = Path.Combine(tmpDir, "plugins", "my-plugin", "skills", "my-skill");
+        var testsDir = Path.Combine(tmpDir, "tests");
+        var evalDir = Path.Combine(testsDir, "my-plugin", "my-skill");
+        Directory.CreateDirectory(skillDir);
+        Directory.CreateDirectory(evalDir);
+        try
+        {
+            await File.WriteAllTextAsync(Path.Combine(skillDir, "SKILL.md"), "---\nname: my-skill\ndescription: test\n---\nBody", TestContext.Current.CancellationToken);
+            await File.WriteAllTextAsync(Path.Combine(evalDir, "eval.yaml"), "scenarios:\n  - name: test\n    prompt: hi\n    assertions:\n      - type: exit_success", TestContext.Current.CancellationToken);
+
+            var skills = await SkillDiscovery.DiscoverSkills(skillDir, testsDir);
+            Assert.Single(skills);
+            Assert.NotNull(skills[0].EvalPath);
+            Assert.Contains("my-plugin", skills[0].EvalPath!);
+        }
+        finally
+        {
+            Directory.Delete(tmpDir, true);
+        }
+    }
+
+    [Fact]
+    public async Task ResolveEvalPathPrefersFlatLayout()
+    {
+        // When both flat and nested exist, flat wins
+        var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
+        var skillDir = Path.Combine(tmpDir, "my-skill");
+        var testsDir = Path.Combine(tmpDir, "tests");
+        var flatEvalDir = Path.Combine(testsDir, "my-skill");
+        var nestedEvalDir = Path.Combine(testsDir, "some-plugin", "my-skill");
+        Directory.CreateDirectory(skillDir);
+        Directory.CreateDirectory(flatEvalDir);
+        Directory.CreateDirectory(nestedEvalDir);
+        try
+        {
+            await File.WriteAllTextAsync(Path.Combine(skillDir, "SKILL.md"), "---\nname: my-skill\ndescription: test\n---\nBody", TestContext.Current.CancellationToken);
+            await File.WriteAllTextAsync(Path.Combine(flatEvalDir, "eval.yaml"), "scenarios:\n  - name: test\n    prompt: hi\n    assertions:\n      - type: exit_success", TestContext.Current.CancellationToken);
+            await File.WriteAllTextAsync(Path.Combine(nestedEvalDir, "eval.yaml"), "scenarios:\n  - name: test\n    prompt: hi\n    assertions:\n      - type: exit_success", TestContext.Current.CancellationToken);
+
+            var skills = await SkillDiscovery.DiscoverSkills(skillDir, testsDir);
+            Assert.Single(skills);
+            Assert.NotNull(skills[0].EvalPath);
+            // Flat path should win
+            Assert.DoesNotContain("some-plugin", skills[0].EvalPath!);
+        }
+        finally
+        {
+            Directory.Delete(tmpDir, true);
+        }
+    }
 }