Skip to content

feat: noise-test evaluation option #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
DeagleGross wants to merge 2 commits into from
Closed

feat: noise-test evaluation option #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d870730
use BPE tokenizer
DeagleGross Mar 7, 2026
362b7b6
noise-test impl
DeagleGross Mar 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 155 additions & 2 deletions eng/skill-validator/src/Commands/ValidateCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ public static RootCommand Create()
var reporterOpt = new Option<string[]>("--reporter") { Description = "Reporter (console, json, junit, markdown). Can be repeated.", AllowMultipleArgumentsPerToken = true };
var noOverfittingCheckOpt = new Option<bool>("--no-overfitting-check") { Description = "Disable LLM-based overfitting analysis (on by default)" };
var overfittingFixOpt = new Option<bool>("--overfitting-fix") { Description = "Generate a fixed eval.yaml with improved rubric items/assertions" };
var noiseSkillsDirOpt = new Option<string?>("--noise-skills-dir") { Description = "Directory containing skills to load as noise. Enables the noise test: re-runs scenarios with all noise skills loaded and measures degradation." };
var noiseMaxDegradationOpt = new Option<double>("--noise-max-degradation") { Description = "Maximum acceptable quality degradation (0-1) in noise test", DefaultValueFactory = _ => 0.2 };

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--noise-max-degradation is documented as a 0–1 value, but the parsed value is accepted without any range validation. Negative values (or values > 1) will lead to confusing pass/fail behavior. Consider validating/clamping to [0,1] during option parsing and producing a clear error message when out of range.

Suggested change
var noiseMaxDegradationOpt = new Option<double>("--noise-max-degradation") { Description = "Maximum acceptable quality degradation (0-1) in noise test", DefaultValueFactory = _ => 0.2 };
var noiseMaxDegradationOpt = new Option<double>("--noise-max-degradation") { Description = "Maximum acceptable quality degradation (0-1) in noise test", DefaultValueFactory = _ => 0.2 };
noiseMaxDegradationOpt.AddValidator(result =>
{
var value = result.GetValueOrDefault<double>();
if (value < 0 || value > 1)
{
result.ErrorMessage = "--noise-max-degradation must be between 0 and 1 (inclusive).";
}
});

Copilot uses AI. Check for mistakes.

var command = new RootCommand("Validate that agent skills meaningfully improve agent performance")
{
Expand All @@ -53,6 +55,8 @@ public static RootCommand Create()
reporterOpt,
noOverfittingCheckOpt,
overfittingFixOpt,
noiseSkillsDirOpt,
noiseMaxDegradationOpt,
};

command.SetAction(async (parseResult, _) =>
Expand Down Expand Up @@ -98,6 +102,8 @@ public static RootCommand Create()
TestsDir = parseResult.GetValue(testsDirOpt),
OverfittingCheck = !parseResult.GetValue(noOverfittingCheckOpt),
OverfittingFix = parseResult.GetValue(overfittingFixOpt),
NoiseSkillsDir = parseResult.GetValue(noiseSkillsDirOpt),
NoiseMaxDegradation = parseResult.GetValue(noiseMaxDegradationOpt),
};

return await Run(config);
Expand Down Expand Up @@ -164,6 +170,14 @@ public static async Task<int> Run(ValidatorConfig config)

Console.WriteLine($"Found {allSkills.Count} skill(s)\n");

// Discover noise skills when --noise-skills-dir is provided
var noiseSkills = new List<SkillInfo>();
if (config.NoiseSkillsDir is not null)
{
noiseSkills.AddRange(await SkillDiscovery.DiscoverSkills(config.NoiseSkillsDir, config.TestsDir));
Console.WriteLine($"Noise test enabled: discovered {noiseSkills.Count} noise skill(s) from {config.NoiseSkillsDir}");
}

// Check per-plugin aggregate description size
var aggregateFailures = CheckAggregateDescriptionLimits(allSkills);
if (aggregateFailures.Count > 0)
Expand All @@ -184,7 +198,7 @@ public static async Task<int> Run(ValidatorConfig config)
// Evaluate skills
spinner.Start($"Evaluating {allSkills.Count} skill(s)...");
var skillTasks = allSkills.Select(skill =>
skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner)));
skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills)));
var settled = await Task.WhenAll(skillTasks.Select(async t =>
{
try { return (Result: await t, Error: (Exception?)null); }
Expand Down Expand Up @@ -280,7 +294,8 @@ internal static List<string> CheckAggregateDescriptionLimits(IReadOnlyList<Skill
SkillInfo skill,
ValidatorConfig config,
bool usePairwise,
Spinner spinner)
Spinner spinner,
IReadOnlyList<SkillInfo> noiseSkills)
{
var prefix = $"[{skill.Name}]";
var log = (string msg) => spinner.Log($"{prefix} {msg}");
Expand Down Expand Up @@ -382,6 +397,28 @@ internal static List<string> CheckAggregateDescriptionLimits(IReadOnlyList<Skill
}
}

// --- Noise test: run scenarios with all skills loaded ---
if (config.NoiseSkillsDir is not null && noiseSkills.Count > 0)
{
try
{
var noiseResult = await ExecuteNoiseTest(skill, noiseSkills, config, usePairwise, spinner);
verdict.NoiseTestResult = noiseResult;
if (!noiseResult.Passed)
{
log($"\x1b[33m\u26a0\ufe0f Noise test: quality degraded by {noiseResult.OverallDegradation * 100:F1}% with {noiseResult.TotalSkillsLoaded} skills loaded\x1b[0m");
Comment on lines +405 to +409

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Noise test results are computed and logged, but they don’t affect the verdict. When noiseResult.Passed is false, verdict.Passed / FailureKind / Reason aren’t updated, so validation can still exit success despite exceeding --noise-max-degradation. Please wire noise-test failure into the verdict (and decide whether exceptions should also fail the skill when noise testing is enabled).

Copilot uses AI. Check for mistakes.
}
else
{
log($"\u2705 Noise test passed ({noiseResult.TotalSkillsLoaded} skills loaded, degradation: {noiseResult.OverallDegradation * 100:F1}%)");
}
}
catch (Exception ex)
{
log($"\u26a0\ufe0f Noise test failed: {ex.Message}");
}
}

var notActivated = comparisons.Where(c => c.SkillActivation is { Activated: false }).ToList();
// Separate unexpected non-activations (expect_activation defaulting to true)
// from expected ones (negative tests with expect_activation: false).
Expand Down Expand Up @@ -598,6 +635,122 @@ private static async Task<RunExecutionResult> ExecuteRun(
return new RunExecutionResult(baseline, withSkillResult, pairwise, skillActivation);
}

// --- Noise test: run scenarios with all discovered skills loaded ---

private static async Task<NoiseTestResult> ExecuteNoiseTest(
SkillInfo targetSkill,
IReadOnlyList<SkillInfo> allSkills,
ValidatorConfig config,
bool usePairwise,
Comment on lines +640 to +644

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ExecuteNoiseTest(...) accepts usePairwise but never uses it, which is misleading given the rest of validation respects --judge-mode. Either remove the parameter or implement pairwise judging for the noise test (and ensure the degradation metric aligns with the chosen judge mode).

Copilot uses AI. Check for mistakes.
Spinner spinner)
{
var prefix = $"[{targetSkill.Name}/noise]";
var log = (string msg) => spinner.Log($"{prefix} {msg}");

var otherSkills = allSkills.Where(s =>
!string.Equals(s.Path, targetSkill.Path, StringComparison.OrdinalIgnoreCase)).ToList();
int totalLoaded = otherSkills.Count + 1; // target + others

log($"🔊 Running noise test with {totalLoaded} skills loaded...");

var noiseScenarios = new List<NoiseScenarioResult>();
using var scenarioLimit = new ConcurrencyLimiter(config.ParallelScenarios);

var tasks = targetSkill.EvalConfig!.Scenarios
.Where(s => s.ExpectActivation) // only test positive scenarios
.Select(scenario => scenarioLimit.RunAsync(async () =>
{
var tag = $"[{targetSkill.Name}/noise/{scenario.Name}]";
var scenarioLog = (string msg) => spinner.Log($"{tag} {msg}");

scenarioLog("running skill-only vs all-skills...");

// Run with target skill only
var skillOnlyMetrics = await AgentRunner.RunAgent(new RunOptions(
scenario, targetSkill, targetSkill.EvalPath, config.Model, config.Verbose, scenarioLog));

// Run with all skills loaded
var allSkillsMetrics = await AgentRunner.RunAgent(new RunOptions(
scenario, targetSkill, targetSkill.EvalPath, config.Model, config.Verbose, scenarioLog, AdditionalSkills: otherSkills));

Comment on lines +672 to +675

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The noise test currently runs only a single skill-only vs all-skills pair per scenario, ignoring config.Runs (which is otherwise used to average runs per scenario). This makes noise results much higher variance than the main evaluation and can surprise users passing --runs N. Consider reusing the same multi-run/averaging approach here, or clearly documenting that noise testing is always single-run.

Copilot uses AI. Check for mistakes.
// Evaluate assertions on both
if (scenario.Assertions is { Count: > 0 })
{
skillOnlyMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(
scenario.Assertions, skillOnlyMetrics.AgentOutput, skillOnlyMetrics.WorkDir);
allSkillsMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(
scenario.Assertions, allSkillsMetrics.AgentOutput, allSkillsMetrics.WorkDir);
}
var soConstraints = AssertionEvaluator.EvaluateConstraints(scenario, skillOnlyMetrics);
var asConstraints = AssertionEvaluator.EvaluateConstraints(scenario, allSkillsMetrics);
skillOnlyMetrics.AssertionResults = [..skillOnlyMetrics.AssertionResults, ..soConstraints];
allSkillsMetrics.AssertionResults = [..allSkillsMetrics.AssertionResults, ..asConstraints];

skillOnlyMetrics.TaskCompleted = scenario.Assertions is { Count: > 0 } || soConstraints.Count > 0
? skillOnlyMetrics.AssertionResults.All(a => a.Passed)
: skillOnlyMetrics.ErrorCount == 0;
allSkillsMetrics.TaskCompleted = scenario.Assertions is { Count: > 0 } || asConstraints.Count > 0
? allSkillsMetrics.AssertionResults.All(a => a.Passed)
: allSkillsMetrics.ErrorCount == 0;

// Judge both runs
var judgeOpts = new JudgeOptions(config.JudgeModel, config.Verbose, config.JudgeTimeout, skillOnlyMetrics.WorkDir, targetSkill.Path);
JudgeResult skillOnlyJudge, allSkillsJudge;
try
{
skillOnlyJudge = await Services.Judge.JudgeRun(scenario, skillOnlyMetrics, judgeOpts);
}
catch
{
skillOnlyJudge = new JudgeResult([], 3, "Judge failed");
}
try
{
allSkillsJudge = await Services.Judge.JudgeRun(scenario, allSkillsMetrics,
judgeOpts with { WorkDir = allSkillsMetrics.WorkDir });
}
catch
{
allSkillsJudge = new JudgeResult([], 3, "Judge failed");
}

var skillOnlyResult = new RunResult(skillOnlyMetrics, skillOnlyJudge);
var allSkillsResult = new RunResult(allSkillsMetrics, allSkillsJudge);

// Compare: skill-only is "baseline", all-skills is "with-skill"
// A positive score means all-skills is *better*, negative means degradation
var comparison = Comparator.CompareScenario(scenario.Name, skillOnlyResult, allSkillsResult);
var degradation = -comparison.ImprovementScore; // positive = degradation

var activation = MetricsCollector.ExtractSkillActivation(
allSkillsMetrics.Events, skillOnlyMetrics.ToolCallBreakdown);

scenarioLog($"✓ degradation: {degradation * 100:F1}%, target skill activated: {activation.Activated}");
Comment on lines +725 to +728

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Noise test logs/reporting label this as “target skill activated”, but MetricsCollector.ExtractSkillActivation(...) only tells you whether any skill/instruction event happened in the run. With multiple skills loaded this will almost always be true even if the target skill wasn’t invoked. Consider deriving a “target activated” boolean by checking whether DetectedSkills contains targetSkill.Name (case-insensitive) rather than using the aggregate Activated flag.

Copilot uses AI. Check for mistakes.

return new NoiseScenarioResult(
scenario.Name,
skillOnlyResult,
allSkillsResult,
degradation,
comparison.Breakdown,
activation,
totalLoaded);
}));

noiseScenarios = (await Task.WhenAll(tasks)).ToList();

double overallDegradation = noiseScenarios.Count > 0
? noiseScenarios.Average(s => s.DegradationScore)
Comment on lines +742 to +743

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

overallDegradation is computed as a simple average of DegradationScore, but DegradationScore can be negative when the all-skills run scores better than the skill-only run. That allows improvements in some scenarios to cancel out degradation in others and can also produce negative “degradation” percentages in reports. Consider clamping per-scenario degradation to a minimum of 0 (or averaging only the positive degradation values) before computing the overall metric / pass-fail.

Suggested change
double overallDegradation = noiseScenarios.Count > 0
? noiseScenarios.Average(s => s.DegradationScore)
var nonNegativeDegradations = noiseScenarios
.Select(s => Math.Max(0, s.DegradationScore))
.ToList();
double overallDegradation = nonNegativeDegradations.Count > 0
? nonNegativeDegradations.Average()

Copilot uses AI. Check for mistakes.
: 0;
bool passed = overallDegradation <= config.NoiseMaxDegradation;

string reason = passed
? $"Quality degradation {overallDegradation * 100:F1}% within threshold of {config.NoiseMaxDegradation * 100:F1}% ({totalLoaded} skills loaded)"
: $"Quality degradation {overallDegradation * 100:F1}% exceeds threshold of {config.NoiseMaxDegradation * 100:F1}% ({totalLoaded} skills loaded)";

return new NoiseTestResult(noiseScenarios, overallDegradation, passed, reason, totalLoaded);
}

private static RunResult AverageResults(List<RunResult> runs)
{
if (runs.Count == 1) return runs[0];
Expand Down
21 changes: 21 additions & 0 deletions eng/skill-validator/src/Models/Models.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ public sealed class SkillVerdict
public IReadOnlyList<string>? ProfileWarnings { get; set; }
public bool SkillNotActivated { get; set; }
public OverfittingResult? OverfittingResult { get; set; }
public NoiseTestResult? NoiseTestResult { get; set; }
}

// --- Overfitting assessment ---
Expand Down Expand Up @@ -274,6 +275,24 @@ public sealed record OverfittingJudgeOptions(
int Timeout,
string WorkDir);

// --- Multi-skill noise test ---

public sealed record NoiseScenarioResult(
string ScenarioName,
RunResult WithSkillOnly,
RunResult WithAllSkills,
double DegradationScore,
MetricBreakdown Breakdown,
SkillActivationInfo? SkillActivation,
int TotalSkillsLoaded);

public sealed record NoiseTestResult(
IReadOnlyList<NoiseScenarioResult> Scenarios,
double OverallDegradation,
bool Passed,
string Reason,
int TotalSkillsLoaded);

// --- Config ---

public sealed record ReporterSpec(ReporterType Type);
Expand Down Expand Up @@ -308,6 +327,8 @@ public sealed record ValidatorConfig
public string? TestsDir { get; init; }
public bool OverfittingCheck { get; init; } = true;
public bool OverfittingFix { get; init; }
public string? NoiseSkillsDir { get; init; }
public double NoiseMaxDegradation { get; init; } = 0.2;
}

public static class DefaultWeights
Expand Down
23 changes: 19 additions & 4 deletions eng/skill-validator/src/Services/AgentRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ public sealed record RunOptions(
string? EvalPath,
string Model,
bool Verbose,
Action<string>? Log = null);
Action<string>? Log = null,
IReadOnlyList<SkillInfo>? AdditionalSkills = null);

public static class AgentRunner
{
Expand Down Expand Up @@ -104,7 +105,8 @@ public static bool CheckPermission(PermissionRequest request, string workDir, st

internal static SessionConfig BuildSessionConfig(
SkillInfo? skill, string model, string workDir,
IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null)
IReadOnlyDictionary<string, MCPServerDef>? mcpServers = null,
IReadOnlyList<SkillInfo>? additionalSkills = null)
{
var skillPath = skill is not null ? Path.GetDirectoryName(skill.Path) : null;

Comment on lines 111 to 112

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SkillInfo.Path is already the skill directory (see SkillDiscovery creating SkillInfo with Path: dirPath). Using Path.GetDirectoryName(skill.Path) moves up one level (often to the shared skills/ folder), which undermines the “target skill only” intent and can accidentally load multiple skills. Use skill.Path directly when building the primary SkillDirectories entry.

Copilot uses AI. Check for mistakes.
Expand All @@ -113,6 +115,19 @@ internal static SessionConfig BuildSessionConfig(
Directory.CreateDirectory(configDir);
_workDirs.Add(configDir);

// Build skill directories list: primary skill + any additional skills
var skillDirs = new List<string>();
if (skillPath is not null) skillDirs.Add(skillPath);
if (additionalSkills is { Count: > 0 })
{
foreach (var s in additionalSkills)
{
var dir = Path.GetDirectoryName(s.Path);
if (dir is not null && !skillDirs.Contains(dir, StringComparer.OrdinalIgnoreCase))
skillDirs.Add(dir);
Comment on lines +123 to +127

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For additional/noise skills, Path.GetDirectoryName(s.Path) has the same issue as the primary skill: since s.Path is already a directory, this collapses many skills to the same parent directory and makes AdditionalSkills de-duping behave incorrectly. Use s.Path directly when adding to skillDirs so the session loads the intended set of skill directories.

Copilot uses AI. Check for mistakes.
}
}

// Convert MCPServerDef records to the SDK's Dictionary<string, object> shape
Dictionary<string, object>? sdkMcp = null;
if (mcpServers is { Count: > 0 })
Expand All @@ -138,7 +153,7 @@ internal static SessionConfig BuildSessionConfig(
Model = model,
Streaming = true,
WorkingDirectory = workDir,
SkillDirectories = skill is not null ? [skillPath!] : [],
SkillDirectories = skillDirs,
ConfigDir = configDir,
McpServers = sdkMcp,
InfiniteSessions = new InfiniteSessionConfig { Enabled = false },
Expand Down Expand Up @@ -172,7 +187,7 @@ public static async Task<RunMetrics> RunAgent(RunOptions options)
var client = await GetSharedClient(options.Verbose);

await using var session = await client.CreateSessionAsync(
BuildSessionConfig(options.Skill, options.Model, workDir, options.Skill?.McpServers));
BuildSessionConfig(options.Skill, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills));

var done = new TaskCompletionSource();
var effectiveTimeout = options.Scenario.Timeout;
Expand Down
54 changes: 54 additions & 0 deletions eng/skill-validator/src/Services/Reporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,25 @@ private static void ReportConsole(IReadOnlyList<SkillVerdict> verdicts, bool ver
Console.WriteLine($" \x1b[2m•\x1b[0m [{item.Classification}] \x1b[2m{item.AssertionSummary}\x1b[0m\n \x1b[2m— {item.Reasoning}\x1b[0m");
}
}

// Noise test results
if (verdict.NoiseTestResult is { } noiseResult)
{
Console.WriteLine();
var noiseIcon = noiseResult.Passed ? "✅" : "⚠️";
var noiseColor = noiseResult.Passed ? "\x1b[32m" : "\x1b[33m";
Console.WriteLine($" 🔊 Noise test ({noiseResult.TotalSkillsLoaded} skills loaded): {noiseColor}{noiseResult.OverallDegradation * 100:F1}% degradation\x1b[0m {noiseIcon}");
Console.WriteLine($" \x1b[2m{noiseResult.Reason}\x1b[0m");

foreach (var ns in noiseResult.Scenarios)
{
var nsIcon = ns.DegradationScore <= 0 ? "\x1b[32m↑\x1b[0m" : "\x1b[33m↓\x1b[0m";
var activated = ns.SkillActivation?.Activated == true ? "✅" : "⚠️ not activated";
Console.WriteLine($" {nsIcon} {ns.ScenarioName} degradation: {ns.DegradationScore * 100:F1}% target skill: {activated}");
Console.WriteLine($" \x1b[2mskill-only: {ns.WithSkillOnly.JudgeResult.OverallScore:F1}/5 → all-skills: {ns.WithAllSkills.JudgeResult.OverallScore:F1}/5\x1b[0m");
Comment on lines +147 to +150

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Console noise reporting shows “target skill: activated” based on ns.SkillActivation?.Activated, but in a multi-skill run that flag represents “some skill/tool activity happened” and does not indicate the target skill was invoked. Consider computing this cell by checking whether ns.SkillActivation?.DetectedSkills contains verdict.SkillName (case-insensitive) and report that instead.

Copilot uses AI. Check for mistakes.
}
}

if (verdict.Scenarios.Count > 0)
{
Console.WriteLine();
Expand Down Expand Up @@ -363,6 +382,41 @@ public static string GenerateMarkdownSummary(
if (anyTimeout)
sb.AppendLine("\n> ⏰ **timeout** — run hit the scenario timeout limit; scoring may be impacted by aborting model execution before it could produce its full output");

// Noise test results
var withNoise = verdicts.Where(v => v.NoiseTestResult is not null).ToList();
if (withNoise.Count > 0)
{
sb.AppendLine();
sb.AppendLine("### Noise Test (Multi-Skill Loading)");
sb.AppendLine();
sb.AppendLine("| Skill | Skills Loaded | Degradation | Verdict |");
sb.AppendLine("|-------|--------------|-------------|---------|");
foreach (var v in withNoise)
{
var nr = v.NoiseTestResult!;
var icon = nr.Passed ? "✅" : "⚠️";
sb.AppendLine($"| {v.SkillName} | {nr.TotalSkillsLoaded} | {nr.OverallDegradation * 100:F1}% | {icon} |");
}

foreach (var v in withNoise)
{
var nr = v.NoiseTestResult!;
if (nr.Scenarios.Count > 0)
{
sb.AppendLine();
sb.AppendLine($"**{v.SkillName}** noise scenarios:");
sb.AppendLine();
sb.AppendLine("| Scenario | Skill-Only | All-Skills | Degradation | Target Activated |");
sb.AppendLine("|----------|-----------|------------|-------------|-----------------|");
foreach (var ns in nr.Scenarios)
{
var activated = ns.SkillActivation?.Activated == true ? "✅" : "⚠️";
sb.AppendLine($"| {ns.ScenarioName} | {ns.WithSkillOnly.JudgeResult.OverallScore:F1}/5 | {ns.WithAllSkills.JudgeResult.OverallScore:F1}/5 | {ns.DegradationScore * 100:F1}% | {activated} |");
Comment on lines +412 to +414

Copilot AI Mar 8, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Markdown noise reporting uses ns.SkillActivation?.Activated for “Target Activated”, which doesn’t actually indicate the target skill was invoked when multiple skills are loaded. Consider computing this cell by checking whether ns.SkillActivation?.DetectedSkills contains the skill being reported (v.SkillName, case-insensitive).

Copilot uses AI. Check for mistakes.
}
}
}
}

sb.AppendLine($"\nModel: {model ?? "unknown"} | Judge: {judgeModel ?? "unknown"}");
return sb.ToString();
}
Expand Down
Loading
Loading