Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .config/dotnet-tools.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
"isRoot": true,
"tools": {
"nbgv": {
"version": "3.10.85",
"version": "3.10.91",
"commands": [
"nbgv"
],
"rollForward": false
}
}
}
}
29 changes: 14 additions & 15 deletions eng/skill-validator/src/Evaluate/AgentRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
using System.Text.Json;
using System.Text.Json.Nodes;
using SkillValidator.Shared;
using GitHub.Copilot.SDK;
using GitHub.Copilot;

namespace SkillValidator.Evaluate;

Expand Down Expand Up @@ -68,14 +68,14 @@ public static async Task<CopilotClient> GetPluginClient(

var options = new CopilotClientOptions
{
LogLevel = verbose ? "info" : "none",
LogLevel = verbose ? CopilotLogLevel.Info : CopilotLogLevel.None,
SessionFs = new SessionFsConfig
{
InitialCwd = Environment.CurrentDirectory,
InitialWorkingDirectory = Environment.CurrentDirectory,
SessionStatePath = "session-state",
Conventions = OperatingSystem.IsWindows()
? GitHub.Copilot.SDK.Rpc.SessionFsSetProviderConventions.Windows
: GitHub.Copilot.SDK.Rpc.SessionFsSetProviderConventions.Posix,
? GitHub.Copilot.Rpc.SessionFsSetProviderConventions.Windows
: GitHub.Copilot.Rpc.SessionFsSetProviderConventions.Posix,
},
};

Expand Down Expand Up @@ -443,22 +443,21 @@ internal static async Task<SessionConfig> BuildSessionConfig(
Streaming = true,
WorkingDirectory = workDir,
SkillDirectories = [..skillDirs, ..noiseDirs],
ConfigDir = configDir,
ConfigDirectory = configDir,
McpServers = sdkMcp,
CustomAgents = customAgents,
InfiniteSessions = new InfiniteSessionConfig { Enabled = false },
// SDK 0.3.0 requires a SessionFsProvider (abstract base class).
// The SDK requires a SessionFsProvider (abstract base class).
// Without this, events.jsonl files are never written and
// session replay data is lost.
CreateSessionFsHandler = _ => new LocalSessionFsHandler(configDir),
CreateSessionFsProvider = _ => new LocalSessionFsHandler(configDir),
OnPermissionRequest = (request, _) =>
{
// SDK 0.2.0: PermissionRequest only has Kind, no path data.
// Permission sandboxing is handled via Hooks.OnPreToolUse instead.
return Task.FromResult(new PermissionRequestResult
{
Kind = PermissionRequestResultKind.Approved,
});
// PermissionRequest carries per-kind data (e.g. Read.Path,
// Write.FileName, Shell.FullCommandText/PossiblePaths), but we
// don't use it here: permission sandboxing is enforced via
// Hooks.OnPreToolUse instead, so this handler approves all.
return Task.FromResult(GitHub.Copilot.Rpc.PermissionDecision.ApproveOnce());
},
Hooks = new SessionHooks
{
Expand Down Expand Up @@ -580,7 +579,7 @@ await BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workD

// Register event handler BEFORE SelectAsync so SubagentSelectedEvent
// from the agent selection is captured in the events list.
session.On(evt =>
session.On<SessionEvent>(evt =>
{
var agentEvent = new AgentEvent(
evt.Type,
Expand Down
7 changes: 2 additions & 5 deletions eng/skill-validator/src/Evaluate/Judge.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using System.Text.Json;
using System.Text.Json.Nodes;
using SkillValidator.Shared;
using GitHub.Copilot.SDK;
using GitHub.Copilot.Rpc;

namespace SkillValidator.Evaluate;

Expand Down Expand Up @@ -47,10 +47,7 @@ public static class Judge
{
// Judge sessions: deny all tool permissions. Judging should be a
// pure LLM task — no file access or tool execution needed.
return Task.FromResult(new PermissionRequestResult
{
Kind = PermissionRequestResultKind.UserNotAvailable,
});
return Task.FromResult(PermissionDecision.UserNotAvailable());
},
cancellationToken: cancellationToken);

Expand Down
14 changes: 6 additions & 8 deletions eng/skill-validator/src/Evaluate/LlmSession.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using GitHub.Copilot.SDK;
using GitHub.Copilot;
using GitHub.Copilot.Rpc;

namespace SkillValidator.Evaluate;

Expand Down Expand Up @@ -30,7 +31,7 @@ internal static async Task<LlmResponse> SendAsync(
int timeoutMs,
bool verbose,
string timeoutLabel = "LLM",
PermissionRequestHandler? onPermissionRequest = null,
Func<PermissionRequest, PermissionInvocation, Task<PermissionDecision>>? onPermissionRequest = null,
CancellationToken cancellationToken = default)
{
var client = await AgentRunner.GetSharedClient(verbose);
Expand All @@ -52,11 +53,8 @@ internal static async Task<LlmResponse> SendAsync(
Content = systemPrompt,
},
InfiniteSessions = new InfiniteSessionConfig { Enabled = false },
CreateSessionFsHandler = _ => new LocalSessionFsHandler(tempConfigDir),
OnPermissionRequest = onPermissionRequest ?? ((_, _) => Task.FromResult(new PermissionRequestResult
{
Kind = PermissionRequestResultKind.UserNotAvailable,
})),
CreateSessionFsProvider = _ => new LocalSessionFsHandler(tempConfigDir),
OnPermissionRequest = onPermissionRequest ?? ((_, _) => Task.FromResult(PermissionDecision.UserNotAvailable())),
});

using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
Expand All @@ -70,7 +68,7 @@ internal static async Task<LlmResponse> SendAsync(
string responseContent = "";
int inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;

session.On(evt =>
session.On<SessionEvent>(evt =>
{
switch (evt)
{
Expand Down
12 changes: 6 additions & 6 deletions eng/skill-validator/src/Evaluate/LocalSessionFsHandler.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System.Collections.Concurrent;
using GitHub.Copilot.SDK;
using GitHub.Copilot.SDK.Rpc;
using GitHub.Copilot;
using GitHub.Copilot.Rpc;

namespace SkillValidator.Evaluate;

Expand Down Expand Up @@ -117,14 +117,14 @@ protected override Task<SessionFsStatResult> StatAsync(string path, Cancellation
throw new FileNotFoundException($"Not found: {path}");
}

protected override Task MkdirAsync(string path, bool recursive, int? mode, CancellationToken cancellationToken)
protected override Task MakeDirectoryAsync(string path, bool recursive, int? mode, CancellationToken cancellationToken)
{
var resolved = ResolvePath(path);
Directory.CreateDirectory(resolved);
return Task.CompletedTask;
}

protected override Task<IList<string>> ReaddirAsync(string path, CancellationToken cancellationToken)
protected override Task<IList<string>> ReadDirectoryAsync(string path, CancellationToken cancellationToken)
{
var resolved = ResolvePath(path);
var entries = new List<string>();
Expand All @@ -136,7 +136,7 @@ protected override Task<IList<string>> ReaddirAsync(string path, CancellationTok
return Task.FromResult<IList<string>>(entries);
}

protected override Task<IList<SessionFsReaddirWithTypesEntry>> ReaddirWithTypesAsync(string path, CancellationToken cancellationToken)
protected override Task<IList<SessionFsReaddirWithTypesEntry>> ReadDirectoryWithTypesAsync(string path, CancellationToken cancellationToken)
{
var resolved = ResolvePath(path);
var entries = new List<SessionFsReaddirWithTypesEntry>();
Expand All @@ -154,7 +154,7 @@ protected override Task<IList<SessionFsReaddirWithTypesEntry>> ReaddirWithTypesA
return Task.FromResult<IList<SessionFsReaddirWithTypesEntry>>(entries);
}

protected override Task RmAsync(string path, bool recursive, bool force, CancellationToken cancellationToken)
protected override Task RemoveAsync(string path, bool recursive, bool force, CancellationToken cancellationToken)
{
var resolved = ResolvePath(path);
if (File.Exists(resolved))
Expand Down
7 changes: 2 additions & 5 deletions eng/skill-validator/src/Evaluate/PairwiseJudge.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using System.Text.Json;
using System.Text.Json.Nodes;
using SkillValidator.Shared;
using GitHub.Copilot.SDK;
using GitHub.Copilot.Rpc;

namespace SkillValidator.Evaluate;

Expand Down Expand Up @@ -87,10 +87,7 @@ public static class PairwiseJudge
{
// Pairwise judge sessions: deny all tool permissions. The judge
// should operate purely on the provided text — no tool execution.
return Task.FromResult(new PermissionRequestResult
{
Kind = PermissionRequestResultKind.UserNotAvailable,
});
return Task.FromResult(PermissionDecision.UserNotAvailable());
},
cancellationToken: cancellationToken);

Expand Down
31 changes: 11 additions & 20 deletions eng/skill-validator/src/SkillValidator.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@
<!-- YamlDotNet 17.x is not fully trim/AOT-compatible; suppress the
assembly-level trim analysis warning so native publish succeeds. -->
<NoWarn>$(NoWarn);IL2104</NoWarn>
<!-- GitHub.Copilot.SDK 1.x marks its permission-decision and session-fs
RPC types ([Experimental] "GHCP001"). These types appear directly in
the SessionFsProvider base-class signatures and the OnPermissionRequest
delegate, so consuming them is unavoidable. Suppress the experimental
diagnostic until the SDK promotes them to stable. -->
<NoWarn>$(NoWarn);GHCP001</NoWarn>

<!-- Packaging -->
<PackageId>Microsoft.DotNet.SkillValidator</PackageId>
Expand Down Expand Up @@ -42,34 +48,19 @@
<PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="2.0.0" />
<PackageReference Include="System.CommandLine" Version="2.0.10" />
<!-- external -->
<PackageReference Include="GitHub.Copilot.SDK" Version="0.3.0" />
<PackageReference Include="YamlDotNet" Version="18.0.0" />
<PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" Version="18.0.0" />
<PackageReference Include="GitHub.Copilot.SDK" Version="1.0.7" />
<PackageReference Include="YamlDotNet" Version="18.1.0" />
<PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" Version="18.1.0" />

<!--
Transitive pin: GitHub.Copilot.SDK still drags in Nerdbank.MessagePack
1.0.2 which has a known high-severity vulnerability (GHSA-2cwq-pwfr-wcw3).
Fixed in 1.1.62. Drop this once the upstream chain bumps past 1.1.62.
-->
<PackageReference Include="Nerdbank.MessagePack" Version="1.2.4" />

<!--
Transitive pin: GitHub.Copilot.SDK -> StreamJsonRpc 2.24.84 drags in
MessagePack 2.5.198 which has a known high-severity vulnerability
(GHSA-hv8m-jj95-wg3x, out-of-bounds read in LZ4 decompression). Patched in
the v2 line as of 2.5.301. Drop this once the upstream chain bumps past it.
-->
<PackageReference Include="MessagePack" Version="2.5.301" />

<!--
Transitive pin: Microsoft.Data.Sqlite 10.0.7 -> SQLitePCLRaw.bundle_e_sqlite3
Transitive pin: Microsoft.Data.Sqlite 10.0.10 -> SQLitePCLRaw.bundle_e_sqlite3
drags in the native SQLitePCLRaw.lib.e_sqlite3 2.1.11, which bundles a
SQLite older than 3.50.2 and carries a known high-severity vulnerability
(GHSA-2m69-gcr7-jv3q / CVE-2025-6965). 3.50.3 ships the patched SQLite
native library and is API-compatible with the 2.1.x managed provider.
Drop this once Microsoft.Data.Sqlite bumps past it.
-->
<PackageReference Include="SQLitePCLRaw.lib.e_sqlite3" Version="3.50.3" />
<PackageReference Include="SQLitePCLRaw.lib.e_sqlite3" Version="3.53.3" />
</ItemGroup>

</Project>
18 changes: 9 additions & 9 deletions eng/skill-validator/tests/Evaluate/RunnerTests.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System.Diagnostics;
using System.Text.Json;
using GitHub.Copilot.SDK;
using GitHub.Copilot;
using SkillValidator.Evaluate;
using SkillValidator.Shared;

Expand Down Expand Up @@ -213,25 +213,25 @@ public async Task SetsWorkingDirectoryToWorkDir()
public async Task SetsConfigDirToUniqueTempDirForSkillIsolation()
{
var config = await AgentRunner.BuildSessionConfig(MockSkill, null, "gpt-4.1", "C:\\tmp\\work");
Assert.NotEqual("C:\\tmp\\work", config.ConfigDir);
Assert.StartsWith(Path.GetTempPath(), config.ConfigDir);
Assert.True(Directory.Exists(config.ConfigDir));
Assert.NotEqual("C:\\tmp\\work", config.ConfigDirectory);
Assert.StartsWith(Path.GetTempPath(), config.ConfigDirectory);
Assert.True(Directory.Exists(config.ConfigDirectory));
}

[Fact]
public async Task SetsConfigDirToUniqueTempDirEvenWithoutSkill()
{
var config = await AgentRunner.BuildSessionConfig(null, null, "gpt-4.1", "C:\\tmp\\work");
Assert.NotEqual("C:\\tmp\\work", config.ConfigDir);
Assert.StartsWith(Path.GetTempPath(), config.ConfigDir);
Assert.NotEqual("C:\\tmp\\work", config.ConfigDirectory);
Assert.StartsWith(Path.GetTempPath(), config.ConfigDirectory);
}

[Fact]
public async Task EachCallGetsUniqueConfigDir()
{
var config1 = await AgentRunner.BuildSessionConfig(null, null, "gpt-4.1", "C:\\tmp\\work");
var config2 = await AgentRunner.BuildSessionConfig(null, null, "gpt-4.1", "C:\\tmp\\work");
Assert.NotEqual(config1.ConfigDir, config2.ConfigDir);
Assert.NotEqual(config1.ConfigDirectory, config2.ConfigDirectory);
}

[Fact]
Expand Down Expand Up @@ -356,7 +356,7 @@ public async Task DropsMcpCwd()
var config = await AgentRunner.BuildSessionConfig(MockSkill, null, "gpt-4.1", "C:\\tmp\\work", mcpServers);
Assert.NotNull(config.McpServers);
var entry = (McpStdioServerConfig)config.McpServers["ok"];
Assert.Null(entry.Cwd);
Assert.Null(entry.WorkingDirectory);
}

[Fact]
Expand Down Expand Up @@ -451,7 +451,7 @@ public async Task PluginRootNullPreservesSkillDirectories()

public class ExtractPathFromToolArgsTests
{
private static PreToolUseHookInput MakeInput(object? toolArgs) =>
private static PreToolUseHookInput MakeInput(JsonElement? toolArgs) =>
new() { ToolArgs = toolArgs };

[Fact]
Expand Down