-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Add validation script and nested AGENTS.md to enforce agent skill quality #53721
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
c0dac1b
63b0755
7664588
735901b
208de5f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| # Agent Skills | ||
|
|
||
| When creating skills, follow: | ||
| - Agent skills specification: https://agentskills.io/specification.md | ||
| - Best practices: https://agentskills.io/skill-creation/best-practices.md | ||
|
|
||
| ## Structure | ||
|
|
||
| ``` | ||
| .github/skills/skill-name/ | ||
| ├── SKILL.md # Required: metadata + instructions | ||
| ├── scripts/ # Optional: executable code | ||
| ├── references/ # Optional: documentation | ||
| ├── assets/ # Optional: templates, resources | ||
| └── ... # Any additional files or directories | ||
| ``` | ||
|
|
||
| ## Quick Checklist | ||
|
|
||
| - [ ] Run `dotnet .github/skills/ValidateSkill.cs <skill-dir>` to validate format. | ||
| - [ ] `description` describes what the skill does and when to use it. Skill body does not include "When to use this skill". | ||
| - [ ] Skill does not explain things the agent already knows. Focus on what's specific to the task at hand. | ||
| - [ ] Deterministic processes use scripts (for example, to fetch and format data from an API). | ||
| - [ ] Scripts use PowerShell or .NET file-based apps, not bash. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| #!/usr/bin/env dotnet | ||
| #:property ManagePackageVersionsCentrally=false | ||
| #:property PublishAot=false | ||
| #:package YamlDotNet@16.3.0 | ||
|
|
||
| using YamlDotNet.Serialization; | ||
| using System.Text.RegularExpressions; | ||
|
|
||
| if (args.Length == 0) | ||
| { | ||
| Console.Error.WriteLine("Usage: dotnet ValidateSkill.cs <path-to-skill-directory>"); | ||
| return 1; | ||
| } | ||
|
|
||
| string skillDir = Path.GetFullPath(args[0]); | ||
| string skillName = Path.GetFileName(skillDir); | ||
|
lbussell marked this conversation as resolved.
Outdated
|
||
| string skillFile = Path.Combine(skillDir, "SKILL.md"); | ||
|
|
||
| // SKILL.md must exist in the skill directory | ||
| if (!File.Exists(skillFile)) | ||
| { | ||
| Console.Error.WriteLine($"SKILL.md not found in {skillDir}"); | ||
| return 1; | ||
| } | ||
|
|
||
| string text = File.ReadAllText(skillFile); | ||
|
|
||
| // SKILL.md must begin with YAML frontmatter delimited by --- | ||
| if (!text.StartsWith("---")) | ||
| { | ||
| Console.Error.WriteLine("No YAML frontmatter found."); | ||
| return 1; | ||
| } | ||
|
|
||
| int endIndex = text.IndexOf("---", 3); | ||
| if (endIndex < 0) | ||
| { | ||
| Console.Error.WriteLine("Unterminated YAML frontmatter."); | ||
| return 1; | ||
| } | ||
|
|
||
| string yaml = text.Substring(3, endIndex - 3).Trim(); | ||
|
lbussell marked this conversation as resolved.
Outdated
|
||
|
|
||
| IDeserializer deserializer = new DeserializerBuilder().Build(); | ||
| Dictionary<string, object> frontmatter = deserializer.Deserialize<Dictionary<string, object>>(yaml); | ||
|
|
||
|
Comment on lines
+47
to
+49
|
||
| // name is required | ||
| if (!frontmatter.TryGetValue("name", out object? nameValue) || nameValue is not string frontmatterName) | ||
| { | ||
| Console.Error.WriteLine("Frontmatter missing 'name' field."); | ||
| return 1; | ||
| } | ||
|
|
||
| // name must be 1-64 characters | ||
| if (frontmatterName.Length == 0 || frontmatterName.Length > 64) | ||
| { | ||
| Console.Error.WriteLine($"Name is {frontmatterName.Length} chars (must be 1-64)."); | ||
| return 1; | ||
| } | ||
|
|
||
| // name: lowercase alphanumeric and hyphens only, no leading/trailing/consecutive hyphens | ||
| if (!Regex.IsMatch(frontmatterName, @"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") | ||
| || frontmatterName.Contains("--")) | ||
| { | ||
| Console.Error.WriteLine($"Invalid name '{frontmatterName}'. Must be lowercase letters, numbers, and hyphens only. Must not start/end with a hyphen or contain consecutive hyphens."); | ||
| return 1; | ||
| } | ||
|
|
||
| // name must match the parent directory name | ||
| if (!string.Equals(skillName, frontmatterName, StringComparison.Ordinal)) | ||
| { | ||
| Console.Error.WriteLine($"Name mismatch: directory is '{skillName}' but SKILL.md name is '{frontmatterName}'."); | ||
| return 1; | ||
| } | ||
|
|
||
| // description is required | ||
| if (!frontmatter.TryGetValue("description", out object? descValue) || descValue is not string description) | ||
| { | ||
| Console.Error.WriteLine("Frontmatter missing 'description' field."); | ||
| return 1; | ||
| } | ||
|
|
||
| // description must be 1-1024 characters | ||
| if (description.Length > 1024) | ||
| { | ||
| Console.Error.WriteLine($"Description is {description.Length} chars (max 1024)."); | ||
|
lbussell marked this conversation as resolved.
Outdated
|
||
| return 1; | ||
| } | ||
|
|
||
| // Keep SKILL.md under 500 lines; move detailed content to references/ or scripts/ | ||
| // See "Progressive Disclosure" at https://agentskills.io/specification.md | ||
| int lineCount = text.Split('\n').Length; | ||
| if (lineCount > 500) | ||
| { | ||
| Console.Error.WriteLine($"SKILL.md is {lineCount} lines (max 500). See \"Progressive Disclosure\" at https://agentskills.io/specification.md"); | ||
| return 1; | ||
|
Comment on lines
+95
to
+99
|
||
| } | ||
|
|
||
| Console.WriteLine($"Skill '{frontmatterName}' is valid."); | ||
| return 0; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| --- | ||
| name: incremental-test | ||
| description: >- | ||
| Run dotnet.Tests incrementally without a full build.cmd rebuild. Use after | ||
| modifying source code in SDK projects to quickly build only changed projects, | ||
| deploy their outputs into the redist SDK layout, and run tests against them. | ||
| --- | ||
|
|
||
| # Incremental Test Runner for dotnet.Tests | ||
|
|
||
| ## Prerequisites | ||
|
|
||
| A full build must have been completed at least once (`build.cmd` or `build.sh`) so the redist SDK layout exists at `artifacts/bin/redist/Debug/dotnet/sdk/<version>/`. | ||
|
|
||
| ## Workflow | ||
|
|
||
| ### Step 1: Build modified projects | ||
|
|
||
| Build each modified project using the repo-local dotnet: | ||
|
|
||
| ``` | ||
| ./.dotnet/dotnet build <path-to-project.csproj> -c Debug | ||
| ``` | ||
|
|
||
| ### Step 2: Copy outputs to the redist SDK layout | ||
|
|
||
| Run the copy script with the project names (matching the directory names under `artifacts/bin/`): | ||
|
|
||
| ```pwsh | ||
| scripts/Copy-ToRedist.ps1 <ProjectName> [<ProjectName2> ...] | ||
| ``` | ||
|
|
||
| For example, after modifying `Microsoft.DotNet.Cli.Utils` and `dotnet`: | ||
|
|
||
| ```pwsh | ||
| scripts/Copy-ToRedist.ps1 Microsoft.DotNet.Cli.Utils dotnet | ||
| ``` | ||
|
|
||
| The script discovers the SDK version directory, copies only DLLs that are already present in the redist layout, and handles satellite resource assemblies. | ||
|
|
||
| ### Step 3: Build the test project (if test code was modified) | ||
|
|
||
| ``` | ||
| ./.dotnet/dotnet build test/dotnet.Tests/dotnet.Tests.csproj | ||
| ``` | ||
|
|
||
| This project outputs directly to `artifacts/bin/redist/Debug/` via `TestHostFolder`. | ||
|
|
||
| ### Step 4: Run the tests | ||
|
|
||
| ``` | ||
| ./.dotnet/dotnet exec artifacts/bin/redist/Debug/dotnet.Tests.dll -method "*TestMethodName*" | ||
| ``` | ||
|
|
||
| Or via `dotnet test`: | ||
|
|
||
| ``` | ||
| ./.dotnet/dotnet test test/dotnet.Tests/dotnet.Tests.csproj --no-build --filter "Name~TestMethodName" | ||
| ``` | ||
|
|
||
| ## Gotchas | ||
|
|
||
| - Only DLLs **already present** in the redist layout are copied. If your change introduces a new shipped assembly, run a full `build.cmd`/`build.sh` instead. | ||
| - Multi-targeting projects (e.g., `net10.0` and `net472`): always use the `net10.0` output. | ||
| - The `dotnet` project builds into `artifacts/bin/dotnet/Debug/net10.0/`, not `artifacts/bin/Cli/dotnet/...`. |
|
lbussell marked this conversation as resolved.
Outdated
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| #!/usr/bin/env pwsh | ||
| <# | ||
| .SYNOPSIS | ||
| Copies build output DLLs into the redist SDK layout for incremental testing. | ||
|
|
||
| .DESCRIPTION | ||
| After building modified projects with `dotnet build`, this script copies their | ||
| output assemblies into the redist SDK layout so that dotnet.Tests can run against them. | ||
|
|
||
| .PARAMETER Projects | ||
| One or more project names whose output DLLs should be copied. | ||
| Use the directory name under artifacts/bin/ (e.g., "Microsoft.DotNet.Cli.Utils", "dotnet"). | ||
|
|
||
| .EXAMPLE | ||
| ./Copy-ToRedist.ps1 Microsoft.DotNet.Cli.Utils dotnet | ||
| #> | ||
|
|
||
| param( | ||
| [Parameter(Mandatory, Position = 0, ValueFromRemainingArguments)] | ||
| [string[]]$Projects | ||
| ) | ||
|
|
||
| $ErrorActionPreference = 'Stop' | ||
|
|
||
| $repoRoot = git rev-parse --show-toplevel | ||
| if ($LASTEXITCODE -ne 0) { throw "Not inside a git repository." } | ||
|
|
||
| $redistSdkBase = Join-Path $repoRoot 'artifacts' 'bin' 'redist' 'Debug' 'dotnet' 'sdk' | ||
|
|
||
| if (-not (Test-Path $redistSdkBase)) { | ||
| throw "Redist SDK layout not found at '$redistSdkBase'. Run a full build first (build.cmd / build.sh)." | ||
| } | ||
|
|
||
| $sdkVersionDir = Get-ChildItem $redistSdkBase -Directory | | ||
| Sort-Object LastWriteTime -Descending | | ||
| Select-Object -First 1 | ||
|
|
||
| if (-not $sdkVersionDir) { | ||
| throw "No SDK version directory found under '$redistSdkBase'." | ||
| } | ||
|
|
||
| $targetDir = $sdkVersionDir.FullName | ||
| Write-Host "Target SDK directory: $targetDir" | ||
|
|
||
| foreach ($project in $Projects) { | ||
| $outputDir = Join-Path $repoRoot 'artifacts' 'bin' $project 'Debug' 'net10.0' | ||
|
|
||
| if (-not (Test-Path $outputDir)) { | ||
| Write-Warning "Build output not found: $outputDir — skipping '$project'." | ||
| continue | ||
| } | ||
|
|
||
| # Find DLLs in the project output | ||
| $dlls = Get-ChildItem $outputDir -Filter '*.dll' -File | ||
|
|
||
| foreach ($dll in $dlls) { | ||
| $targetPath = Join-Path $targetDir $dll.Name | ||
|
|
||
| # Safety: only copy DLLs that already exist in the redist layout | ||
| if (-not (Test-Path $targetPath)) { | ||
| continue | ||
| } | ||
|
|
||
| Copy-Item $dll.FullName $targetPath -Force | ||
| Write-Host " Copied $($dll.Name)" | ||
| } | ||
|
|
||
| # Copy satellite resource assemblies (e.g., cs/, de/, etc.) | ||
| $cultureDirs = Get-ChildItem $outputDir -Directory | Where-Object { | ||
| Test-Path (Join-Path $_.FullName '*.resources.dll') | ||
| } | ||
|
|
||
| foreach ($cultureDir in $cultureDirs) { | ||
| $targetCultureDir = Join-Path $targetDir $cultureDir.Name | ||
| if (-not (Test-Path $targetCultureDir)) { continue } | ||
|
|
||
| $resourceDlls = Get-ChildItem $cultureDir.FullName -Filter '*.resources.dll' -File | ||
| foreach ($resDll in $resourceDlls) { | ||
| $resTarget = Join-Path $targetCultureDir $resDll.Name | ||
| if (-not (Test-Path $resTarget)) { continue } | ||
|
|
||
| Copy-Item $resDll.FullName $resTarget -Force | ||
| Write-Host " Copied $($cultureDir.Name)/$($resDll.Name)" | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Write-Host "Done." |
Uh oh!
There was an error while loading. Please reload this page.