From 043ad56d770cd682bb8d2d708de83642d838d7b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 23 Mar 2026 19:01:25 +0100 Subject: [PATCH 1/3] Address comments from skill validation --- .../skills/migrate-mstest-v1v2-to-v3/SKILL.md | 60 +++++------ .../skills/migrate-mstest-v3-to-v4/SKILL.md | 102 +++++++++--------- .../skills/migrate-vstest-to-mtp/SKILL.md | 76 ++++++++----- .../skills/mtp-hot-reload/SKILL.md | 12 +-- plugins/dotnet-test/skills/run-tests/SKILL.md | 10 +- .../skills/writing-mstest-tests/SKILL.md | 53 +++++---- 6 files changed, 172 insertions(+), 141 deletions(-) diff --git a/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md b/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md index 200411b4e3..3e573012f0 100644 --- a/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md @@ -8,20 +8,20 @@ description: > updating MSTest packages from 1.x/2.x to 3.x. USE FOR: upgrading from MSTest v1 assembly references (Microsoft.VisualStudio.QualityTools.UnitTestFramework) or MSTest v2 NuGet - (MSTest.TestFramework 1.x–2.x) to MSTest v3, fixing assertion overload + (MSTest.TestFramework 1.x-2.x) to MSTest v3, fixing assertion overload errors (AreEqual/AreNotEqual), updating DataRow constructors, replacing .testsettings with .runsettings, timeout behavior changes, target framework - compatibility (.NET 5 dropped — use .NET 6+; .NET Fx < 4.6.2 dropped), + compatibility (.NET 5 dropped -- use .NET 6+; .NET Fx < 4.6.2 dropped), adopting MSTest.Sdk. - First step toward MSTest v4 — after this, use migrate-mstest-v3-to-v4. + First step toward MSTest v4 -- after this, use migrate-mstest-v3-to-v4. DO NOT USE FOR: migrating to MSTest v4 (use migrate-mstest-v3-to-v4), migrating between frameworks (MSTest to xUnit/NUnit), or general .NET upgrades unrelated to MSTest. --- -# MSTest v1/v2 → v3 Migration +# MSTest v1/v2 -> v3 Migration -Migrate a test project from MSTest v1 (assembly references) or MSTest v2 (NuGet 1.x–2.x) to MSTest v3. MSTest v3 is **not binary compatible** with v1/v2 — libraries compiled against v1/v2 must be recompiled. +Migrate a test project from MSTest v1 (assembly references) or MSTest v2 (NuGet 1.x-2.x) to MSTest v3. MSTest v3 is **not binary compatible** with v1/v2 -- libraries compiled against v1/v2 must be recompiled. ## When to Use @@ -34,7 +34,7 @@ Migrate a test project from MSTest v1 (assembly references) or MSTest v2 (NuGet ## When Not to Use - Project already uses MSTest v3 (3.x packages) -- Upgrading v3 to v4 — use `migrate-mstest-v3-to-v4` +- Upgrading v3 to v4 -- use `migrate-mstest-v3-to-v4` - Migrating between frameworks (MSTest to xUnit/NUnit) ## Inputs @@ -56,31 +56,31 @@ MSTest v3 introduces these breaking changes from v1/v2. Address only the ones re | `DataRow` max 16 constructor parameters (early v3) | Compile error if >16 args; fixed in later v3 versions | Update to latest 3.x, or refactor test / wrap extra params in array | | `.testsettings` / `` no longer supported | Settings silently ignored | Delete `.testsettings`, create `.runsettings` with equivalent config | | Timeout behavior unified across .NET Core / Framework | Tests with `[Timeout]` may behave differently | Verify timeout values; adjust if needed | -| Dropped target frameworks: .NET 5, .NET Fx < 4.6.2, netstandard1.0, UWP < 16299, WinUI < 18362 | Build error | Update TFM: .NET 5 → net8.0 (LTS) or net6.0+, netfx → net462+, netstandard1.0 → netstandard2.0. Note: net6.0, net8.0, net9.0 are all supported | +| Dropped target frameworks: .NET 5, .NET Fx < 4.6.2, netstandard1.0, UWP < 16299, WinUI < 18362 | Build error | Update TFM: .NET 5 -> net8.0 (LTS) or net6.0+, netfx -> net462+, netstandard1.0 -> netstandard2.0. Note: net6.0, net8.0, net9.0 are all supported | | Not binary compatible with v1/v2 | Libraries compiled against v1/v2 must be recompiled | Recompile all dependencies against v3 | ## Response Guidelines - **Focused fix requests** (user has specific compilation errors after upgrading): Address only the relevant breaking change from the table above. Show a concise before/after fix. Do not walk through the full migration workflow. - **Specific feature migration** (user asks about one aspect like .testsettings, DataRow, or assertions): Address only that specific aspect with a concrete fix. Do not walk through the entire migration workflow or unrelated breaking changes. -- **"What to expect" questions** (user asks about breaking changes before upgrading): Present only the breaking changes that are clearly relevant to the user's visible code and configuration. For each, give a one-line fix summary. Do not include every possible breaking change — only the ones that apply. Do not walk through the full workflow. +- **"What to expect" questions** (user asks about breaking changes before upgrading): Present only the breaking changes that are clearly relevant to the user's visible code and configuration. For each, give a one-line fix summary. Do not include every possible breaking change -- only the ones that apply. Do not walk through the full workflow. - **Full migration requests** (user wants complete migration): Follow the complete workflow below. -- **Comparison questions** (user asks about v1 vs v2 differences): Explain concisely — v1 uses assembly references and requires removing them first; v2 uses NuGet and just needs a version bump. Both converge on the same v3 packages and breaking changes. +- **Comparison questions** (user asks about v1 vs v2 differences): Explain concisely -- v1 uses assembly references and requires removing them first; v2 uses NuGet and just needs a version bump. Both converge on the same v3 packages and breaking changes. ## Migration Paths - **MSTest v1 (assembly reference to QualityTools)**: Remove the assembly reference (Step 2), add v3 NuGet packages (Step 3), fix breaking changes (Step 5). -- **MSTest v2 (NuGet packages 1.x–2.x)**: Update package versions to 3.x (Step 3), fix breaking changes (Step 5). No assembly reference removal needed. +- **MSTest v2 (NuGet packages 1.x-2.x)**: Update package versions to 3.x (Step 3), fix breaking changes (Step 5). No assembly reference removal needed. -Both paths converge at Step 3 — the same v3 packages and breaking changes apply regardless of starting version. +Both paths converge at Step 3 -- the same v3 packages and breaking changes apply regardless of starting version. ## Workflow ### Step 1: Assess the project 1. Identify which MSTest version is currently in use: - - **Assembly reference**: Look for `Microsoft.VisualStudio.QualityTools.UnitTestFramework` in project references → MSTest v1 - - **NuGet packages**: Check `MSTest.TestFramework` and `MSTest.TestAdapter` package versions → v1 if 1.x, v2 if 2.x + - **Assembly reference**: Look for `Microsoft.VisualStudio.QualityTools.UnitTestFramework` in project references -> MSTest v1 + - **NuGet packages**: Check `MSTest.TestFramework` and `MSTest.TestAdapter` package versions -> v1 if 1.x, v2 if 2.x 2. Check if the project uses a `.testsettings` file (indicated by `` in test configuration) 3. Check if the target framework is dropped in v3 (see Step 4) 4. Run a clean build to establish a baseline of existing errors/warnings @@ -91,14 +91,14 @@ If the project uses MSTest v1 via assembly references: 1. Remove the reference to `Microsoft.VisualStudio.QualityTools.UnitTestFramework.dll` - In SDK-style projects, remove the `` element from the `.csproj` - - In non-SDK-style projects, remove via Visual Studio Solution Explorer → References → right-click → Remove + - In non-SDK-style projects, remove via Visual Studio Solution Explorer -> References -> right-click -> Remove 2. Save the project file ### Step 3: Update packages to MSTest v3 Choose one of these approaches: -**Option A — Install the MSTest metapackage (recommended):** +**Option A -- Install the MSTest metapackage (recommended):** Remove individual `MSTest.TestFramework` and `MSTest.TestAdapter` package references and replace with the unified `MSTest` metapackage: @@ -108,7 +108,7 @@ Remove individual `MSTest.TestFramework` and `MSTest.TestAdapter` package refere Also ensure `Microsoft.NET.Test.Sdk` is referenced (or update individual `MSTest.TestFramework` + `MSTest.TestAdapter` packages to 3.8.0 if you prefer not using the metapackage). -**Option B — Use MSTest.Sdk (SDK-style projects only):** +**Option B -- Use MSTest.Sdk (SDK-style projects only):** Change `` to ``. MSTest.Sdk automatically provides MSTest.TestFramework, MSTest.TestAdapter, MSTest.Analyzers, and Microsoft.NET.Test.Sdk. @@ -137,27 +137,27 @@ MSTest v3 supports .NET 6+, .NET Core 3.1, .NET Framework 4.6.2+, .NET Standard Run `dotnet build` and fix errors using the Breaking Changes Summary above. Key fixes: -**Assertion overloads** — MSTest v3 removed `Assert.AreEqual(object, object)` and `Assert.AreNotEqual(object, object)`. Add explicit generic type parameters: +**Assertion overloads** -- MSTest v3 removed `Assert.AreEqual(object, object)` and `Assert.AreNotEqual(object, object)`. Add explicit generic type parameters: ```csharp // Before (v1/v2) // After (v3) -Assert.AreEqual(expected, actual); → Assert.AreEqual(expected, actual); -Assert.AreNotEqual(a, b); → Assert.AreNotEqual(a, b); -Assert.AreSame(expected, actual); → Assert.AreSame(expected, actual); +Assert.AreEqual(expected, actual); -> Assert.AreEqual(expected, actual); +Assert.AreNotEqual(a, b); -> Assert.AreNotEqual(a, b); +Assert.AreSame(expected, actual); -> Assert.AreSame(expected, actual); ``` -**DataRow strict type matching** — argument types must exactly match parameter types. Implicit conversions that worked in v2 fail in v3: +**DataRow strict type matching** -- argument types must exactly match parameter types. Implicit conversions that worked in v2 fail in v3: ```csharp -// Error: 1L (long) won't convert to int parameter → fix: use 1 (int) -// Error: 1.0 (double) won't convert to float parameter → fix: use 1.0f (float) +// Error: 1L (long) won't convert to int parameter -> fix: use 1 (int) +// Error: 1.0 (double) won't convert to float parameter -> fix: use 1.0f (float) ``` -**Timeout behavior** — unified across .NET Core and .NET Framework. Verify `[Timeout]` values still work. +**Timeout behavior** -- unified across .NET Core and .NET Framework. Verify `[Timeout]` values still work. ### Step 6: Replace .testsettings with .runsettings -The `.testsettings` file and `` are no longer supported in MSTest v3. **Delete the `.testsettings` file** and create a `.runsettings` file — do not keep both. +The `.testsettings` file and `` are no longer supported in MSTest v3. **Delete the `.testsettings` file** and create a `.runsettings` file -- do not keep both. Key mappings: @@ -165,15 +165,15 @@ Key mappings: |---|---| | `TestTimeout` property | `30000` | | Deployment config | `true` or remove | -| Assembly resolution settings | Remove — not needed in modern .NET | +| Assembly resolution settings | Remove -- not needed in modern .NET | | Data collectors | `` section | > **Important**: Map timeout to `` (per-test), **not** `` (session-wide). Remove `` entirely. ### Step 7: Verify -1. Run `dotnet build` — confirm zero errors and review any new warnings -2. Run `dotnet test` — confirm all tests pass +1. Run `dotnet build` -- confirm zero errors and review any new warnings +2. Run `dotnet test` -- confirm all tests pass 3. Compare test results (pass/fail counts) to the pre-migration baseline 4. Check that no tests were silently dropped due to discovery changes @@ -181,7 +181,7 @@ Key mappings: - [ ] MSTest v3 packages (or MSTest.Sdk) correctly referenced; v1/v2 references removed - [ ] Project builds with zero errors -- [ ] All tests pass (`dotnet test`) — compare pass/fail counts to pre-migration baseline +- [ ] All tests pass (`dotnet test`) -- compare pass/fail counts to pre-migration baseline - [ ] `.testsettings` replaced with `.runsettings` (if applicable) ## Next Step @@ -192,5 +192,5 @@ After v3 migration, use `migrate-mstest-v3-to-v4` for MSTest v4. | Pitfall | Solution | |---------|----------| -| Missing `Microsoft.NET.Test.Sdk` | Add package reference — required for test discovery with VSTest | +| Missing `Microsoft.NET.Test.Sdk` | Add package reference -- required for test discovery with VSTest | | MSTest.Sdk tests not found by `vstest.console` | MSTest.Sdk defaults to Microsoft.Testing.Platform; add explicit `Microsoft.NET.Test.Sdk` for VSTest compatibility | diff --git a/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md b/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md index 2985540051..3ff330d315 100644 --- a/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md @@ -7,20 +7,20 @@ description: > after updating MSTest packages from 3.x to 4.x. Also use for target framework compatibility (e.g. net6.0/net7.0 support with MSTest v4). USE FOR: upgrading MSTest packages from 3.x to 4.x, fixing source breaking - changes (Execute → ExecuteAsync, CallerInfo constructor, ClassCleanupBehavior + changes (Execute -> ExecuteAsync, CallerInfo constructor, ClassCleanupBehavior removal, TestContext.Properties, Assert API changes, ExpectedExceptionAttribute removal, TestTimeout enum removal), resolving behavioral changes (TreatDiscoveryWarningsAsErrors, TestContext lifecycle, TestCase.Id changes, - MSTest.Sdk MTP changes), handling dropped TFMs (net5.0–net7.0 dropped — only + MSTest.Sdk MTP changes), handling dropped TFMs (net5.0-net7.0 dropped -- only net8.0+, net462, uap10.0 supported). DO NOT USE FOR: migrating from MSTest v1/v2 to v3 (use migrate-mstest-v1v2-to-v3 first), migrating between test frameworks, or general .NET upgrades unrelated to MSTest. --- -# MSTest v3 → v4 Migration +# MSTest v3 -> v4 Migration -Migrate a test project from MSTest v3 to MSTest v4. The outcome is a project using MSTest v4 that builds cleanly, passes tests, and accounts for every source-incompatible and behavioral change. MSTest v4 is **not binary compatible** with MSTest v3 — any library compiled against v3 must be recompiled against v4. +Migrate a test project from MSTest v3 to MSTest v4. The outcome is a project using MSTest v4 that builds cleanly, passes tests, and accounts for every source-incompatible and behavioral change. MSTest v4 is **not binary compatible** with MSTest v3 -- any library compiled against v3 must be recompiled against v4. ## When to Use @@ -32,8 +32,8 @@ Migrate a test project from MSTest v3 to MSTest v4. The outcome is a project usi ## When Not to Use -- The project already uses MSTest v4 and builds cleanly — migration is done -- Upgrading from MSTest v1 or v2 — use `migrate-mstest-v1v2-to-v3` first, then return here +- The project already uses MSTest v4 and builds cleanly -- migration is done +- Upgrading from MSTest v1 or v2 -- use `migrate-mstest-v1v2-to-v3` first, then return here - The project does not use MSTest - Migrating between test frameworks (e.g., MSTest to xUnit or NUnit) @@ -48,24 +48,24 @@ Migrate a test project from MSTest v3 to MSTest v4. The outcome is a project usi ## Response Guidelines - **Focused fix requests** (user has specific compilation errors after upgrading): Address only the relevant breaking changes from Step 3. Show concrete before/after code using the user's actual types and method names. Mention any related analyzer that could have caught this earlier (e.g., MSTEST0006 for ExpectedException). Do not walk through the entire migration workflow. -- **"What to expect" questions** (user asks about breaking changes before upgrading): Present ALL major breaking changes from the Step 3 quick-lookup table — not just the ones visible in the current code. For each, provide a one-line fix summary. Also mention key behavioral changes from Step 4 (especially TestCase.Id history impact and TreatDiscoveryWarningsAsErrors default). If project code is available, highlight which changes apply directly. +- **"What to expect" questions** (user asks about breaking changes before upgrading): Present ALL major breaking changes from the Step 3 quick-lookup table -- not just the ones visible in the current code. For each, provide a one-line fix summary. Also mention key behavioral changes from Step 4 (especially TestCase.Id history impact and TreatDiscoveryWarningsAsErrors default). If project code is available, highlight which changes apply directly. - **Full migration requests** (user wants complete migration): Follow the complete workflow below. - **Behavioral/runtime symptom reports** (user describes test execution differences without build errors): Match described symptoms to the behavioral changes table in Step 4. Provide targeted, symptom-specific advice. Mention other behavioral changes the user should watch for. Do not walk through source breaking changes unless the user also has build errors. -- **CI/test-discovery issues** (tests not discovered, vstest.console stopped working, CI pipeline failures after upgrading): Focus on §4.5 (MSTest.Sdk defaults to MTP mode, which does not include Microsoft.NET.Test.Sdk — needed for vstest.console) and §4.4 (TreatDiscoveryWarningsAsErrors). Explain the root cause clearly and give both fix options (add Microsoft.NET.Test.Sdk package or switch to `dotnet test`). Do not walk through the full migration workflow. +- **CI/test-discovery issues** (tests not discovered, vstest.console stopped working, CI pipeline failures after upgrading): Focus on 4.5 (MSTest.Sdk defaults to MTP mode, which does not include Microsoft.NET.Test.Sdk -- needed for vstest.console) and 4.4 (TreatDiscoveryWarningsAsErrors). Explain the root cause clearly and give both fix options (add Microsoft.NET.Test.Sdk package or switch to `dotnet test`). Do not walk through the full migration workflow. - **Explanatory questions** (user asks "is this a known change?", "what else should I watch out for?"): Explain the relevant changes and advise. Mention related changes the user might encounter next. Do not prescribe a full migration procedure. ## Workflow -> **Commit strategy:** Commit at each logical boundary — after updating packages (Step 2), after resolving source breaking changes (Step 3), after addressing behavioral changes (Step 4). This keeps each commit focused and reviewable. +> **Commit strategy:** Commit at each logical boundary -- after updating packages (Step 2), after resolving source breaking changes (Step 3), after addressing behavioral changes (Step 4). This keeps each commit focused and reviewable. ### Step 1: Assess the project 1. Identify the current MSTest version by checking package references for `MSTest`, `MSTest.TestFramework`, `MSTest.TestAdapter`, or `MSTest.Sdk` in `.csproj`, `Directory.Build.props`, or `Directory.Packages.props`. 2. Confirm the project is on MSTest v3 (3.x). If on v1 or v2, use `migrate-mstest-v1v2-to-v3` first. -3. Check target framework(s) — MSTest v4 drops support for .NET Core 3.1 through .NET 7. Supported target frameworks are: **net8.0**, **net9.0**, **net462** (.NET Framework 4.6.2+), **uap10.0.16299** (UWP), **net9.0-windows10.0.17763.0** (modern UWP), and **net8.0-windows10.0.18362.0** (WinUI). -4. Check for custom `TestMethodAttribute` subclasses — these require changes in v4. -5. Check for usages of `ExpectedExceptionAttribute` — removed in v4 (deprecated since v3 with analyzer MSTEST0006). -6. Check for usages of `Assert.ThrowsException` (deprecated) — removed in v4. +3. Check target framework(s) -- MSTest v4 drops support for .NET Core 3.1 through .NET 7. Supported target frameworks are: **net8.0**, **net9.0**, **net462** (.NET Framework 4.6.2+), **uap10.0.16299** (UWP), **net9.0-windows10.0.17763.0** (modern UWP), and **net8.0-windows10.0.18362.0** (WinUI). +4. Check for custom `TestMethodAttribute` subclasses -- these require changes in v4. +5. Check for usages of `ExpectedExceptionAttribute` -- removed in v4 (deprecated since v3 with analyzer MSTEST0006). +6. Check for usages of `Assert.ThrowsException` (deprecated) -- removed in v4. 7. Run a clean build to establish a baseline of existing errors/warnings. ### Step 2: Update packages to MSTest v4 @@ -97,23 +97,23 @@ Work through compilation errors systematically. Use this quick-lookup table to i | Error / Pattern in code | Breaking change | Fix | |---|---|---| -| Custom `TestMethodAttribute` overrides `Execute` | Execute removed | Change to `ExecuteAsync` returning `Task` (§3.1) | -| `[TestMethod("name")]` or custom attribute constructor | CallerInfo params added | Use `DisplayName = "name"` named param; propagate CallerInfo in subclasses (§3.2) | -| `ClassCleanupBehavior.EndOfClass` | Enum removed | Remove argument: just `[ClassCleanup]` (§3.3) | -| `TestContext.Properties.Contains("key")` | `Properties` is `IDictionary` | Change to `ContainsKey("key")` (§3.4) | -| `[Timeout(TestTimeout.Infinite)]` | `TestTimeout` enum removed | Replace with `[Timeout(int.MaxValue)]` (§3.5) | -| `TestContext.ManagedType` | Property removed | Use `FullyQualifiedTestClassName` (§3.6) | -| `Assert.AreEqual(a, b, "msg {0}", arg)` | Message+params overloads removed | Use string interpolation: `$"msg {arg}"` (§3.7) | -| `Assert.ThrowsException(...)` | Renamed | Replace with `Assert.ThrowsExactly(...)` or `Assert.Throws(...)` (§3.7) | -| `Assert.IsInstanceOfType(obj, out var t)` | Out parameter removed | Use `var t = Assert.IsInstanceOfType(obj)` (§3.7) | -| `[ExpectedException(typeof(T))]` | Attribute removed | Move assertion into test body: `Assert.ThrowsExactly(() => ...)` (§3.8) | -| Project targets net5.0, net6.0, or net7.0 | TFM dropped | Change to net8.0 or net9.0 (§3.9) | +| Custom `TestMethodAttribute` overrides `Execute` | Execute removed | Change to `ExecuteAsync` returning `Task` (3.1) | +| `[TestMethod("name")]` or custom attribute constructor | CallerInfo params added | Use `DisplayName = "name"` named param; propagate CallerInfo in subclasses (3.2) | +| `ClassCleanupBehavior.EndOfClass` | Enum removed | Remove argument: just `[ClassCleanup]` (3.3) | +| `TestContext.Properties.Contains("key")` | `Properties` is `IDictionary` | Change to `ContainsKey("key")` (3.4) | +| `[Timeout(TestTimeout.Infinite)]` | `TestTimeout` enum removed | Replace with `[Timeout(int.MaxValue)]` (3.5) | +| `TestContext.ManagedType` | Property removed | Use `FullyQualifiedTestClassName` (3.6) | +| `Assert.AreEqual(a, b, "msg {0}", arg)` | Message+params overloads removed | Use string interpolation: `$"msg {arg}"` (3.7) | +| `Assert.ThrowsException(...)` | Renamed | Replace with `Assert.ThrowsExactly(...)` or `Assert.Throws(...)` (3.7) | +| `Assert.IsInstanceOfType(obj, out var t)` | Out parameter removed | Use `var t = Assert.IsInstanceOfType(obj)` (3.7) | +| `[ExpectedException(typeof(T))]` | Attribute removed | Move assertion into test body: `Assert.ThrowsExactly(() => ...)` (3.8) | +| Project targets net5.0, net6.0, or net7.0 | TFM dropped | Change to net8.0 or net9.0 (3.9) | > **Important**: Scan the entire project for ALL patterns above before starting fixes. Multiple breaking changes often coexist in the same project. -#### 3.1 TestMethodAttribute.Execute → ExecuteAsync +#### 3.1 TestMethodAttribute.Execute -> ExecuteAsync -If you have custom `TestMethodAttribute` subclasses that override `Execute`, change to `ExecuteAsync`. This change was made because the v3 synchronous `Execute` API caused deadlocks when test code used `async`/`await` internally — the synchronous wrapper would block the thread while the async operation needed that same thread to complete. +If you have custom `TestMethodAttribute` subclasses that override `Execute`, change to `ExecuteAsync`. This change was made because the v3 synchronous `Execute` API caused deadlocks when test code used `async`/`await` internally -- the synchronous wrapper would block the thread while the async operation needed that same thread to complete. ```csharp // Before (v3) @@ -126,7 +126,7 @@ public sealed class MyTestMethodAttribute : TestMethodAttribute } } -// After (v4) — Option A: wrap synchronous logic with Task.FromResult +// After (v4) -- Option A: wrap synchronous logic with Task.FromResult public sealed class MyTestMethodAttribute : TestMethodAttribute { public override Task ExecuteAsync(ITestMethod testMethod) @@ -136,7 +136,7 @@ public sealed class MyTestMethodAttribute : TestMethodAttribute } } -// After (v4) — Option B: make properly async +// After (v4) -- Option B: make properly async public sealed class MyTestMethodAttribute : TestMethodAttribute { public override async Task ExecuteAsync(ITestMethod testMethod) @@ -239,10 +239,10 @@ Assert.AreEqual(expected, actual, $"Expected {expected} but got {actual}"); // Before (v3) Assert.ThrowsException(() => DoSomething()); -// After (v4) — exact type match (same behavior as old ThrowsException) +// After (v4) -- exact type match (same behavior as old ThrowsException) Assert.ThrowsExactly(() => DoSomething()); -// After (v4) — also catches derived exception types +// After (v4) -- also catches derived exception types Assert.Throws(() => DoSomething()); ``` @@ -279,7 +279,7 @@ public void TestMethod() } ``` -**When the test has setup code before the throwing call**, wrap only the throwing call in the lambda — keep Arrange/Act separation clear: +**When the test has setup code before the throwing call**, wrap only the throwing call in the lambda -- keep Arrange/Act separation clear: ```csharp // Before (v3) @@ -324,7 +324,7 @@ public async Task FetchData_BadUrl_Throws() #### 3.9 Dropped target frameworks -MSTest v4 supports: **net8.0**, **net9.0**, **net462** (.NET Framework 4.6.2+), **uap10.0.16299** (UWP), **net9.0-windows10.0.17763.0** (modern UWP), and **net8.0-windows10.0.18362.0** (WinUI). All other frameworks are dropped — including net5.0, net6.0, net7.0, and netcoreapp3.1. +MSTest v4 supports: **net8.0**, **net9.0**, **net462** (.NET Framework 4.6.2+), **uap10.0.16299** (UWP), **net9.0-windows10.0.17763.0** (modern UWP), and **net8.0-windows10.0.18362.0** (WinUI). All other frameworks are dropped -- including net5.0, net6.0, net7.0, and netcoreapp3.1. If the test project targets an unsupported framework, update `TargetFramework`: @@ -360,12 +360,12 @@ These changes won't cause build errors but may affect test runtime behavior. | Symptom | Cause | Fix | |---|---|---| -| Tests show as new in Azure DevOps / test history lost | `TestCase.Id` generation changed (§4.3) | No code fix; history will re-baseline | -| `TestContext.TestName` throws in `[ClassInitialize]` | v4 enforces lifecycle scope (§4.2) | Move access to `[TestInitialize]` or test methods | -| Tests not discovered / discovery failures | `TreatDiscoveryWarningsAsErrors` now true (§4.4) | Fix warnings, or set to false in .runsettings | -| Tests hang that didn't before | AppDomain disabled by default (§4.1) | Set `DisableAppDomain` to false in .runsettings `RunConfiguration` | -| vstest.console can't find tests with MSTest.Sdk | MSTest.Sdk defaults to MTP; `Microsoft.NET.Test.Sdk` only added in VSTest mode (§4.5) | Add explicit package reference or switch to `dotnet test` | -| New warnings from analyzers | Analyzer severities upgraded (§4.6) | Fix warnings or suppress in .editorconfig | +| Tests show as new in Azure DevOps / test history lost | `TestCase.Id` generation changed (4.3) | No code fix; history will re-baseline | +| `TestContext.TestName` throws in `[ClassInitialize]` | v4 enforces lifecycle scope (4.2) | Move access to `[TestInitialize]` or test methods | +| Tests not discovered / discovery failures | `TreatDiscoveryWarningsAsErrors` now true (4.4) | Fix warnings, or set to false in .runsettings | +| Tests hang that didn't before | AppDomain disabled by default (4.1) | Set `DisableAppDomain` to false in .runsettings `RunConfiguration` | +| vstest.console can't find tests with MSTest.Sdk | MSTest.Sdk defaults to MTP; `Microsoft.NET.Test.Sdk` only added in VSTest mode (4.5) | Add explicit package reference or switch to `dotnet test` | +| New warnings from analyzers | Analyzer severities upgraded (4.6) | Fix warnings or suppress in .editorconfig | #### 4.1 DisableAppDomain defaults to true @@ -382,10 +382,10 @@ AppDomains are disabled by default. On .NET Framework, when running inside testh #### 4.2 TestContext throws when used incorrectly MSTest v4 now throws when accessing test-specific properties in the wrong lifecycle stage: -- `TestContext.FullyQualifiedTestClassName` — cannot be accessed in `[AssemblyInitialize]` -- `TestContext.TestName` — cannot be accessed in `[AssemblyInitialize]` or `[ClassInitialize]` +- `TestContext.FullyQualifiedTestClassName` -- cannot be accessed in `[AssemblyInitialize]` +- `TestContext.TestName` -- cannot be accessed in `[AssemblyInitialize]` or `[ClassInitialize]` -**Fix**: Move any code that accesses `TestContext.TestName` from `[ClassInitialize]` to `[TestInitialize]` or individual test methods, where per-test context is available. Do not replace `TestName` with `FullyQualifiedTestClassName` as a workaround — they have different semantics. +**Fix**: Move any code that accesses `TestContext.TestName` from `[ClassInitialize]` to `[TestInitialize]` or individual test methods, where per-test context is available. Do not replace `TestName` with `FullyQualifiedTestClassName` as a workaround -- they have different semantics. #### 4.3 TestCase.Id generation changed @@ -407,9 +407,9 @@ v4 uses stricter defaults. Discovery warnings are now treated as errors, which m #### 4.5 MSTest.Sdk and vstest.console compatibility -MSTest.Sdk defaults to Microsoft.Testing.Platform (MTP) mode. In MTP mode, MSTest.Sdk does **not** add a reference to `Microsoft.NET.Test.Sdk` — it only adds it in VSTest mode. This is not a v4-specific change; it applies to MSTest.Sdk v3 as well. Without `Microsoft.NET.Test.Sdk`, `vstest.console` cannot discover or run tests and will silently find zero tests. This commonly surfaces during migration when a CI pipeline uses `vstest.console` but the project uses MSTest.Sdk in its default MTP mode. +MSTest.Sdk defaults to Microsoft.Testing.Platform (MTP) mode. In MTP mode, MSTest.Sdk does **not** add a reference to `Microsoft.NET.Test.Sdk` -- it only adds it in VSTest mode. This is not a v4-specific change; it applies to MSTest.Sdk v3 as well. Without `Microsoft.NET.Test.Sdk`, `vstest.console` cannot discover or run tests and will silently find zero tests. This commonly surfaces during migration when a CI pipeline uses `vstest.console` but the project uses MSTest.Sdk in its default MTP mode. -**Option A — Switch to VSTest mode**: Set the `UseVSTest` property. MSTest.Sdk will then automatically add `Microsoft.NET.Test.Sdk`: +**Option A -- Switch to VSTest mode**: Set the `UseVSTest` property. MSTest.Sdk will then automatically add `Microsoft.NET.Test.Sdk`: ```xml @@ -420,7 +420,7 @@ MSTest.Sdk defaults to Microsoft.Testing.Platform (MTP) mode. In MTP mode, MSTes ``` -**Option B — Switch CI to `dotnet test`**: Replace `vstest.console` invocations in your CI pipeline with `dotnet test`. This works natively with MTP and is the recommended long-term approach for MSTest.Sdk projects. +**Option B -- Switch CI to `dotnet test`**: Replace `vstest.console` invocations in your CI pipeline with `dotnet test`. This works natively with MTP and is the recommended long-term approach for MSTest.Sdk projects. If you need VSTest during a transition period, Option A works without changing CI pipelines. @@ -434,8 +434,8 @@ Review and fix any new warnings, or suppress them in `.editorconfig` if intentio ### Step 5: Verify -1. Run `dotnet build` — confirm zero errors and review any new warnings -2. Run `dotnet test` — confirm all tests pass +1. Run `dotnet build` -- confirm zero errors and review any new warnings +2. Run `dotnet test` -- confirm all tests pass 3. Compare test results (pass/fail counts) to the pre-migration baseline 4. If using Azure DevOps test tracking, be aware that `TestCase.Id` changes may affect history continuity 5. Check that no tests were silently dropped due to stricter discovery @@ -456,8 +456,8 @@ Review and fix any new warnings, or suppress them in `.editorconfig` if intentio ## Related Skills -- `writing-mstest-tests` — for modern MSTest v4 assertion APIs and test authoring best practices -- `run-tests` — for running tests after migration +- `writing-mstest-tests` -- for modern MSTest v4 assertion APIs and test authoring best practices +- `run-tests` -- for running tests after migration ## Common Pitfalls @@ -466,11 +466,11 @@ Review and fix any new warnings, or suppress them in `.editorconfig` if intentio | Custom `TestMethodAttribute` still overrides `Execute` | Change to `ExecuteAsync` returning `Task` | | `TestMethodAttribute("display name")` no longer compiles | Use `TestMethodAttribute(DisplayName = "display name")` | | `ClassCleanupBehavior` enum not found | Remove the enum argument; `[ClassCleanup]` now always runs at end of class. For end-of-assembly cleanup, use `[AssemblyCleanup]` | -| `TestContext.Properties.Contains` missing | Use `ContainsKey` — `Properties` is now `IDictionary` | +| `TestContext.Properties.Contains` missing | Use `ContainsKey` -- `Properties` is now `IDictionary` | | `ExpectedException` attribute not found | Replace with `Assert.ThrowsExactly(() => ...)` inside the test body | | `Assert.ThrowsException` not found | Replace with `Assert.ThrowsExactly` (or `Assert.Throws` for derived types) | | `Assert.AreEqual` with format string args fails | Use string interpolation: `$"message {value}"` | | Tests hang that didn't before | AppDomain is disabled by default; on .NET Fx in testhost it is re-enabled automatically | -| Azure DevOps test history breaks | Expected — `TestCase.Id` generation changed; no code fix, results will re-baseline | +| Azure DevOps test history breaks | Expected -- `TestCase.Id` generation changed; no code fix, results will re-baseline | | Discovery warnings now fail the run | `TreatDiscoveryWarningsAsErrors` is true by default; fix the discovery warnings | -| Net6.0/net7.0 targets don't compile | Update to net8.0 — MSTest v4 supports net8.0, net9.0, net462, uap10.0.16299, modern UWP, and WinUI | +| Net6.0/net7.0 targets don't compile | Update to net8.0 -- MSTest v4 supports net8.0, net9.0, net462, uap10.0.16299, modern UWP, and WinUI | diff --git a/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md b/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md index 30c18fb660..dfe6b7e83a 100644 --- a/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md @@ -16,11 +16,11 @@ description: > projects. --- -# VSTest → Microsoft.Testing.Platform Migration +# VSTest -> Microsoft.Testing.Platform Migration Migrate a .NET test solution from VSTest to Microsoft.Testing.Platform (MTP). The outcome is a solution where all test projects run on MTP, `dotnet test` works correctly, and CI/CD pipelines are updated. -> **Important**: Do not mix VSTest-based and MTP-based .NET test projects in the same solution or run configuration — this is an unsupported scenario. +> **Important**: Do not mix VSTest-based and MTP-based .NET test projects in the same solution or run configuration -- this is an unsupported scenario. ## When to Use @@ -33,31 +33,40 @@ Migrate a .NET test solution from VSTest to Microsoft.Testing.Platform (MTP). Th ## When Not to Use -- The project already runs on Microsoft.Testing.Platform — migration is done -- Migrating between test frameworks (e.g., MSTest to xUnit.net) — different effort entirely -- The project builds UWP or packaged WinUI test projects — MTP does not support these yet -- The solution mixes .NET and non-.NET test adapters (e.g., JavaScript or C++ adapters) — VSTest is required -- Upgrading MSTest versions — use `migrate-mstest-v1v2-to-v3` or `migrate-mstest-v3-to-v4` +- The project already runs on Microsoft.Testing.Platform -- migration is done +- Migrating between test frameworks (e.g., MSTest to xUnit.net) -- different effort entirely +- The project builds UWP or packaged WinUI test projects -- MTP does not support these yet +- The solution mixes .NET and non-.NET test adapters (e.g., JavaScript or C++ adapters) -- VSTest is required +- Upgrading MSTest versions -- use `migrate-mstest-v1v2-to-v3` or `migrate-mstest-v3-to-v4` + +## Inputs + +| Input | Required | Description | +|-------|----------|-------------| +| Project or solution path | Yes | The `.csproj`, `.sln`, or `.slnx` entry point containing test projects | +| Test framework | No | MSTest, NUnit, xUnit.net v2, or xUnit.net v3. Auto-detected from package references | +| .NET SDK version | No | Determines `dotnet test` integration mode. Auto-detected via `dotnet --version` | +| CI/CD pipeline files | No | Paths to pipeline definitions that invoke `vstest.console` or `dotnet test` | ## Workflow ### Step 1: Assess the solution -1. Identify the test framework for each test project — see [references/platform-detection.md](references/platform-detection.md) for the package-to-framework mapping. Key indicators: +1. Identify the test framework for each test project -- see [references/platform-detection.md](references/platform-detection.md) for the package-to-framework mapping. Key indicators: - **MSTest**: References `MSTest` or `MSTest.TestAdapter`, or uses `MSTest.Sdk` (with `` not set to `false`). Note: `MSTest.TestFramework` alone is a library dependency, not a test project. - **NUnit**: References `NUnit3TestAdapter` - **xUnit.net**: References `xunit` and `xunit.runner.visualstudio` -2. Check the .NET SDK version (`dotnet --version`) — this determines how `dotnet test` integrates with MTP -3. Check whether a `Directory.Build.props` file exists at the solution or repo root — all MTP properties should go there for consistency +2. Check the .NET SDK version (`dotnet --version`) -- this determines how `dotnet test` integrates with MTP +3. Check whether a `Directory.Build.props` file exists at the solution or repo root -- all MTP properties should go there for consistency 4. Check for `vstest.console.exe` usage in CI scripts or pipeline definitions 5. Check for VSTest-specific `dotnet test` arguments in CI scripts: `--filter`, `--logger`, `--collect`, `--settings`, `--blame*` 6. Run `dotnet test` to establish a baseline of test pass/fail counts ### Step 2: Set up Directory.Build.props -> **Critical**: Always set MTP properties in `Directory.Build.props` at the solution or repo root — never per-project. This prevents inconsistent configuration where some projects use VSTest and others use MTP (an unsupported scenario). +> **Critical**: Always set MTP properties in `Directory.Build.props` at the solution or repo root -- never per-project. This prevents inconsistent configuration where some projects use VSTest and others use MTP (an unsupported scenario). -> **Note**: MTP requires test projects to have `Exe`. Only `MSTest.Sdk` sets this automatically. For all other setups (MSTest NuGet packages with `EnableMSTestRunner`, NUnit with `EnableNUnitRunner`, xUnit.net with `YTest.MTP.XUnit2`), you must set `Exe` explicitly — either per-project or in `Directory.Build.props` with a condition that targets only test projects. +> **Note**: MTP requires test projects to have `Exe`. Only `MSTest.Sdk` sets this automatically. For all other setups (MSTest NuGet packages with `EnableMSTestRunner`, NUnit with `EnableNUnitRunner`, xUnit.net with `YTest.MTP.XUnit2`), you must set `Exe` explicitly -- either per-project or in `Directory.Build.props` with a condition that targets only test projects. ### Step 3: Enable the framework-specific MTP runner @@ -65,7 +74,7 @@ Each framework has its own opt-in property. Add these in `Directory.Build.props` #### MSTest -**Option A — MSTest NuGet packages (3.2.0+):** +**Option A -- MSTest NuGet packages (3.2.0+):** ```xml @@ -76,9 +85,9 @@ Each framework has its own opt-in property. Add these in `Directory.Build.props` Ensure the project references MSTest 3.2.0 or later. If the version is already 3.2.0+, no MSTest version upgrade is needed for MTP migration. -**Option B — MSTest.Sdk:** +**Option B -- MSTest.Sdk:** -When using `MSTest.Sdk`, MTP is enabled by default — no `EnableMSTestRunner` or `OutputType Exe` property is needed (the SDK sets both automatically). The only action is: if the project has `true`, **remove it**. That property forces the project to use VSTest instead of MTP. +When using `MSTest.Sdk`, MTP is enabled by default -- no `EnableMSTestRunner` or `OutputType Exe` property is needed (the SDK sets both automatically). The only action is: if the project has `true`, **remove it**. That property forces the project to use VSTest instead of MTP. #### NUnit @@ -101,7 +110,7 @@ Requires `NUnit3TestAdapter` **5.0.0** or later. #### xUnit.net -Add a reference to `YTest.MTP.XUnit2` — this package provides MTP support for xUnit.net v2 projects without requiring an upgrade to xunit.v3. You must also set `OutputType` to `Exe`: +Add a reference to `YTest.MTP.XUnit2` -- this package provides MTP support for xUnit.net v2 projects without requiring an upgrade to xunit.v3. You must also set `OutputType` to `Exe`: ```xml @@ -146,7 +155,7 @@ Use the native MTP mode by adding a `test` section to `global.json`: } ``` -In this mode, `dotnet test` arguments are passed directly — for example, `dotnet test --report-trx`. +In this mode, `dotnet test` arguments are passed directly -- for example, `dotnet test --report-trx`. > **Important**: `global.json` does not support trailing commas. Ensure the JSON is strictly valid. @@ -188,16 +197,16 @@ VSTest-specific arguments must be translated to MTP equivalents. Build-related a **MSTest, NUnit, and xUnit.net v2 (with `YTest.MTP.XUnit2`)**: The VSTest `--filter` syntax is identical on both VSTest and MTP. No changes needed. -**xUnit.net v3 (native MTP)**: xUnit.net v3 does NOT support the VSTest `--filter` syntax on MTP. See the **VSTest → MTP filter translation** section in [references/filter-syntax.md](references/filter-syntax.md) for the complete translation table. Key translation example: +**xUnit.net v3 (native MTP)**: xUnit.net v3 does NOT support the VSTest `--filter` syntax on MTP. See the **VSTest -> MTP filter translation** section in [references/filter-syntax.md](references/filter-syntax.md) for the complete translation table. Key translation example: ```shell # VSTest dotnet test --filter "FullyQualifiedName~IntegrationTests&Category=Smoke" -# xUnit.net v3 MTP — using individual filters (AND behavior) +# xUnit.net v3 MTP -- using individual filters (AND behavior) dotnet test -- --filter-class *IntegrationTests* --filter-trait "Category=Smoke" -# xUnit.net v3 MTP — using query language (assembly/namespace/class/method[trait]) +# xUnit.net v3 MTP -- using query language (assembly/namespace/class/method[trait]) dotnet test -- --filter-query "/*/*/*IntegrationTests*/*[Category=Smoke]" ``` @@ -281,29 +290,40 @@ VSTest silently succeeds when zero tests are discovered. MTP fails with **exit c Once migration is complete and verified, remove packages that are only needed for VSTest: -- `Microsoft.NET.Test.Sdk` — not needed for MTP (MSTest.Sdk v4 already omits it by default) -- `xunit.runner.visualstudio` — only needed for VSTest discovery of xUnit.net (not needed when using `YTest.MTP.XUnit2`) -- `NUnit3TestAdapter` VSTest-only features — the adapter is still needed but only for the MTP runner +- `Microsoft.NET.Test.Sdk` -- not needed for MTP (MSTest.Sdk v4 already omits it by default) +- `xunit.runner.visualstudio` -- only needed for VSTest discovery of xUnit.net (not needed when using `YTest.MTP.XUnit2`) +- `NUnit3TestAdapter` VSTest-only features -- the adapter is still needed but only for the MTP runner > **Note**: If you need to maintain VSTest compatibility during a transition period, keep these packages. ### Step 10: Verify -1. Run `dotnet build` — confirm zero errors -2. Run `dotnet test` — confirm all tests pass +1. Run `dotnet build` -- confirm zero errors +2. Run `dotnet test` -- confirm all tests pass 3. Compare test pass/fail counts to the pre-migration baseline -4. Run the test executable directly (e.g., `./bin/Debug/net8.0/MyTests.exe`) — confirm it works +4. Run the test executable directly (e.g., `./bin/Debug/net8.0/MyTests.exe`) -- confirm it works 5. Verify CI pipeline produces the expected test result artifacts (TRX files, code coverage, crash dumps) 6. Test that Test Explorer in Visual Studio (17.14+) or VS Code discovers and runs tests +## Validation + +- [ ] All test projects use MTP runner (no VSTest-only configuration remains) +- [ ] `dotnet build` completes with zero errors +- [ ] `dotnet test` passes all tests and test counts match pre-migration baseline +- [ ] Test executable runs directly (e.g., `./bin/Debug/net8.0/MyTests.exe`) +- [ ] CI pipeline produces expected test result artifacts (TRX files, code coverage, crash dumps) +- [ ] Test Explorer in Visual Studio or VS Code discovers and runs tests +- [ ] No `vstest.console.exe` invocations remain in CI scripts +- [ ] `Exe` is set for all non-MSTest.Sdk test projects + ## Common Pitfalls | Pitfall | Solution | |---------|----------| -| Mixing VSTest and MTP projects in the same solution | Migrate all test projects together — mixed mode is unsupported | +| Mixing VSTest and MTP projects in the same solution | Migrate all test projects together -- mixed mode is unsupported | | `dotnet test` arguments ignored on .NET 9 and earlier | Use `--` to separate build args from MTP args: `dotnet test -- --report-trx` | | Exit code 8 on CI without failures | MTP fails when zero tests run; use `--ignore-exit-code 8` or fix test discovery | -| MSTest.Sdk v4 + vstest.console no longer works | MSTest.Sdk v4 no longer adds `Microsoft.NET.Test.Sdk` — add it explicitly or switch to `dotnet test` | +| MSTest.Sdk v4 + vstest.console no longer works | MSTest.Sdk v4 no longer adds `Microsoft.NET.Test.Sdk` -- add it explicitly or switch to `dotnet test` | | Missing `Exe` | Required for all setups except MSTest.Sdk (which sets it automatically) | ## Next Steps diff --git a/plugins/dotnet-test/skills/mtp-hot-reload/SKILL.md b/plugins/dotnet-test/skills/mtp-hot-reload/SKILL.md index 080129639c..0d6ce60e89 100644 --- a/plugins/dotnet-test/skills/mtp-hot-reload/SKILL.md +++ b/plugins/dotnet-test/skills/mtp-hot-reload/SKILL.md @@ -28,7 +28,7 @@ Set up and use Microsoft Testing Platform hot reload to rapidly iterate fixes on - User needs to write new tests from scratch (use general coding assistance) - User needs to diagnose why a test is failing (use diagnostic skills) - User wants Visual Studio Test Explorer hot reload (different feature, built into VS) -- Project uses VSTest — hot reload requires Microsoft Testing Platform (MTP) +- Project uses VSTest -- hot reload requires Microsoft Testing Platform (MTP) - User needs CI/CD pipeline configuration ## Inputs @@ -56,13 +56,13 @@ Install the `Microsoft.Testing.Extensions.HotReload` package: dotnet add package Microsoft.Testing.Extensions.HotReload ``` -> **Note**: When using `Microsoft.Testing.Platform.MSBuild` (included transitively by MSTest, NUnit, and xUnit runners), the extension is auto-registered when you install its NuGet package — no code changes needed. +> **Note**: When using `Microsoft.Testing.Platform.MSBuild` (included transitively by MSTest, NUnit, and xUnit runners), the extension is auto-registered when you install its NuGet package -- no code changes needed. ### Step 3: Enable hot reload Hot reload is activated by setting the `TESTINGPLATFORM_HOTRELOAD_ENABLED` environment variable to `1`. -**Option A — Set it in the shell before running tests:** +**Option A -- Set it in the shell before running tests:** ```shell # PowerShell @@ -72,7 +72,7 @@ $env:TESTINGPLATFORM_HOTRELOAD_ENABLED = "1" export TESTINGPLATFORM_HOTRELOAD_ENABLED=1 ``` -**Option B — Add it to `launchSettings.json` (recommended for repeatable use):** +**Option B -- Add it to `launchSettings.json` (recommended for repeatable use):** Create or update `Properties/launchSettings.json` in the test project: @@ -97,7 +97,7 @@ Run the test project directly (not through `dotnet test`) to use hot reload in c dotnet run --project ``` -To filter to specific failing tests, pass the filter after `--`. The syntax depends on the test framework — see [references/filter-syntax.md](references/filter-syntax.md) for full details. Quick examples: +To filter to specific failing tests, pass the filter after `--`. The syntax depends on the test framework -- see [references/filter-syntax.md](references/filter-syntax.md) for full details. Quick examples: | Framework | Filter syntax | |-----------|--------------| @@ -140,5 +140,5 @@ Once all tests pass: | Using `dotnet test` instead of `dotnet run` | Hot reload requires `dotnet run --project ` to run the test host directly in console mode | | Project uses VSTest, not MTP | Hot reload requires MTP. Migrate to MTP first or use VS Test Explorer hot reload | | Forgetting to set the environment variable | Set `TESTINGPLATFORM_HOTRELOAD_ENABLED=1` before running | -| Expecting Test Explorer integration | Console mode only — no VS/VS Code Test Explorer support | +| Expecting Test Explorer integration | Console mode only -- no VS/VS Code Test Explorer support | | Making unsupported code changes (rude edits) | Some changes (adding new types, changing method signatures) require a restart. Stop and re-run | diff --git a/plugins/dotnet-test/skills/run-tests/SKILL.md b/plugins/dotnet-test/skills/run-tests/SKILL.md index 56399b5b6a..dc32526284 100644 --- a/plugins/dotnet-test/skills/run-tests/SKILL.md +++ b/plugins/dotnet-test/skills/run-tests/SKILL.md @@ -50,7 +50,7 @@ Detect the test platform and framework, run tests, and apply filters using `dotn | MTP | 8 or 9 | `dotnet test [] -- ` | | MTP | 10+ | `dotnet test --project ` | -**Detection files to always check** (in order): `global.json` → `.csproj` → `Directory.Build.props` → `Directory.Packages.props` +**Detection files to always check** (in order): `global.json` -> `.csproj` -> `Directory.Build.props` -> `Directory.Packages.props` ### Step 1: Detect the test platform and framework @@ -171,9 +171,9 @@ These alternative invocations accept MTP command line arguments directly (no `-- See [references/filter-syntax.md](references/filter-syntax.md) for the complete filter syntax for each platform and framework combination. Key points: - **VSTest** (MSTest, xUnit v2, NUnit): `dotnet test --filter ` with `=`, `!=`, `~`, `!~` operators -- **MTP — MSTest and NUnit**: Same `--filter` syntax as VSTest; pass after `--` on SDK 8/9, directly on SDK 10+ -- **MTP — xUnit v3**: Uses `--filter-class`, `--filter-method`, `--filter-trait` (not VSTest expression syntax) -- **MTP — TUnit**: Uses `--treenode-filter` with path-based syntax +- **MTP -- MSTest and NUnit**: Same `--filter` syntax as VSTest; pass after `--` on SDK 8/9, directly on SDK 10+ +- **MTP -- xUnit v3**: Uses `--filter-class`, `--filter-method`, `--filter-trait` (not VSTest expression syntax) +- **MTP -- TUnit**: Uses `--treenode-filter` with path-based syntax ## Validation @@ -188,7 +188,7 @@ See [references/filter-syntax.md](references/filter-syntax.md) for the complete | Pitfall | Solution | |---------|----------| | Missing `Microsoft.NET.Test.Sdk` in a VSTest project | Tests won't be discovered. Add `` | -| Using VSTest `--filter` syntax with xUnit v3 on MTP | xUnit v3 on MTP uses `--filter-class`, `--filter-method`, etc. — not the VSTest expression syntax | +| Using VSTest `--filter` syntax with xUnit v3 on MTP | xUnit v3 on MTP uses `--filter-class`, `--filter-method`, etc. -- not the VSTest expression syntax | | Passing MTP args without `--` on .NET SDK 8/9 | Before .NET 10, MTP args must go after `--`: `dotnet test -- --report-trx` | | Using `--` for MTP args on .NET SDK 10+ | On .NET 10+, MTP args are passed directly: `dotnet test --report-trx` (using `--` still works but is unnecessary) | | Multi-TFM project runs tests for all frameworks | Use `--framework ` to target a specific framework | diff --git a/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md b/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md index 7818322108..ec490f8530 100644 --- a/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md +++ b/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md @@ -116,7 +116,7 @@ Assert.IsNull(value); Assert.IsNotNull(value); ``` -#### Exception testing — use `Assert.Throws` instead of `[ExpectedException]` +#### Exception testing -- use `Assert.Throws` instead of `[ExpectedException]` ```csharp // Synchronous @@ -142,7 +142,7 @@ Assert.IsEmpty(collection); Assert.IsNotEmpty(collection); ``` -Replace `Assert.IsTrue(x != null)` with `Assert.IsNotNull(x)` — specialized assertions give better failure messages than generic `Assert.IsTrue`. +Replace `Assert.IsTrue(x != null)` with `Assert.IsNotNull(x)` -- specialized assertions give better failure messages than generic `Assert.IsTrue`. #### String assertions @@ -156,11 +156,11 @@ Assert.MatchesRegex(@"\d{3}-\d{4}", phoneNumber); #### Type assertions ```csharp -// MSTest 3.x — out parameter +// MSTest 3.x -- out parameter Assert.IsInstanceOfType(result, out var typed); typed.Handle(); -// MSTest 4.x — returns directly +// MSTest 4.x -- returns directly var typed = Assert.IsInstanceOfType(result); ``` @@ -200,7 +200,7 @@ public void ApplyDiscount_ReturnsExpectedPrice(decimal price, int percent, decim Assert.AreEqual(expected, result); } -// ValueTuple — preferred (MSTest 3.7+) +// ValueTuple -- preferred (MSTest 3.7+) public static IEnumerable<(decimal price, int percent, decimal expected)> DiscountTestData => [ (100m, 10, 90m), @@ -222,7 +222,7 @@ public static IEnumerable set `TestContext` property -> `[TestInitialize]` + - With constructor injection of `TestContext`: Constructor (receives `TestContext`) -> `[TestInitialize]` 4. Test method -5. `[TestCleanup]` → `DisposeAsync` → `Dispose` — per test -6. `[ClassCleanup]` — once per class -7. `[AssemblyCleanup]` — once per assembly +5. `[TestCleanup]` -> `DisposeAsync` -> `Dispose` -- per test +6. `[ClassCleanup]` -- once per class +7. `[AssemblyCleanup]` -- once per assembly ### Step 6: Apply cancellation and timeout patterns @@ -310,17 +310,28 @@ public void LocalOnly_InteractiveTest() { } public sealed class DatabaseIntegrationTests { } ``` +## Validation + +- [ ] Test classes are `sealed` +- [ ] Test methods follow `MethodName_Scenario_ExpectedBehavior` naming +- [ ] `Assert.ThrowsExactly` used instead of `[ExpectedException]` +- [ ] Specialized assertions used instead of `Assert.IsTrue` (e.g., `Assert.IsNotNull`, `Assert.AreEqual`) +- [ ] DynamicData uses ValueTuple return types instead of `IEnumerable` +- [ ] Sync initialization done in the constructor, not `[TestInitialize]` +- [ ] `TestContext.CancellationToken` passed to async calls in tests with `[Timeout]` +- [ ] Project builds with zero errors and all tests pass + ## Common Pitfalls | Pitfall | Solution | |---------|----------| -| `Assert.AreEqual(actual, expected)` — swapped arguments | Always put expected first: `Assert.AreEqual(expected, actual)`. Failure messages show "Expected: X, Actual: Y" so wrong order makes messages confusing | -| `[ExpectedException]` — obsolete, cannot assert message | Use `Assert.Throws` or `Assert.ThrowsExactly` | -| `items.Single()` — unclear exception on failure | Use `Assert.ContainsSingle(items)` for better failure messages | -| Hard cast `(MyType)result` — unclear exception | Use `Assert.IsInstanceOfType(result)` | +| `Assert.AreEqual(actual, expected)` -- swapped arguments | Always put expected first: `Assert.AreEqual(expected, actual)`. Failure messages show "Expected: X, Actual: Y" so wrong order makes messages confusing | +| `[ExpectedException]` -- obsolete, cannot assert message | Use `Assert.Throws` or `Assert.ThrowsExactly` | +| `items.Single()` -- unclear exception on failure | Use `Assert.ContainsSingle(items)` for better failure messages | +| Hard cast `(MyType)result` -- unclear exception | Use `Assert.IsInstanceOfType(result)` | | `IEnumerable` for DynamicData | Use `IEnumerable<(T1, T2, ...)>` ValueTuples for type safety | -| Sync setup in `[TestInitialize]` | Initialize in the constructor instead — enables `readonly` fields and satisfies nullability analyzers | +| Sync setup in `[TestInitialize]` | Initialize in the constructor instead -- enables `readonly` fields and satisfies nullability analyzers | | `CancellationToken.None` in async tests | Use `TestContext.CancellationToken` for cooperative timeout | -| `public TestContext? TestContext { get; set; }` | Drop the `?` — MSTest suppresses CS8618 for this property | -| `TestContext TestContext { get; set; } = null!` | Remove `= null!` — unnecessary, MSTest handles assignment | +| `public TestContext? TestContext { get; set; }` | Drop the `?` -- MSTest suppresses CS8618 for this property | +| `TestContext TestContext { get; set; } = null!` | Remove `= null!` -- unnecessary, MSTest handles assignment | | Non-sealed test classes | Seal test classes by default for performance | From afc8f06c993c3cde98ebeae3d11d535bc997a085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 23 Mar 2026 19:18:01 +0100 Subject: [PATCH 2/3] 2nd round of improvement --- .agents/skills/create-skill-test/SKILL.md | 30 ++++---- .../shared/compiled/performance.lock.md | 70 +++++++++---------- .../skills/exp-test-tagging/SKILL.md | 40 +++++------ .../skills/msbuild-server/SKILL.md | 55 +++++++++++---- .../resolve-project-references/SKILL.md | 50 +++++++++---- .../skills/convert-to-cpm/SKILL.md | 36 +++++----- .../dotnet-test/skills/crap-score/SKILL.md | 14 ++-- .../skills/migrate-mstest-v1v2-to-v3/SKILL.md | 2 +- .../skills/migrate-mstest-v3-to-v4/SKILL.md | 6 +- .../skills/migrate-vstest-to-mtp/SKILL.md | 2 +- plugins/dotnet-test/skills/run-tests/SKILL.md | 3 +- .../skills/test-anti-patterns/SKILL.md | 40 +++++------ .../skills/writing-mstest-tests/SKILL.md | 2 + 13 files changed, 198 insertions(+), 152 deletions(-) diff --git a/.agents/skills/create-skill-test/SKILL.md b/.agents/skills/create-skill-test/SKILL.md index d9e97330c0..b64215a280 100644 --- a/.agents/skills/create-skill-test/SKILL.md +++ b/.agents/skills/create-skill-test/SKILL.md @@ -43,7 +43,7 @@ tests///eval.yaml tests//agent./eval.yaml ``` -For skills, verify the skill exists at `plugins//skills//SKILL.md`. For agents, verify the agent exists at `plugins//agents/.agent.md`. Read the target content to understand what it does — this is critical for writing non-overfitted rubric items. +For skills, verify the skill exists at `plugins//skills//SKILL.md`. For agents, verify the agent exists at `plugins//agents/.agent.md`. Read the target content to understand what it does -- this is critical for writing non-overfitted rubric items. ### Step 2: Create the test directory and eval.yaml @@ -52,11 +52,11 @@ Create the directory and file: ``` # For skills: tests/// -└── eval.yaml ++-- eval.yaml # For agents: tests//agent./ -└── eval.yaml ++-- eval.yaml ``` The `agent.` prefix disambiguates agent test directories from skill test directories that might share the same name. @@ -172,7 +172,7 @@ Assertions are hard pass/fail checks. Use them for objective, binary-verifiable | `file_not_exists` | `path` | No file matching glob exists | | `file_contains` | `path`, `value` | File at glob path contains text | | `file_not_contains` | `path`, `value` | File at glob path does NOT contain text | -| `exit_success` | — | Agent produced non-empty output | +| `exit_success` | -- | Agent produced non-empty output | #### Assertion guidelines @@ -198,11 +198,11 @@ The overfitting judge classifies each rubric item: #### Rubric writing rules -1. **Test outcomes, not methods.** Write "Identified the root cause of the build failure" — not "Replayed the binlog using `dotnet build /flp`." +1. **Test outcomes, not methods.** Write "Identified the root cause of the build failure" -- not "Replayed the binlog using `dotnet build /flp`." 2. **Allow alternative approaches.** If multiple valid solutions exist, the rubric item should accept any of them. 3. **Never reference the skill by name** or use phrasing copied directly from the SKILL.md. 4. **Don't test pre-existing LLM knowledge.** If the LLM already knows something (common APIs, standard syntax, basic escaping), testing for it adds no signal. -5. **Test findings, not diagnostic steps.** Write "Correctly determined that the root cause is a missing PackageReference" — not "Used `dotnet restore` to check package resolution." +5. **Test findings, not diagnostic steps.** Write "Correctly determined that the root cause is a missing PackageReference" -- not "Used `dotnet restore` to check package resolution." 6. **Each item should be independently evaluable.** Avoid compound items that test multiple things. #### Examples @@ -232,19 +232,19 @@ max_turns: 10 # Maximum agent iterations max_tokens: 5000 # Maximum token budget ``` -Use constraints sparingly — only when the scenario specifically requires or forbids certain agent behaviors. +Use constraints sparingly -- only when the scenario specifically requires or forbids certain agent behaviors. ### Step 8: Add non-activation scenarios with `expect_activation: false` -Many skills have clear boundaries — situations where the skill should recognize it does not apply and decline gracefully. Test these boundaries using `expect_activation: false`. +Many skills have clear boundaries -- situations where the skill should recognize it does not apply and decline gracefully. Test these boundaries using `expect_activation: false`. #### How `expect_activation: false` works When a scenario has `expect_activation: false`: 1. **All three runs still execute** (baseline, skilled-isolated, skilled-plugin) and assertions are evaluated on each. The flag does not change which runs are performed. -2. **Activation verdict is inverted** — if the skill is not activated for this prompt, the evaluator reports it as `ℹ️ not activated (expected)` instead of treating it as a failure. -3. **The scenario is excluded from the noise test** — the multi-skill activation test only runs positive (`expect_activation: true`) scenarios. +2. **Activation verdict is inverted** -- if the skill is not activated for this prompt, the evaluator reports it as `[Info] not activated (expected)` instead of treating it as a failure. +3. **The scenario is excluded from the noise test** -- the multi-skill activation test only runs positive (`expect_activation: true`) scenarios. #### When to use non-activation scenarios @@ -255,7 +255,7 @@ Add `expect_activation: false` scenarios when the skill has explicit "When Not t | **Wrong input format** | Skill handles Android tombstones; scenario provides an iOS crash log | | **Out-of-scope request** | Skill collects dumps; scenario asks to *analyze* a dump | | **Incompatible project type** | Skill converts PackageReference to CPM; scenario has packages.config | -| **Wrong framework version** | Skill migrates .NET 8→9; scenario provides a .NET 8 app and asks for .NET 10 migration | +| **Wrong framework version** | Skill migrates .NET 8 to 9; scenario provides a .NET 8 app and asks for .NET 10 migration | | **Prerequisite not met** | Skill requires a specific file format that isn't present | #### Example: Wrong input format @@ -317,9 +317,9 @@ Add `expect_activation: false` scenarios when the skill has explicit "When Not t Non-activation rubric items typically verify three things: -1. **Recognition** — The agent identified *why* the skill doesn't apply. -2. **Restraint** — The agent did NOT attempt the skill's workflow (no file modifications, no tool installs). -3. **Redirection** — The agent suggested the correct alternative approach or next step. +1. **Recognition** -- The agent identified *why* the skill doesn't apply. +2. **Restraint** -- The agent did NOT attempt the skill's workflow (no file modifications, no tool installs). +3. **Redirection** -- The agent suggested the correct alternative approach or next step. ### Step 9: Validate the eval.yaml @@ -350,7 +350,7 @@ dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- evaluate \ ```yaml scenarios: - name: "" - prompt: "" + prompt: "" setup: copy_test_files: true assertions: diff --git a/agentic-workflows/dotnet-msbuild/shared/compiled/performance.lock.md b/agentic-workflows/dotnet-msbuild/shared/compiled/performance.lock.md index 10eaa47783..dc0c8c633d 100644 --- a/agentic-workflows/dotnet-msbuild/shared/compiled/performance.lock.md +++ b/agentic-workflows/dotnet-msbuild/shared/compiled/performance.lock.md @@ -449,6 +449,8 @@ Is your no-op build slow (> 10s per project)? - **Root causes**: too many assembly references, network-based reference paths, large assembly search paths - **Fixes**: reduce reference count, use `false` for RAR-heavy analysis, set `true` for diagnostic - **Advanced**: `` and `` +- **Key insight**: RAR runs unconditionally even on incremental builds because users may have installed targeting packs or GACed assemblies (see dotnet/msbuild#2015). With .NET Core micro-assemblies, the reference count is often very high. +- **Reduce transitive references**: Set `true` to avoid pulling in the full transitive closure (note: projects may need to add direct references for any types they consume). Use `ReferenceOutputAssembly="false"` on ProjectReferences that are only needed at build time (not API surface). Trim unused PackageReferences. ### 2. Roslyn Analyzers and Source Generators @@ -473,23 +475,30 @@ Is your no-op build slow (> 10s per project)? ### 4. Excessive File I/O (Copy tasks) - **Symptoms**: Copy task shows high aggregate time -- **Root causes**: copying thousands of files, copying across network drives -- **Fixes**: use hardlinks (`true`), reduce CopyToOutputDirectory items, use `true` when appropriate +- **Root causes**: copying thousands of files, copying across network drives, Copy task unintentionally running once per item (per-file) instead of as a single batch (see dotnet/msbuild#12884) +- **Fixes**: use hardlinks (`true`), reduce CopyToOutputDirectory items, use `true` when appropriate, set `true`, consider `--artifacts-path` (.NET 8+) for centralized output layout +- **Dev Drive**: On Windows, switching to a Dev Drive (ReFS with copy-on-write and reduced Defender scans) can significantly reduce file I/O overhead for Copy-heavy builds. Recommend for both dev machines and self-hosted CI agents. ### 5. Evaluation Overhead - **Symptoms**: build starts slow before any compilation +- **Root causes**: complex Directory.Build.props, wildcard globs scanning large directories, NuGetSdkResolver overhead (adds 180-400ms per project evaluation even when restored — see dotnet/msbuild#4025) +- **Fixes**: reduce Directory.Build.props complexity, use `false` for legacy projects with explicit file lists, avoid NuGet-based SDK resolvers if possible - See: `eval-performance` skill for detailed guidance ### 6. NuGet Restore in Build - **Symptoms**: restore runs every build even when unnecessary -- **Fix**: separate restore from build: `dotnet restore` then `dotnet build --no-restore` +- **Fixes**: + - Separate restore from build: `dotnet restore` then `dotnet build --no-restore` + - Enable static graph evaluation: `true` in Directory.Build.props — can save significant time in large builds (results are workload-dependent) -### 7. Large Project Count +### 7. Large Project Count and Graph Shape -- **Symptoms**: many small projects, each takes minimal time but overhead adds up +- **Symptoms**: many small projects, each takes minimal time but overhead adds up; deep dependency chains serialize the build - **Consider**: project consolidation, or use `/graph` mode for better scheduling +- **Graph shape matters**: a wide dependency graph (few levels, many parallel branches) builds faster than a deep one (many levels, serialized). Refactoring from deep to wide can yield significant improvements in both clean and incremental build times. +- **Actions**: look for unnecessary project dependencies, consider splitting a bottleneck project into two, or merging small leaf projects ## Using Binlog Replay for Performance Analysis @@ -524,12 +533,24 @@ Step-by-step workflow using text log replay: ## Quick Wins Checklist - [ ] Use `/maxcpucount` (or `-m`) for parallel builds -- [ ] Separate restore from build (`--no-restore`) -- [ ] Enable hardlinks for Copy -- [ ] Disable analyzers in dev inner loop +- [ ] Separate restore from build (`dotnet restore` then `dotnet build --no-restore`) +- [ ] Enable static graph restore (`true`) +- [ ] Enable hardlinks for Copy (`true`) +- [ ] Disable analyzers conditionally in dev inner loop: `false` +- [ ] Enable reference assemblies (`true`) - [ ] Check for broken incremental builds (see `incremental-build` skill) - [ ] Check for bin/obj clashes (see `check-bin-obj-clash` skill) - [ ] Use graph build (`/graph`) for multi-project solutions +- [ ] Use `--artifacts-path` (.NET 8+) for centralized output layout +- [ ] Enable Dev Drive (ReFS) on Windows dev machines and self-hosted CI + +## Impact Categorization + +When reporting findings, categorize by impact to help prioritize fixes: + +- 🔴 **HIGH IMPACT** (do first): Items consuming >10% of total build time, or a single target >50% of build time +- 🟡 **MEDIUM IMPACT**: Items consuming 2-10% of build time +- 🟢 **QUICK WINS**: Easy changes with modest impact (e.g., property flags in Directory.Build.props) --- @@ -814,6 +835,8 @@ Step-by-step: --- +## eval-performance + ## MSBuild Evaluation Phases For a comprehensive overview of MSBuild's evaluation and execution model, see [Build process overview](https://learn.microsoft.com/en-us/visualstudio/msbuild/build-process-overview). @@ -868,31 +891,6 @@ Key insight: evaluation happens BEFORE any targets run. Slow evaluation = slow b ## Multiple Evaluations - A project evaluated multiple times = wasted work -- Common causes: referenced from multiple other projects with different global properties -- Each unique set of global properties = separate evaluation -- Diagnosis: `grep 'Evaluation started.*ProjectName' full.log` → if count > 1, check for differing global properties -- Fix: normalize global properties, use graph build (`/graph`) - -## TreatAsLocalProperty - -- Prevents property values from flowing to child projects via MSBuild task -- Overuse: declaring many TreatAsLocalProperty entries adds evaluation overhead -- Correct use: only when you genuinely need to override an inherited property - -## Property Function Cost - -- Property functions execute during evaluation -- Most are cheap (string operations) -- Expensive: `$([System.IO.File]::ReadAllText(...))` during evaluation — reads file on every evaluation -- Expensive: network calls, heavy computation -- Rule: property functions should be fast and side-effect-free - -## Optimization Checklist - -- [ ] Check preprocessed output size: `dotnet msbuild -pp:full.xml` -- [ ] Verify evaluation count: should be 1 per project per TFM -- [ ] Exclude large directories from globs -- [ ] Avoid file I/O in property functions during evaluation -- [ ] Minimize import depth -- [ ] Use graph build to reduce redundant evaluations -- [ ] Check for unnecessary UsingTask declarations \ No newline at end of file +- Common causes: referenced from multiple other projects + +[truncated] \ No newline at end of file diff --git a/plugins/dotnet-experimental/skills/exp-test-tagging/SKILL.md b/plugins/dotnet-experimental/skills/exp-test-tagging/SKILL.md index 7e180cc18e..bb3dfe7fad 100644 --- a/plugins/dotnet-experimental/skills/exp-test-tagging/SKILL.md +++ b/plugins/dotnet-experimental/skills/exp-test-tagging/SKILL.md @@ -46,7 +46,7 @@ Use exactly these trait names and values. Do not invent new trait values outside | `security` | Verifies authentication, authorization, input sanitization, or secrets handling | Tests for SQL injection, XSS, CSRF, unauthorized access, token validation, permission checks | | `concurrency` | Validates thread safety, parallelism, or async correctness | Uses `Task.WhenAll`, locks, `Parallel.ForEach`, `SemaphoreSlim`, reproduces race conditions | | `resilience` | Tests retry logic, timeouts, circuit breakers, or graceful degradation | Asserts behavior under transient failures, network drops, or service unavailability (e.g., Polly policies) | -| `destructive` | Mutates shared or external state that is hard to roll back | Deletes records, drops resources, modifies global config — useful for CI isolation decisions | +| `destructive` | Mutates shared or external state that is hard to roll back | Deletes records, drops resources, modifies global config -- useful for CI isolation decisions | | `configuration` | Verifies settings loading, defaults, environment behavior | Tests missing config keys, invalid values, environment variable fallbacks, options validation | | `flaky` | Known to intermittently fail (meta-tag for test health tracking) | Mark tests the team knows are unreliable; used to quarantine or prioritize stabilization | @@ -80,23 +80,23 @@ Record which tests already have tags to avoid duplication. For each test method without traits, analyze: -1. **Method name** — names containing `Invalid`, `Fail`, `Error`, `Throw`, `Reject`, `BadInput`, `Null`, `Negative` suggest `negative` -2. **Assertion type** — `Assert.ThrowsException`, `Assert.Throws`, `Should().Throw()` suggest `negative` -3. **Input values** — `null`, `""`, `0`, `-1`, `int.MaxValue`, `int.MinValue`, empty collections suggest `boundary` -4. **Setup complexity** — minimal setup with basic assertions suggests `smoke`; external dependencies suggest `integration` -5. **Comments and names** — references to issue numbers or "regression" / "bug" / "fix for #..." suggest `regression` -6. **Timing assertions** — `Stopwatch`, `BenchmarkDotNet`, elapsed-time checks suggest `performance` -7. **Feature centrality** — tests on primary public API entry points or critical user workflows suggest `critical-path` -8. **Security patterns** — validates auth, checks permissions, sanitizes input, tests for injection, handles tokens/secrets suggest `security` -9. **Parallel/async constructs** — `Task.WhenAll`, `Parallel.ForEach`, locks, `SemaphoreSlim`, `ConcurrentDictionary`, race condition names suggest `concurrency` -10. **Fault injection** — simulates failures, tests retries, timeouts, or circuit breakers suggest `resilience` -11. **State mutation** — deletes external records, drops resources, modifies shared/global state suggest `destructive` -12. **Full-stack flow** — test spans entry point through data layer to final response, covering a complete user scenario suggest `end-to-end` -13. **Config/settings** — loads configuration, tests missing keys, validates options, checks environment variables suggest `configuration` -14. **Known instability** — test has `[Ignore]`/`[Skip]` comments about flakiness, or names contain "flaky"/"intermittent" suggest `flaky` -15. **Default** — if the test verifies a normal success path, tag `positive` - -When in doubt between `positive` and `negative`, read the assertion: if it asserts success → `positive`; if it asserts failure → `negative`. +1. **Method name** -- names containing `Invalid`, `Fail`, `Error`, `Throw`, `Reject`, `BadInput`, `Null`, `Negative` suggest `negative` +2. **Assertion type** -- `Assert.ThrowsException`, `Assert.Throws`, `Should().Throw()` suggest `negative` +3. **Input values** -- `null`, `""`, `0`, `-1`, `int.MaxValue`, `int.MinValue`, empty collections suggest `boundary` +4. **Setup complexity** -- minimal setup with basic assertions suggests `smoke`; external dependencies suggest `integration` +5. **Comments and names** -- references to issue numbers or "regression" / "bug" / "fix for #..." suggest `regression` +6. **Timing assertions** -- `Stopwatch`, `BenchmarkDotNet`, elapsed-time checks suggest `performance` +7. **Feature centrality** -- tests on primary public API entry points or critical user workflows suggest `critical-path` +8. **Security patterns** -- validates auth, checks permissions, sanitizes input, tests for injection, handles tokens/secrets suggest `security` +9. **Parallel/async constructs** -- `Task.WhenAll`, `Parallel.ForEach`, locks, `SemaphoreSlim`, `ConcurrentDictionary`, race condition names suggest `concurrency` +10. **Fault injection** -- simulates failures, tests retries, timeouts, or circuit breakers suggest `resilience` +11. **State mutation** -- deletes external records, drops resources, modifies shared/global state suggest `destructive` +12. **Full-stack flow** -- test spans entry point through data layer to final response, covering a complete user scenario suggest `end-to-end` +13. **Config/settings** -- loads configuration, tests missing keys, validates options, checks environment variables suggest `configuration` +14. **Known instability** -- test has `[Ignore]`/`[Skip]` comments about flakiness, or names contain "flaky"/"intermittent" suggest `flaky` +15. **Default** -- if the test verifies a normal success path, tag `positive` + +When in doubt between `positive` and `negative`, read the assertion: if it asserts success -> `positive`; if it asserts failure -> `negative`. ### Step 4: Apply trait attributes @@ -151,7 +151,7 @@ After tagging, produce a summary table: | destructive | 1 | 1.3% | | configuration | 2 | 2.6% | | flaky | 1 | 1.3% | -| **Total tests** | **78** | — | +| **Total tests** | **78** | -- | Note: Percentages exceed 100% because tests can have multiple traits. ``` @@ -174,7 +174,7 @@ Include observations such as: | Pitfall | Solution | |---------|----------| | Guessing traits without reading the test body | Always read assertions and setup to classify accurately | -| Tagging a test only as `boundary` without `positive`/`negative` | Every test should also be `positive` or `negative` — `boundary` is additive | +| Tagging a test only as `boundary` without `positive`/`negative` | Every test should also be `positive` or `negative` -- `boundary` is additive | | Using `TestCategory` syntax in an xUnit project | Match the attribute style to the detected framework | | Duplicating an existing category attribute | Check for pre-existing traits in Step 2 before adding | | Over-tagging as `critical-path` | Reserve for tests on primary public entry points, not every helper | diff --git a/plugins/dotnet-msbuild/skills/msbuild-server/SKILL.md b/plugins/dotnet-msbuild/skills/msbuild-server/SKILL.md index 9821c10114..b1e8c66dca 100644 --- a/plugins/dotnet-msbuild/skills/msbuild-server/SKILL.md +++ b/plugins/dotnet-msbuild/skills/msbuild-server/SKILL.md @@ -1,23 +1,37 @@ --- name: msbuild-server -description: "Guide for using MSBuild Server to improve CLI build performance. Activate when developers report slow incremental builds from the command line, or when CLI builds are noticeably slower than IDE builds. Covers MSBUILDUSESERVER=1 environment variable for persistent server-based caching. Do not activate for IDE-based builds (Visual Studio already uses a long-lived process)." +description: "Guide for using MSBuild Server to improve CLI build performance. Only activate in MSBuild/.NET build context. Activate when developers report slow incremental builds from the command line, or when CLI builds are noticeably slower than IDE builds. Covers MSBUILDUSESERVER=1 environment variable for persistent server-based caching. Do not activate for IDE-based builds (Visual Studio already uses a long-lived process)." --- -## MSBuild Server for CLI Caching +# MSBuild Server for CLI Caching -### When to Use +Use the MSBuild Server to cache evaluation results across CLI builds, matching the performance advantage Visual Studio gets from its long-lived MSBuild process. + +## When to Use - Small incremental builds from CLI (`dotnet build`) are slower than expected - Developers notice that VS builds are faster than CLI builds for the same project - CI agents run many sequential builds of the same repo -### How It Works +## When Not to Use + +- IDE-based builds (Visual Studio already uses a long-lived MSBuild process) +- One-off builds where cold-start overhead is acceptable +- Build correctness issues are suspected (disable the server to isolate the problem) + +## Inputs + +| Input | Required | Description | +|-------|----------|-------------| +| Shell context | No | The shell where the environment variable will be set (bash, PowerShell, or Windows persistent) | + +## Workflow -By default, each `dotnet build` invocation starts a fresh MSBuild process. The MSBuild Server keeps a long-lived process that caches evaluation results, resolved assemblies, and other state across builds — similar to how Visual Studio reuses its MSBuild nodes. +### Step 1: Confirm CLI context -### Setup +Verify the developer is building from the command line (`dotnet build`), not from Visual Studio or another IDE. The MSBuild Server provides no benefit inside an IDE. -Set the environment variable: +### Step 2: Set the environment variable ```bash # Bash / CI @@ -30,14 +44,25 @@ $env:MSBUILDUSESERVER = "1" setx MSBUILDUSESERVER 1 ``` -### Expected Impact +### Step 3: Validate improvement + +Run two sequential builds of the same project and compare times: + +1. First build (cold): `dotnet build` -- server starts, no cache benefit +2. Second build (warm): `dotnet build` -- should be noticeably faster + +The most noticeable improvement is in repos with many projects or complex `Directory.Build.props` chains. + +## Validation -- **First build**: no change (server starts cold) -- **Subsequent incremental builds**: faster due to cached evaluation, resolved references, and warm JIT -- Most noticeable in repos with many projects or complex `Directory.Build.props` chains +- [ ] `MSBUILDUSESERVER=1` is set in the shell +- [ ] Second sequential build is faster than the first +- [ ] `dotnet build-server shutdown` followed by a rebuild confirms the server restarts cleanly -### Limitations +## Common Pitfalls -- The server process persists in the background — uses some memory -- If builds behave unexpectedly, try `dotnet build-server shutdown` to reset -- Not all MSBuild features are compatible with server mode +| Pitfall | Solution | +|---------|----------| +| Expecting improvement in Visual Studio | VS already uses long-lived MSBuild nodes; the server adds no benefit | +| Build correctness issues after enabling | Run `dotnet build-server shutdown` to reset; if issues persist, disable the server | +| Server process using unexpected memory | The server persists in background; shut down with `dotnet build-server shutdown` when idle | diff --git a/plugins/dotnet-msbuild/skills/resolve-project-references/SKILL.md b/plugins/dotnet-msbuild/skills/resolve-project-references/SKILL.md index 55fd0f1ff1..98b3c288b5 100644 --- a/plugins/dotnet-msbuild/skills/resolve-project-references/SKILL.md +++ b/plugins/dotnet-msbuild/skills/resolve-project-references/SKILL.md @@ -1,38 +1,58 @@ --- name: resolve-project-references -description: "Guide for interpreting ResolveProjectReferences time in MSBuild performance summaries. Activate when ResolveProjectReferences appears as the most expensive target and developers are trying to optimize it directly. Explains that the reported time includes wait time for dependent project builds and is misleading. Guides users to focus on task self-time instead. Do not activate for general build performance — use build-perf-diagnostics instead." +description: "Guide for interpreting ResolveProjectReferences time in MSBuild performance summaries. Only activate in MSBuild/.NET build context. Activate when ResolveProjectReferences appears as the most expensive target and developers are trying to optimize it directly. Explains that the reported time includes wait time for dependent project builds and is misleading. Guides users to focus on task self-time instead. Do not activate for general build performance -- use build-perf-diagnostics instead." --- -## Misleading ResolveProjectReferences Time +# Misleading ResolveProjectReferences Time -### The Problem +Prevent misguided optimization of `ResolveProjectReferences` by explaining that its reported time is wall-clock wait time, not CPU work. -When reviewing the **Target Performance Summary** from a diagnostic build log, `ResolveProjectReferences` often appears as the single most expensive target — sometimes consuming 50-80% of total build time. This is **misleading**. +## When to Use -### Why It's Misleading +- `ResolveProjectReferences` appears as the most expensive target in the Target Performance Summary +- A developer is trying to optimize `ResolveProjectReferences` directly +- Build performance analysis shows a single target consuming 50-80% of total build time -The reported time for `ResolveProjectReferences` includes **waiting for dependent projects to build** while the MSBuild node is yielded (see dotnet/msbuild#3135). During this wait, the node may be doing useful work on other projects. +## When Not to Use -The target itself does very little work — it just triggers the build of referenced projects and waits for them to complete. The "time" attributed to it is wall-clock wait time, not CPU work. +- General build performance optimization (use `build-perf-diagnostics` instead) +- The bottleneck is clearly a different target (e.g., `Csc`, `ResolveAssemblyReference`) +- The user has not yet captured a binlog or performance summary -### Correct Diagnosis +## Inputs -1. **Use Task Performance Summary instead of Target Performance Summary** for an accurate picture of where time is actually spent -2. **Focus on self-time** of actual tasks: `Csc` (compilation), `ResolveAssemblyReference` (RAR), `Copy`, etc. -3. **Do not optimize ResolveProjectReferences directly** — optimize the targets/tasks it's waiting on +| Input | Required | Description | +|-------|----------|-------------| +| Build log or binlog | Yes | A diagnostic build log or binlog containing the Target Performance Summary | -### How to Get Task Performance Summary +## Workflow + +### Step 1: Confirm the misleading symptom + +Verify that `ResolveProjectReferences` appears as the top target in the **Target** Performance Summary. This is the misleading metric. + +### Step 2: Explain why it is misleading + +The reported time includes **waiting for dependent projects to build** while the MSBuild node is yielded (see dotnet/msbuild#3135). During this wait, the node may be doing useful work on other projects. The target itself does very little work. + +### Step 3: Redirect to task self-time + +Guide the user to use the **Task** Performance Summary instead: ```bash dotnet msbuild build.binlog -noconlog -fl "-flp:v=diag;logfile=full.log;performancesummary" grep "Task Performance Summary" -A 50 full.log ``` -### What to Optimize Instead - -Look at the tasks consuming the most cumulative time: +Focus on self-time of actual tasks: - **Csc**: see `build-perf-diagnostics` skill (Section 2: Roslyn Analyzers) - **ResolveAssemblyReference**: see `build-perf-diagnostics` skill (Section 1: RAR) - **Copy**: see `build-perf-diagnostics` skill (Section 4: File I/O) - **Serialization bottlenecks**: see `build-parallelism` skill + +## Validation + +- [ ] Task Performance Summary was used instead of Target Performance Summary +- [ ] `ResolveProjectReferences` was not set as the optimization target +- [ ] A concrete task (e.g., `Csc`, `Copy`, `ResolveAssemblyReference`) was identified as the true bottleneck diff --git a/plugins/dotnet-nuget/skills/convert-to-cpm/SKILL.md b/plugins/dotnet-nuget/skills/convert-to-cpm/SKILL.md index c0c1abcdb8..5bba5d66a8 100644 --- a/plugins/dotnet-nuget/skills/convert-to-cpm/SKILL.md +++ b/plugins/dotnet-nuget/skills/convert-to-cpm/SKILL.md @@ -7,7 +7,7 @@ description: > properties from Directory.Build.props into Directory.Packages.props, resolving version conflicts or mismatches across a solution or repository, updating or bumping or syncing package versions across projects. Also activate when packages are out of sync, drifting, - or inconsistent — even without the user mentioning CPM. Provides baseline build capture, + or inconsistent -- even without the user mentioning CPM. Provides baseline build capture, version conflict resolution, build validation with binlog comparison, and a structured post-conversion report. DO NOT USE FOR: packages.config projects (must migrate to PackageReference first) or repositories that already have CPM fully enabled. @@ -22,7 +22,7 @@ Migrate .NET projects from per-project package versioning to NuGet Central Packa - The user wants to adopt Central Package Management for a .NET repository, solution, or project - Package versions are scattered across many `.csproj`, `.fsproj`, or `.vbproj` files and the user wants a single source of truth - The user mentions `Directory.Packages.props`, CPM, or centralizing NuGet versions -- The user wants to update, bump, upgrade, align, or sync a NuGet package version across multiple projects — CPM is the recommended approach for managing shared package versions, so suggest converting to CPM as part of the update if the projects use `PackageReference` and CPM is not already enabled +- The user wants to update, bump, upgrade, align, or sync a NuGet package version across multiple projects -- CPM is the recommended approach for managing shared package versions, so suggest converting to CPM as part of the update if the projects use `PackageReference` and CPM is not already enabled - Package versions are out of sync, conflicting, or mismatched across projects and the user wants to resolve or unify them ## When Not to Use @@ -36,7 +36,7 @@ Migrate .NET projects from per-project package versioning to NuGet Central Packa | Input | Required | Description | |-------|----------|-------------| | Scope | Yes | A project file, solution file, or directory containing .NET projects to convert | -| Version conflict strategy | No | How to resolve cases where the same package has different versions across projects. When conflicts are detected, do not assume a default strategy — ask the user which strategy to use or explicitly confirm a proposed strategy before proceeding. | +| Version conflict strategy | No | How to resolve cases where the same package has different versions across projects. When conflicts are detected, do not assume a default strategy -- ask the user which strategy to use or explicitly confirm a proposed strategy before proceeding. | ## Workflow @@ -52,7 +52,7 @@ If the scope is unclear, ask the user. ### Step 2: Establish baseline build -Before making any changes, verify the scope builds successfully and capture a baseline binlog and package list. Run `dotnet clean`, then `dotnet build -bl:baseline.binlog`, then `dotnet package list --format json > baseline-packages.json`. Read [baseline-comparison.md](references/baseline-comparison.md) for the full procedure and fallback options. If the baseline build fails, stop and inform the user — the scope must build cleanly before conversion. Do not delete `baseline.binlog` or `baseline-packages.json` — they are needed for the post-conversion comparison and report. +Before making any changes, verify the scope builds successfully and capture a baseline binlog and package list. Run `dotnet clean`, then `dotnet build -bl:baseline.binlog`, then `dotnet package list --format json > baseline-packages.json`. Read [baseline-comparison.md](references/baseline-comparison.md) for the full procedure and fallback options. If the baseline build fails, stop and inform the user -- the scope must build cleanly before conversion. Do not delete `baseline.binlog` or `baseline-packages.json` -- they are needed for the post-conversion comparison and report. ### Step 3: Check for existing CPM @@ -68,7 +68,7 @@ Present audit results to the user before proceeding, including: - A table of each package, its version(s), and which projects use it - Any version conflicts, security advisories, or complexities requiring decisions -When version conflicts exist, present each one individually with the affected projects, the distinct versions found, and the resolution options (align to highest, use `VersionOverride`, etc.) with their trade-offs. Do not upgrade any package beyond the highest version already in use across the scope — this avoids introducing version incompatibilities or breaking changes that are unrelated to the CPM conversion itself. Note any known security advisories or other upgrade opportunities as follow-up items for the user to address after the conversion is complete. Ask the user to decide on each conflict before proceeding. Read [audit-complexities.md § Same package with different versions](references/audit-complexities.md) for the resolution workflow and presentation format. +When version conflicts exist, present each one individually with the affected projects, the distinct versions found, and the resolution options (align to highest, use `VersionOverride`, etc.) with their trade-offs. Do not upgrade any package beyond the highest version already in use across the scope -- this avoids introducing version incompatibilities or breaking changes that are unrelated to the CPM conversion itself. Note any known security advisories or other upgrade opportunities as follow-up items for the user to address after the conversion is complete. Ask the user to decide on each conflict before proceeding. Read [audit-complexities.md - Same package with different versions](references/audit-complexities.md) for the resolution workflow and presentation format. ### Step 5: Create or update Directory.Packages.props @@ -79,8 +79,8 @@ Create the file with `dotnet new packagesprops` (.NET 8+) or manually. Add a `

` that now has a corresponding ``. Also update any shared `.props`/`.targets` files identified in step 4. - Preserve all other attributes (`PrivateAssets`, `IncludeAssets`, `ExcludeAssets`, `GeneratePathProperty`, `Aliases`) -- Preserve conditional `` elements — only remove the `Version` attribute within them -- Retain each file's existing indentation style (spaces vs. tabs, indentation depth) and blank lines — do not reformat or reorganize unchanged lines +- Preserve conditional `` elements -- only remove the `Version` attribute within them +- Retain each file's existing indentation style (spaces vs. tabs, indentation depth) and blank lines -- do not reformat or reorganize unchanged lines - Use `VersionOverride` (with user confirmation) when a project needs a different version than the central one ### Step 7: Handle MSBuild version properties @@ -95,7 +95,7 @@ Run a clean restore and build, capturing a post-conversion binlog and package li ### Step 9: Post-conversion report -**You must create a `convert-to-cpm.md` file** alongside the binlog and JSON artifacts. Do not skip this step or substitute inline chat output for the file — the user needs a persistent, shareable document. This file should be self-contained and shareable — suitable for a pull request description, a team review, or a record of what was done. Structure the report with the following sections: +**You must create a `convert-to-cpm.md` file** alongside the binlog and JSON artifacts. Do not skip this step or substitute inline chat output for the file -- the user needs a persistent, shareable document. This file should be self-contained and shareable -- suitable for a pull request description, a team review, or a record of what was done. Structure the report with the following sections: #### Section 1: Conversion overview @@ -110,24 +110,24 @@ If any version conflicts were encountered, list each one with: - What the user decided (aligned to highest, used `VersionOverride`, etc.) - The practical impact: which projects now resolve a different version than before, and which are unchanged -If no conflicts were found, state that all packages had consistent versions across projects — this is a positive signal worth noting. +If no conflicts were found, state that all packages had consistent versions across projects -- this is a positive signal worth noting. -#### Section 3: Package comparison — baseline vs. result +#### Section 3: Package comparison -- baseline vs. result Compare `baseline-packages.json` and `after-cpm-packages.json` per project. See [baseline-comparison.md](references/baseline-comparison.md) for the comparison procedure. Present two tables: -- **Changes table**: Packages where the resolved version changed, a `VersionOverride` was introduced, or a package was added/removed. Include a status column explaining what changed and why (e.g., "VersionOverride — project retains pinned version", "Aligned to highest version"). +- **Changes table**: Packages where the resolved version changed, a `VersionOverride` was introduced, or a package was added/removed. Include a status column explaining what changed and why (e.g., "VersionOverride -- project retains pinned version", "Aligned to highest version"). - **Unchanged table**: All other packages, confirming they resolve identically to baseline. -If there are no changes at all, state that the conversion is fully version-neutral — this is the ideal outcome and provides reassurance. +If there are no changes at all, state that the conversion is fully version-neutral -- this is the ideal outcome and provides reassurance. #### Section 4: Risk assessment Provide a clear confidence statement: -- **✅ Low risk** — Conversion is version-neutral; all packages resolve to the same versions as baseline. The build and restore succeeded. Recommend running `dotnet test` as a final check. -- **⚠️ Moderate risk** — Some packages changed versions (e.g., minor/patch alignment). List the affected packages and projects. Recommend reviewing the changes table and running `dotnet test` to verify no regressions. -- **🔴 High risk** — Major version changes were applied, or packages were added/removed unexpectedly. Recommend careful review, running `dotnet test`, and comparing binlogs before merging. +- **[Low risk]** -- Conversion is version-neutral; all packages resolve to the same versions as baseline. The build and restore succeeded. Recommend running `dotnet test` as a final check. +- **[Moderate risk]** -- Some packages changed versions (e.g., minor/patch alignment). List the affected packages and projects. Recommend reviewing the changes table and running `dotnet test` to verify no regressions. +- **[High risk]** -- Major version changes were applied, or packages were added/removed unexpectedly. Recommend careful review, running `dotnet test`, and comparing binlogs before merging. Call out any specific warnings: `VersionOverride` usage that partially undermines centralization, or MSBuild property removal that could affect other build logic. @@ -146,9 +146,9 @@ Present follow-up items as a numbered checklist so the user can track them. List the artifacts produced during conversion and explain how to use them: -- **`baseline.binlog`** and **`after-cpm.binlog`** — MSBuild binary logs captured before and after conversion. These are available for manual validation and troubleshooting if needed. -- **`baseline-packages.json`** and **`after-cpm-packages.json`** — Machine-readable snapshots of resolved package versions per project, used to produce the comparison tables above. -- **`convert-to-cpm.md`** — This report file, suitable for use as a pull request description or team review artifact. +- **`baseline.binlog`** and **`after-cpm.binlog`** -- MSBuild binary logs captured before and after conversion. These are available for manual validation and troubleshooting if needed. +- **`baseline-packages.json`** and **`after-cpm-packages.json`** -- Machine-readable snapshots of resolved package versions per project, used to produce the comparison tables above. +- **`convert-to-cpm.md`** -- This report file, suitable for use as a pull request description or team review artifact. Recommend the user run `dotnet test` to validate runtime behavior beyond build success. If any version conflicts were resolved by aligning to the highest version, recommend reviewing the release notes for the affected packages. diff --git a/plugins/dotnet-test/skills/crap-score/SKILL.md b/plugins/dotnet-test/skills/crap-score/SKILL.md index cf53ae75a2..f6babbd335 100644 --- a/plugins/dotnet-test/skills/crap-score/SKILL.md +++ b/plugins/dotnet-test/skills/crap-score/SKILL.md @@ -27,11 +27,11 @@ Where: | CRAP Score | Risk Level | Interpretation | |------------|------------|----------------| | < 5 | Low | Simple and well-tested | -| 5–15 | Moderate | Acceptable for most code | -| 15–30 | High | Needs more tests or simplification | +| 5-15 | Moderate | Acceptable for most code | +| 15-30 | High | Needs more tests or simplification | | > 30 | Critical | Refactor and add coverage urgently | -A method with 100% coverage has CRAP = complexity (the minimum). A method with 0% coverage has CRAP = complexity² + complexity. +A method with 100% coverage has CRAP = complexity (the minimum). A method with 0% coverage has CRAP = complexity^2 + complexity. ## When to Use @@ -126,16 +126,16 @@ Include: For high-CRAP methods, suggest one or both: -1. **Add tests** — identify uncovered branches and suggest specific test cases -2. **Reduce complexity** — suggest extract-method refactoring for deeply nested logic +1. **Add tests** -- identify uncovered branches and suggest specific test cases +2. **Reduce complexity** -- suggest extract-method refactoring for deeply nested logic Calculate the **coverage needed** to bring a method below a CRAP threshold of 15: $$\text{cov}_{\text{needed}} = 1 - \left(\frac{15 - \text{comp}}{\text{comp}^2}\right)^{1/3}$$ -This formula only applies when $\text{comp} < 15$. When $\text{comp} \geq 15$, the minimum possible CRAP score (at 100% coverage) is $\text{comp}$ itself, which already meets or exceeds the threshold. In that case, **coverage alone cannot bring the CRAP score below the threshold** — the method must be refactored to reduce its cyclomatic complexity first. +This formula only applies when comp < 15. When comp >= 15, the minimum possible CRAP score (at 100% coverage) is comp itself, which already meets or exceeds the threshold. In that case, **coverage alone cannot bring the CRAP score below the threshold** -- the method must be refactored to reduce its cyclomatic complexity first. -Report this as: "To bring `ProcessOrder` (complexity 12) below CRAP 15, increase coverage from 45% to at least 72%." For methods where complexity alone exceeds the threshold, report: "`ComplexMethod` (complexity 18) cannot reach CRAP < 15 through testing alone — reduce complexity by extracting sub-methods." +Report this as: "To bring `ProcessOrder` (complexity 12) below CRAP 15, increase coverage from 45% to at least 72%." For methods where complexity alone exceeds the threshold, report: "`ComplexMethod` (complexity 18) cannot reach CRAP < 15 through testing alone -- reduce complexity by extracting sub-methods." ## Validation diff --git a/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md b/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md index 3e573012f0..4982a8befd 100644 --- a/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md @@ -11,7 +11,7 @@ description: > (MSTest.TestFramework 1.x-2.x) to MSTest v3, fixing assertion overload errors (AreEqual/AreNotEqual), updating DataRow constructors, replacing .testsettings with .runsettings, timeout behavior changes, target framework - compatibility (.NET 5 dropped -- use .NET 6+; .NET Fx < 4.6.2 dropped), + compatibility (.NET 5 dropped -- use .NET 6+; .NET Fx older than 4.6.2 dropped), adopting MSTest.Sdk. First step toward MSTest v4 -- after this, use migrate-mstest-v3-to-v4. DO NOT USE FOR: migrating to MSTest v4 (use migrate-mstest-v3-to-v4), diff --git a/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md b/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md index 3ff330d315..597c660b6a 100644 --- a/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md @@ -1,7 +1,7 @@ --- name: migrate-mstest-v3-to-v4 description: > - Migrate an MSTest v3 test project to MSTest v4 (latest). Use when user says + Migrate an MSTest v3 test project to MSTest v4. Use when user says "upgrade to MSTest v4", "update to latest MSTest", "MSTest 4 migration", "MSTest v4 breaking changes", "MSTest v4 compatibility", or has build errors after updating MSTest packages from 3.x to 4.x. Also use for target @@ -11,8 +11,8 @@ description: > removal, TestContext.Properties, Assert API changes, ExpectedExceptionAttribute removal, TestTimeout enum removal), resolving behavioral changes (TreatDiscoveryWarningsAsErrors, TestContext lifecycle, TestCase.Id changes, - MSTest.Sdk MTP changes), handling dropped TFMs (net5.0-net7.0 dropped -- only - net8.0+, net462, uap10.0 supported). + MSTest.Sdk MTP changes), handling dropped TFMs (net5.0-net7.0 dropped, + only net8.0+, net462, uap10.0 supported). DO NOT USE FOR: migrating from MSTest v1/v2 to v3 (use migrate-mstest-v1v2-to-v3 first), migrating between test frameworks, or general .NET upgrades unrelated to MSTest. diff --git a/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md b/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md index dfe6b7e83a..19584873ea 100644 --- a/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md @@ -113,7 +113,7 @@ Requires `NUnit3TestAdapter` **5.0.0** or later. Add a reference to `YTest.MTP.XUnit2` -- this package provides MTP support for xUnit.net v2 projects without requiring an upgrade to xunit.v3. You must also set `OutputType` to `Exe`: ```xml - + ``` ```xml diff --git a/plugins/dotnet-test/skills/run-tests/SKILL.md b/plugins/dotnet-test/skills/run-tests/SKILL.md index dc32526284..e9bd6957dd 100644 --- a/plugins/dotnet-test/skills/run-tests/SKILL.md +++ b/plugins/dotnet-test/skills/run-tests/SKILL.md @@ -133,6 +133,7 @@ These flags apply to MTP on both SDK versions. On SDK 8/9, pass after `--`; on S |------|-------------| | `--no-build` | Skip build, use previously built output | | `--framework ` | Target a specific framework in multi-TFM projects | +| `--results-directory

` | Directory for test result output | | `--diagnostic` | Enable diagnostic logging for the test platform | | `--diagnostic-output-directory ` | Directory for diagnostic log output | @@ -193,4 +194,4 @@ See [references/filter-syntax.md](references/filter-syntax.md) for the complete | Using `--` for MTP args on .NET SDK 10+ | On .NET 10+, MTP args are passed directly: `dotnet test --report-trx` (using `--` still works but is unnecessary) | | Multi-TFM project runs tests for all frameworks | Use `--framework ` to target a specific framework | | `global.json` runner setting ignored | Requires .NET 10+ SDK. On older SDKs, use `` MSBuild property instead | -| TUnit `--treenode-filter` not recognized | TUnit is MTP-only and requires `dotnet run`, not `dotnet test` with VSTest | +| TUnit `--treenode-filter` not recognized | TUnit is MTP-only. On .NET SDK 10+ use `dotnet test`; on older SDKs use `dotnet run` since VSTest-mode `dotnet test` does not support TUnit | diff --git a/plugins/dotnet-test/skills/test-anti-patterns/SKILL.md b/plugins/dotnet-test/skills/test-anti-patterns/SKILL.md index 804d1f8422..8c4e9d9fa9 100644 --- a/plugins/dotnet-test/skills/test-anti-patterns/SKILL.md +++ b/plugins/dotnet-test/skills/test-anti-patterns/SKILL.md @@ -36,23 +36,23 @@ Analyze .NET test code for anti-patterns, code smells, and quality issues that u Read the test files the user wants reviewed. If the user points to a directory or project, scan for all test files (files containing `[TestClass]`, `[TestMethod]`, `[Fact]`, `[Test]`, or `[Theory]` attributes). -If production code is available, read it too — this is critical for detecting tests that are coupled to implementation details rather than behavior. +If production code is available, read it too -- this is critical for detecting tests that are coupled to implementation details rather than behavior. ### Step 2: Scan for anti-patterns Check each test file against the anti-pattern catalog below. Report findings grouped by severity. -#### Critical — Tests that give false confidence +#### Critical -- Tests that give false confidence | Anti-Pattern | What to Look For | |---|---| | **No assertions** | Test methods that execute code but never assert anything. A passing test without assertions proves nothing. | | **Swallowed exceptions** | `try { ... } catch { }` or `catch (Exception)` without rethrowing or asserting. Failures are silently hidden. | -| **Assert in catch block only** | `try { Act(); } catch (Exception ex) { Assert.Fail(ex.Message); }` — use `Assert.ThrowsException` or equivalent instead. The test passes when no exception is thrown even if the result is wrong. | +| **Assert in catch block only** | `try { Act(); } catch (Exception ex) { Assert.Fail(ex.Message); }` -- use `Assert.ThrowsException` or equivalent instead. The test passes when no exception is thrown even if the result is wrong. | | **Always-true assertions** | `Assert.IsTrue(true)`, `Assert.AreEqual(x, x)`, or conditions that can never fail. | | **Commented-out assertions** | Assertions that were disabled but the test still runs, giving the illusion of coverage. | -#### High — Tests likely to cause pain +#### High -- Tests likely to cause pain | Anti-Pattern | What to Look For | |---|---| @@ -62,18 +62,18 @@ Check each test file against the anti-pattern catalog below. Report findings gro | **Implementation coupling** | Testing private methods via reflection, asserting on internal state, verifying exact method call counts on collaborators instead of observable behavior. | | **Broad exception assertions** | `Assert.ThrowsException(...)` instead of the specific exception type. Also: `[ExpectedException(typeof(Exception))]`. | -#### Medium — Maintainability and clarity issues +#### Medium -- Maintainability and clarity issues | Anti-Pattern | What to Look For | |---|---| | **Poor naming** | Test names like `Test1`, `TestMethod`, names that don't describe the scenario or expected outcome. Good: `Add_NegativeNumber_ThrowsArgumentException`. | -| **Magic values** | Unexplained numbers or strings in arrange/assert: `Assert.AreEqual(42, result)` — what does 42 mean? | -| **Duplicate tests** | Three or more test methods with near-identical bodies that differ only in a single input value. Should be data-driven (`[DataRow]`, `[Theory]`, `[TestCase]`). Note: Two tests covering distinct boundary conditions (e.g., zero vs. negative) are NOT duplicates — separate tests for different edge cases provide clearer failure diagnostics and are a valid practice. | +| **Magic values** | Unexplained numbers or strings in arrange/assert: `Assert.AreEqual(42, result)` -- what does 42 mean? | +| **Duplicate tests** | Three or more test methods with near-identical bodies that differ only in a single input value. Should be data-driven (`[DataRow]`, `[Theory]`, `[TestCase]`). Note: Two tests covering distinct boundary conditions (e.g., zero vs. negative) are NOT duplicates -- separate tests for different edge cases provide clearer failure diagnostics and are a valid practice. | | **Giant tests** | Test methods exceeding ~30 lines or testing multiple behaviors at once. Hard to diagnose when they fail. | | **Assertion messages that repeat the assertion** | `Assert.AreEqual(expected, actual, "Expected and actual are not equal")` adds no information. Messages should describe the business meaning. | | **Missing AAA separation** | Arrange, Act, Assert phases are interleaved or indistinguishable. | -#### Low — Style and hygiene +#### Low -- Style and hygiene | Anti-Pattern | What to Look For | |---|---| @@ -87,7 +87,7 @@ Check each test file against the anti-pattern catalog below. Report findings gro Before reporting, re-check each finding against these severity rules: - **Critical/High**: Only for issues that cause tests to give false confidence or be unreliable. A test that always passes regardless of correctness is Critical. Flaky shared state is High. -- **Medium**: Only for issues that actively harm maintainability — 5+ nearly-identical tests, truly meaningless names like `Test1`. +- **Medium**: Only for issues that actively harm maintainability -- 5+ nearly-identical tests, truly meaningless names like `Test1`. - **Low**: Cosmetic naming mismatches, minor style preferences, assertion messages that could be better. When in doubt, rate Low. - **Not an issue**: Separate tests for distinct boundary conditions (zero vs. negative vs. null). Explicit per-test setup instead of `[TestInitialize]` (this *improves* isolation). Tests that are short and clear but could theoretically be consolidated. @@ -97,22 +97,22 @@ IMPORTANT: If the tests are well-written, say so clearly up front. Do not inflat Present findings in this structure: -1. **Summary** — Total issues found, broken down by severity (Critical / High / Medium / Low). If tests are well-written, lead with that assessment. -2. **Critical and High findings** — List each with: +1. **Summary** -- Total issues found, broken down by severity (Critical / High / Medium / Low). If tests are well-written, lead with that assessment. +2. **Critical and High findings** -- List each with: - The anti-pattern name - The specific location (file, method name, line) - A brief explanation of why it's a problem - A concrete fix (show before/after code when helpful) -3. **Medium and Low findings** — Summarize in a table unless the user wants full detail -4. **Positive observations** — Call out things the tests do well (sealed class, specific exception types, data-driven tests, clear AAA structure, proper use of fakes, good naming). Don't only report negatives. +3. **Medium and Low findings** -- Summarize in a table unless the user wants full detail +4. **Positive observations** -- Call out things the tests do well (sealed class, specific exception types, data-driven tests, clear AAA structure, proper use of fakes, good naming). Don't only report negatives. -### Step 4: Prioritize recommendations +### Step 5: Prioritize recommendations If there are many findings, recommend which to fix first: -1. **Critical** — Fix immediately, these tests may be giving false confidence -2. **High** — Fix soon, these cause flakiness or maintenance burden -3. **Medium/Low** — Fix opportunistically during related edits +1. **Critical** -- Fix immediately, these tests may be giving false confidence +2. **High** -- Fix soon, these cause flakiness or maintenance burden +3. **Medium/Low** -- Fix opportunistically during related edits ## Validation @@ -127,10 +127,10 @@ If there are many findings, recommend which to fix first: | Pitfall | Solution | |---------|----------| | Reporting style issues as critical | Naming and formatting are Medium/Low, never Critical | -| Suggesting rewrites instead of targeted fixes | Show minimal diffs — change the assertion, not the whole test | +| Suggesting rewrites instead of targeted fixes | Show minimal diffs -- change the assertion, not the whole test | | Flagging intentional design choices | If `Thread.Sleep` is in an integration test testing actual timing, that's not an anti-pattern. Consider context. | | Inventing false positives on clean code | If tests follow best practices, say so. A review finding "0 Critical, 0 High, 1 Low" is perfectly valid. Don't inflate findings to justify the review. | | Flagging separate boundary tests as duplicates | Two tests for zero and negative inputs test different edge cases. Only flag as duplicates when 3+ tests have truly identical bodies differing by a single value. | -| Rating cosmetic issues as Medium | Naming mismatches (e.g., method name says `ArgumentException` but asserts `ArgumentOutOfRangeException`) are Low, not Medium — the test still works correctly. | -| Ignoring the test framework | xUnit uses `[Fact]`/`[Theory]`, NUnit uses `[Test]`/`[TestCase]`, MSTest uses `[TestMethod]`/`[DataRow]` — use correct terminology | +| Rating cosmetic issues as Medium | Naming mismatches (e.g., method name says `ArgumentException` but asserts `ArgumentOutOfRangeException`) are Low, not Medium -- the test still works correctly. | +| Ignoring the test framework | xUnit uses `[Fact]`/`[Theory]`, NUnit uses `[Test]`/`[TestCase]`, MSTest uses `[TestMethod]`/`[DataRow]` -- use correct terminology | | Missing the forest for the trees | If 80% of tests have no assertions, lead with that systemic issue rather than listing every instance | diff --git a/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md b/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md index ec490f8530..22ca8167d3 100644 --- a/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md +++ b/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md @@ -282,6 +282,8 @@ public async Task FetchData_ReturnsWithinTimeout() #### Retry flaky tests (MSTest 3.9+) +Use only for genuinely flaky external dependencies (network, file system), not to paper over race conditions or shared state issues. + ```csharp [TestMethod] [Retry(3)] From 2ea48926d66b0d984ecfe77a18955ff49ece4078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 30 Mar 2026 10:46:53 +0200 Subject: [PATCH 3/3] Further improvements --- plugins/dotnet-test/skills/crap-score/SKILL.md | 2 ++ .../skills/migrate-mstest-v1v2-to-v3/SKILL.md | 1 + .../skills/migrate-mstest-v3-to-v4/SKILL.md | 3 ++- .../dotnet-test/skills/writing-mstest-tests/SKILL.md | 10 +++++++++- tests/dotnet-test/run-tests/eval.yaml | 2 +- tests/dotnet-test/test-anti-patterns/eval.yaml | 1 + tests/dotnet-test/writing-mstest-tests/eval.yaml | 4 ++++ 7 files changed, 20 insertions(+), 3 deletions(-) diff --git a/plugins/dotnet-test/skills/crap-score/SKILL.md b/plugins/dotnet-test/skills/crap-score/SKILL.md index f6babbd335..1f5e526d29 100644 --- a/plugins/dotnet-test/skills/crap-score/SKILL.md +++ b/plugins/dotnet-test/skills/crap-score/SKILL.md @@ -58,6 +58,8 @@ A method with 100% coverage has CRAP = complexity (the minimum). A method with 0 ### Step 1: Collect code coverage data +If no coverage data exists yet (no Cobertura XML available), **always run `dotnet test` with coverage collection first** and mention the exact command in your response. Do not skip this step -- CRAP scores require coverage data. + Check the test project's `.csproj` for the coverage package, then run the appropriate command: | Coverage Package | Command | Output Location | diff --git a/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md b/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md index 4982a8befd..85377105d9 100644 --- a/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-mstest-v1v2-to-v3/SKILL.md @@ -61,6 +61,7 @@ MSTest v3 introduces these breaking changes from v1/v2. Address only the ones re ## Response Guidelines +- **Always identify the current version first**: Before recommending any migration steps, explicitly state the current MSTest version detected in the project (e.g., "Your project uses MSTest v2 (2.2.10)" or "This is an MSTest v1 project using QualityTools assembly references"). This grounds the migration advice and confirms you've read the project files. - **Focused fix requests** (user has specific compilation errors after upgrading): Address only the relevant breaking change from the table above. Show a concise before/after fix. Do not walk through the full migration workflow. - **Specific feature migration** (user asks about one aspect like .testsettings, DataRow, or assertions): Address only that specific aspect with a concrete fix. Do not walk through the entire migration workflow or unrelated breaking changes. - **"What to expect" questions** (user asks about breaking changes before upgrading): Present only the breaking changes that are clearly relevant to the user's visible code and configuration. For each, give a one-line fix summary. Do not include every possible breaking change -- only the ones that apply. Do not walk through the full workflow. diff --git a/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md b/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md index 597c660b6a..f42fb20c57 100644 --- a/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md +++ b/plugins/dotnet-test/skills/migrate-mstest-v3-to-v4/SKILL.md @@ -47,7 +47,8 @@ Migrate a test project from MSTest v3 to MSTest v4. The outcome is a project usi ## Response Guidelines -- **Focused fix requests** (user has specific compilation errors after upgrading): Address only the relevant breaking changes from Step 3. Show concrete before/after code using the user's actual types and method names. Mention any related analyzer that could have caught this earlier (e.g., MSTEST0006 for ExpectedException). Do not walk through the entire migration workflow. +- **Always identify the current version first**: Before recommending any migration steps, explicitly state the current MSTest version detected in the project (e.g., "Your project uses MSTest v3 (3.8.0)"). This confirms you've read the project files and grounds the migration advice. +- **Focused fix requests** (user has specific compilation errors after upgrading): Address only the relevant breaking changes from Step 3. **Always provide concrete fixed code** using the user's actual types and method names — show a complete, copy-pasteable code snippet, not just a description of what to change. For custom `TestMethodAttribute` subclasses, show the full fixed class including CallerInfo propagation to the base constructor. Mention any related analyzer that could have caught this earlier (e.g., MSTEST0006 for ExpectedException). Do not walk through the entire migration workflow. - **"What to expect" questions** (user asks about breaking changes before upgrading): Present ALL major breaking changes from the Step 3 quick-lookup table -- not just the ones visible in the current code. For each, provide a one-line fix summary. Also mention key behavioral changes from Step 4 (especially TestCase.Id history impact and TreatDiscoveryWarningsAsErrors default). If project code is available, highlight which changes apply directly. - **Full migration requests** (user wants complete migration): Follow the complete workflow below. - **Behavioral/runtime symptom reports** (user describes test execution differences without build errors): Match described symptoms to the behavioral changes table in Step 4. Provide targeted, symptom-specific advice. Mention other behavioral changes the user should watch for. Do not walk through source breaking changes unless the user also has build errors. diff --git a/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md b/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md index 22ca8167d3..ecf03c520c 100644 --- a/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md +++ b/plugins/dotnet-test/skills/writing-mstest-tests/SKILL.md @@ -142,7 +142,15 @@ Assert.IsEmpty(collection); Assert.IsNotEmpty(collection); ``` -Replace `Assert.IsTrue(x != null)` with `Assert.IsNotNull(x)` -- specialized assertions give better failure messages than generic `Assert.IsTrue`. +Replace generic `Assert.IsTrue` with specialized assertions -- they give better failure messages: + +| Instead of | Use | +|---|---| +| `Assert.IsTrue(list.Count > 0)` | `Assert.IsNotEmpty(list)` | +| `Assert.IsTrue(list.Count() == 3)` | `Assert.HasCount(3, list)` | +| `Assert.IsTrue(x != null)` | `Assert.IsNotNull(x)` | +| `list.Single(predicate)` + `Assert.IsNotNull` | `Assert.ContainsSingle(list)` | +| `Assert.IsTrue(list.Contains(item))` | `Assert.Contains(item, list)` | #### String assertions diff --git a/tests/dotnet-test/run-tests/eval.yaml b/tests/dotnet-test/run-tests/eval.yaml index 6c0f03238e..45a8b0e021 100644 --- a/tests/dotnet-test/run-tests/eval.yaml +++ b/tests/dotnet-test/run-tests/eval.yaml @@ -44,7 +44,7 @@ scenarios: - "Uses '-- --report-trx' (with separator) instead of just '--report-trx'" - "Does not suggest the VSTest-style '--logger trx' for an MTP project" expect_tools: ["bash"] - timeout: 300 + timeout: 360 - name: "Run tests with blame-hang on MTP project (SDK 10)" prompt: "My tests are occasionally hanging. I want to run them with a hang timeout of 5 minutes to detect the culprit. Here's my project." diff --git a/tests/dotnet-test/test-anti-patterns/eval.yaml b/tests/dotnet-test/test-anti-patterns/eval.yaml index 392ed7c645..02053e6e14 100644 --- a/tests/dotnet-test/test-anti-patterns/eval.yaml +++ b/tests/dotnet-test/test-anti-patterns/eval.yaml @@ -144,6 +144,7 @@ scenarios: - "Identified that AddUser_NullUser uses broad Assert.ThrowsException instead of the specific ArgumentNullException" - "Identified that the static HttpClient field is never disposed and appears unused" - "Provided concrete fixes for the critical and high severity findings" + reject_tools: ["bash", "edit", "create"] timeout: 120 - name: "Detect flakiness indicators and test coupling" diff --git a/tests/dotnet-test/writing-mstest-tests/eval.yaml b/tests/dotnet-test/writing-mstest-tests/eval.yaml index de775673f8..4a0382e06a 100644 --- a/tests/dotnet-test/writing-mstest-tests/eval.yaml +++ b/tests/dotnet-test/writing-mstest-tests/eval.yaml @@ -111,6 +111,7 @@ scenarios: } } ``` + reject_tools: ["bash", "edit", "create"] assertions: - type: output_matches pattern: "Assert\\.AreEqual\\(5" @@ -163,6 +164,7 @@ scenarios: service.CreateUser(null, "John"); } ``` + reject_tools: ["bash", "edit", "create"] assertions: - type: output_matches pattern: "Assert\\.ThrowsExactly|Assert\\.Throws" @@ -197,6 +199,7 @@ scenarios: Assert.IsTrue(admin != null); } ``` + reject_tools: ["bash", "edit", "create"] assertions: - type: output_matches pattern: "Assert\\.HasCount|Assert\\.AreEqual\\(3" @@ -227,6 +230,7 @@ scenarios: Assert.AreEqual("email", handler.Type); } ``` + reject_tools: ["bash", "edit", "create"] assertions: - type: output_matches pattern: "IsInstanceOfType"