From cbbfa79d6187d6db26837f2f98c915f0405d5a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 1 Jul 2026 12:23:41 +0200 Subject: [PATCH 1/7] Consolidate dotnet-test-frameworks into test-analysis-extensions Remove the orphaned dotnet-test-frameworks reference skill, whose content was a duplicate subset of test-analysis-extensions/extensions/dotnet.md. Nothing actually loaded it. Redirect the remaining references and drop its eval tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../skills/exp-test-maintainability/SKILL.md | 2 +- plugins/dotnet-test/README.md | 1 - .../skills/dotnet-test-frameworks/SKILL.md | 139 ---- .../extensions/dotnet.md | 2 - .../dotnet-test-frameworks/eval.vally.yaml | 637 ------------------ .../dotnet-test-frameworks/eval.yaml | 556 --------------- 6 files changed, 1 insertion(+), 1336 deletions(-) delete mode 100644 plugins/dotnet-test/skills/dotnet-test-frameworks/SKILL.md delete mode 100644 tests/dotnet-test/dotnet-test-frameworks/eval.vally.yaml delete mode 100644 tests/dotnet-test/dotnet-test-frameworks/eval.yaml diff --git a/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md b/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md index 64d780b2e3..4b6b597e67 100644 --- a/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md +++ b/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md @@ -36,7 +36,7 @@ Analyze .NET test code for maintainability issues: duplicated boilerplate, copy- ### Step 1: Gather the test code -Read all test files the user provides or references. If the user points to a directory or project, scan for all test files — see the `dotnet-test-frameworks` skill for framework-specific markers. +Read all test files the user provides or references. If the user points to a directory or project, scan for all test files — see the `test-analysis-extensions` skill's `extensions/dotnet.md` reference file for framework-specific markers. ### Step 2: Identify maintainability issues diff --git a/plugins/dotnet-test/README.md b/plugins/dotnet-test/README.md index 61a49cbcff..7020e3326f 100644 --- a/plugins/dotnet-test/README.md +++ b/plugins/dotnet-test/README.md @@ -71,7 +71,6 @@ For non-.NET languages, use the native coverage tool: `coverage.py`/`pytest-cov` | **test-analysis-extensions** | Language-specific guidance loaded by the polyglot analysis skills (test markers, assertion APIs, sleeps, skips, mystery-guest indicators, integration markers, tag-support capability) | | **platform-detection** *(.NET)* | Detect VSTest vs MTP and identify the test framework from project files | | **filter-syntax** *(.NET)* | Test filter syntax reference for VSTest and MTP across all frameworks | -| **dotnet-test-frameworks** *(.NET)* | Framework detection patterns, assertion APIs, skip annotations, and lifecycle methods (kept for backward compatibility with .NET-only skills like `writing-mstest-tests`) | ## Agents diff --git a/plugins/dotnet-test/skills/dotnet-test-frameworks/SKILL.md b/plugins/dotnet-test/skills/dotnet-test-frameworks/SKILL.md deleted file mode 100644 index ddc5cd5d88..0000000000 --- a/plugins/dotnet-test/skills/dotnet-test-frameworks/SKILL.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -name: dotnet-test-frameworks -description: "Reference data for .NET test framework detection patterns, assertion APIs, skip annotations, setup/teardown methods, and common test smell indicators across MSTest, xUnit, NUnit, and TUnit. Loaded by test analysis skills (test-anti-patterns) as framework-specific lookup tables." -user-invocable: false -disable-model-invocation: true -license: MIT ---- - -# .NET Test Framework Reference - -Language-specific detection patterns for .NET test frameworks (MSTest, xUnit, NUnit, TUnit). - -## Test File Identification - -| Framework | Test class markers | Test method markers | -| --------- | ------------------ | ------------------- | -| MSTest | `[TestClass]` | `[TestMethod]`, `[DataTestMethod]` | -| xUnit | *(none — convention-based)* | `[Fact]`, `[Theory]` | -| NUnit | `[TestFixture]` | `[Test]`, `[TestCase]`, `[TestCaseSource]` | -| TUnit | *(none — convention-based)* | `[Test]` | - -## Assertion APIs by Framework - -| Category | MSTest | xUnit | NUnit | TUnit | -| -------- | ------ | ----- | ----- | ----- | -| Equality | `Assert.AreEqual` | `Assert.Equal` | `Assert.That(x, Is.EqualTo(y))` | `await Assert.That(x).IsEqualTo(y)` | -| Boolean | `Assert.IsTrue` / `Assert.IsFalse` | `Assert.True` / `Assert.False` | `Assert.That(x, Is.True)` | `await Assert.That(x).IsTrue()` / `await Assert.That(x).IsFalse()` | -| Null | `Assert.IsNull` / `Assert.IsNotNull` | `Assert.Null` / `Assert.NotNull` | `Assert.That(x, Is.Null)` | `await Assert.That(x).IsNull()` / `await Assert.That(x).IsNotNull()` | -| Exception | `Assert.Throws()` / `Assert.ThrowsExactly()` | `Assert.Throws()` | `Assert.That(() => ..., Throws.TypeOf())` | `await Assert.That(() => ...).Throws()` / `await Assert.That(() => ...).ThrowsExactly()` | -| Collection | `CollectionAssert.Contains` | `Assert.Contains` | `Assert.That(col, Has.Member(x))` | `await Assert.That(col).Contains(x)` | -| String | `StringAssert.Contains` | `Assert.Contains(str, sub)` | `Assert.That(str, Does.Contain(sub))` | `await Assert.That(str).Contains(sub)` | -| Type | `Assert.IsInstanceOfType` | `Assert.IsAssignableFrom` | `Assert.That(x, Is.InstanceOf())` | `await Assert.That(x).IsAssignableTo()` (use `await Assert.That(x).IsTypeOf()` for exact-type check) | -| Inconclusive | `Assert.Inconclusive()` | *skip via `[Fact(Skip)]`* | `Assert.Inconclusive()` | `Skip.Test("reason")` (no true inconclusive state) | -| Fail | `Assert.Fail()` | `Assert.Fail()` (.NET 10+) | `Assert.Fail()` | `Assert.Fail()` | - -**TUnit-specific:** assertions are async and **must be awaited** — a forgotten `await` causes the assertion to never run, and the test passes silently. A built-in analyzer warns when `await` is missing. Multiple assertions can be combined with `.And` / `.Or` chaining or grouped via `Assert.Multiple()`. - -Third-party assertion libraries: `Should*` (Shouldly), `.Should()` (FluentAssertions / AwesomeAssertions), `Verify()` (Verify). TUnit also ships an optional `TUnit.Assertions.Should` package providing FluentAssertions-style `value.Should().BeEqualTo(...)` on top of the same infrastructure. - -## Sleep/Delay Patterns - -| Pattern | Example | -| ------- | ------- | -| Thread sleep | `Thread.Sleep(2000)` | -| Task delay | `await Task.Delay(1000)` | -| SpinWait | `SpinWait.SpinUntil(() => condition, timeout)` | - -## Skip/Ignore Annotations - -| Framework | Annotation | With reason | -| --------- | ---------- | ----------- | -| MSTest | `[Ignore]` | `[Ignore("reason")]` | -| xUnit | `[Fact(Skip = "reason")]` | *(reason is required)* | -| NUnit | `[Ignore("reason")]` | *(reason is required)* | -| TUnit | `[Skip("reason")]` | *(reason is required; also valid at class and assembly scope, e.g. `[assembly: Skip("…")]`. Dynamic in-test skipping via `Skip.Test("reason")`.)* | -| Conditional | `#if false` / `#if NEVER` | *(no reason possible)* | - -## Exception Handling — Idiomatic Alternatives - -When a test uses `try`/`catch` to verify exceptions, suggest the framework-native alternative: - -**MSTest:** - -```csharp -// Instead of try/catch (matches exact type): -var ex = Assert.ThrowsExactly( - () => processor.ProcessOrder(emptyOrder)); -Assert.AreEqual("Order must contain at least one item", ex.Message); - -// Or (also matches derived types): -var ex = Assert.Throws( - () => processor.ProcessOrder(emptyOrder)); -Assert.AreEqual("Order must contain at least one item", ex.Message); -``` - -**xUnit:** - -```csharp -var ex = Assert.Throws( - () => processor.ProcessOrder(emptyOrder)); -Assert.Equal("Order must contain at least one item", ex.Message); -``` - -**NUnit:** - -```csharp -var ex = Assert.Throws( - () => processor.ProcessOrder(emptyOrder)); -Assert.That(ex.Message, Is.EqualTo("Order must contain at least one item")); -``` - -**TUnit:** - -```csharp -await Assert.That(() => processor.ProcessOrder(emptyOrder)) - .Throws() - .WithMessage("Order must contain at least one item"); - -// Or, for exact-type matching (no derived types): -await Assert.That(() => processor.ProcessOrder(emptyOrder)) - .ThrowsExactly(); -``` - -## Mystery Guest — Common .NET Patterns - -| Smell indicator | What to look for | -| --------------- | ---------------- | -| File system | `File.ReadAllText`, `File.Exists`, `File.WriteAllBytes`, `Directory.GetFiles`, `Path.Combine` with hard-coded paths | -| Database | `SqlConnection`, `DbContext` (without in-memory provider), `SqlCommand` | -| Network | `HttpClient` without `HttpMessageHandler` override, `WebRequest`, `TcpClient` | -| Environment | `Environment.GetEnvironmentVariable`, `Environment.CurrentDirectory` | -| Acceptable | `MemoryStream`, `StringReader`, `InMemory` database providers, custom `DelegatingHandler` | - -## Integration Test Markers - -Recognize these as integration tests (adjust smell severity accordingly): - -- Class name contains `Integration`, `E2E`, `EndToEnd`, or `Acceptance` -- `[TestCategory("Integration")]` (MSTest) -- `[Trait("Category", "Integration")]` (xUnit) -- `[Category("Integration")]` (NUnit, TUnit) -- Project name ending in `.IntegrationTests` or `.E2ETests` - -## Setup/Teardown Methods - -| Framework | Setup | Teardown | -| --------- | ----- | -------- | -| MSTest | `[TestInitialize]` or constructor | `[TestCleanup]` or `IDisposable.Dispose` / `IAsyncDisposable.DisposeAsync` | -| xUnit | constructor | `IDisposable.Dispose` / `IAsyncDisposable.DisposeAsync` | -| NUnit | `[SetUp]` | `[TearDown]` | -| TUnit | `[Before(Test)]` or constructor | `[After(Test)]` or `IDisposable.Dispose` / `IAsyncDisposable.DisposeAsync` | -| MSTest (class) | `[ClassInitialize]` | `[ClassCleanup]` | -| NUnit (class) | `[OneTimeSetUp]` | `[OneTimeTearDown]` | -| xUnit (class) | `IClassFixture` | fixture's `Dispose` | -| TUnit (class) | `[Before(Class)]` | `[After(Class)]` | -| TUnit (assembly) | `[Before(Assembly)]` | `[After(Assembly)]` | -| TUnit (session) | `[Before(TestSession)]` | `[After(TestSession)]` | - -**TUnit-specific:** `[BeforeEvery(Test)]` / `[AfterEvery(Test)]` (and the `Class` / `Assembly` variants) run for every test/class/assembly across the whole test run — useful for global cross-cutting hooks. Hooks may optionally accept a context object (`TestContext`, `ClassHookContext`, etc.) and/or a `CancellationToken`. diff --git a/plugins/dotnet-test/skills/test-analysis-extensions/extensions/dotnet.md b/plugins/dotnet-test/skills/test-analysis-extensions/extensions/dotnet.md index fe2193e5e8..54a6ce332d 100644 --- a/plugins/dotnet-test/skills/test-analysis-extensions/extensions/dotnet.md +++ b/plugins/dotnet-test/skills/test-analysis-extensions/extensions/dotnet.md @@ -2,8 +2,6 @@ Reference data for analyzing .NET test code. Used by the polyglot test analysis skills (`assertion-quality`, `test-anti-patterns`, `test-gap-analysis`, `test-smell-detection`, `test-tagging`). -> See also: the standalone `dotnet-test-frameworks` skill, which carries the same data and is loaded by .NET-only skills. - ## Capability tags | Capability | Support | diff --git a/tests/dotnet-test/dotnet-test-frameworks/eval.vally.yaml b/tests/dotnet-test/dotnet-test-frameworks/eval.vally.yaml deleted file mode 100644 index f6103efee4..0000000000 --- a/tests/dotnet-test/dotnet-test-frameworks/eval.vally.yaml +++ /dev/null @@ -1,637 +0,0 @@ -name: dotnet-test-frameworks -description: Evaluates the dotnet-test/dotnet-test-frameworks skill -type: capability -config: - timeout: 2m -stimuli: - - name: Cross-framework assertion equivalence mapping - expect_activation: false - prompt: | - I have these MSTest assertions in my test file and I need to know their equivalents in both xUnit and NUnit. Please provide a complete mapping for all six: - - ```csharp - Assert.AreEqual(expected, actual); - Assert.IsTrue(condition); - Assert.IsNull(obj); - Assert.IsInstanceOfType(obj, typeof(MyClass)); - CollectionAssert.Contains(list, item); - StringAssert.Contains(str, "substring"); - ``` - graders: - - type: output-contains - config: - substring: Assert.Equal - - type: output-matches - config: - pattern: Assert\.True|Assert\.False - - type: output-matches - config: - pattern: Assert\.Null|Assert\.NotNull - - type: output-matches - config: - pattern: Assert\.IsAssignableFrom|Assert\.IsType - - type: output-matches - config: - pattern: Assert\.Contains - - type: output-matches - config: - pattern: Assert\.That\(.*Is\.EqualTo - - type: output-matches - config: - pattern: Is\.True|Is\.False - - type: output-matches - config: - pattern: Is\.Null - - type: output-matches - config: - pattern: Has\.Member|Does\.Contain - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Provided correct xUnit equivalents for all six MSTest assertions - - Provided correct NUnit equivalents using the constraint model (Assert.That with Is/Has/Does syntax) - - Covered collection assertion equivalents for both target frameworks - - Covered string assertion equivalents for both target frameworks - - - name: Identify TUnit framework and its unique attributes - expect_activation: false - prompt: | - I inherited a project with this test file and I'm not sure what testing framework it uses. - Can you identify it and explain the key attributes — in particular whether the test class - needs any marker attribute, and what `[ClassDataSource]` actually does? - - ```csharp - using TUnit.Core; - - [ClassDataSource] - public class OrderTests - { - private readonly DatabaseFixture _db; - - public OrderTests(DatabaseFixture db) - { - _db = db; - } - - [Test] - public async Task CreateOrder_WithValidItems_Succeeds() - { - var order = new Order { Items = new[] { "Widget" } }; - var result = await _db.OrderService.CreateAsync(order); - await Assert.That(result.Id).IsNotNull(); - } - - [Test] - [Skip("Waiting for payment gateway sandbox")] - public async Task ProcessPayment_ChargesCorrectAmount() - { - // TODO: implement - } - } - ``` - graders: - - type: output-matches - config: - pattern: "[Tt][Uu]nit" - - type: output-matches - config: - pattern: ClassDataSource - - type: output-matches - config: - pattern: \[Skip - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Correctly identified the framework as TUnit - - Clarified that TUnit test classes do NOT require a class-level marker attribute (convention-based, like xUnit) — `[ClassDataSource]` is a fixture / data source, not the class marker - - Explained that `[ClassDataSource]` injects a shared fixture instance into the class constructor - - Identified `[Test]` as the test method marker - - Noted the [Skip] attribute with required reason string as TUnit's mechanism for skipping tests - - Mentioned that TUnit assertions are async and must be awaited (e.g. `await Assert.That(...).IsNotNull()`) - - - name: Replace try-catch with framework-native exception assertions - expect_activation: false - prompt: | - My team lead says these tests should use proper exception assertions instead of try/catch. - Can you refactor each one using its framework's native approach? - - MSTest test: - ```csharp - [TestMethod] - public void ProcessOrder_EmptyOrder_ThrowsException() - { - try - { - var processor = new OrderProcessor(); - processor.ProcessOrder(new Order()); - Assert.Fail("Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - Assert.AreEqual("Order must contain at least one item", ex.Message); - } - } - ``` - - xUnit test: - ```csharp - [Fact] - public void ProcessOrder_EmptyOrder_ThrowsException() - { - try - { - var processor = new OrderProcessor(); - processor.ProcessOrder(new Order()); - Assert.True(false, "Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - Assert.Equal("Order must contain at least one item", ex.Message); - } - } - ``` - - NUnit test: - ```csharp - [Test] - public void ProcessOrder_EmptyOrder_ThrowsException() - { - try - { - var processor = new OrderProcessor(); - processor.ProcessOrder(new Order()); - Assert.Fail("Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - Assert.That(ex.Message, Is.EqualTo("Order must contain at least one item")); - } - } - ``` - graders: - - type: output-matches - config: - pattern: \[TestMethod\].*Assert\.Throws(Exactly)?<|Assert\.Throws(Exactly)?<.*AreEqual - - type: output-matches - config: - pattern: \[Fact\].*Assert\.Throws<|Assert\.Throws<.*Assert\.Equal - - type: output-matches - config: - pattern: Throws\.TypeOf|Assert\.That\(.*Throws - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Replaced the MSTest try/catch with Assert.ThrowsExactly or Assert.Throws - - Replaced the xUnit try/catch with Assert.Throws - - Replaced the NUnit try/catch with the constraint-model pattern or Assert.Throws - - Preserved exception message verification in all three refactored versions - - - name: Skip annotations across all four frameworks - expect_activation: false - prompt: | - I maintain a solution with test projects using MSTest, xUnit, NUnit, and TUnit. - I need to temporarily skip some tests in each project with a reason message explaining why. - What's the correct attribute to use in each framework? - graders: - - type: output-matches - config: - pattern: MSTest - - type: output-matches - config: - pattern: xUnit - - type: output-matches - config: - pattern: NUnit - - type: output-matches - config: - pattern: TUnit - - type: output-matches - config: - pattern: \[Ignore\( - - type: output-matches - config: - pattern: Skip\s*=\s*" - - type: output-matches - config: - pattern: \[Skip\( - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Provided the correct skip attribute for MSTest ([Ignore] with optional reason) - - Provided the correct skip mechanism for xUnit (Skip property on [Fact] or [Theory]) - - Provided the correct skip attribute for NUnit ([Ignore] with required reason) - - Provided the correct skip attribute for TUnit ([Skip] with required reason) - - - name: Convert NUnit lifecycle methods to xUnit equivalents - expect_activation: false - prompt: | - I'm migrating this NUnit test class to xUnit. How should I convert the setup and teardown methods? - - ```csharp - [TestFixture] - public class CustomerRepositoryTests - { - private DbContext _context; - private CustomerRepository _repo; - - [OneTimeSetUp] - public void ClassSetup() - { - Database.Initialize(); - } - - [OneTimeTearDown] - public void ClassTeardown() - { - Database.Cleanup(); - } - - [SetUp] - public void TestSetup() - { - _context = new TestDbContext(); - _repo = new CustomerRepository(_context); - } - - [TearDown] - public void TestTeardown() - { - _context.Dispose(); - } - - [Test] - public void FindById_ExistingCustomer_ReturnsCustomer() - { - var customer = _repo.FindById(1); - Assert.That(customer, Is.Not.Null); - Assert.That(customer.Name, Is.EqualTo("Alice")); - } - } - ``` - graders: - - type: output-matches - config: - pattern: IClassFixture - - type: output-matches - config: - pattern: IDisposable|Dispose - - type: output-matches - config: - pattern: \[Fact\] - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Converted per-test [SetUp]/[TearDown] to constructor and IDisposable.Dispose - - Converted class-level [OneTimeSetUp]/[OneTimeTearDown] to an IClassFixture with the fixture's constructor and - Dispose - - Removed [TestFixture] and converted [Test] to [Fact] - - - name: Identify integration tests by markers and code patterns - expect_activation: false - prompt: | - I have a test solution with hundreds of tests. I need to separate integration tests from - unit tests for CI pipeline optimization. Here are samples from three projects — which ones - are integration tests and what indicators tell you that? - - Project A (MSTest): - ```csharp - [TestClass] - [TestCategory("Integration")] - public class PaymentGatewayTests - { - [TestMethod] - public async Task ChargeCard_ValidCard_ReturnsSuccess() - { - var client = new HttpClient(); - var gateway = new PaymentGateway(client); - var result = await gateway.ChargeAsync("4111111111111111", 99.99m); - Assert.IsTrue(result.Success); - } - } - ``` - - Project B (xUnit): - ```csharp - public class OrderApiTests - { - [Fact] - [Trait("Category", "Integration")] - public async Task CreateOrder_ReturnsCreatedStatus() - { - using var factory = new WebApplicationFactory(); - var client = factory.CreateClient(); - var response = await client.PostAsJsonAsync("/orders", new { Item = "Widget" }); - Assert.Equal(HttpStatusCode.Created, response.StatusCode); - } - } - ``` - - Project C (NUnit): - ```csharp - [TestFixture] - public class DatabaseQueryTests - { - [Test] - public void GetCustomer_ById_ReturnsExpectedName() - { - using var conn = new SqlConnection("Server=localhost;..."); - conn.Open(); - var name = conn.QuerySingle("SELECT Name FROM Customers WHERE Id = 1"); - Assert.That(name, Is.EqualTo("Alice")); - } - } - ``` - - Project D (TUnit): - ```csharp - using TUnit.Core; - - public class InventoryServiceTests - { - [Test] - public async Task ReserveStock_AgainstLiveWarehouse_ReducesAvailableQuantity() - { - using var conn = new SqlConnection("Server=warehouse-db;..."); - await conn.OpenAsync(); - var service = new InventoryService(conn); - var ok = await service.ReserveAsync(sku: "WIDGET-1", quantity: 5); - await Assert.That(ok).IsTrue(); - } - } - ``` - graders: - - type: output-matches - config: - pattern: TestCategory.*Integration|\[TestCategory\("Integration"\)\] - - type: output-matches - config: - pattern: Trait.*Category.*Integration|\[Trait\( - - type: output-matches - config: - pattern: SqlConnection|database - - type: output-matches - config: - pattern: \[Category\("Integration"\)\]|Category.*Integration.*NUnit|NUnit.*Category - - type: output-matches - config: - pattern: "[Tt][Uu]nit" - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Identified Project A as integration test via the [TestCategory("Integration")] marker and direct HttpClient usage - - Identified Project B as integration test via the [Trait("Category", "Integration")] marker - - Identified Project C as integration test based on direct database access (SqlConnection), even without an explicit - category marker - - Identified Project D as an integration test based on direct database access against a live warehouse server - - Recommended adding a framework-appropriate integration category marker to Project C (NUnit `[Category("Integration")]`) - and Project D (TUnit `[Category("Integration")]`) - - - name: Convert cross-framework assertions to TUnit syntax - expect_activation: false - prompt: | - I'm porting a small test class from MSTest to TUnit. Can you rewrite each of these - assertions in the TUnit equivalent? Please show me the exact line each one becomes. - - ```csharp - Assert.AreEqual(expected, actual); - Assert.IsTrue(order.IsValid); - Assert.IsNull(result.Error); - Assert.IsInstanceOfType(handler, typeof(IOrderHandler)); - CollectionAssert.Contains(processedIds, 42); - var ex = Assert.ThrowsException(() => processor.Run()); - Assert.AreEqual("Order is empty", ex.Message); - ``` - graders: - - type: output-matches - config: - pattern: await\s+Assert\.That - - type: output-matches - config: - pattern: \.IsEqualTo\( - - type: output-matches - config: - pattern: \.IsTrue\(\) - - type: output-matches - config: - pattern: \.IsNull\(\) - - type: output-matches - config: - pattern: \.IsAssignableTo<|\.IsTypeOf< - - type: output-matches - config: - pattern: \.Contains\( - - type: output-matches - config: - pattern: \.Throws(Exactly)? - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Used `await Assert.That(...)` for every converted assertion (TUnit assertions are async and must be awaited) - - Mapped equality, boolean, null, type, and collection-contains assertions to the correct TUnit fluent member - (IsEqualTo / IsTrue / IsNull / IsAssignableTo or IsTypeOf / Contains) - - Converted the exception assertion to `await Assert.That(() => ...).Throws()` (or `ThrowsExactly()`) and - preserved the message check (e.g. via `.WithMessage(...)` or a follow-up awaited assertion on the exception) - - Did not silently drop the `await` on any assertion or leave NUnit/xUnit-style `Assert.Throws` syntax in the - TUnit output - - - name: Diagnose silently-passing TUnit test with missing await - expect_activation: false - prompt: | - This TUnit test is green in CI, but I'm certain `CalculateTotal` is returning the wrong - value (I added a deliberate bug that returns 0 instead of the sum). Why is the test still - passing, and how do I fix it? - - ```csharp - using TUnit.Core; - - public class CartTests - { - [Test] - public async Task CalculateTotal_TwoItems_ReturnsSum() - { - var cart = new Cart(); - cart.Add(new Item(price: 10m)); - cart.Add(new Item(price: 32m)); - - var total = cart.CalculateTotal(); - - Assert.That(total).IsEqualTo(42m); - } - } - ``` - graders: - - type: output-matches - config: - pattern: await - - type: output-matches - config: - pattern: (never (run|execute|awaited)|not (run|awaited|executed)|silently|discard|fire[- ]and[- ]forget|unobserved) - - type: output-matches - config: - pattern: await\s+Assert\.That\(total\)\.IsEqualTo - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Correctly identified the missing `await` on the `Assert.That(...).IsEqualTo(...)` expression as the root cause - - Explained that TUnit assertions are async and produce a task — without `await`, the assertion is never observed - and the test passes regardless of the actual value - - "Provided the corrected line: `await Assert.That(total).IsEqualTo(42m);`" - - Optionally mentioned the built-in TUnit analyzer that warns when an assertion is not awaited, or suggested - treating that analyzer warning as an error in CI - - Did NOT misdiagnose the failure as a bug in `Cart.CalculateTotal`, a comparison-precision issue (decimal vs - double), or a missing test discovery problem - - - name: Refactor TUnit try/catch to native exception assertion - expect_activation: false - prompt: | - My team lead wants this TUnit test to use the framework's native exception assertion - instead of try/catch, and to verify both the exception type and the message in one - idiomatic expression. Please refactor it. - - ```csharp - using TUnit.Core; - - public class OrderProcessorTests - { - [Test] - public async Task ProcessOrder_EmptyOrder_ThrowsInvalidOperation() - { - var processor = new OrderProcessor(); - try - { - processor.ProcessOrder(new Order()); - Assert.Fail("Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - await Assert.That(ex.Message).IsEqualTo("Order must contain at least one item"); - } - } - } - ``` - graders: - - type: output-matches - config: - pattern: await\s+Assert\.That\( - - type: output-matches - config: - pattern: \.Throws(Exactly)? - - type: output-matches - config: - pattern: (WithMessage|IsEqualTo).*Order must contain at least one item|Order must contain at least one item - - type: output-not-matches - config: - pattern: try\s*\{[\s\S]*catch\s*\(InvalidOperationException - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Removed the try/catch block entirely and replaced it with a single awaited TUnit exception assertion on the - throwing delegate - - Used `Throws()` (or `ThrowsExactly()`) to assert the - exception type - - Verified the message in an idiomatic way (e.g. `.WithMessage("Order must contain at least one item")` chained on - the throw assertion, or an awaited follow-up assertion on the captured exception) - - Kept the `await` on every assertion call in the refactored test - - - name: TUnit lifecycle hooks at test, class, assembly, and session scope - expect_activation: false - prompt: | - In a TUnit test project, I need code that runs: - - 1. Before every individual test in a class (e.g. reset a shared in-memory database). - 2. Once before the first test in a class and once after the last (e.g. open / dispose - a class-scoped fixture). - 3. Once before any test in the assembly starts and once after they all finish - (e.g. start / stop an in-process WireMock server for the whole assembly). - 4. Once at the very start of the whole test run and once at the very end - (e.g. apply EF Core migrations and tear down the database for the entire session). - - What attributes / methods does TUnit use for each of these, and where do they go? - A short C# sketch for each scope would be ideal. - graders: - - type: output-matches - config: - pattern: \[Before\(Test\)\] - - type: output-matches - config: - pattern: \[After\(Test\)\] - - type: output-matches - config: - pattern: \[Before\(Class\)\] - - type: output-matches - config: - pattern: \[After\(Class\)\] - - type: output-matches - config: - pattern: \[Before\(Assembly\)\] - - type: output-matches - config: - pattern: \[After\(Assembly\)\] - - type: output-matches - config: - pattern: \[Before\(TestSession\)\] - - type: output-matches - config: - pattern: \[After\(TestSession\)\] - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - Provided per-test hooks using `[Before(Test)]` / `[After(Test)]` (or noted the constructor + IAsyncDisposable - alternative) on instance methods of the test class - - Provided per-class hooks using `[Before(Class)]` / `[After(Class)]` on static methods of the test class - - Provided per-assembly hooks using `[Before(Assembly)]` / `[After(Assembly)]` on static methods, noting they - apply to every test in the assembly - - Provided per-session hooks using `[Before(TestSession)]` / `[After(TestSession)]` on static methods, noting they - run exactly once across the whole test run - - Did NOT confuse TUnit's scoped `[Before(...)]` / `[After(...)]` attributes with NUnit's - `[SetUp]`/`[TearDown]`/`[OneTimeSetUp]`/`[OneTimeTearDown]` or xUnit's `IClassFixture` / `IAsyncLifetime` - - Optionally mentioned that hook methods may accept a context object (e.g. `TestContext`, `ClassHookContext`) - and/or a `CancellationToken`, or that `[BeforeEvery(Test)]` / `[AfterEvery(Test)]` (and the `Class`/`Assembly` - variants) run for every test/class/assembly across the run - - - name: TUnit skip mechanisms — attribute, assembly-wide, and dynamic - expect_activation: false - prompt: | - In a TUnit test project I need three different ways to skip tests, each with a clear - reason message: - - 1. Skip one specific test method (it's waiting on a payment-gateway sandbox). - 2. Skip every test in an entire assembly when that assembly is built in a special - "smoke" configuration. - 3. Inside a test method, decide at runtime to skip the test if the current machine - is not joined to the corporate VPN. - - What does each one look like in TUnit? - graders: - - type: output-matches - config: - pattern: \[Skip\( - - type: output-matches - config: - pattern: assembly\s*:\s*Skip - - type: output-matches - config: - pattern: Skip\.Test\( - - type: exit-success - - type: prompt - - type: pairwise - rubric: - - For the single-test case, used the `[Skip("reason")]` attribute on the test method with a required reason string - - >- - For the assembly-wide case, used an assembly-level attribute such as `[assembly: Skip("…")]` - (or equivalent class-level `[Skip(...)]` on a base class) rather than annotating each test - individually - - For the runtime case, used the dynamic `Skip.Test("reason")` call inside the test method, after the VPN check, - and explained that this is TUnit's equivalent of a runtime skip (distinct from a true "inconclusive" state) - - Did NOT propose xUnit's `[Fact(Skip = "…")]`, MSTest's `Assert.Inconclusive()`, or NUnit's `Assume.That(...)` as - TUnit answers diff --git a/tests/dotnet-test/dotnet-test-frameworks/eval.yaml b/tests/dotnet-test/dotnet-test-frameworks/eval.yaml deleted file mode 100644 index 6f611500a6..0000000000 --- a/tests/dotnet-test/dotnet-test-frameworks/eval.yaml +++ /dev/null @@ -1,556 +0,0 @@ -scenarios: - - name: "Cross-framework assertion equivalence mapping" - expect_activation: false - prompt: | - I have these MSTest assertions in my test file and I need to know their equivalents in both xUnit and NUnit. Please provide a complete mapping for all six: - - ```csharp - Assert.AreEqual(expected, actual); - Assert.IsTrue(condition); - Assert.IsNull(obj); - Assert.IsInstanceOfType(obj, typeof(MyClass)); - CollectionAssert.Contains(list, item); - StringAssert.Contains(str, "substring"); - ``` - assertions: - - type: "output_contains" - value: "Assert.Equal" - - type: "output_matches" - pattern: "Assert\\.True|Assert\\.False" - - type: "output_matches" - pattern: "Assert\\.Null|Assert\\.NotNull" - - type: "output_matches" - pattern: "Assert\\.IsAssignableFrom|Assert\\.IsType" - - type: "output_matches" - pattern: "Assert\\.Contains" - - type: "output_matches" - pattern: "Assert\\.That\\(.*Is\\.EqualTo" - - type: "output_matches" - pattern: "Is\\.True|Is\\.False" - - type: "output_matches" - pattern: "Is\\.Null" - - type: "output_matches" - pattern: "Has\\.Member|Does\\.Contain" - - type: "exit_success" - rubric: - - "Provided correct xUnit equivalents for all six MSTest assertions" - - "Provided correct NUnit equivalents using the constraint model (Assert.That with Is/Has/Does syntax)" - - "Covered collection assertion equivalents for both target frameworks" - - "Covered string assertion equivalents for both target frameworks" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Identify TUnit framework and its unique attributes" - expect_activation: false - prompt: | - I inherited a project with this test file and I'm not sure what testing framework it uses. - Can you identify it and explain the key attributes — in particular whether the test class - needs any marker attribute, and what `[ClassDataSource]` actually does? - - ```csharp - using TUnit.Core; - - [ClassDataSource] - public class OrderTests - { - private readonly DatabaseFixture _db; - - public OrderTests(DatabaseFixture db) - { - _db = db; - } - - [Test] - public async Task CreateOrder_WithValidItems_Succeeds() - { - var order = new Order { Items = new[] { "Widget" } }; - var result = await _db.OrderService.CreateAsync(order); - await Assert.That(result.Id).IsNotNull(); - } - - [Test] - [Skip("Waiting for payment gateway sandbox")] - public async Task ProcessPayment_ChargesCorrectAmount() - { - // TODO: implement - } - } - ``` - assertions: - - type: "output_matches" - pattern: "[Tt][Uu]nit" - - type: "output_matches" - pattern: "ClassDataSource" - - type: "output_matches" - pattern: "\\[Skip" - - type: "exit_success" - rubric: - - "Correctly identified the framework as TUnit" - - "Clarified that TUnit test classes do NOT require a class-level marker attribute (convention-based, like xUnit) — `[ClassDataSource]` is a fixture / data source, not the class marker" - - "Explained that `[ClassDataSource]` injects a shared fixture instance into the class constructor" - - "Identified `[Test]` as the test method marker" - - "Noted the [Skip] attribute with required reason string as TUnit's mechanism for skipping tests" - - "Mentioned that TUnit assertions are async and must be awaited (e.g. `await Assert.That(...).IsNotNull()`)" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Replace try-catch with framework-native exception assertions" - expect_activation: false - prompt: | - My team lead says these tests should use proper exception assertions instead of try/catch. - Can you refactor each one using its framework's native approach? - - MSTest test: - ```csharp - [TestMethod] - public void ProcessOrder_EmptyOrder_ThrowsException() - { - try - { - var processor = new OrderProcessor(); - processor.ProcessOrder(new Order()); - Assert.Fail("Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - Assert.AreEqual("Order must contain at least one item", ex.Message); - } - } - ``` - - xUnit test: - ```csharp - [Fact] - public void ProcessOrder_EmptyOrder_ThrowsException() - { - try - { - var processor = new OrderProcessor(); - processor.ProcessOrder(new Order()); - Assert.True(false, "Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - Assert.Equal("Order must contain at least one item", ex.Message); - } - } - ``` - - NUnit test: - ```csharp - [Test] - public void ProcessOrder_EmptyOrder_ThrowsException() - { - try - { - var processor = new OrderProcessor(); - processor.ProcessOrder(new Order()); - Assert.Fail("Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - Assert.That(ex.Message, Is.EqualTo("Order must contain at least one item")); - } - } - ``` - assertions: - - type: "output_matches" - pattern: "\\[TestMethod\\].*Assert\\.Throws(Exactly)?<|Assert\\.Throws(Exactly)?<.*AreEqual" - - type: "output_matches" - pattern: "Assert\\.AreEqual\\([^)]*\\.Message" - - type: "output_matches" - pattern: "\\[Fact\\].*Assert\\.Throws<|Assert\\.Throws<.*Assert\\.Equal" - - type: "output_matches" - pattern: "Throws\\.TypeOf|Assert\\.That\\(.*Throws" - - type: "exit_success" - rubric: - - "Replaced the MSTest try/catch with Assert.ThrowsExactly or Assert.Throws" - - "Replaced the xUnit try/catch with Assert.Throws" - - "Replaced the NUnit try/catch with the constraint-model pattern or Assert.Throws" - - "Preserved exception message verification in all three refactored versions (e.g. `Assert.AreEqual(\"...\", ex.Message)` for MSTest)" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Skip annotations across all four frameworks" - expect_activation: false - prompt: | - I maintain a solution with test projects using MSTest, xUnit, NUnit, and TUnit. - I need to temporarily skip some tests in each project with a reason message explaining why. - What's the correct attribute to use in each framework? - assertions: - - type: "output_matches" - pattern: "MSTest" - - type: "output_matches" - pattern: "xUnit" - - type: "output_matches" - pattern: "NUnit" - - type: "output_matches" - pattern: "TUnit" - - type: "output_matches" - pattern: "\\[Ignore\\(" - - type: "output_matches" - pattern: "Skip\\s*=\\s*\"" - - type: "output_matches" - pattern: "\\[Skip\\(" - - type: "exit_success" - rubric: - - "Provided the correct skip attribute for MSTest ([Ignore] with optional reason)" - - "Provided the correct skip mechanism for xUnit (Skip property on [Fact] or [Theory])" - - "Provided the correct skip attribute for NUnit ([Ignore] with required reason)" - - "Provided the correct skip attribute for TUnit ([Skip] with required reason)" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Convert NUnit lifecycle methods to xUnit equivalents" - expect_activation: false - prompt: | - I'm migrating this NUnit test class to xUnit. How should I convert the setup and teardown methods? - - ```csharp - [TestFixture] - public class CustomerRepositoryTests - { - private DbContext _context; - private CustomerRepository _repo; - - [OneTimeSetUp] - public void ClassSetup() - { - Database.Initialize(); - } - - [OneTimeTearDown] - public void ClassTeardown() - { - Database.Cleanup(); - } - - [SetUp] - public void TestSetup() - { - _context = new TestDbContext(); - _repo = new CustomerRepository(_context); - } - - [TearDown] - public void TestTeardown() - { - _context.Dispose(); - } - - [Test] - public void FindById_ExistingCustomer_ReturnsCustomer() - { - var customer = _repo.FindById(1); - Assert.That(customer, Is.Not.Null); - Assert.That(customer.Name, Is.EqualTo("Alice")); - } - } - ``` - assertions: - - type: "output_matches" - pattern: "IClassFixture" - - type: "output_matches" - pattern: "IDisposable|Dispose" - - type: "output_matches" - pattern: "\\[Fact\\]" - - type: "exit_success" - rubric: - - "Converted per-test [SetUp]/[TearDown] to constructor and IDisposable.Dispose" - - "Converted class-level [OneTimeSetUp]/[OneTimeTearDown] to an IClassFixture with the fixture's constructor and Dispose" - - "Removed [TestFixture] and converted [Test] to [Fact]" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Identify integration tests by markers and code patterns" - expect_activation: false - prompt: | - I have a test solution with hundreds of tests. I need to separate integration tests from - unit tests for CI pipeline optimization. Here are samples from three projects — which ones - are integration tests and what indicators tell you that? - - Project A (MSTest): - ```csharp - [TestClass] - [TestCategory("Integration")] - public class PaymentGatewayTests - { - [TestMethod] - public async Task ChargeCard_ValidCard_ReturnsSuccess() - { - var client = new HttpClient(); - var gateway = new PaymentGateway(client); - var result = await gateway.ChargeAsync("4111111111111111", 99.99m); - Assert.IsTrue(result.Success); - } - } - ``` - - Project B (xUnit): - ```csharp - public class OrderApiTests - { - [Fact] - [Trait("Category", "Integration")] - public async Task CreateOrder_ReturnsCreatedStatus() - { - using var factory = new WebApplicationFactory(); - var client = factory.CreateClient(); - var response = await client.PostAsJsonAsync("/orders", new { Item = "Widget" }); - Assert.Equal(HttpStatusCode.Created, response.StatusCode); - } - } - ``` - - Project C (NUnit): - ```csharp - [TestFixture] - public class DatabaseQueryTests - { - [Test] - public void GetCustomer_ById_ReturnsExpectedName() - { - using var conn = new SqlConnection("Server=localhost;..."); - conn.Open(); - var name = conn.QuerySingle("SELECT Name FROM Customers WHERE Id = 1"); - Assert.That(name, Is.EqualTo("Alice")); - } - } - ``` - - Project D (TUnit): - ```csharp - using TUnit.Core; - - public class InventoryServiceTests - { - [Test] - public async Task ReserveStock_AgainstLiveWarehouse_ReducesAvailableQuantity() - { - using var conn = new SqlConnection("Server=warehouse-db;..."); - await conn.OpenAsync(); - var service = new InventoryService(conn); - var ok = await service.ReserveAsync(sku: "WIDGET-1", quantity: 5); - await Assert.That(ok).IsTrue(); - } - } - ``` - assertions: - - type: "output_matches" - pattern: "TestCategory.*Integration|\\[TestCategory\\(\"Integration\"\\)\\]" - - type: "output_matches" - pattern: "Trait.*Category.*Integration|\\[Trait\\(" - - type: "output_matches" - pattern: "SqlConnection|database" - - type: "output_matches" - pattern: "\\[Category\\(\"Integration\"\\)\\]|Category.*Integration.*NUnit|NUnit.*Category" - - type: "output_matches" - pattern: "[Tt][Uu]nit" - - type: "exit_success" - rubric: - - "Identified Project A as integration test via the [TestCategory(\"Integration\")] marker and direct HttpClient usage" - - "Identified Project B as integration test via the [Trait(\"Category\", \"Integration\")] marker" - - "Identified Project C as integration test based on direct database access (SqlConnection), even without an explicit category marker" - - "Identified Project D as an integration test based on direct database access against a live warehouse server" - - "Recommended adding a framework-appropriate integration category marker to Project C (NUnit `[Category(\"Integration\")]`) and Project D (TUnit `[Category(\"Integration\")]`)" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Convert cross-framework assertions to TUnit syntax" - expect_activation: false - prompt: | - I'm porting a small test class from MSTest to TUnit. Can you rewrite each of these - assertions in the TUnit equivalent? Please show me the exact line each one becomes. - - ```csharp - Assert.AreEqual(expected, actual); - Assert.IsTrue(order.IsValid); - Assert.IsNull(result.Error); - Assert.IsInstanceOfType(handler, typeof(IOrderHandler)); - CollectionAssert.Contains(processedIds, 42); - var ex = Assert.ThrowsException(() => processor.Run()); - Assert.AreEqual("Order is empty", ex.Message); - ``` - assertions: - - type: "output_matches" - pattern: "await\\s+Assert\\.That" - - type: "output_matches" - pattern: "\\.IsEqualTo\\(" - - type: "output_matches" - pattern: "\\.IsTrue\\(\\)" - - type: "output_matches" - pattern: "\\.IsNull\\(\\)" - - type: "output_matches" - pattern: "\\.IsAssignableTo<|\\.IsTypeOf<" - - type: "output_matches" - pattern: "\\.Contains\\(" - - type: "output_matches" - pattern: "\\.Throws(Exactly)?" - - type: "exit_success" - rubric: - - "Used `await Assert.That(...)` for every converted assertion (TUnit assertions are async and must be awaited)" - - "Mapped equality, boolean, null, type, and collection-contains assertions to the correct TUnit fluent member (IsEqualTo / IsTrue / IsNull / IsAssignableTo or IsTypeOf / Contains)" - - "Converted the exception assertion to `await Assert.That(() => ...).Throws()` (or `ThrowsExactly()`) and preserved the message check (e.g. via `.WithMessage(...)` or a follow-up awaited assertion on the exception)" - - "Did not silently drop the `await` on any assertion or leave NUnit/xUnit-style `Assert.Throws` syntax in the TUnit output" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Diagnose silently-passing TUnit test with missing await" - expect_activation: false - prompt: | - This TUnit test is green in CI, but I'm certain `CalculateTotal` is returning the wrong - value (I added a deliberate bug that returns 0 instead of the sum). Why is the test still - passing, and how do I fix it? - - ```csharp - using TUnit.Core; - - public class CartTests - { - [Test] - public async Task CalculateTotal_TwoItems_ReturnsSum() - { - var cart = new Cart(); - cart.Add(new Item(price: 10m)); - cart.Add(new Item(price: 32m)); - - var total = cart.CalculateTotal(); - - Assert.That(total).IsEqualTo(42m); - } - } - ``` - assertions: - - type: "output_matches" - pattern: "await" - - type: "output_matches" - pattern: "(never (run|execute|awaited)|not (run|awaited|executed)|silently|discard|fire[- ]and[- ]forget|unobserved)" - - type: "output_matches" - pattern: "await\\s+Assert\\.That\\(total\\)\\.IsEqualTo" - - type: "exit_success" - rubric: - - "Correctly identified the missing `await` on the `Assert.That(...).IsEqualTo(...)` expression as the root cause" - - "Explained that TUnit assertions are async and produce a task — without `await`, the assertion is never observed and the test passes regardless of the actual value" - - "Provided the corrected line: `await Assert.That(total).IsEqualTo(42m);`" - - "Optionally mentioned the built-in TUnit analyzer that warns when an assertion is not awaited, or suggested treating that analyzer warning as an error in CI" - - "Did NOT misdiagnose the failure as a bug in `Cart.CalculateTotal`, a comparison-precision issue (decimal vs double), or a missing test discovery problem" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "Refactor TUnit try/catch to native exception assertion" - expect_activation: false - prompt: | - My team lead wants this TUnit test to use the framework's native exception assertion - instead of try/catch, and to verify both the exception type and the message in one - idiomatic expression. Please refactor it. - - ```csharp - using TUnit.Core; - - public class OrderProcessorTests - { - [Test] - public async Task ProcessOrder_EmptyOrder_ThrowsInvalidOperation() - { - var processor = new OrderProcessor(); - try - { - processor.ProcessOrder(new Order()); - Assert.Fail("Expected exception was not thrown"); - } - catch (InvalidOperationException ex) - { - await Assert.That(ex.Message).IsEqualTo("Order must contain at least one item"); - } - } - } - ``` - assertions: - - type: "output_matches" - pattern: "await\\s+Assert\\.That\\(" - - type: "output_matches" - pattern: "\\.Throws(Exactly)?" - - type: "output_matches" - pattern: "(WithMessage|IsEqualTo).*Order must contain at least one item|Order must contain at least one item" - - type: "output_not_matches" - pattern: "try\\s*\\{[\\s\\S]*catch\\s*\\(InvalidOperationException" - - type: "exit_success" - rubric: - - "Removed the try/catch block entirely and replaced it with a single awaited TUnit exception assertion on the throwing delegate" - - "Used `Throws()` (or `ThrowsExactly()`) to assert the exception type" - - "Verified the message in an idiomatic way (e.g. `.WithMessage(\"Order must contain at least one item\")` chained on the throw assertion, or an awaited follow-up assertion on the captured exception)" - - "Kept the `await` on every assertion call in the refactored test" - reject_tools: ["bash", "edit", "create"] - timeout: 120 - - - name: "TUnit lifecycle hooks at test, class, assembly, and session scope" - expect_activation: false - prompt: | - In a TUnit test project, I need code that runs: - - 1. Before every individual test in a class (e.g. reset a shared in-memory database). - 2. Once before the first test in a class and once after the last (e.g. open / dispose - a class-scoped fixture). - 3. Once before any test in the assembly starts and once after they all finish - (e.g. start / stop an in-process WireMock server for the whole assembly). - 4. Once at the very start of the whole test run and once at the very end - (e.g. apply EF Core migrations and tear down the database for the entire session). - - What attributes / methods does TUnit use for each of these, and where do they go? - A short C# sketch for each scope would be ideal. - assertions: - - type: "output_matches" - pattern: "\\[Before\\(Test\\)\\]" - - type: "output_matches" - pattern: "\\[After\\(Test\\)\\]" - - type: "output_matches" - pattern: "\\[Before\\(Class\\)\\]" - - type: "output_matches" - pattern: "\\[After\\(Class\\)\\]" - - type: "output_matches" - pattern: "\\[Before\\(Assembly\\)\\]" - - type: "output_matches" - pattern: "\\[After\\(Assembly\\)\\]" - - type: "output_matches" - pattern: "\\[Before\\(TestSession\\)\\]" - - type: "output_matches" - pattern: "\\[After\\(TestSession\\)\\]" - - type: "exit_success" - rubric: - - "Provided per-test hooks using `[Before(Test)]` / `[After(Test)]` (or noted the constructor + IAsyncDisposable alternative) on instance methods of the test class" - - "Provided per-class hooks using `[Before(Class)]` / `[After(Class)]` on static methods of the test class" - - "Provided per-assembly hooks using `[Before(Assembly)]` / `[After(Assembly)]` on static methods, noting they apply to every test in the assembly" - - "Provided per-session hooks using `[Before(TestSession)]` / `[After(TestSession)]` on static methods, noting they run exactly once across the whole test run" - - "Did NOT confuse TUnit's scoped `[Before(...)]` / `[After(...)]` attributes with NUnit's `[SetUp]`/`[TearDown]`/`[OneTimeSetUp]`/`[OneTimeTearDown]` or xUnit's `IClassFixture` / `IAsyncLifetime`" - - "Optionally mentioned that hook methods may accept a context object (e.g. `TestContext`, `ClassHookContext`) and/or a `CancellationToken`, or that `[BeforeEvery(Test)]` / `[AfterEvery(Test)]` (and the `Class`/`Assembly` variants) run for every test/class/assembly across the run" - reject_tools: ["bash", "edit", "create"] - timeout: 180 - - - name: "TUnit skip mechanisms — attribute, assembly-wide, and dynamic" - expect_activation: false - prompt: | - In a TUnit test project I need three different ways to skip tests, each with a clear - reason message: - - 1. Skip one specific test method (it's waiting on a payment-gateway sandbox). - 2. Skip every test in an entire assembly when that assembly is built in a special - "smoke" configuration. - 3. Inside a test method, decide at runtime to skip the test if the current machine - is not joined to the corporate VPN. - - What does each one look like in TUnit? - assertions: - - type: "output_matches" - pattern: "\\[Skip\\(" - - type: "output_matches" - pattern: "assembly\\s*:\\s*Skip" - - type: "output_matches" - pattern: "Skip\\.Test\\(" - - type: "exit_success" - rubric: - - "For the single-test case, used the `[Skip(\"reason\")]` attribute on the test method with a required reason string" - - "For the assembly-wide case, used an assembly-level attribute such as `[assembly: Skip(\"…\")]` (or equivalent class-level `[Skip(...)]` on a base class) rather than annotating each test individually" - - "For the runtime case, used the dynamic `Skip.Test(\"reason\")` call inside the test method, after the VPN check, and explained that this is TUnit's equivalent of a runtime skip (distinct from a true \"inconclusive\" state)" - - "Did NOT propose xUnit's `[Fact(Skip = \"…\")]`, MSTest's `Assert.Inconclusive()`, or NUnit's `Assume.That(...)` as TUnit answers" - reject_tools: ["bash", "edit", "create"] - timeout: 120 From 18199830fc3433dfa8bacd391f5eef77e65a986d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 1 Jul 2026 13:14:30 +0200 Subject: [PATCH 2/7] Make exp-test-maintainability Step 1 self-contained and fix activation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address PR review: exp-test-maintainability (dotnet-experimental plugin) must not point at test-analysis-extensions (dotnet-test plugin) — inline the .NET framework markers instead. Also strengthen the description/When-to-Use triggers so the 'each new case needs a whole new method / suggest a better structure' scenario reliably activates the skill instead of routing to a test-writing skill. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../skills/exp-test-maintainability/SKILL.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md b/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md index 4b6b597e67..80ba9b0633 100644 --- a/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md +++ b/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md @@ -1,6 +1,6 @@ --- name: exp-test-maintainability -description: "Detects duplicate boilerplate, copy-paste tests, and structural maintainability issues across .NET test suites. Use when the user asks to reduce repetition, consolidate similar test methods, convert copy-paste tests to data-driven parameterized tests, suggest a better test structure, or identify refactoring opportunities. Identifies repeated construction, assertion patterns, copy-paste methods convertible to DataRow/Theory/TestCase, redundant setup/teardown, and shared infrastructure. Produces an analysis report with concrete before/after suggestions. Works with MSTest, xUnit, NUnit, and TUnit. DO NOT USE FOR: writing new tests (use writing-mstest-tests), reviewing test quality or anti-patterns (use test-anti-patterns), or deep mock auditing (use exp-mock-usage-analysis)." +description: "Detects duplicate boilerplate, copy-paste tests, and structural maintainability issues across .NET test suites. Use when the user asks to reduce repetition, consolidate similar test methods, convert copy-paste tests to data-driven parameterized tests, suggest a better test structure, parameterize repetitive test methods, avoid writing a whole new test method for each case, reduce the number of near-identical test methods, or identify refactoring opportunities. Also use when the user has many similar test methods that differ only by input/expected values and wants a cleaner structure — even if they phrase it as \"adding more test cases\". Identifies repeated construction, assertion patterns, copy-paste methods convertible to DataRow/Theory/TestCase, redundant setup/teardown, and shared infrastructure. Produces an analysis report with concrete before/after suggestions. Works with MSTest, xUnit, NUnit, and TUnit. DO NOT USE FOR: writing brand-new tests for untested code (use writing-mstest-tests), reviewing test quality or anti-patterns (use test-anti-patterns), or deep mock auditing (use exp-mock-usage-analysis)." license: MIT --- @@ -16,6 +16,7 @@ Analyze .NET test code for maintainability issues: duplicated boilerplate, copy- - User asks for refactoring opportunities in a test suite - User wants to identify shared setup or teardown candidates - User asks "what patterns repeat across my tests?" +- User has many similar test methods that differ only by input/expected values and wants a better structure — even when phrased as "each new case needs its own method" or "I want to add more test cases" - User wants to centralize test data, introduce builders or helpers ## When Not to Use @@ -36,7 +37,14 @@ Analyze .NET test code for maintainability issues: duplicated boilerplate, copy- ### Step 1: Gather the test code -Read all test files the user provides or references. If the user points to a directory or project, scan for all test files — see the `test-analysis-extensions` skill's `extensions/dotnet.md` reference file for framework-specific markers. +Read all test files the user provides or references. If the user points to a directory or project, scan for all test files using these framework markers: + +| Framework | Test class markers | Test method markers | +|-----------|--------------------|---------------------| +| MSTest | `[TestClass]` | `[TestMethod]`, `[DataTestMethod]` | +| xUnit | *(none — convention-based)* | `[Fact]`, `[Theory]` | +| NUnit | `[TestFixture]` | `[Test]`, `[TestCase]`, `[TestCaseSource]` | +| TUnit | *(none — convention-based)* | `[Test]` | ### Step 2: Identify maintainability issues From 00a9ba32580db91b57ea771b1241cb5139f4f6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 1 Jul 2026 13:26:20 +0200 Subject: [PATCH 3/7] Revert exp-test-maintainability description broadening; keep self-contained markers The description broadening regressed activation (3/4 -> 0/4 across a variance-dominated single eval run, CV up to 924%). Description is the activation lever, so restore the original wording and keep only the review-comment fix (inlined .NET framework markers in Step 1). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../skills/exp-test-maintainability/SKILL.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md b/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md index 80ba9b0633..583b14ae6c 100644 --- a/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md +++ b/plugins/dotnet-experimental/skills/exp-test-maintainability/SKILL.md @@ -1,6 +1,6 @@ --- name: exp-test-maintainability -description: "Detects duplicate boilerplate, copy-paste tests, and structural maintainability issues across .NET test suites. Use when the user asks to reduce repetition, consolidate similar test methods, convert copy-paste tests to data-driven parameterized tests, suggest a better test structure, parameterize repetitive test methods, avoid writing a whole new test method for each case, reduce the number of near-identical test methods, or identify refactoring opportunities. Also use when the user has many similar test methods that differ only by input/expected values and wants a cleaner structure — even if they phrase it as \"adding more test cases\". Identifies repeated construction, assertion patterns, copy-paste methods convertible to DataRow/Theory/TestCase, redundant setup/teardown, and shared infrastructure. Produces an analysis report with concrete before/after suggestions. Works with MSTest, xUnit, NUnit, and TUnit. DO NOT USE FOR: writing brand-new tests for untested code (use writing-mstest-tests), reviewing test quality or anti-patterns (use test-anti-patterns), or deep mock auditing (use exp-mock-usage-analysis)." +description: "Detects duplicate boilerplate, copy-paste tests, and structural maintainability issues across .NET test suites. Use when the user asks to reduce repetition, consolidate similar test methods, convert copy-paste tests to data-driven parameterized tests, suggest a better test structure, or identify refactoring opportunities. Identifies repeated construction, assertion patterns, copy-paste methods convertible to DataRow/Theory/TestCase, redundant setup/teardown, and shared infrastructure. Produces an analysis report with concrete before/after suggestions. Works with MSTest, xUnit, NUnit, and TUnit. DO NOT USE FOR: writing new tests (use writing-mstest-tests), reviewing test quality or anti-patterns (use test-anti-patterns), or deep mock auditing (use exp-mock-usage-analysis)." license: MIT --- @@ -16,7 +16,6 @@ Analyze .NET test code for maintainability issues: duplicated boilerplate, copy- - User asks for refactoring opportunities in a test suite - User wants to identify shared setup or teardown candidates - User asks "what patterns repeat across my tests?" -- User has many similar test methods that differ only by input/expected values and wants a better structure — even when phrased as "each new case needs its own method" or "I want to add more test cases" - User wants to centralize test data, introduce builders or helpers ## When Not to Use From f308a93a19db34fd4ac53887ac224b26a7239957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 1 Jul 2026 13:36:23 +0200 Subject: [PATCH 4/7] Fix scenario 1 eval: use project fixture so the skill activates Scenario 1 pasted a self-contained snippet inline and asked to 'suggest a better structure', which the base model answers directly without loading the skill -> NOT ACTIVATED with no headroom. Convert it to the project-fixture pattern used by the activating scenarios (3 & 4): move the repetitive InputValidatorTests into a Validation.Tests fixture and ask the model to analyze the project. Mirror the change in eval.vally.yaml. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../exp-test-maintainability/eval.vally.yaml | 101 ++---------------- .../exp-test-maintainability/eval.yaml | 101 ++---------------- .../Validation.Tests/InputValidatorTests.cs | 88 +++++++++++++++ .../Validation.Tests/Validation.Tests.csproj | 11 ++ 4 files changed, 115 insertions(+), 186 deletions(-) create mode 100644 tests/dotnet-experimental/exp-test-maintainability/fixtures/data-driven-candidate/Validation.Tests/InputValidatorTests.cs create mode 100644 tests/dotnet-experimental/exp-test-maintainability/fixtures/data-driven-candidate/Validation.Tests/Validation.Tests.csproj diff --git a/tests/dotnet-experimental/exp-test-maintainability/eval.vally.yaml b/tests/dotnet-experimental/exp-test-maintainability/eval.vally.yaml index 1d27d64fc3..afaf8d6996 100644 --- a/tests/dotnet-experimental/exp-test-maintainability/eval.vally.yaml +++ b/tests/dotnet-experimental/exp-test-maintainability/eval.vally.yaml @@ -6,99 +6,14 @@ config: stimuli: - name: Recommend data-driven patterns with display names for unclear parameters prompt: | - I need to add more test cases for our validation logic but each new case needs a whole new method. - Can you suggest a better structure? - - ```csharp - using Microsoft.VisualStudio.TestTools.UnitTesting; - - namespace Validation.Tests; - - [TestClass] - public sealed class InputValidatorTests - { - [TestMethod] - public void Validate_EmptyString_ReturnsFalse() - { - var validator = new InputValidator(); - var result = validator.Validate(""); - Assert.IsFalse(result.IsValid); - Assert.AreEqual("Input cannot be empty", result.ErrorMessage); - } - - [TestMethod] - public void Validate_TooShort_ReturnsFalse() - { - var validator = new InputValidator(); - var result = validator.Validate("ab"); - Assert.IsFalse(result.IsValid); - Assert.AreEqual("Input must be at least 3 characters", result.ErrorMessage); - } - - [TestMethod] - public void Validate_TooLong_ReturnsFalse() - { - var validator = new InputValidator(); - var result = validator.Validate(new string('x', 256)); - Assert.IsFalse(result.IsValid); - Assert.AreEqual("Input must not exceed 255 characters", result.ErrorMessage); - } - - [TestMethod] - public void Validate_ContainsSpecialChars_ReturnsFalse() - { - var validator = new InputValidator(); - var result = validator.Validate("hello