From fc3b2a3361a3bd3c60fefb61fd27af534528e6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 20 Apr 2026 11:12:07 +0200 Subject: [PATCH 1/5] Improve migrate-mstest-v1v2-to-v3 evals to differentiate from baseline Five scenarios were performing at or below stock Copilot because prompts gave away answers and rubric items tested knowledge the baseline LLM can deduce from the code alone. Changes per scenario: - Fix DataRow type mismatch: added double-to-float mismatch, vaguer prompt, rubric tests both mismatch types and latest-3.x knowledge - Migrate .testsettings to .runsettings: added code coverage data collector to fixture, rubric tests correct TestTimeout mapping ( not ) and collector mapping - Migrate MSTest v2 NuGet to v3: vaguer prompt (forces model to read csproj), rubric tests metapackage/MSTest.Sdk recommendation and Microsoft.NET.Test.Sdk dependency awareness - Fix Assert.AreEqual overload errors: removed exact error code from prompt, added mixed-type and interface-reference cases, rubric tests correct generic type parameter selection - Handle dropped TFM: changed to multi-target (net5.0;net462; netcoreapp3.1), rubric tests that model knows which TFMs are still supported and doesn't upgrade them indiscriminately --- .../migrate-mstest-v1v2-to-v3/eval.yaml | 123 +++++++++++------- 1 file changed, 77 insertions(+), 46 deletions(-) diff --git a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml index a55f1fc963..8fde201e9d 100644 --- a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml +++ b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml @@ -31,8 +31,8 @@ scenarios: - name: "Migrate MSTest v2 NuGet project to v3" prompt: | - My test project uses MSTest.TestFramework 2.2.10 and MSTest.TestAdapter 2.2.10 on .NET 6. - I want to upgrade to MSTest v3. What breaking changes should I expect? + I want to upgrade my MSTest test project to the latest major version. + Can you review my project and tell me what I need to change? setup: files: - path: "TestProject.csproj" @@ -41,15 +41,14 @@ scenarios: source: "fixtures/v2-nuget/UserServiceTests.cs" assertions: - type: "output_matches" - pattern: "(2\\.2\\.10|v2|version 2)" + pattern: "(3\\.[0-9]+|v3|version 3|MSTest\\.Sdk)" - type: "output_matches" - pattern: "(3\\.[0-9]+|v3|version 3)" + pattern: "(AreEqual|Assert|overload|generic)" rubric: - - "Identifies the project as MSTest v2 from the 2.2.10 package versions" - - "Recommends updating MSTest packages to 3.x or switching to the MSTest metapackage" - - "Warns about Assert.AreEqual/AreNotEqual object overload removal" - - "Mentions DataRow constructor changes and strict type matching" - - "Mentions timeout behavior unification across .NET Core and .NET Framework" + - "Recommends the MSTest metapackage or MSTest.Sdk as the upgrade path, not just bumping individual package versions" + - "Warns about the Assert.AreEqual/AreNotEqual object overload removal visible in UserServiceTests.cs" + - "Identifies the DataRow 1L (long) to int type mismatch and explains that v3 enforces strict type matching" + - "Mentions that Microsoft.NET.Test.Sdk must be kept as a dependency (or notes MSTest.Sdk provides it automatically)" timeout: 240 # ============================================================================ @@ -58,9 +57,9 @@ scenarios: - name: "Fix Assert.AreEqual object overload errors after v3 upgrade" prompt: | - I just upgraded to MSTest v3 and I'm getting compilation errors like: - "error CS1503: Argument 1: cannot convert from 'object' to 'string'" - on Assert.AreEqual calls. My tests compare objects. How do I fix this? + After updating my MSTest NuGet packages, several of my tests no longer compile. + I'm seeing errors on various Assert calls. Can you review my test file and fix + the compilation issues? setup: files: - path: "TestProject.csproj" @@ -83,39 +82,54 @@ scenarios: public class ComparisonTests { [TestMethod] - public void CompareObjects_AreEqual() + public void CompareStrings_AreEqual() { - object expected = GetExpected(); - object actual = GetActual(); + object expected = GetExpectedName(); + object actual = GetActualName(); Assert.AreEqual(expected, actual); } [TestMethod] - public void CompareObjects_AreNotEqual() + public void CompareIntegers_AreNotEqual() { - object a = "hello"; - object b = "world"; + object a = GetCount(); + object b = GetOtherCount(); Assert.AreNotEqual(a, b); } [TestMethod] public void CompareReferences_AreSame() { - object obj = new object(); - Assert.AreSame(obj, obj); + var svc = GetService(); + Assert.AreSame(svc, GetService()); } - private static object GetExpected() => 42; - private static object GetActual() => 42; + [TestMethod] + public void MixedComparison_DifferentTypes() + { + object result = Compute(); + Assert.AreEqual(42, result); + Assert.AreNotEqual(null, result); + } + + private static object GetExpectedName() => "Alice"; + private static object GetActualName() => "Alice"; + private static object GetCount() => 1; + private static object GetOtherCount() => 2; + private static IService GetService() => new ServiceImpl(); + private static object Compute() => 42; } + + public interface IService { } + public class ServiceImpl : IService { } assertions: - type: "output_matches" pattern: "(generic|AreEqual<||type.?param)" rubric: - - "Explains that MSTest v3 removed the Assert.AreEqual(object, object) overload" - - "Recommends adding explicit generic type parameters: Assert.AreEqual(expected, actual)" - - "Applies the same fix pattern to AreNotEqual, AreSame, and AreNotSame" - - "Does not suggest downgrading to MSTest v2 as a solution" + - "Explains that MSTest v3 removed the non-generic Assert.AreEqual(object, object) overload family" + - "Recommends adding explicit generic type parameters and chooses the correct type for each case (e.g., Assert.AreEqual for mixed types, Assert.AreSame or the concrete type for interface references)" + - "Applies the fix to AreNotEqual and AreSame calls, not just AreEqual" + - "Does not suggest downgrading MSTest as a solution" timeout: 180 # ============================================================================ @@ -124,8 +138,9 @@ scenarios: - name: "Migrate from .testsettings to .runsettings" prompt: | - My MSTest project has a .testsettings file that configures test deployment and timeout. - After upgrading to MSTest v3, it seems to be ignored. How do I migrate? + My MSTest project has a .testsettings file that configures test deployment, timeout, + and a code coverage data collector. After a package update, the settings seem to be + ignored. How do I migrate this configuration? setup: files: - path: "TestProject.csproj" @@ -154,6 +169,12 @@ scenarios: + + + + + assertions: - type: "output_matches" @@ -162,9 +183,10 @@ scenarios: pattern: "(testsettings|legacy|no longer supported)" rubric: - "Explains that .testsettings and LegacySettings are no longer supported in MSTest v3" - - "Provides guidance on creating a .runsettings file as a replacement" - - "Shows how to configure timeout and other settings in .runsettings XML format" - - "Does not suggest keeping the .testsettings file alongside .runsettings" + - "Provides a .runsettings file that maps the TestTimeout to (per-test), not to " + - "Maps the code coverage data collector into the section of .runsettings" + - "Recommends deleting the .testsettings file entirely rather than keeping it alongside .runsettings" + - "Notes that AssemblyResolution settings can be removed as they are not needed in modern .NET" timeout: 180 # ============================================================================ @@ -173,9 +195,9 @@ scenarios: - name: "Fix DataRow type mismatch errors after v3 upgrade" prompt: | - After upgrading to MSTest v3, many of my data-driven tests fail to compile with errors - about DataRow constructors. I have tests with lots of parameters and some implicit - type conversions. What changed and how do I fix it? + After updating my test project's NuGet packages, several data-driven tests + no longer compile. The errors are on the DataRow attributes. Can you review + my test file and fix the issues? setup: files: - path: "TestProject.csproj" @@ -205,6 +227,14 @@ scenarios: Assert.IsNotNull(name); } + [TestMethod] + [DataRow(10.0, 3.5f)] + [DataRow(20.0, 7.2f)] + public void CalculateDiscount(float price, float rate) + { + Assert.IsTrue(rate > 0); + } + [TestMethod] [DataRow(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)] public void ManyParameters(int a, int b, int c, int d, int e, int f, @@ -216,12 +246,12 @@ scenarios: } assertions: - type: "output_matches" - pattern: "(DataRow|type.?match|constructor|16|param)" + pattern: "(DataRow|type.?match|constructor|strict)" rubric: - - "Explains that MSTest v3 requires DataRow values to match parameter types exactly" - - "Identifies the 1L (long) vs int mismatch and recommends fixing to 1 (int)" - - "Explains the 16-parameter limit on DataRow constructors and suggests refactoring or wrapping in an array" - - "Does not suggest downgrading as a fix" + - "Explains that MSTest v3 enforces strict type matching in DataRow -- implicit conversions that worked in v2 now fail" + - "Fixes the long-to-int mismatch (1L → 1) and the double-to-float mismatch (10.0 → 10.0f)" + - "Explains the 16-parameter constructor limit and suggests refactoring or updating to the latest 3.x where it may be resolved" + - "Does not suggest downgrading MSTest as a fix" timeout: 240 # ============================================================================ @@ -266,15 +296,15 @@ scenarios: - name: "Handle dropped target framework during v3 migration" prompt: | - My test project targets .NET 5.0 and uses MSTest v2. I want to upgrade to MSTest v3. - Will there be any framework compatibility issues? + My test project multi-targets several frameworks and uses MSTest v2. + I want to upgrade to MSTest v3. Will I run into compatibility issues? setup: files: - path: "TestProject.csproj" content: | - net5.0 + net5.0;net462;netcoreapp3.1 false @@ -285,11 +315,12 @@ scenarios: assertions: - type: "output_matches" - pattern: "(net5\\.0|dropped|unsupported|net6\\.0|net8\\.0)" + pattern: "(net5\\.0|dropped|unsupported|net8\\.0)" rubric: - - "Identifies that .NET 5.0 is no longer supported by MSTest v3" - - "Recommends upgrading to .NET 6.0 or later (preferably .NET 8.0 as a current LTS)" - - "Does not suggest staying on .NET 5.0 with MSTest v3" + - "Identifies that .NET 5.0 is dropped in MSTest v3 and recommends replacing it with .NET 8.0 (LTS) or .NET 6+" + - "Correctly states that .NET Framework 4.6.2 is still supported by MSTest v3 and does not need to change" + - "Correctly states that .NET Core 3.1 is still supported by MSTest v3 and does not need to change" + - "Does not recommend upgrading all TFMs indiscriminately -- only the unsupported one" timeout: 180 # ============================================================================ From 59bc6678a5dd074cc270a30f01ac1136bd5835d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 20 Apr 2026 11:26:51 +0200 Subject: [PATCH 2/5] Update tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml index 8fde201e9d..20bdf7e52d 100644 --- a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml +++ b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml @@ -43,7 +43,7 @@ scenarios: - type: "output_matches" pattern: "(3\\.[0-9]+|v3|version 3|MSTest\\.Sdk)" - type: "output_matches" - pattern: "(AreEqual|Assert|overload|generic)" + pattern: "(Assert\\.(AreEqual|AreNotEqual)(<[^>]+>)?|object overload|overload removal|remove[ds]? the object overload|generic type parameter[s]?)" rubric: - "Recommends the MSTest metapackage or MSTest.Sdk as the upgrade path, not just bumping individual package versions" - "Warns about the Assert.AreEqual/AreNotEqual object overload removal visible in UserServiceTests.cs" From b46f176f3ab68a56b7514549681fcbf346fc0f14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 20 Apr 2026 11:37:04 +0200 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml index 20bdf7e52d..0d52e0f3f6 100644 --- a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml +++ b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml @@ -31,7 +31,7 @@ scenarios: - name: "Migrate MSTest v2 NuGet project to v3" prompt: | - I want to upgrade my MSTest test project to the latest major version. + I want to upgrade my MSTest v2 test project to the next major version. Can you review my project and tell me what I need to change? setup: files: @@ -101,7 +101,7 @@ scenarios: public void CompareReferences_AreSame() { var svc = GetService(); - Assert.AreSame(svc, GetService()); + Assert.AreSame(svc, svc); } [TestMethod] From 2f7fced40bbd4b99e9535bd2a61bd2f267876b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 20 Apr 2026 11:49:02 +0200 Subject: [PATCH 4/5] Update tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml index 0d52e0f3f6..99ccc16ad3 100644 --- a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml +++ b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml @@ -250,7 +250,7 @@ scenarios: rubric: - "Explains that MSTest v3 enforces strict type matching in DataRow -- implicit conversions that worked in v2 now fail" - "Fixes the long-to-int mismatch (1L → 1) and the double-to-float mismatch (10.0 → 10.0f)" - - "Explains the 16-parameter constructor limit and suggests refactoring or updating to the latest 3.x where it may be resolved" + - "Explains the 16-parameter constructor limit and suggests refactoring; if the project is on an earlier 3.x release, it may also suggest updating to the latest 3.x where the limitation may be resolved" - "Does not suggest downgrading MSTest as a fix" timeout: 240 From 8764937d5108722c94783cc282e6555d4bb4b18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 20 Apr 2026 19:06:17 +0200 Subject: [PATCH 5/5] Improve prompt keywords for skill activation Add MSTest-specific keywords to prompts that were failing to activate the migrate-mstest-v1v2-to-v3 skill: - Fix DataRow: 'test project NuGet packages' -> 'MSTest packages to a newer major version' (had zero MSTest keywords before) - Fix Assert.AreEqual: add 'to the latest major version' upgrade context - Migrate .testsettings: 'package update' -> 'upgrading MSTest packages' - Handle dropped TFM: 'upgrade' -> 'migrate', add 'currently' for clarity --- .../migrate-mstest-v1v2-to-v3/eval.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml index 375ca81288..013eb84b7e 100644 --- a/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml +++ b/tests/dotnet-test/migrate-mstest-v1v2-to-v3/eval.yaml @@ -62,9 +62,9 @@ scenarios: - name: "Fix Assert.AreEqual object overload errors after v3 upgrade" prompt: | - After updating my MSTest NuGet packages, several of my tests no longer compile. - I'm seeing errors on various Assert calls. Can you review my test file and fix - the compilation issues? + After upgrading my MSTest NuGet packages to the latest major version, several + of my tests no longer compile. I'm seeing errors on various Assert calls. Can + you review my test file and fix the compilation issues? setup: files: - path: "TestProject.csproj" @@ -144,8 +144,8 @@ scenarios: - name: "Migrate from .testsettings to .runsettings" prompt: | My MSTest project has a .testsettings file that configures test deployment, timeout, - and a code coverage data collector. After a package update, the settings seem to be - ignored. How do I migrate this configuration? + and a code coverage data collector. After upgrading MSTest packages, the settings + seem to be ignored. How do I migrate this configuration? setup: files: - path: "TestProject.csproj" @@ -200,9 +200,9 @@ scenarios: - name: "Fix DataRow type mismatch errors after v3 upgrade" prompt: | - After updating my test project's NuGet packages, several data-driven tests - no longer compile. The errors are on the DataRow attributes. Can you review - my test file and fix the issues? + After upgrading my MSTest packages to a newer major version, several + data-driven tests no longer compile. The errors are on the DataRow + attributes. Can you review my test file and fix the issues? setup: files: - path: "TestProject.csproj" @@ -301,8 +301,8 @@ scenarios: - name: "Handle dropped target framework during v3 migration" prompt: | - My test project multi-targets several frameworks and uses MSTest v2. - I want to upgrade to MSTest v3. Will I run into compatibility issues? + My test project multi-targets several frameworks and currently uses MSTest v2. + I want to migrate to MSTest v3. Will I run into any compatibility issues? setup: files: - path: "TestProject.csproj"