Unify acceptance test data sources into [TestMatrix] and [CompatibilityMatrix]

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs

@@ -11,7 +11,7 @@ public class ArgumentProcessorTests : AcceptanceTestBase
 {
 
     [TestMethod]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -32,7 +32,7 @@ public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo r
     }
 
     [TestMethod]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/BlameDataCollectorTests.cs

@@ -40,7 +40,7 @@ public BlameDataCollectorTests()
 
     [TestMethod]
     [TestCategory("Windows-Review")]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -55,8 +55,8 @@ public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInf
 
     [TestMethod]
     [TestCategory("Windows-Review")]
-    [NetFullTargetFrameworkDataSource(useCoreRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: NetFx, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo)
     {
 
@@ -79,7 +79,7 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo)
 
     [TestMethod]
     [TestCategory("Windows-Review")]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInfo runnerInfo)
     {
 
@@ -103,7 +103,7 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf
     [TestMethod]
     [TestCategory("Windows-Review")]
     // This tests .net runner and .net framework runner, together with .net framework testhost.
-    [NetFullTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx)]
     public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlwaysIsEnabled(RunnerInfo runnerInfo)
     {
 
@@ -125,7 +125,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw
     }
 
     [TestMethod]
-    [NetCoreRunner("net481;net11.0")]
+    [TestMatrix(console: Net)]
     public void HangDumpOnTimeout(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -147,7 +147,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo)
 
     [TestMethod]
     // .NET testhost does not support dump on exit
-    [NetFullTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx)]
 
     public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo)
     {
@@ -169,7 +169,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo)
 
     [TestMethod]
     // .NET tfms do not support dump on exit, but runner does
-    [NetFullTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx)]
 
     public void CrashDumpOnExit(RunnerInfo runnerInfo)
     {
@@ -190,7 +190,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetCoreRunner("net481;net11.0")]
+    [TestMatrix(console: Net)]
     public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -210,7 +210,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetCoreRunner(NET)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CrashDumpChildProcesses(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -224,7 +224,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetCoreRunner("net481;net11.0")]
+    [TestMatrix(console: Net)]
     public void HangDumpChildProcesses(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -240,7 +240,7 @@ public void HangDumpChildProcesses(RunnerInfo runnerInfo)
 
     [TestMethod]
     [DoNotParallelize] // Modifies the test asset's runtimeconfig.json on disk.
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void HangDumpShouldNotHangWhenTestHostFailsToStart(RunnerInfo runnerInfo)
     {
         // When testhost can't start (e.g. wrong runtime version), the inactivity timer
@@ -281,8 +281,8 @@ public void HangDumpShouldNotHangWhenTestHostFailsToStart(RunnerInfo runnerInfo)
     [TestMethod]
     [TestCategory("Windows-Review")]
     [DoNotParallelize] // Installs/uninstalls procdump as machine-wide postmortem debugger via HKLM registry.
-    [NetFullTargetFrameworkDataSource]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx)]
+    [TestMatrix(testHost: Net)]
     public void BlameDataCollectorAeDebuggerShouldCollectDump(RunnerInfo runnerInfo)
     {
         // For convenience skip locally, but never skip in CI. If this cannot pass in CI we are not testing it at all.

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CodeCoverageTests.cs

@@ -48,8 +48,8 @@ public enum SettingsType
 public class CodeCoverageTests : CodeCoverageAcceptanceTestBase
 {
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CollectCodeCoverageWithCollectOptionForx86(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -67,8 +67,8 @@ public void CollectCodeCoverageWithCollectOptionForx86(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CollectCodeCoverageWithCollectOptionForx64(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -86,8 +86,8 @@ public void CollectCodeCoverageWithCollectOptionForx64(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CollectCodeCoverageX86WithRunSettings(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -105,8 +105,8 @@ public void CollectCodeCoverageX86WithRunSettings(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CollectCodeCoverageX64WithRunSettings(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -124,8 +124,8 @@ public void CollectCodeCoverageX64WithRunSettings(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CodeCoverageShouldAvoidExclusionsX86(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -146,8 +146,8 @@ public void CodeCoverageShouldAvoidExclusionsX86(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CodeCoverageShouldAvoidExclusionsX64(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -168,8 +168,8 @@ public void CodeCoverageShouldAvoidExclusionsX64(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CollectCodeCoverageSpecifyOutputFormatXml(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()
@@ -187,8 +187,8 @@ public void CollectCodeCoverageSpecifyOutputFormatXml(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: NetFx)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CollectCodeCoverageSpecifyOutputFormatCoberturaOverrideRunSettingsConfiguration(RunnerInfo runnerInfo)
     {
         var parameters = new TestParameters()

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CreateNoNewWindowTests.cs

@@ -14,7 +14,7 @@ public class CreateNoNewWindowTests : AcceptanceTestBase
     [TestMethod]
     // CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost.
     [TestCategory("Windows-Review")]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)]
+    [TestMatrix(testHost: NetFx)]
     public void WhenCreateNoNewWindowIsFalse_DiagShowsCreateNoWindowFalse(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -38,7 +38,7 @@ public void WhenCreateNoNewWindowIsFalse_DiagShowsCreateNoWindowFalse(RunnerInfo
     [TestMethod]
     // CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost.
     [TestCategory("Windows-Review")]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)]
+    [TestMatrix(testHost: NetFx)]
     public void WhenCreateNoNewWindowIsTrue_DiagShowsCreateNoWindowTrue(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -62,7 +62,7 @@ public void WhenCreateNoNewWindowIsTrue_DiagShowsCreateNoWindowTrue(RunnerInfo r
     [TestMethod]
     // CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost.
     [TestCategory("Windows-Review")]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)]
+    [TestMatrix(testHost: NetFx)]
     public void WhenCreateNoNewWindowIsNotSet_DefaultIsTrue(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectionTests.cs

@@ -20,7 +20,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
 public class DataCollectionTests : AcceptanceTestBase
 {
     [TestMethod]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -43,7 +43,7 @@ public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -66,7 +66,7 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner
     }
 
     [TestMethod]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -80,7 +80,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo
     [TestMethod]
     // .NET Framework testhost-specific assembly loading; not applicable to the netcore testhost.
     [TestCategory("Windows-Review")]
-    [NetFullTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx)]
     public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -92,7 +92,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn
     }
 
     [TestMethod]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: Net)]
     public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectorTests.Coverlet.cs

@@ -14,7 +14,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
 public class DataCollectorTestsCoverlets : AcceptanceTestBase
 {
     [TestMethod]
-    [NetCoreRunner(HOST_NET)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void RunCoverletCoverage(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DebugAssertTests.cs

@@ -11,7 +11,7 @@ public class DebugAssertTests : AcceptanceTestBase
 {
     [TestMethod]
     // this is core only, there is nothing we can do about TPDebug.Assert crashing the process on framework
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(RunnerInfo runnerInfo)
     {
         // when debugging this test in case it starts failing, be aware that the default behavior of TPDebug.Assert

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DifferentTestFrameworkSimpleTests.cs

@@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
 public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase
 {
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [TestMatrix(testHost: NetFx, inProcess: true)]
     public void NonDllRunAllTestExecution(RunnerInfo runnerInfo)
     {
         // This used to test Chutzpah, to prove that we can run tests that are not shipped in dlls.
@@ -36,7 +36,7 @@ public void NonDllRunAllTestExecution(RunnerInfo runnerInfo)
     [TestMethod]
     // vstest.console is x64 now, but x86 run "in process" run should still succeed by being run in x86 testhost
     // Skip .NET (Core) tests because we test them below.
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useCoreRunner: false)]
+    [TestMatrix(console: NetFx, testHost: NetFx, inProcess: true)]
     public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -48,7 +48,7 @@ public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo)
     [TestCategory("Windows-Review")]
     // vstest.console is 64-bit now, run in process to test the 64-bit native dll
     // Skip .NET (Core) tests because we test them below.
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useCoreRunner: false)]
+    [TestMatrix(console: NetFx, testHost: NetFx, inProcess: true)]
     public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -60,16 +60,16 @@ public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo)
     // C++ tests cannot run in .NET Framework host under .NET Core, because we only ship .NET Standard CPP adapter in .NET Core
     // We also don't test x86 for .NET Core, because the resolver there does not switch between x86 and x64 correctly, it just uses the parent process bitness.
     // We run this on netcore31 and not the default netcore21 because netcore31 is the minimum tfm that has the runtime features we need, such as additionaldeps.
-    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false, useCoreRunner: true)]
+    [TestMatrix(console: Net, testHost: Net)]
     public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
         CppRunAllTests("x64");
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx, inProcess: true)]
+    [TestMatrix(testHost: Net)]
     public void NUnitRunAllTestExecution(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -84,8 +84,8 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
-    [NetCoreTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx, inProcess: true)]
+    [TestMatrix(testHost: Net)]
     public void XUnitRunAllTestExecution(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DisableAppdomainTests.cs

@@ -17,7 +17,7 @@ public class DisableAppdomainTests : AcceptanceTestBase
     [TestMethod]
     [TestCategory("Windows")]
     // Run in .NET Framework testhost, disabling appdomain will force running out of process in all cases.
-    [NetFullTargetFrameworkDataSource(inProcess: true)]
+    [TestMatrix(testHost: NetFx, inProcess: true)]
     public void DisableAppdomainTest(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
@@ -30,7 +30,7 @@ public void DisableAppdomainTest(RunnerInfo runnerInfo)
 
     [TestMethod]
     [TestCategory("Windows")]
-    [NetFullTargetFrameworkDataSource]
+    [TestMatrix(testHost: NetFx)]
     public void NewtonSoftDependencyWithDisableAppdomainTest(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);