diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/.runsettings b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/.runsettings index 823b5bb2d5..cd8b6eb4da 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/.runsettings +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/.runsettings @@ -1,5 +1,2 @@ - - true - \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs index 0f296846c5..8f2fe25d03 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs @@ -7,13 +7,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] -[TestCategory("Windows-Review")] public class ArgumentProcessorTests : AcceptanceTestBase { [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -34,8 +32,7 @@ public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo r } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CreateNoNewWindowTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CreateNoNewWindowTests.cs index f7acd3b94a..5122bec933 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CreateNoNewWindowTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CreateNoNewWindowTests.cs @@ -12,6 +12,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class CreateNoNewWindowTests : AcceptanceTestBase { [TestMethod] + // CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)] public void WhenCreateNoNewWindowIsFalse_DiagShowsCreateNoWindowFalse(RunnerInfo runnerInfo) { @@ -34,6 +36,8 @@ public void WhenCreateNoNewWindowIsFalse_DiagShowsCreateNoWindowFalse(RunnerInfo } [TestMethod] + // CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)] public void WhenCreateNoNewWindowIsTrue_DiagShowsCreateNoWindowTrue(RunnerInfo runnerInfo) { @@ -56,6 +60,8 @@ public void WhenCreateNoNewWindowIsTrue_DiagShowsCreateNoWindowTrue(RunnerInfo r } [TestMethod] + // CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)] public void WhenCreateNoNewWindowIsNotSet_DefaultIsTrue(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectionTests.cs index 26e068f9cb..af0b35edcc 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectionTests.cs @@ -20,7 +20,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class DataCollectionTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) { @@ -44,7 +43,6 @@ public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runnerInfo) { @@ -80,6 +78,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo } [TestMethod] + // .NET Framework testhost-specific assembly loading; not applicable to the netcore testhost. [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(RunnerInfo runnerInfo) @@ -93,7 +92,6 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs index 3680b3c9bc..23d047a76f 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs @@ -18,7 +18,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class DiscoveryTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void DiscoverAllTests(RunnerInfo runnerInfo) { @@ -32,7 +31,6 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useVsixRunner: true)] [NetCoreTargetFrameworkDataSource] [TestCategory("Smoke")] public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) @@ -55,8 +53,7 @@ public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) { var dummyFilePath = Path.Combine(TempDirectory.Path, $"{Guid.NewGuid()}.txt"); @@ -74,7 +71,6 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(RunnerInfo runnerInfo) { @@ -119,8 +115,7 @@ public void TypesToLoadAttributeTests() } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void DiscoverTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs index 64634bde47..7602917081 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs @@ -16,7 +16,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class DotnetTestMSBuildOutputTests : AcceptanceTestBase { [TestMethod] - // patched dotnet is not published on non-windows systems + // Special characters (~, !, |, %) don't survive the MSBuildLogger output round-trip on non-Windows terminals. [TestCategory("Windows-Review")] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(RunnerInfo runnerInfo) @@ -54,8 +54,6 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru } [TestMethod] - // patched dotnet is not published on non-windows systems - [TestCategory("Windows-Review")] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo) { @@ -74,8 +72,6 @@ public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo) [TestMethod] - // patched dotnet is not published on non-windows systems - [TestCategory("Windows-Review")] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void MSBuildLoggerCanBeDisabledByEnvironmentVariableProperty(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs index 4fd4b85bb4..920a1c7f9b 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs @@ -16,8 +16,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] public class ExecutionTests : AcceptanceTestBase { - //TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [MSTestCompatibilityDataSource] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) @@ -35,6 +35,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [TestHostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) @@ -52,6 +53,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru } [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo) @@ -67,9 +69,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(R } [TestMethod] - [TestCategory("Windows-Review")] [TestCategory("Smoke")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useVsixRunner: true)] [NetCoreTargetFrameworkDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations3(RunnerInfo runnerInfo) { @@ -87,7 +87,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations3(R // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test // frameworks have any added value over using 2 mstest dlls? [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInfo) { @@ -134,7 +133,6 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { @@ -156,7 +154,6 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { @@ -173,6 +170,9 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) } [TestMethod] + // Asserts the testhost-specific stack overflow message; the .NET Framework variant requires the + // .NET Framework testhost, which is only available on Windows. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) @@ -198,6 +198,9 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn } [TestMethod] + // Asserts the testhost-specific unhandled exception message; the .NET Framework variant requires the + // .NET Framework testhost, which is only available on Windows. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) @@ -292,8 +295,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -312,8 +314,7 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -366,8 +367,7 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FilePatternParserTests.cs index 7eddb74cc2..4365382eb3 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FilePatternParserTests.cs @@ -12,7 +12,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class FilePatternParserTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { @@ -32,7 +31,6 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(RunnerInfo runnerInfo) { @@ -58,7 +56,6 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo runnerInfo) { @@ -84,7 +81,6 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FrameworkTests.cs index c08561abe9..24a4da4add 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FrameworkTests.cs @@ -13,7 +13,6 @@ public class FrameworkTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { @@ -27,7 +26,6 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { @@ -69,6 +67,7 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) [TestMethod] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] + // The .NET (Core) runner produces a different framework-incompatible warning on non-Windows, so keep this Windows-only. [TestCategory("Windows-Review")] public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/LoggerTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/LoggerTests.cs index 1922937371..86717def09 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/LoggerTests.cs @@ -16,8 +16,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class LoggerTests : AcceptanceTestBase { [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -37,8 +36,7 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -58,8 +56,7 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void HtmlLoggerWithFriendlyNameContainsExpectedContent(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -106,8 +103,7 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -147,8 +143,7 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAndTreatNoTestsAsErrorIsTrue(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -170,8 +165,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAndTreatNoTestsAsErrorIsFalse(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -193,7 +187,6 @@ public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAn } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedWhenDataCollectorLogsError(RunnerInfo runnerInfo) { @@ -222,7 +215,6 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedWhenDataCollectorLog } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void TrxLoggerResultSummaryOutcomeValueShouldBeCompletedWhenDataCollectorLogsErrorAndTreatErrorMessagesAsWarningsIsTrue(RunnerInfo runnerInfo) { @@ -315,7 +307,6 @@ private static void IsFileAndContentEqual(string filePath) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void TrxLoggerShouldNotDoubleCountDataDrivenTestResults(RunnerInfo runnerInfo) { @@ -347,7 +338,6 @@ public void TrxLoggerShouldNotDoubleCountDataDrivenTestResults(RunnerInfo runner } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void TrxLoggerShouldPlaceTrxFileInSubdirectoryWhenLogFileNameContainsPath(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/MultitargetingTestHostTests.cs index 4731d3586a..163238e783 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/MultitargetingTestHostTests.cs @@ -12,6 +12,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class MultitargetingTestHostTests : AcceptanceTestBase { [TestMethod] + // Multitargeting is exercised against .NET Framework testhosts (net481), which only exist on Windows, + // so both data sources produce zero rows on Linux/macOS. [TestCategory("Windows-Review")] // the underlying test is using xUnit to avoid AppDomain enhancements in MSTest that make this pass even without multitargetting // xUnit supports net452 onwards, so that is why this starts at net452, I also don't test all framework versions diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/PortableNugetPackageTests.cs index 1e8006421e..1b42e0b8f0 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/PortableNugetPackageTests.cs @@ -24,7 +24,6 @@ public static void ClassInit(TestContext _) } [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { @@ -39,7 +38,6 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void DiscoverAllTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RecursiveResourcesLookupTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RecursiveResourcesLookupTests.cs index d5fcc7756b..6bdf038ef8 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RecursiveResourcesLookupTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RecursiveResourcesLookupTests.cs @@ -11,7 +11,8 @@ public class RecursiveResourcesLookupTests : AcceptanceTestBase { [TestMethod] // This only fails on .NET Framework, and it fails in testhost, so no need to double check with - // two different runners. + // two different runners. The NetFull data source is empty on Linux/macOS. + [TestCategory("Windows-Review")] [Ignore("Temporarily ignore until solving https://github.com/microsoft/testfx/issues/2692")] [NetFullTargetFrameworkDataSource(useCoreRunner: false)] public void RunsToCompletionWhenJapaneseResourcesAreLookedUpForMSCorLib(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ResultsDirectoryTests.cs index c59053faef..76e6937772 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ResultsDirectoryTests.cs @@ -13,7 +13,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class ResultsDirectoryTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) { @@ -34,7 +33,6 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void ResultsDirectoryRelativePathShouldWork(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs index 58ac39da92..2816b7e9b4 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs @@ -313,8 +313,6 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo /// /// [TestMethod] - // patched dotnet is not published on non-windows systems - [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSourceAttribute(useDesktopRunner: false)] [NetCoreTargetFrameworkDataSourceAttribute(useDesktopRunner: false)] public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializationCompatibilityTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializationCompatibilityTests.cs index 79b808c014..9cb64cea20 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializationCompatibilityTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializationCompatibilityTests.cs @@ -38,6 +38,7 @@ public class SerializationCompatibilityTests : AcceptanceTestBase /// Verifies that discovery request/response messages serialize correctly across the version boundary. /// [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource()] public void DiscoverTests_LatestRunner_WithOlderTesthosts(RunnerInfo runnerInfo) @@ -73,6 +74,7 @@ public void DiscoverTests_LatestRunner_WithOlderTesthosts(RunnerInfo runnerInfo) /// Verifies that older runners can understand discovery responses from the new STJ-based testhost. /// [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [TestHostCompatibilityDataSource] public void DiscoverTests_OlderRunners_WithLatestTesthost(RunnerInfo runnerInfo) @@ -109,6 +111,7 @@ public void DiscoverTests_OlderRunners_WithLatestTesthost(RunnerInfo runnerInfo) /// Verifies that test run messages (start, result, complete) serialize correctly across versions. /// [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunTests_LatestRunner_WithOlderTesthosts(RunnerInfo runnerInfo) @@ -148,6 +151,7 @@ public void RunTests_LatestRunner_WithOlderTesthosts(RunnerInfo runnerInfo) /// Verifies that older runners can process execution results from the new STJ-based testhost. /// [TestMethod] + // Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [TestHostCompatibilityDataSource] public void RunTests_OlderRunners_WithLatestTesthost(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializerSelectionTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializerSelectionTests.cs index 9558c0f162..b21d517bd2 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializerSelectionTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/SerializerSelectionTests.cs @@ -25,6 +25,9 @@ public void OnNetCoreRunner_ShouldUseSystemTextJson(RunnerInfo runnerInfo) } [TestMethod] + // The .NET Framework runner (and its Jsonite serializer) only runs on Windows; the core counterpart + // is covered by OnNetCoreRunner_ShouldUseSystemTextJson. + [TestCategory("Windows-Review")] [NetFrameworkRunner(Net481TargetFramework)] public void OnNetFrameworkRunner_ShouldUseJsonite(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TelemetryTests.cs index ec7f1ad8c3..a972b9a62c 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TelemetryTests.cs @@ -20,7 +20,6 @@ public class TelemetryTests : AcceptanceTestBase private const string LOG_TELEMETRY_PATH = "VSTEST_LOGTELEMETRY_PATH"; [TestMethod] - [NetFullTargetFrameworkDataSourceAttribute(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void RunTestsShouldPublishMetrics(RunnerInfo runnerInfo) { @@ -30,7 +29,6 @@ public void RunTestsShouldPublishMetrics(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSourceAttribute(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void DiscoverTestsShouldPublishMetrics(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TestCaseFilterTests.cs index 0afd627f46..32323d9a54 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TestCaseFilterTests.cs @@ -10,7 +10,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class TestCaseFilterTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSourceAttribute(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) { @@ -27,7 +26,6 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) { @@ -44,7 +42,6 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) { @@ -61,7 +58,6 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) { @@ -80,7 +76,6 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) { @@ -97,7 +92,6 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) { @@ -114,7 +108,6 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) { @@ -135,7 +128,6 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) /// this command should provide same results as /TestCaseFilter:"FullyQualifiedName~UnitTest1". /// [TestMethod] - [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerInfo runnerInfo) { @@ -179,7 +171,6 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSourceAttribute(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithNoneTestCategoryFilterMatchesUncategorizedTests(RunnerInfo runnerInfo) { @@ -199,7 +190,6 @@ public void RunSelectedTestsWithNoneTestCategoryFilterMatchesUncategorizedTests( } [TestMethod] - [NetFullTargetFrameworkDataSourceAttribute(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] public void RunSelectedTestsWithNoneTestCategoryNotEqualFilterMatchesCategorizedTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/.runsettings b/test/Microsoft.TestPlatform.Library.IntegrationTests/.runsettings index 823b5bb2d5..cd8b6eb4da 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/.runsettings +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/.runsettings @@ -1,5 +1,2 @@ - - true - \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/FilterSourceIntegrationTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/FilterSourceIntegrationTests.cs index a4d78efa44..f7a9c725ac 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/FilterSourceIntegrationTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/FilterSourceIntegrationTests.cs @@ -16,7 +16,6 @@ namespace Microsoft.TestPlatform.Library.IntegrationTests; public class FilterSourceIntegrationTests : AcceptanceTestBase { [TestMethod] - [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void FilterSourcePackage_AllTestsPass(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/CustomTestHostLauncherTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/CustomTestHostLauncherTests.cs index 55d68513a8..9a3f86259b 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/CustomTestHostLauncherTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/CustomTestHostLauncherTests.cs @@ -35,6 +35,7 @@ public void Cleanup() } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [WrapperCompatibilityDataSource()] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) @@ -57,6 +58,7 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [TestCategory("Feature")] [WrapperCompatibilityDataSource] diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs index 32fa1eb801..d85cb80977 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs @@ -22,6 +22,7 @@ namespace Microsoft.TestPlatform.Library.IntegrationTests.TranslationLayerTests; [TestClass] +// This test runs the packaged .NET Framework vstest.console.exe, which cannot start on Linux/macOS. [TestCategory("Windows-Review")] public class DataCollectorAttachmentProcessor : AcceptanceTestBase { diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index 23ebdf9ca3..cd1c7ad13a 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -68,7 +68,8 @@ public void RunTestsWithNunitAdapter(RunnerInfo runnerInfo) } [TestMethod] - // there are logs in the diagnostic log, it is failing with NullReferenceException because path is null + // The xUnit adapter produces no results on Linux/macOS (diagnostic log shows a NullReferenceException because path is null), + // so the run returns an empty sequence and .First() throws. Keep this Windows-only. [TestCategory("Windows-Review")] [NetCoreTargetFrameworkDataSource] public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) @@ -104,8 +105,8 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) } [TestMethod] - [TestCategory("Windows-Review")] // TODO: this does not work with netcore testhost, why? + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void RunTestsWithNonDllAdapter(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DiscoverTests.cs index 5a96e743cd..190079f3a3 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/DiscoverTests.cs @@ -44,6 +44,7 @@ public void Cleanup() } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [WrapperCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) @@ -62,6 +63,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [WrapperCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) @@ -87,7 +89,6 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI [TestMethod] [TestCategory("Smoke")] [NetCoreTargetFrameworkDataSource] - [NetFullTargetFrameworkDataSource(useVsixRunner: true)] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedIn(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -184,6 +185,8 @@ public void DiscoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscover // Normally we test on two runner, against single .NET Testhost, but because source navigation happens in testhost // it is better to test against both desktop and core runners to make sure source navigation discovery works in both scenarios. // We run .NET Runner -> .NET Testhost and .NET Framework Runner -> .NET Frameworks Testhost. + // The .NET Framework runner/testhost is not available on Linux/macOS. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(useCoreRunner: false)] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/LiveUnitTestingTests.cs index 21afa15e53..38e5f2a347 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -37,6 +37,7 @@ public void Cleanup() [TestMethod] // Touches appdomain settings, preferring .NET Framework testhost here. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void DiscoverTestsUsingLiveUnitTesting(RunnerInfo runnerInfo) { @@ -62,6 +63,7 @@ public void DiscoverTestsUsingLiveUnitTesting(RunnerInfo runnerInfo) [TestMethod] // Touches appdomain settings, preferring .NET Framework testhost here. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void RunTestsWithLiveUnitTesting(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTests.cs index 5e394623b2..7975ecd58e 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTests.cs @@ -46,6 +46,7 @@ public void Cleanup() } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [WrapperCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo) @@ -65,7 +66,6 @@ public void RunAllTests(RunnerInfo runnerInfo) [TestMethod] [NetCoreTargetFrameworkDataSource] - [NetFullTargetFrameworkDataSource(useVsixRunner: true)] [TestCategory("Smoke")] public void RunAllTestsFromDlls(RunnerInfo runnerInfo) { @@ -83,6 +83,7 @@ public void RunAllTestsFromDlls(RunnerInfo runnerInfo) } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [WrapperCompatibilityDataSource()] public void RunAllTestsWithMixedTFMsWillRunTestsFromAllProvidedDllEvenWhenTheyMixTFMs(RunnerInfo runnerInfo) @@ -171,6 +172,9 @@ public void RunTestsWithTelemetryOptedOut(RunnerInfo runnerInfo) [TestMethod] // This is testing the behavior of crash in testhost, run on different testhost, and just .NET runner. + // The assertion below branches on the .NET Framework-specific stack overflow message, and the + // .NET Framework testhost is only available on Windows, so this runs as Windows-Review. + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) @@ -195,8 +199,6 @@ public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) } [TestMethod] - [TestCategory("Windows-Review")] - [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTestsWithFilterTests.cs index decb300853..3dc564da17 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -37,6 +37,7 @@ public void Cleanup() } [TestMethod] + // WrapperCompatibilityDataSource includes the .NET Framework runner, which is not available on Linux/macOS. [TestCategory("Windows-Review")] [WrapperCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/SerializeTestRunTests.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/SerializeTestRunTests.cs index 1c470a41b6..87f27d6787 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/SerializeTestRunTests.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/SerializeTestRunTests.cs @@ -17,8 +17,6 @@ namespace Microsoft.TestPlatform.Library.IntegrationTests.TranslationLayerTests; [TestClass] // TODO: this comment seems inaccurate and would mean all our linux and macos tests are broken? -// We need to dogfood the package built in this repo *-dev and we pack tha tp only on windows -[TestCategory("Windows-Review")] public class SerialTestRunDecoratorTests : AcceptanceTestBase { private IVsTestConsoleWrapper? _vstestConsoleWrapper; @@ -51,7 +49,6 @@ public void Cleanup() [TestMethod] // This is testhost concept, try it on combination of testhosts, and .NET Runner. [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] - [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] public void DiscoverTestsAndRunTestsSequentially(RunnerInfo runnerInfo) { // Arrange @@ -73,7 +70,6 @@ public void DiscoverTestsAndRunTestsSequentially(RunnerInfo runnerInfo) [TestMethod] // This is testhost concept, try it on combination of testhosts, and .NET Runner. [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] - [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] public void DiscoverTestsAndRunTestsSequentially_DisabledByFeatureFlag(RunnerInfo runnerInfo) { // Arrange @@ -95,7 +91,6 @@ public void DiscoverTestsAndRunTestsSequentially_DisabledByFeatureFlag(RunnerInf [TestMethod] [NetCoreTargetFrameworkDataSource] - [NetFullTargetFrameworkDataSource] public void DiscoverTestsAndRunTestsSequentially_IsNotSupportedForSources(RunnerInfo runnerInfo) { // Arrange diff --git a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/TargetFrameworkTestHostDemultiplexer.cs b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/TargetFrameworkTestHostDemultiplexer.cs index bb553f04ca..c3b2d7d1b9 100644 --- a/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/TargetFrameworkTestHostDemultiplexer.cs +++ b/test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/TargetFrameworkTestHostDemultiplexer.cs @@ -16,8 +16,6 @@ namespace Microsoft.TestPlatform.Library.IntegrationTests.TranslationLayerTests; [TestClass] -// We need to dogfood the package built in this repo *-dev and we pack tha tp only on windows -[TestCategory("Windows-Review")] public class TargetFrameworkTestHostDemultiplexer : AcceptanceTestBase { private IVsTestConsoleWrapper? _vstestConsoleWrapper; @@ -40,19 +38,16 @@ public void Cleanup() [TestMethod] [NetCoreTargetFrameworkDataSource] - [NetFullTargetFrameworkDataSource] public void ExecuteContainerInMultiHost(RunnerInfo runnerInfo) => ExecuteContainerInMultiHost(runnerInfo, 3); [TestMethod] [NetCoreTargetFrameworkDataSource] - [NetFullTargetFrameworkDataSource] public void ExecuteContainerInMultiHost_MoreHostsThanTests(RunnerInfo runnerInfo) => ExecuteContainerInMultiHost(runnerInfo, 20); [TestMethod] [NetCoreTargetFrameworkDataSource] - [NetFullTargetFrameworkDataSource] public void ExecuteSingleContainerInDefaultSingleHost(RunnerInfo runnerInfo) => ExecuteContainerInMultiHost(runnerInfo, -1); diff --git a/test/vstest.console.UnitTests/Processors/AeDebuggerArgumentProcessorTest.cs b/test/vstest.console.UnitTests/Processors/AeDebuggerArgumentProcessorTest.cs index c8b1645c71..bac053c84a 100644 --- a/test/vstest.console.UnitTests/Processors/AeDebuggerArgumentProcessorTest.cs +++ b/test/vstest.console.UnitTests/Processors/AeDebuggerArgumentProcessorTest.cs @@ -18,6 +18,7 @@ namespace vstest.console.UnitTests.Processors; [TestClass] +// AeDebug (post-mortem debugger) is a Windows-only feature and these tests rely on Windows path semantics. [TestCategory("Windows-Review")] public class AeDebuggerArgumentProcessorTest { diff --git a/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs index bed998f47f..4e9ebdb118 100644 --- a/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs @@ -393,7 +393,6 @@ public void InitializeShouldCreateEntryForBlameAlongWithCollectHangDumpParameter } [TestMethod] - [TestCategory("Windows-Review")] public void InitializeMonitorPostmortemDebuggerShouldGenerateCorrectConfiguration() { var runsettingsString = string.Format(CultureInfo.CurrentCulture, _defaultRunSettings, ""); @@ -434,7 +433,6 @@ public void InitializeMonitorPostmortemDebuggerShouldGenerateCorrectConfiguratio } [TestMethod] - [TestCategory("Windows-Review")] public void InitializeMonitorPostmortemDebuggerShouldGenerateCorrectConfigurationAlsoIfIncomplete() { var runsettingsString = string.Format(CultureInfo.CurrentCulture, _defaultRunSettings, ""); diff --git a/test/vstest.console.UnitTests/Processors/ShowDeprecateDotnetVStestMessageArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/ShowDeprecateDotnetVStestMessageArgumentProcessorTests.cs index 59537431ab..ee748fa37c 100644 --- a/test/vstest.console.UnitTests/Processors/ShowDeprecateDotnetVStestMessageArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/ShowDeprecateDotnetVStestMessageArgumentProcessorTests.cs @@ -7,7 +7,6 @@ namespace vstest.console.UnitTests.Processors; [TestClass] -[TestCategory("Windows-Review")] public class ShowDeprecateDotnetVStestMessageArgumentProcessorTests { [TestMethod]