Skip to content

Remove Windows-Review from cross-platform tests #16113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jakub Jareš (nohwnd) merged 8 commits into from
Jun 29, 2026
Merged

Remove Windows-Review from cross-platform tests #16113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ec1c5c1
Remove Windows-Review restriction from cross-platform tests
nohwnd Jun 12, 2026
7111f82
Merge remote-tracking branch 'origin/main' into remove-windows-review…
nohwnd Jun 23, 2026
908b9d2
Keep Windows-Review on AeDebugger tests
nohwnd Jun 24, 2026
97c8d34
Keep Windows-Review on integration tests that need the .NET Framework…
nohwnd Jun 24, 2026
b174f6d
Keep Windows-Review on RunTestsWithXunitAdapter (fails on Linux)
nohwnd Jun 24, 2026
bd7f77c
Make tests with empty data sources fail instead of masking them
nohwnd Jun 26, 2026
deeb812
Re-add Windows-Review to the .NET Framework-only tests the audit missed
nohwnd Jun 26, 2026
0854a6c
Remove the empty .NET Framework data source from cross-platform tests
nohwnd Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/.runsettings
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,2 @@
<RunSettings>
<MSTestV2>
<ConsiderEmptyDataSourceAsInconclusive>true</ConsiderEmptyDataSourceAsInconclusive>
</MSTestV2>
</RunSettings>
Comment thread
nohwnd marked this conversation as resolved.
7 changes: 2 additions & 5 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@
namespace Microsoft.TestPlatform.AcceptanceTests;

[TestClass]
[TestCategory("Windows-Review")]
public class ArgumentProcessorTests : AcceptanceTestBase
{

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
Comment thread
nohwnd marked this conversation as resolved.
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand All @@ -34,8 +32,7 @@ public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo r
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand Down
6 changes: 6 additions & 0 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/CreateNoNewWindowTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
public class CreateNoNewWindowTests : AcceptanceTestBase
{
[TestMethod]
// CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost.
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)]
public void WhenCreateNoNewWindowIsFalse_DiagShowsCreateNoWindowFalse(RunnerInfo runnerInfo)
{
Expand All @@ -34,6 +36,8 @@ public void WhenCreateNoNewWindowIsFalse_DiagShowsCreateNoWindowFalse(RunnerInfo
}

[TestMethod]
// CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost.
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)]
public void WhenCreateNoNewWindowIsTrue_DiagShowsCreateNoWindowTrue(RunnerInfo runnerInfo)
{
Expand All @@ -56,6 +60,8 @@ public void WhenCreateNoNewWindowIsTrue_DiagShowsCreateNoWindowTrue(RunnerInfo r
}

[TestMethod]
// CreateNoNewWindow maps to the Windows-only process CreateNoWindow flag and only runs on the .NET Framework testhost.
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: false)]
public void WhenCreateNoNewWindowIsNotSet_DefaultIsTrue(RunnerInfo runnerInfo)
{
Expand Down
4 changes: 1 addition & 3 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DataCollectionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
public class DataCollectionTests : AcceptanceTestBase
{
[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo)
{
Expand All @@ -44,7 +43,6 @@ public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -80,6 +78,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo
}

[TestMethod]
// .NET Framework testhost-specific assembly loading; not applicable to the netcore testhost.
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
Expand All @@ -93,7 +92,6 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo)
{
Expand Down
9 changes: 2 additions & 7 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
public class DiscoveryTests : AcceptanceTestBase
{
[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void DiscoverAllTests(RunnerInfo runnerInfo)
{
Expand All @@ -32,7 +31,6 @@ public void DiscoverAllTests(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useVsixRunner: true)]
[NetCoreTargetFrameworkDataSource]
[TestCategory("Smoke")]
public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo)
Expand All @@ -55,8 +53,7 @@ public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo)
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
Comment thread
nohwnd marked this conversation as resolved.
{
var dummyFilePath = Path.Combine(TempDirectory.Path, $"{Guid.NewGuid()}.txt");
Expand All @@ -74,7 +71,6 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -119,8 +115,7 @@ public void TypesToLoadAttributeTests()
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void DiscoverTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand Down
6 changes: 1 addition & 5 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
public class DotnetTestMSBuildOutputTests : AcceptanceTestBase
{
[TestMethod]
// patched dotnet is not published on non-windows systems
// Special characters (~, !, |, %) don't survive the MSBuildLogger output round-trip on non-Windows terminals.
[TestCategory("Windows-Review")]
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(RunnerInfo runnerInfo)
Expand Down Expand Up @@ -54,8 +54,6 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru
}

[TestMethod]
// patched dotnet is not published on non-windows systems
[TestCategory("Windows-Review")]
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo)
{
Expand All @@ -74,8 +72,6 @@ public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo)


[TestMethod]
// patched dotnet is not published on non-windows systems
[TestCategory("Windows-Review")]
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void MSBuildLoggerCanBeDisabledByEnvironmentVariableProperty(RunnerInfo runnerInfo)
{
Expand Down
24 changes: 12 additions & 12 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
[TestClass]
public class ExecutionTests : AcceptanceTestBase
{
//TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them?
[TestMethod]
// Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS.
[TestCategory("Windows-Review")]
[MSTestCompatibilityDataSource]
public void RunMultipleTestAssemblies(RunnerInfo runnerInfo)
Comment on lines 19 to 23

@nohwnd Jakub Jareš (nohwnd) Jun 26, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

makes sense, and that's exactly why this one keeps [TestCategory("Windows-Review")] — I re-added it in 97c8d34. The compat-matrix net48 desktop runner can't start on Linux/macOS, so the test stays Windows-only. ubuntu + macOS are green in build 1480079.

Expand All @@ -35,6 +35,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo)
}

[TestMethod]
// Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS.
[TestCategory("Windows-Review")]
[TestHostCompatibilityDataSource]
public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo)
Expand All @@ -52,6 +53,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru
}

[TestMethod]
// Compatibility matrix includes the .NET Framework runner/testhost, which is not available on Linux/macOS.
[TestCategory("Windows-Review")]
[RunnerCompatibilityDataSource]
public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo)
Expand All @@ -67,9 +69,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(R
}

[TestMethod]
[TestCategory("Windows-Review")]
[TestCategory("Smoke")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useVsixRunner: true)]
[NetCoreTargetFrameworkDataSource]
public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations3(RunnerInfo runnerInfo)
{
Expand All @@ -87,7 +87,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations3(R
// the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test
// frameworks have any added value over using 2 mstest dlls?
[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -134,7 +133,6 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void TestSessionTimeOutTests(RunnerInfo runnerInfo)
{
Expand All @@ -156,7 +154,6 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo)
{
Expand All @@ -173,6 +170,9 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo)
}

[TestMethod]
// Asserts the testhost-specific stack overflow message; the .NET Framework variant requires the
// .NET Framework testhost, which is only available on Windows.
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo)
Expand All @@ -198,6 +198,9 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn
}

[TestMethod]
// Asserts the testhost-specific unhandled exception message; the .NET Framework variant requires the
// .NET Framework testhost, which is only available on Windows.
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo)
Expand Down Expand Up @@ -292,8 +295,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
Comment thread
nohwnd marked this conversation as resolved.
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand All @@ -312,8 +314,7 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand Down Expand Up @@ -366,8 +367,7 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(RunnerInfo runnerInfo)
Comment thread
nohwnd marked this conversation as resolved.
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand Down
4 changes: 0 additions & 4 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FilePatternParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests;
public class FilePatternParserTests : AcceptanceTestBase
{
[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo)
{
Expand All @@ -32,7 +31,6 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(RunnerInfo runnerInfo)
{
Expand All @@ -58,7 +56,6 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo runnerInfo)
{
Expand All @@ -84,7 +81,6 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerInfo)
{
Expand Down
3 changes: 1 addition & 2 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/FrameworkTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ public class FrameworkTests : AcceptanceTestBase
{

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo)
{
Expand All @@ -27,7 +26,6 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo)
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -69,6 +67,7 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo)
[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
// The .NET (Core) runner produces a different framework-incompatible warning on non-Windows, so keep this Windows-only.
[TestCategory("Windows-Review")]
public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInfo runnerInfo)
{
Expand Down
Loading
Loading