Migrate from VSTest to Microsoft.Testing.Platform #8498
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| <!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. --> | ||
|
danmoseley marked this conversation as resolved.
|
||
| <Project> | ||
|
|
||
| <Target Name="RunTests" | ||
| Outputs="%(TestToRun.ResultsStdOutPath)" | ||
| Condition="'@(TestToRun)' != ''"> | ||
| <Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Test" /> | ||
|
|
||
| <PropertyGroup> | ||
| <_TestResultDirectory>$([System.IO.Path]::GetDirectoryName('%(TestToRun.ResultsTrxPath)'))</_TestResultDirectory> | ||
| <_TestResultTrxFileName>$([System.IO.Path]::GetFileName('%(TestToRun.ResultsTrxPath)'))</_TestResultTrxFileName> | ||
| <_TestResultXmlFileName>$([System.IO.Path]::GetFileName('%(TestToRun.ResultsXmlFileName)'))</_TestResultXmlFileName> | ||
| <_TestResultHtmlFileName>$([System.IO.Path]::GetFileName('%(TestToRun.ResultsHtmlFileName)'))</_TestResultHtmlFileName> | ||
| </PropertyGroup> | ||
|
|
||
| <PropertyGroup> | ||
| <_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment> | ||
| <_TestAssembly>%(TestToRun.Identity)</_TestAssembly> | ||
| <_TestAssembly>$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe'))</_TestAssembly> | ||
| <_TestRuntime>%(TestToRun.TestRuntime)</_TestRuntime> | ||
| <_TestTimeout>%(TestToRun.TestTimeout)</_TestTimeout> | ||
|
radical marked this conversation as resolved.
|
||
| <_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments> | ||
|
|
||
| <_TestRunner>$(_TestAssembly)</_TestRunner> | ||
| <_TestRunnerArgs>$(_TestRunnerAdditionalArguments) --results-directory "$(_TestResultDirectory)" --report-xunit --report-xunit-filename "$(_TestResultXmlFileName)" --report-xunit-html --report-xunit-html-filename "$(_TestResultHtmlFileName)"</_TestRunnerArgs> | ||
| </PropertyGroup> | ||
|
|
||
| <PropertyGroup Condition="'$(_TestRuntime)' == 'Core'"> | ||
| <_TestRunnerArgs>$(_TestRunnerArgs) --auto-reporters off</_TestRunnerArgs> | ||
| </PropertyGroup> | ||
|
|
||
| <PropertyGroup> | ||
| <_TestRunnerCommand>"$(_TestRunner)" $(_TestRunnerArgs)</_TestRunnerCommand> | ||
|
|
||
| <!-- | ||
| Redirect std output of the runner. | ||
| Note that xUnit outputs failure info to both STDOUT (stack trace, message) and STDERR (failed test name) | ||
| --> | ||
| <_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestToRun.ResultsStdOutPath)" 2>&1</_TestRunnerCommand> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <_OutputFiles Include="%(TestToRun.ResultsXmlPath)" /> | ||
| <_OutputFiles Include="%(TestToRun.ResultsHtmlPath)" /> | ||
| <_OutputFiles Include="%(TestToRun.ResultsStdOutPath)" /> | ||
| </ItemGroup> | ||
|
|
||
| <MakeDir Directories="@(_OutputFiles->'%(RootDir)%(Directory)')"/> | ||
| <Delete Files="@(_OutputFiles)" /> | ||
|
|
||
| <Message Text="Running tests: $(_TestAssembly) [$(_TestEnvironment)]" Importance="high"/> | ||
| <Exec Command='$(_TestRunnerCommand)' | ||
| LogStandardErrorAsError="false" | ||
| WorkingDirectory="$(_TargetDir)" | ||
| IgnoreExitCode="true" | ||
| Timeout="$(_TestTimeout)" | ||
| EnvironmentVariables="DOTNET_ROOT=$(DotNetRoot);DOTNET_ROOT_X86=$(DotNetRoot)x86" | ||
| ContinueOnError="WarnAndContinue"> | ||
| <Output TaskParameter="ExitCode" PropertyName="_TestErrorCode" /> | ||
| </Exec> | ||
|
|
||
| <!-- | ||
| Add command line to the log. | ||
| --> | ||
| <WriteLinesToFile File="%(TestToRun.ResultsStdOutPath)" | ||
| Overwrite="false" | ||
| Lines=";=== COMMAND LINE ===;$(_TestRunnerCommand)" | ||
| Condition="'$(TestCaptureOutput)' != 'false'" /> | ||
|
|
||
| <!-- | ||
| Report test status. | ||
| --> | ||
| <Message Text="Tests succeeded: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" /> | ||
|
|
||
| <PropertyGroup> | ||
| <_ResultsFileToDisplay>%(TestToRun.ResultsHtmlPath)</_ResultsFileToDisplay> | ||
| <_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay> | ||
| </PropertyGroup> | ||
|
|
||
| <!-- | ||
| Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project | ||
| we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961. | ||
| --> | ||
| <Error Text="Tests failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' != '0' and '$(_ErrorOnTestFailure)' != 'false'" File="XUnit" /> | ||
|
|
||
| <ItemGroup> | ||
| <FileWrites Include="@(_OutputFiles)"/> | ||
| </ItemGroup> | ||
| </Target> | ||
|
|
||
| </Project> | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,35 +1,2 @@ | ||
| # Filter out failing (wrong framework, platform, runtime or activeissue) tests | ||
| --filter-not-trait "category=failing" | ||
|
danmoseley marked this conversation as resolved.
Outdated
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>$(DefaultTargetFramework)</TargetFramework> | ||
| <TestingPlatformCommandLineArguments>$(TestingPlatformCommandLineArguments) --ignore-exit-code 8</TestingPlatformCommandLineArguments> | ||
|
danmoseley marked this conversation as resolved.
Contributor
There was a problem hiding this comment. Youssef Fahmy (@Youssef1313) this doesn't appear to be used in Microsoft.Testing.Platform.targets, which results in the tests to fail Should this be tucked on to
Member
Author
There was a problem hiding this comment. Igor Velikorossov (@RussKie)
Contributor
There was a problem hiding this comment. What would be the right way to fix this?
Member
Author
There was a problem hiding this comment. I think you can simply append
Member
Author
There was a problem hiding this comment. Hmm, what I'm suggesting won't probably be possible without my PR on Arcade side. For now I think you could duplicate the
Member
Author
There was a problem hiding this comment. Or simply just rename Aspire.Components.Common.Tests to end with TestUtilities if it's really not intended to be a test project, then you don't need the --ignore-exit-code 8 at all.
Contributor
There was a problem hiding this comment.
Yep, working on it
Contributor
There was a problem hiding this comment. I don't think we can do it easily (if at all).
Member
Author
There was a problem hiding this comment. Igor Velikorossov (@RussKie) You need to reference only |
||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,26 +1,4 @@ | ||
| --show-live-output on | ||
|
Contributor
There was a problem hiding this comment. if we wanted to experiment with parallel tests runs for a particular library, we'd put it in a file like this presumably? (right now none may be stable for parallel runs, nor diagnosable, but for traditional unit tests it may be possible)
Member
Author
There was a problem hiding this comment. Either that, or add it to Note: the current file is no longer really runsettings, it's a response file. RunSettings is actually not supported by core MTP (there is limited support for it through VSTestBridge, but xUnit doesn't use the bridge).
Member
Author
There was a problem hiding this comment. You can also do
Contributor
There was a problem hiding this comment. why do we still have .runsettings if it's not really supported?
Member
Author
There was a problem hiding this comment. The file name is just The original intent is very unclear to me, there are too many .runsettings scattered and it's not claer which is used for what and when. If you can clarify the original intent, maybe there can be a good room to clean it up more.
Contributor
There was a problem hiding this comment. I cannot clarify. Possibly Ankit Jain (@radical) can or perhaps it's just happenstance, this isn't a mature repo. Any cleanup welcome!
Member
Author
There was a problem hiding this comment. Okay, I went ahead and cleaned up all the Any customizations that are needed for a specific project can be done via
Member
There was a problem hiding this comment. We have various runsettings to have different logging and timeout setup, and the files themselves weren't compostable. This was useful for local vs CI runs. I did start moving to overriding on the command line. But cleaning all this up to use command line parameters via msbuild properties will be great!! |
||
|
|
||
| # Filter out failing (wrong framework, platform, runtime or activeissue) tests | ||
| --filter-not-trait "category=failing" | ||
|
danmoseley marked this conversation as resolved.
Outdated
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this really how normal developers are supposed to execute tests using the Microsoft.Testing.Platform? In general, we don't suggest to people to use
dotnet exec. You candotnet XXX.dllor just run the platform-specific executable${{ env.TEST_ASSEMBLY_NAME }}[.exe]Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
All the three approaches will work (dotnet exec dll, dotnet dll, or the executable directly). If you have preference let me know and I can refactor.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What's the preference from the test team?
I guess I would prefer invoking the executable directly.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm reworking the infra with expectation that we'd be running
build.cmd/shfor all build, test, package, etc. needs.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That's not a correct expectation, as for nuget testing, and tests run on helix we run just the binaries, and thus no arcade or repository is available.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We won't be running on Helix at all.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We are using
dotnet execbecause the tests are being run without the project file available. But since the test assembly is executable now, we can directly run that.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I tried to move over to running executable directly, but it's not in the zip file when using
ZipTestArchive. I have no idea why. Maybe we are zipping too early or something. I'm doing dotnet test.dll for now. We can follow-up with investigation to include the exe in the zip.