Revert to MTP and disable server capability as a workaround

.github/workflows/run-tests.yml

@@ -13,10 +13,10 @@ on:
         # relative the repo root
         required: false
         type: string
-      testSessionTimeoutMs:
+      testSessionTimeout:
         required: false
         type: string
-        default: "900000"
+        default: "15m"
       testHangTimeout:
         required: false
         type: string
@@ -220,18 +220,14 @@ jobs:
           TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
           TestsRunningOutsideOfRepo: true
         run: >
-          dotnet test -s .runsettings -v:n ${{ env.TEST_ASSEMBLY_NAME }}.dll
-          -l "console;verbosity=normal"
-          -l "trx;LogFilePrefix=${{ inputs.testShortName }}"
-          -l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
-          --blame
-          --blame-hang-timeout ${{ inputs.testHangTimeout }}
-          --blame-crash
+          dotnet ${{ env.TEST_ASSEMBLY_NAME }}.dll
+          --report-trx --report-trx-filename "${{ inputs.testShortName }}.trx"
+          --hangdump --hangdump-timeout ${{ inputs.testHangTimeout }}
+          --crashdump
           --results-directory ${{ github.workspace }}/testresults
+          --filter-not-trait "category=failing"
+          --timeout ${{ inputs.testSessionTimeout }}
           ${{ inputs.extraTestArgs }}
-          --
-          RunConfiguration.CollectSourceInformation=true
-          RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }}
 
       - name: Run tests
         if: ${{ ! inputs.requiresNugets }}
@@ -240,24 +236,24 @@ jobs:
           CI: false
           DCP_DIAGNOSTICS_LOG_LEVEL: debug
           DCP_DIAGNOSTICS_LOG_FOLDER: ${{ github.workspace }}/testresults/dcp
+          # During restore and build, we use -ci, which causes NUGET_PACKAGES to point to a local cache (Arcade behavior).
+          # In this step, we are not using Arcade, but want to make sure that MSBuild is able to evaluate correctly.
+          # So, we manually set NUGET_PACKAGES
+          NUGET_PACKAGES: ${{ github.workspace }}/.packages
         run: >
           ${{ env.DOTNET_SCRIPT }} test ${{ env.TEST_PROJECT_PATH }}
           /p:ContinuousIntegrationBuild=true
-          -s eng/testing/.runsettings
-          -l "console;verbosity=normal"
-          -l "trx;LogFilePrefix=${{ inputs.testShortName }}"
-          -l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
-          "--blame"
-          --blame-hang-timeout ${{ inputs.testHangTimeout }}
-          --blame-crash
-          --results-directory testresults
+          /p:TrxFileNamePrefix="${{ inputs.testShortName }}"
+          -bl:${{ github.workspace }}/testresults/test.binlog
           --no-restore
           --no-build
-          -bl:${{ github.workspace }}/testresults/test.binlog
-          ${{ inputs.extraTestArgs }}
           --
-          RunConfiguration.CollectSourceInformation=true
-          RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }}
+          --report-trx
+          --hangdump --hangdump-timeout ${{ inputs.testHangTimeout }}
+          --crashdump
+          --results-directory ${{ github.workspace }}/testresults
+          --timeout ${{ inputs.testSessionTimeout }}
+          ${{ inputs.extraTestArgs }}
 
       # Save the result of the previous steps - success or failure
       # in the form of a file result-success/result-failure -{name}.rst

.github/workflows/tests.yml

@@ -77,7 +77,7 @@ jobs:
     with:
       testShortName: ${{ matrix.shortname }}
       os: "ubuntu-latest"
-      extraTestArgs: "--filter \"(quarantined!=true)\""
+      extraTestArgs: "--filter-not-trait \"quarantined=true\""
 
   integrations_test_win:
     uses: ./.github/workflows/run-tests.yml
@@ -90,7 +90,7 @@ jobs:
     with:
       testShortName: ${{ matrix.shortname }}
       os: "windows-latest"
-      extraTestArgs: "--filter \"(quarantined!=true)\""
+      extraTestArgs: "--filter-not-trait \"quarantined=true\""
 
   templates_test_lin:
     name: Templates Linux
@@ -103,10 +103,9 @@ jobs:
       testShortName: ${{ matrix.shortname }}
       os: "ubuntu-latest"
       testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
-      testSessionTimeoutMs: 1200000
+      testSessionTimeout: 20m
       testHangTimeout: 12m
-      # append '.' to the name so only the test class with exactly that name is run
-      extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\""
+      extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
       requiresNugets: true
       requiresTestSdk: true
 
@@ -121,10 +120,9 @@ jobs:
       testShortName: ${{ matrix.shortname }}
       os: "windows-latest"
       testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
-      testSessionTimeoutMs: 1200000
+      testSessionTimeout: 20m
       testHangTimeout: 12m
-      # append '.' to the name so only the test class with exactly that name is run
-      extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\""
+      extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
       requiresNugets: true
       requiresTestSdk: true
 

eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets

@@ -58,14 +58,13 @@
     <PropertyGroup>
       <_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment>
       <_TestAssembly>%(TestToRun.Identity)</_TestAssembly>
+      <_TestAssembly Condition="'$(OS)'=='Windows_NT'">$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe'))</_TestAssembly>
+      <_TestAssembly Condition="'$(OS)'!='Windows_NT'">$(_TestAssembly.TrimEnd('.dll'))</_TestAssembly>
       <_TestTimeout>%(TestToRun.TestTimeout)</_TestTimeout>
-      <_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments>
 
       <_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\</_TargetDir>
 
-      <_TestRunnerCommand>&quot;$(DotNetTool)&quot; test $(_TestAssembly) --list-tests --logger:"console%3Bverbosity=normal" "--Framework:%(TestToRun.TargetFrameworkIdentifier),Version=%(TestToRun.TargetFrameworkVersion)"</_TestRunnerCommand>
-      <_TestRunnerCommand Condition="'$(VSTestRunSettingsFile)' != ''">$(_TestRunnerCommand) "--settings:$(VSTestRunSettingsFile)"</_TestRunnerCommand>
-      <_TestRunnerCommand Condition="'$(_TestRunnerAdditionalArguments)' != ''">$(_TestRunnerCommand) $(_TestRunnerAdditionalArguments)</_TestRunnerCommand>
+      <_TestRunnerCommand>$(_TestAssembly) --list-tests $(_QuarantinedTestRunAdditionalArgs)</_TestRunnerCommand>
 
       <!--
         Redirect std output of the runner.
@@ -92,53 +91,27 @@
           WorkingDirectory="$(_TargetDir)"
           IgnoreExitCode="true"
           Timeout="$(_TestTimeout)"
+          EnvironmentVariables="DOTNET_ROOT=$(DotNetRoot);DOTNET_ROOT_X86=$(DotNetRoot)x86"
           ContinueOnError="WarnAndContinue">
       <Output TaskParameter="ExitCode" PropertyName="_TestErrorCode" />
     </Exec>
 
-    <!--
-      Report test status.
-      -->
-    <Message Text="Search complete: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />
-
     <PropertyGroup>
       <_ResultsFileToDisplay>%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay>
     </PropertyGroup>
 
     <!--
-      Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project
-      we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961.
-    -->
-    <Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' != '0'" File="QuarantinedTestRunsheetBuilder" />
-
-    <!--
-      Read the test result and see if the string "No test matches the given testcase filter" is present.
-      This indicates that there are no quarantined tests to run.
+      Report test status.
       -->
-    <ReadLinesFromFile File="$(_ResultsFileToDisplay)">
-      <Output TaskParameter="Lines" ItemName="FileLines" />
-    </ReadLinesFromFile>
-
-    <ItemGroup>
-      <Lines Include="@(FileLines)" >
-        <Sanitized>$([System.String]::Copy('%(FileLines.Identity)').Replace('`', ''))</Sanitized>
-      </Lines>
-
-      <_UnhandledExceptionLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Unhandled exception`)) " />
-      <_ExceptionDiscoveringTestsLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Exception discovering tests`)) " />
-      <_NoTestMatchesLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`No test matches the given testcase filter`)) " />
-    </ItemGroup>
+    <Message Text="Search complete, no quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '8'" Importance="high" />
+    <Message Text="💡 Search complete, quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />
+    <Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition=" '$(_TestErrorCode)' != '0' and '$(_TestErrorCode)' != '8' " File="QuarantinedTestRunsheetBuilder" />
 
     <PropertyGroup>
-      <_FailedToEnumerateTests Condition=" '@(_UnhandledExceptionLines)' != '' or '@(_ExceptionDiscoveringTestsLines)' != '' ">true</_FailedToEnumerateTests>
-
       <_HasQuarantinedTests>true</_HasQuarantinedTests>
-      <_HasQuarantinedTests Condition=" '@(_NoTestMatchesLines)' != '' ">false</_HasQuarantinedTests>
+      <_HasQuarantinedTests Condition=" '$(_TestErrorCode)' == '8' ">false</_HasQuarantinedTests>
     </PropertyGroup>
 
-    <Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_FailedToEnumerateTests)' == 'true'" File="QuarantinedTestRunsheetBuilder" />
-    <Message Text="💡 Quarantined tests found in $(_TestAssembly) [$(_TestEnvironment)]" Condition=" '$(_HasQuarantinedTests)' == 'true' " Importance="high" />
-
     <!--
       Generate test runsheet, if there are quarantined tests.
       -->
@@ -160,6 +133,12 @@
       <_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux>
       <_TestCommand>-restore -build -test -projects &quot;$(_RelativeTestProjectPath)&quot; /bl:&quot;$(_RelativeTestBinLog)&quot; -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false</_TestCommand>
 
+      <!--
+        Some quarantinted test may only be executable on Windows or Linux, however we can't possibly know that at this time.
+        The MTP runner will return exit code 8 if no tests are found, and we need to ignore it instead of failing the test.
+        -->
+      <_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true</_TestCommand>
+
       <!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
       <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>
 

eng/Testing.props

@@ -9,24 +9,20 @@
     <RunOnAzdoCILinux>true</RunOnAzdoCILinux>
     <RunOnAzdoHelixWindows>true</RunOnAzdoHelixWindows>
     <RunOnAzdoHelixLinux>true</RunOnAzdoHelixLinux>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(UseVSTestRunner)' != 'true'">
-    <_QuarantinedTestRunAdditionalArgs>-trait "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
-    <_NonQuarantinedTestRunAdditionalArgs>-notrait "quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
-  </PropertyGroup>
 
-  <PropertyGroup Condition="'$(UseVSTestRunner)' == 'true'">
-    <_QuarantinedTestRunAdditionalArgs>--filter "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
-    <_NonQuarantinedTestRunAdditionalArgs>--filter "quarantined!=true"</_NonQuarantinedTestRunAdditionalArgs>
+    <_QuarantinedTestRunAdditionalArgs>--filter-trait "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
+    <_NonQuarantinedTestRunAdditionalArgs>--filter-not-trait "quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
   </PropertyGroup>
 
   <PropertyGroup>
     <BlameHangTimeout Condition="'$(BlameHangTimeout)' == '' and '$(MSBuildProjectName)' == 'Aspire.Hosting.SqlServer.Tests'">20m</BlameHangTimeout>
     <BlameHangTimeout Condition="'$(BlameHangTimeout)' == ''">10m</BlameHangTimeout>
-    <_BlameArgs>--blame --blame-hang-timeout $(BlameHangTimeout) --blame-crash</_BlameArgs>
+    <_BlameArgs>--hangdump --hangdump-timeout $(BlameHangTimeout) --crashdump</_BlameArgs>
+
+    <TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --filter-not-trait &quot;category=failing&quot;</TestRunnerAdditionalArguments>
+    <TestRunnerAdditionalArguments Condition=" '$(IgnoreZeroTestResult)' == 'true' ">$(TestRunnerAdditionalArguments) --ignore-exit-code 8</TestRunnerAdditionalArguments>
 
-    <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == ''">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs)</TestRunnerAdditionalArguments>
-    <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs)</TestRunnerAdditionalArguments>
+    <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' != 'true'">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
+    <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
   </PropertyGroup>
 </Project>

eng/Versions.props

@@ -12,10 +12,11 @@
     <DotNetRuntimePreviousVersionForTesting>8.0.13</DotNetRuntimePreviousVersionForTesting>
     <!-- dotnet 8.0 versions for running tests - used for templates tests -->
     <DotNetSdkPreviousVersionForTesting>8.0.406</DotNetSdkPreviousVersionForTesting>
-    <UseVSTestRunner>true</UseVSTestRunner>
     <XunitV3Version>2.0.0</XunitV3Version>
     <XUnitAnalyzersVersion>1.20.0</XUnitAnalyzersVersion>
     <XunitRunnerVisualStudioVersion>3.0.2</XunitRunnerVisualStudioVersion>
+    <MicrosoftTestingPlatformVersion>1.6.3</MicrosoftTestingPlatformVersion>
+    <MicrosoftNETTestSdkVersion>17.13.0</MicrosoftNETTestSdkVersion>
     <!-- Enable to remove prerelease label. -->
     <StabilizePackageVersion Condition="'$(StabilizePackageVersion)' == ''">false</StabilizePackageVersion>
     <DotNetFinalVersionKind Condition="'$(StabilizePackageVersion)' == 'true'">release</DotNetFinalVersionKind>

eng/Xunit3/Xunit3.targets

@@ -4,15 +4,25 @@
     <PackageVersion Include="xunit.v3.core" Version="$(XunitV3Version)" />
     <PackageVersion Include="xunit.analyzers" Version="$(XunitAnalyzersVersion)" />
     <PackageVersion Include="xunit.v3.assert" Version="$(XunitV3Version)" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioVersion)" />
     <PackageVersion Include="xunit.v3.runner.console" Version="$(XunitV3Version)" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioVersion)" />
+    <PackageVersion Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformVersion)" />
+    <PackageVersion Include="Microsoft.Testing.Platform.MSBuild" Version="$(MicrosoftTestingPlatformVersion)" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.HangDump" Version="$(MicrosoftTestingPlatformVersion)" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.CrashDump" Version="$(MicrosoftTestingPlatformVersion)" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="$(MicrosoftTestingPlatformVersion)" />
 
     <PackageReference Include="xunit.v3" />
     <PackageReference Include="xunit.v3.core" />
     <PackageReference Include="xunit.analyzers" />
     <PackageReference Include="xunit.v3.assert" />
     <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="Microsoft.Testing.Platform" />
+    <PackageReference Include="Microsoft.Testing.Platform.MSBuild" />
+    <PackageReference Include="Microsoft.Testing.Extensions.HangDump" />
+    <PackageReference Include="Microsoft.Testing.Extensions.CrashDump" />
+    <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
   </ItemGroup>
 
-  <Import Project="..\tools\VSTest.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
-</Project>
+  <Import Project="Microsoft.Testing.Platform.targets" />
+</Project>

tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj

@@ -9,6 +9,11 @@
     <IsTestUtilityProject>true</IsTestUtilityProject>
     <SkipTests>true</SkipTests>
     <DeployOutsideOfRepoSupportFiles>false</DeployOutsideOfRepoSupportFiles>
+
+    <!-- https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-exit-codes -->
+    <!-- Exit code 8 is "zero tests ran" -->
+    <!-- Currently, none of the tests in this project run in CI. All are ignored -->
+    <TestingPlatformCommandLineArguments>$(TestingPlatformCommandLineArguments) --ignore-exit-code 8</TestingPlatformCommandLineArguments>
   </PropertyGroup>
 
   <ItemGroup>