diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3d2300a9545..c88a972da65 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -13,10 +13,10 @@ on: # relative the repo root required: false type: string - testSessionTimeout: + testSessionTimeoutMs: required: false type: string - default: "15m" + default: "900000" testHangTimeout: required: false type: string @@ -220,14 +220,18 @@ jobs: TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs TestsRunningOutsideOfRepo: true run: > - dotnet ${{ env.TEST_ASSEMBLY_NAME }}.dll - --report-trx --report-trx-filename "${{ inputs.testShortName }}.trx" - --hangdump --hangdump-timeout ${{ inputs.testHangTimeout }} - --crashdump + dotnet test -s .runsettings -v:n ${{ env.TEST_ASSEMBLY_NAME }}.dll + -l "console;verbosity=normal" + -l "trx;LogFilePrefix=${{ inputs.testShortName }}" + -l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" + --blame + --blame-hang-timeout ${{ inputs.testHangTimeout }} + --blame-crash --results-directory ${{ github.workspace }}/testresults - --filter-not-trait "category=failing" - --timeout ${{ inputs.testSessionTimeout }} ${{ inputs.extraTestArgs }} + -- + RunConfiguration.CollectSourceInformation=true + RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }} - name: Run tests if: ${{ ! inputs.requiresNugets }} @@ -236,24 +240,24 @@ jobs: CI: false DCP_DIAGNOSTICS_LOG_LEVEL: debug DCP_DIAGNOSTICS_LOG_FOLDER: ${{ github.workspace }}/testresults/dcp - # During restore and build, we use -ci, which causes NUGET_PACKAGES to point to a local cache (Arcade behavior). - # In this step, we are not using Arcade, but want to make sure that MSBuild is able to evaluate correctly. - # So, we manually set NUGET_PACKAGES - NUGET_PACKAGES: ${{ github.workspace }}/.packages run: > ${{ env.DOTNET_SCRIPT }} test ${{ env.TEST_PROJECT_PATH }} /p:ContinuousIntegrationBuild=true - /p:TrxFileNamePrefix="${{ inputs.testShortName }}" - -bl:${{ github.workspace }}/testresults/test.binlog + -s eng/testing/.runsettings + -l "console;verbosity=normal" + -l "trx;LogFilePrefix=${{ inputs.testShortName }}" + -l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" + "--blame" + --blame-hang-timeout ${{ inputs.testHangTimeout }} + --blame-crash + --results-directory testresults --no-restore --no-build - -- - --report-trx - --hangdump --hangdump-timeout ${{ inputs.testHangTimeout }} - --crashdump - --results-directory ${{ github.workspace }}/testresults - --timeout ${{ inputs.testSessionTimeout }} + -bl:${{ github.workspace }}/testresults/test.binlog ${{ inputs.extraTestArgs }} + -- + RunConfiguration.CollectSourceInformation=true + RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }} # Save the result of the previous steps - success or failure # in the form of a file result-success/result-failure -{name}.rst diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f40b71ec06..c85e92e8c6b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,7 @@ jobs: with: testShortName: ${{ matrix.shortname }} os: "ubuntu-latest" - extraTestArgs: "--filter-not-trait \"quarantined=true\"" + extraTestArgs: "--filter \"(quarantined!=true)\"" integrations_test_win: uses: ./.github/workflows/run-tests.yml @@ -90,7 +90,7 @@ jobs: with: testShortName: ${{ matrix.shortname }} os: "windows-latest" - extraTestArgs: "--filter-not-trait \"quarantined=true\"" + extraTestArgs: "--filter \"(quarantined!=true)\"" templates_test_lin: name: Templates Linux @@ -103,9 +103,10 @@ jobs: testShortName: ${{ matrix.shortname }} os: "ubuntu-latest" testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m + testSessionTimeoutMs: 1200000 testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" + # append '.' to the name so only the test class with exactly that name is run + extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\"" requiresNugets: true requiresTestSdk: true @@ -120,9 +121,10 @@ jobs: testShortName: ${{ matrix.shortname }} os: "windows-latest" testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m + testSessionTimeoutMs: 1200000 testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" + # append '.' to the name so only the test class with exactly that name is run + extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\"" requiresNugets: true requiresTestSdk: true diff --git a/eng/.runsettings b/eng/.runsettings new file mode 100644 index 00000000000..955954d2c90 --- /dev/null +++ b/eng/.runsettings @@ -0,0 +1,33 @@ + + + + + category!=failing + + + + + + + + Minimal + + + + + + + + + + + + + + + + + + + diff --git a/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets b/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets index 015fa4ef272..13790a2cc80 100644 --- a/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets +++ b/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets @@ -58,13 +58,14 @@ <_TestEnvironment>%(TestToRun.EnvironmentDisplay) <_TestAssembly>%(TestToRun.Identity) - <_TestAssembly Condition="'$(OS)'=='Windows_NT'">$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe')) - <_TestAssembly Condition="'$(OS)'!='Windows_NT'">$(_TestAssembly.TrimEnd('.dll')) <_TestTimeout>%(TestToRun.TestTimeout) + <_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments) <_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\ - <_TestRunnerCommand>$(_TestAssembly) --list-tests $(_QuarantinedTestRunAdditionalArgs) + <_TestRunnerCommand>"$(DotNetTool)" test $(_TestAssembly) --list-tests --logger:"console%3Bverbosity=normal" "--Framework:%(TestToRun.TargetFrameworkIdentifier),Version=%(TestToRun.TargetFrameworkVersion)" + <_TestRunnerCommand Condition="'$(VSTestRunSettingsFile)' != ''">$(_TestRunnerCommand) "--settings:$(VSTestRunSettingsFile)" + <_TestRunnerCommand Condition="'$(_TestRunnerAdditionalArguments)' != ''">$(_TestRunnerCommand) $(_TestRunnerAdditionalArguments) + + <_ResultsFileToDisplay>%(TestToRun.ResultsStdOutPath) + + + - - - + + + + + + + $([System.String]::Copy('%(FileLines.Identity)').Replace('`', '')) + + + <_UnhandledExceptionLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Unhandled exception`)) " /> + <_ExceptionDiscoveringTestsLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Exception discovering tests`)) " /> + <_NoTestMatchesLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`No test matches the given testcase filter`)) " /> + + <_FailedToEnumerateTests Condition=" '@(_UnhandledExceptionLines)' != '' or '@(_ExceptionDiscoveringTestsLines)' != '' ">true + <_HasQuarantinedTests>true - <_HasQuarantinedTests Condition=" '$(_TestErrorCode)' == '8' ">false + <_HasQuarantinedTests Condition=" '@(_NoTestMatchesLines)' != '' ">false + + + @@ -133,12 +160,6 @@ <_TestRunnerLinux>./eng/build.sh <_TestCommand>-restore -build -test -projects "$(_RelativeTestProjectPath)" /bl:"$(_RelativeTestBinLog)" -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false - - <_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true - <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"')) diff --git a/eng/Testing.props b/eng/Testing.props index e36c38ffcd1..f43be60cb0a 100644 --- a/eng/Testing.props +++ b/eng/Testing.props @@ -9,19 +9,23 @@ true true true + + + + <_QuarantinedTestRunAdditionalArgs>-trait "quarantined=true" + <_NonQuarantinedTestRunAdditionalArgs>-notrait "quarantined=true" + - <_QuarantinedTestRunAdditionalArgs>--filter-trait "quarantined=true" - <_NonQuarantinedTestRunAdditionalArgs>--filter-not-trait "quarantined=true" + + <_QuarantinedTestRunAdditionalArgs>--filter "quarantined=true" + <_NonQuarantinedTestRunAdditionalArgs>--filter "quarantined!=true" 10m - <_BlameArgs>--hangdump --hangdump-timeout $(BlameHangTimeout) --crashdump - - $(TestRunnerAdditionalArguments) --filter-not-trait "category=failing" - $(TestRunnerAdditionalArguments) --ignore-exit-code 8 + <_BlameArgs>--blame --blame-hang-timeout $(BlameHangTimeout) --blame-crash - $(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(_BlameArgs) - $(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(_BlameArgs) + $(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs) + $(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs) diff --git a/eng/Versions.props b/eng/Versions.props index 3ff66d11ee9..913611b4984 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -12,10 +12,10 @@ 8.0.13 8.0.406 + true 2.0.0 1.20.0 3.0.2 - 1.6.3 false release diff --git a/eng/Xunit3/Xunit3.targets b/eng/Xunit3/Xunit3.targets index 71b6dccb5c6..2cda46b44b6 100644 --- a/eng/Xunit3/Xunit3.targets +++ b/eng/Xunit3/Xunit3.targets @@ -4,23 +4,15 @@ + - - - - - - - - - - + - - + + \ No newline at end of file diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings new file mode 100644 index 00000000000..88a92c090ab --- /dev/null +++ b/eng/testing/.runsettings @@ -0,0 +1,35 @@ + + + + + 1200000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + + + + + + + + + + + + + diff --git a/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj b/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj index 8bd5357c0c0..9bee666eb8e 100644 --- a/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj +++ b/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj @@ -9,11 +9,6 @@ true true false - - - - - $(TestingPlatformCommandLineArguments) --ignore-exit-code 8 diff --git a/tests/Aspire.EndToEnd.Tests/.runsettings b/tests/Aspire.EndToEnd.Tests/.runsettings new file mode 100644 index 00000000000..348b55730ea --- /dev/null +++ b/tests/Aspire.EndToEnd.Tests/.runsettings @@ -0,0 +1,26 @@ + + + + true + + + + 900000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + diff --git a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj index a074d720d09..5694b015626 100644 --- a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj +++ b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj @@ -20,6 +20,7 @@ BUILD_FOR_TESTS_RUNNING_OUTSIDE_OF_REPO;$(DefineConstants) xunit.runner.json + $(MSBuildThisFileDirectory).runsettings $(TestArchiveTestsDirForEndToEndTests) diff --git a/tests/Aspire.Hosting.Containers.Tests/.runsettings b/tests/Aspire.Hosting.Containers.Tests/.runsettings new file mode 100644 index 00000000000..591a45e7ab7 --- /dev/null +++ b/tests/Aspire.Hosting.Containers.Tests/.runsettings @@ -0,0 +1,23 @@ + + + + + 900000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + diff --git a/tests/Aspire.Hosting.Keycloak.Tests/.runsettings b/tests/Aspire.Hosting.Keycloak.Tests/.runsettings new file mode 100644 index 00000000000..591a45e7ab7 --- /dev/null +++ b/tests/Aspire.Hosting.Keycloak.Tests/.runsettings @@ -0,0 +1,23 @@ + + + + + 900000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + diff --git a/tests/Aspire.Playground.Tests/.runsettings b/tests/Aspire.Playground.Tests/.runsettings new file mode 100644 index 00000000000..1eb28e08584 --- /dev/null +++ b/tests/Aspire.Playground.Tests/.runsettings @@ -0,0 +1,26 @@ + + + + true + + + + 1200000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + diff --git a/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj b/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj index d07a3033a95..2b048c12806 100644 --- a/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj +++ b/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj @@ -17,6 +17,7 @@ $(MSBuildThisFileDirectory)..\..\playground\ $(MSBuildThisFileDirectory)..\Shared\ + $(MSBuildThisFileDirectory).runsettings true diff --git a/tests/Aspire.Templates.Tests/.runsettings b/tests/Aspire.Templates.Tests/.runsettings new file mode 100644 index 00000000000..1eb28e08584 --- /dev/null +++ b/tests/Aspire.Templates.Tests/.runsettings @@ -0,0 +1,26 @@ + + + + true + + + + 1200000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj index 9fdc76892f8..3c4a6f83018 100644 --- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj +++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj @@ -7,6 +7,7 @@ true xunit.runner.json + $(MSBuildThisFileDirectory).runsettings $(TestArchiveTestsDirForTemplateTests) true diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index beb44b89dcd..8c0aabbcfa9 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -5,6 +5,8 @@ + + $(RepositoryEngineeringDir)testing\.runsettings $([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'tests')) $([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'templates-tests')) $([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'e2e-tests')) @@ -14,8 +16,14 @@ $(ArtifactsTestResultsDir) + + Exe + + + + false diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index bf1b4461737..a18ad0736dc 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -4,12 +4,17 @@ + + $(RunSettingsFilePath) + + true $(RepoRoot)tests\helix\xunit.runner.json $(RepositoryEngineeringDir)testing\xunit.runner.json + @@ -36,7 +41,7 @@ - + @@ -61,8 +66,7 @@ - - + diff --git a/tests/Shared/RepoTesting/Aspire.RepoTesting.props b/tests/Shared/RepoTesting/Aspire.RepoTesting.props index 4bf72ed654f..f4c225f2e54 100644 --- a/tests/Shared/RepoTesting/Aspire.RepoTesting.props +++ b/tests/Shared/RepoTesting/Aspire.RepoTesting.props @@ -14,12 +14,6 @@ Xunit3 - - true - true - false - Exe - diff --git a/tests/Shared/RepoTesting/Aspire.RepoTesting.targets b/tests/Shared/RepoTesting/Aspire.RepoTesting.targets index 4f3c9834786..ac54f9b7fd3 100644 --- a/tests/Shared/RepoTesting/Aspire.RepoTesting.targets +++ b/tests/Shared/RepoTesting/Aspire.RepoTesting.targets @@ -41,9 +41,6 @@ true $(IntermediateOutputPath)Directory.Packages.Versions.props - - $(TestingPlatformCommandLineArguments) --filter-not-trait "category=failing" - $(TestingPlatformCommandLineArguments) --report-trx-filename "$(TrxFileNamePrefix)_$(TargetFramework)_$([System.DateTime]::UtcNow.ToString('yyyyMMddhhmmss')).trx" @@ -70,11 +67,8 @@ - - - - - + + diff --git a/tests/Shared/RepoTesting/Directory.Packages.Helix.props b/tests/Shared/RepoTesting/Directory.Packages.Helix.props index 3a74f01ca7e..a36826bcfe9 100644 --- a/tests/Shared/RepoTesting/Directory.Packages.Helix.props +++ b/tests/Shared/RepoTesting/Directory.Packages.Helix.props @@ -86,5 +86,7 @@ + + diff --git a/tests/Shared/TemplatesTesting/BuildEnvironment.cs b/tests/Shared/TemplatesTesting/BuildEnvironment.cs index 66b33a2683c..e68638df5a3 100644 --- a/tests/Shared/TemplatesTesting/BuildEnvironment.cs +++ b/tests/Shared/TemplatesTesting/BuildEnvironment.cs @@ -173,6 +173,8 @@ public BuildEnvironment(bool useSystemDotNet = false, string sdkDirName = "dotne // Avoid using the msbuild terminal logger, so the output can be read // in the tests EnvVars["_MSBUILDTLENABLED"] = "0"; + // .. and disable new output style for vstest + EnvVars["VsTestUseMSBuildOutput"] = "false"; EnvVars["SkipAspireWorkloadManifest"] = "true"; DotNet = Path.Combine(sdkForTemplatePath!, "dotnet"); diff --git a/tests/helix/send-to-helix-basictests.targets b/tests/helix/send-to-helix-basictests.targets index 29b05b8a3e1..fcfb8bdc47d 100644 --- a/tests/helix/send-to-helix-basictests.targets +++ b/tests/helix/send-to-helix-basictests.targets @@ -21,7 +21,8 @@ - <_TestRunCommandArguments Include="--filter-not-trait "quarantined=true"" /> + <_TestRunCommandArguments Condition="'$(OS)' != 'Windows_NT'" Include="--filter "quarantined!=true" -- RunConfiguration.TestSessionTimeout=$TEST_TIMEOUT" /> + <_TestRunCommandArguments Condition="'$(OS)' == 'Windows_NT'" Include="--filter "quarantined!=true" -- RunConfiguration.TestSessionTimeout=%TEST_TIMEOUT%" /> @@ -36,9 +37,16 @@ <_DefaultWorkItems TestName="$([System.Text.RegularExpressions.Regex]::Replace(%(FileName), $(TargetFrameworkSuffixRegex), '$1'))" /> <_DefaultWorkItems TestNameSuffix="$([System.Text.RegularExpressions.Regex]::Replace(%(FileName), $(TargetFrameworkSuffixRegex), '$2'))" /> + + <_DefaultWorkItems TimeoutMs="900000" /> + + <_DefaultWorkItems Condition="$([System.String]::new('%(FileName)').StartsWith('Aspire.Hosting.Elasticsearch.Tests'))" TimeoutMs="3600000" /> + <_DefaultWorkItems Condition="$([System.String]::new('%(FileName)').StartsWith('Aspire.Hosting.Oracle.Tests'))" TimeoutMs="1200000" /> + <_DefaultWorkItems Condition="$([System.String]::new('%(FileName)').StartsWith('Aspire.Pomelo.EntityFrameworkCore.MySql.Tests'))" TimeoutMs="1200000" /> + %(Identity) - $(_EnvVarSetKeyword) "TEST_NAME=%(TestName)" $(_ShellCommandSeparator) $(_EnvVarSetKeyword) $(_ShellCommandSeparator) $(_EnvVarSetKeyword) "TEST_NAME_SUFFIX=%(TestNameSuffix)" $(_ShellCommandSeparator) $(_EnvVarSetKeyword) "CODE_COV_FILE_SUFFIX=%(TestNameSuffix)" + $(_EnvVarSetKeyword) "TEST_NAME=%(TestName)" $(_ShellCommandSeparator) $(_EnvVarSetKeyword) TEST_TIMEOUT=%(TimeoutMs) $(_ShellCommandSeparator) $(_EnvVarSetKeyword) "TEST_NAME_SUFFIX=%(TestNameSuffix)" $(_ShellCommandSeparator) $(_EnvVarSetKeyword) "CODE_COV_FILE_SUFFIX=%(TestNameSuffix)" $(_TestRunCommand) $(_workItemTimeout) diff --git a/tests/helix/send-to-helix-buildonhelixtests.targets b/tests/helix/send-to-helix-buildonhelixtests.targets index 19e0c189d19..096bf419beb 100644 --- a/tests/helix/send-to-helix-buildonhelixtests.targets +++ b/tests/helix/send-to-helix-buildonhelixtests.targets @@ -31,14 +31,14 @@ <_TestRunCommandArguments Remove="@(_TestRunCommandArguments)" /> <_TestBlameArguments Remove="@(_TestBlameArguments)" /> - <_TestBlameArguments Include="--crashdump" /> - <_TestBlameArguments Include="--crashdump-type Full" /> + <_TestBlameArguments Include="--blame-crash" /> + <_TestBlameArguments Include="--blame-crash-dump-type full" /> - <_TestRunCommandArguments Include="dotnet test -- --results-directory $(_HelixLogsPath) --report-trx --report-trx-filename TestResults.trx" /> + <_TestRunCommandArguments Include="dotnet test -s .runsettings --results-directory $(_HelixLogsPath) -v:n" /> <_TestRunCommandArguments Include="@(_TestBlameArguments, ' ')" /> - <_TestRunCommandArguments Include="--filter-not-trait "quarantined=true"" /> + <_TestRunCommandArguments Include="--filter "quarantined!=true"" /> diff --git a/tests/helix/send-to-helix-endtoendtests.targets b/tests/helix/send-to-helix-endtoendtests.targets index 1e9b2d70f33..a64f35da2a4 100644 --- a/tests/helix/send-to-helix-endtoendtests.targets +++ b/tests/helix/send-to-helix-endtoendtests.targets @@ -21,7 +21,7 @@ Text="Could not find EndToEnd tests at %24(_E2ETestsArchivePath)=$(_E2ETestsArchivePath)" /> - <_TestRunCommandArguments Include="--filter-not-trait "quarantined=true" --filter-trait "scenario=$(_TestScenarioEnvVar)"" /> + <_TestRunCommandArguments Include="--filter "(quarantined!=true)&(scenario=$(_TestScenarioEnvVar))"" /> diff --git a/tests/helix/send-to-helix-inner.proj b/tests/helix/send-to-helix-inner.proj index 3ad04b6656e..61095f4a741 100644 --- a/tests/helix/send-to-helix-inner.proj +++ b/tests/helix/send-to-helix-inner.proj @@ -58,19 +58,13 @@ - <_TestBlameArguments Include="--hangdump" /> - <_TestBlameArguments Include="--hangdump-type Full" /> - <_TestBlameArguments Include="--hangdump-timeout 10m" /> - <_TestBlameArguments Include="--crashdump" /> - <_TestBlameArguments Include="--crashdump-type Full" /> - - - <_TestRunCommandArguments Include="$(_SetExecutableBitCommand)dotnet exec $(_TestNameEnvVar).dll --results-directory:$(_HelixLogsPath) --report-trx --report-trx-filename TestResults.trx --filter-not-trait "category=failing" --ignore-exit-code 8" /> + <_TestBlameArguments Include="--blame-hang" /> + <_TestBlameArguments Include="--blame-hang-dump-type full" /> + <_TestBlameArguments Include="--blame-hang-timeout 10m" /> + <_TestBlameArguments Include="--blame-crash" /> + <_TestBlameArguments Include="--blame-crash-dump-type full" /> + + <_TestRunCommandArguments Include="$(_SetExecutableBitCommand)dotnet test -s .runsettings $(_TestNameEnvVar).dll --ResultsDirectory:$(_HelixLogsPath) -v:n" /> <_TestRunCommandArguments Include="@(_TestBlameArguments, ' ')" /> diff --git a/tests/helix/send-to-helix-templatestests.targets b/tests/helix/send-to-helix-templatestests.targets index f731f3b41c4..cdd6cee0918 100644 --- a/tests/helix/send-to-helix-templatestests.targets +++ b/tests/helix/send-to-helix-templatestests.targets @@ -59,8 +59,8 @@ To fix that, append '.' to the name so only the test class with exactly that name is run. --> - %(PreCommands) $(_ShellCommandSeparator) set "TEST_ARGS=--filter-not-trait quarantined=true --filter-not-trait category=failing --filter-class %(Identity)" - %(PreCommands) $(_ShellCommandSeparator) export "TEST_ARGS=--filter-not-trait quarantined=true --filter-not-trait category=failing --filter-class %(Identity)" + %(PreCommands) $(_ShellCommandSeparator) set "TEST_ARGS=--filter quarantined^^!=true^&category^^!=failing^&FullyQualifiedName~%(Identity)." + %(PreCommands) $(_ShellCommandSeparator) export "TEST_ARGS=--filter quarantined!=true&category!=failing&FullyQualifiedName~%(Identity)." $(_TestRunCommand) $(_workItemTimeout)