Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 24 additions & 20 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ on:
# relative the repo root
required: false
type: string
testSessionTimeout:
testSessionTimeoutMs:
required: false
type: string
default: "15m"
default: "900000"
testHangTimeout:
required: false
type: string
Expand Down Expand Up @@ -220,14 +220,18 @@ jobs:
TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
TestsRunningOutsideOfRepo: true
run: >
dotnet ${{ env.TEST_ASSEMBLY_NAME }}.dll
--report-trx --report-trx-filename "${{ inputs.testShortName }}.trx"
--hangdump --hangdump-timeout ${{ inputs.testHangTimeout }}
--crashdump
dotnet test -s .runsettings -v:n ${{ env.TEST_ASSEMBLY_NAME }}.dll
-l "console;verbosity=normal"
-l "trx;LogFilePrefix=${{ inputs.testShortName }}"
-l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
--blame
--blame-hang-timeout ${{ inputs.testHangTimeout }}
--blame-crash
--results-directory ${{ github.workspace }}/testresults
--filter-not-trait "category=failing"
--timeout ${{ inputs.testSessionTimeout }}
${{ inputs.extraTestArgs }}
--
RunConfiguration.CollectSourceInformation=true
RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }}

- name: Run tests
if: ${{ ! inputs.requiresNugets }}
Expand All @@ -236,24 +240,24 @@ jobs:
CI: false
DCP_DIAGNOSTICS_LOG_LEVEL: debug
DCP_DIAGNOSTICS_LOG_FOLDER: ${{ github.workspace }}/testresults/dcp
# During restore and build, we use -ci, which causes NUGET_PACKAGES to point to a local cache (Arcade behavior).
# In this step, we are not using Arcade, but want to make sure that MSBuild is able to evaluate correctly.
# So, we manually set NUGET_PACKAGES
NUGET_PACKAGES: ${{ github.workspace }}/.packages
run: >
${{ env.DOTNET_SCRIPT }} test ${{ env.TEST_PROJECT_PATH }}
/p:ContinuousIntegrationBuild=true
/p:TrxFileNamePrefix="${{ inputs.testShortName }}"
-bl:${{ github.workspace }}/testresults/test.binlog
-s eng/testing/.runsettings
-l "console;verbosity=normal"
-l "trx;LogFilePrefix=${{ inputs.testShortName }}"
-l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
"--blame"
--blame-hang-timeout ${{ inputs.testHangTimeout }}
--blame-crash
--results-directory testresults
--no-restore
--no-build
--
--report-trx
--hangdump --hangdump-timeout ${{ inputs.testHangTimeout }}
--crashdump
--results-directory ${{ github.workspace }}/testresults
--timeout ${{ inputs.testSessionTimeout }}
-bl:${{ github.workspace }}/testresults/test.binlog
${{ inputs.extraTestArgs }}
--
RunConfiguration.CollectSourceInformation=true
RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }}

# Save the result of the previous steps - success or failure
# in the form of a file result-success/result-failure -{name}.rst
Expand Down
14 changes: 8 additions & 6 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ jobs:
with:
testShortName: ${{ matrix.shortname }}
os: "ubuntu-latest"
extraTestArgs: "--filter-not-trait \"quarantined=true\""
extraTestArgs: "--filter \"(quarantined!=true)\""

integrations_test_win:
uses: ./.github/workflows/run-tests.yml
Expand All @@ -90,7 +90,7 @@ jobs:
with:
testShortName: ${{ matrix.shortname }}
os: "windows-latest"
extraTestArgs: "--filter-not-trait \"quarantined=true\""
extraTestArgs: "--filter \"(quarantined!=true)\""

templates_test_lin:
name: Templates Linux
Expand All @@ -103,9 +103,10 @@ jobs:
testShortName: ${{ matrix.shortname }}
os: "ubuntu-latest"
testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
testSessionTimeout: 20m
testSessionTimeoutMs: 1200000
testHangTimeout: 12m
extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
# append '.' to the name so only the test class with exactly that name is run
extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\""
requiresNugets: true
requiresTestSdk: true

Expand All @@ -120,9 +121,10 @@ jobs:
testShortName: ${{ matrix.shortname }}
os: "windows-latest"
testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
testSessionTimeout: 20m
testSessionTimeoutMs: 1200000
testHangTimeout: 12m
extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
# append '.' to the name so only the test class with exactly that name is run
extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\""
requiresNugets: true
requiresTestSdk: true

Expand Down
33 changes: 33 additions & 0 deletions eng/.runsettings
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="utf-8"?>
<RunSettings>
<RunConfiguration>
<!-- Filter out failing (wrong framework, platform, runtime or activeissue) tests -->
<TestCaseFilter>category!=failing</TestCaseFilter>
</RunConfiguration>
<LoggerRunSettings>
<Loggers>
<Logger friendlyName="trx" />
<Logger friendlyName="html" />
<Logger friendlyName="console">
<Configuration>
<Verbosity>Minimal</Verbosity>
</Configuration>
</Logger>
</Loggers>
<Logger friendlyName="blame" enabled="True" />
</LoggerRunSettings>
<DataCollectionRunSettings>
<DataCollectors>
<!-- Enables blame -->
<DataCollector friendlyName="blame" enabled="True">
<Configuration>
<!-- Enables crash dump, with dump type "Full" or "Mini". -->
<CollectDump DumpType="Full" />
<!-- Enables hang dump or testhost and its child processes
when a test hangs for more than 10 minutes. -->
<CollectDumpOnTestSessionHang TestTimeout="10min" HangDumpType="Full" />
</Configuration>
</DataCollector>
</DataCollectors>
</DataCollectionRunSettings>
</RunSettings>
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,14 @@
<PropertyGroup>
<_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment>
<_TestAssembly>%(TestToRun.Identity)</_TestAssembly>
<_TestAssembly Condition="'$(OS)'=='Windows_NT'">$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe'))</_TestAssembly>
<_TestAssembly Condition="'$(OS)'!='Windows_NT'">$(_TestAssembly.TrimEnd('.dll'))</_TestAssembly>
<_TestTimeout>%(TestToRun.TestTimeout)</_TestTimeout>
<_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments>

<_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\</_TargetDir>

<_TestRunnerCommand>$(_TestAssembly) --list-tests $(_QuarantinedTestRunAdditionalArgs)</_TestRunnerCommand>
<_TestRunnerCommand>&quot;$(DotNetTool)&quot; test $(_TestAssembly) --list-tests --logger:"console%3Bverbosity=normal" "--Framework:%(TestToRun.TargetFrameworkIdentifier),Version=%(TestToRun.TargetFrameworkVersion)"</_TestRunnerCommand>
<_TestRunnerCommand Condition="'$(VSTestRunSettingsFile)' != ''">$(_TestRunnerCommand) "--settings:$(VSTestRunSettingsFile)"</_TestRunnerCommand>
<_TestRunnerCommand Condition="'$(_TestRunnerAdditionalArguments)' != ''">$(_TestRunnerCommand) $(_TestRunnerAdditionalArguments)</_TestRunnerCommand>

<!--
Redirect std output of the runner.
Expand All @@ -91,27 +92,53 @@
WorkingDirectory="$(_TargetDir)"
IgnoreExitCode="true"
Timeout="$(_TestTimeout)"
EnvironmentVariables="DOTNET_ROOT=$(DotNetRoot);DOTNET_ROOT_X86=$(DotNetRoot)x86"
ContinueOnError="WarnAndContinue">
<Output TaskParameter="ExitCode" PropertyName="_TestErrorCode" />
</Exec>

<!--
Report test status.
-->
<Message Text="Search complete: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />

<PropertyGroup>
<_ResultsFileToDisplay>%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay>
</PropertyGroup>

<!--
Report test status.
Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project
we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961.
-->
<Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' != '0'" File="QuarantinedTestRunsheetBuilder" />

<!--
Read the test result and see if the string "No test matches the given testcase filter" is present.
This indicates that there are no quarantined tests to run.
-->
<Message Text="Search complete, no quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '8'" Importance="high" />
<Message Text="💡 Search complete, quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />
<Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition=" '$(_TestErrorCode)' != '0' and '$(_TestErrorCode)' != '8' " File="QuarantinedTestRunsheetBuilder" />
<ReadLinesFromFile File="$(_ResultsFileToDisplay)">
<Output TaskParameter="Lines" ItemName="FileLines" />
</ReadLinesFromFile>

<ItemGroup>
<Lines Include="@(FileLines)" >
<Sanitized>$([System.String]::Copy('%(FileLines.Identity)').Replace('`', ''))</Sanitized>
</Lines>

<_UnhandledExceptionLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Unhandled exception`)) " />
<_ExceptionDiscoveringTestsLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Exception discovering tests`)) " />
<_NoTestMatchesLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`No test matches the given testcase filter`)) " />
</ItemGroup>

<PropertyGroup>
<_FailedToEnumerateTests Condition=" '@(_UnhandledExceptionLines)' != '' or '@(_ExceptionDiscoveringTestsLines)' != '' ">true</_FailedToEnumerateTests>

<_HasQuarantinedTests>true</_HasQuarantinedTests>
<_HasQuarantinedTests Condition=" '$(_TestErrorCode)' == '8' ">false</_HasQuarantinedTests>
<_HasQuarantinedTests Condition=" '@(_NoTestMatchesLines)' != '' ">false</_HasQuarantinedTests>
</PropertyGroup>

<Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_FailedToEnumerateTests)' == 'true'" File="QuarantinedTestRunsheetBuilder" />
<Message Text="💡 Quarantined tests found in $(_TestAssembly) [$(_TestEnvironment)]" Condition=" '$(_HasQuarantinedTests)' == 'true' " Importance="high" />

<!--
Generate test runsheet, if there are quarantined tests.
-->
Expand All @@ -133,12 +160,6 @@
<_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux>
<_TestCommand>-restore -build -test -projects &quot;$(_RelativeTestProjectPath)&quot; /bl:&quot;$(_RelativeTestBinLog)&quot; -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false</_TestCommand>

<!--
Some quarantinted test may only be executable on Windows or Linux, however we can't possibly know that at this time.
The MTP runner will return exit code 8 if no tests are found, and we need to ignore it instead of failing the test.
-->
<_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true</_TestCommand>

<!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
<_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>

Expand Down
20 changes: 12 additions & 8 deletions eng/Testing.props
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,23 @@
<RunOnAzdoCILinux>true</RunOnAzdoCILinux>
<RunOnAzdoHelixWindows>true</RunOnAzdoHelixWindows>
<RunOnAzdoHelixLinux>true</RunOnAzdoHelixLinux>
</PropertyGroup>

<PropertyGroup Condition="'$(UseVSTestRunner)' != 'true'">
<_QuarantinedTestRunAdditionalArgs>-trait "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
<_NonQuarantinedTestRunAdditionalArgs>-notrait "quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
</PropertyGroup>

<_QuarantinedTestRunAdditionalArgs>--filter-trait "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
<_NonQuarantinedTestRunAdditionalArgs>--filter-not-trait "quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
<PropertyGroup Condition="'$(UseVSTestRunner)' == 'true'">
<_QuarantinedTestRunAdditionalArgs>--filter "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
<_NonQuarantinedTestRunAdditionalArgs>--filter "quarantined!=true"</_NonQuarantinedTestRunAdditionalArgs>
</PropertyGroup>

<PropertyGroup>
<BlameHangTimeout Condition="'$(BlameHangTimeout)' == ''">10m</BlameHangTimeout>
<_BlameArgs>--hangdump --hangdump-timeout $(BlameHangTimeout) --crashdump</_BlameArgs>

<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --filter-not-trait &quot;category=failing&quot;</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition=" '$(IgnoreZeroTestResult)' == 'true' ">$(TestRunnerAdditionalArguments) --ignore-exit-code 8</TestRunnerAdditionalArguments>
<_BlameArgs>--blame --blame-hang-timeout $(BlameHangTimeout) --blame-crash</_BlameArgs>

<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' != 'true'">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == ''">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs)</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs)</TestRunnerAdditionalArguments>
</PropertyGroup>
</Project>
2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
<DotNetRuntimePreviousVersionForTesting>8.0.13</DotNetRuntimePreviousVersionForTesting>
<!-- dotnet 8.0 versions for running tests - used for templates tests -->
<DotNetSdkPreviousVersionForTesting>8.0.406</DotNetSdkPreviousVersionForTesting>
<UseVSTestRunner>true</UseVSTestRunner>
<XunitV3Version>2.0.0</XunitV3Version>
<XUnitAnalyzersVersion>1.20.0</XUnitAnalyzersVersion>
<XunitRunnerVisualStudioVersion>3.0.2</XunitRunnerVisualStudioVersion>
<MicrosoftTestingPlatformVersion>1.6.3</MicrosoftTestingPlatformVersion>
<!-- Enable to remove prerelease label. -->
<StabilizePackageVersion Condition="'$(StabilizePackageVersion)' == ''">false</StabilizePackageVersion>
<DotNetFinalVersionKind Condition="'$(StabilizePackageVersion)' == 'true'">release</DotNetFinalVersionKind>
Expand Down
16 changes: 4 additions & 12 deletions eng/Xunit3/Xunit3.targets
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,15 @@
<PackageVersion Include="xunit.v3.core" Version="$(XunitV3Version)" />
<PackageVersion Include="xunit.analyzers" Version="$(XunitAnalyzersVersion)" />
<PackageVersion Include="xunit.v3.assert" Version="$(XunitV3Version)" />
<PackageVersion Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioVersion)" />
<PackageVersion Include="xunit.v3.runner.console" Version="$(XunitV3Version)" />
<PackageVersion Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformVersion)" />
<PackageVersion Include="Microsoft.Testing.Platform.MSBuild" Version="$(MicrosoftTestingPlatformVersion)" />
<PackageVersion Include="Microsoft.Testing.Extensions.HangDump" Version="$(MicrosoftTestingPlatformVersion)" />
<PackageVersion Include="Microsoft.Testing.Extensions.CrashDump" Version="$(MicrosoftTestingPlatformVersion)" />
<PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="$(MicrosoftTestingPlatformVersion)" />

<PackageReference Include="xunit.v3" />
<PackageReference Include="xunit.v3.core" />
<PackageReference Include="xunit.analyzers" />
<PackageReference Include="xunit.v3.assert" />
<PackageReference Include="Microsoft.Testing.Platform" />
<PackageReference Include="Microsoft.Testing.Platform.MSBuild" />
<PackageReference Include="Microsoft.Testing.Extensions.HangDump" />
<PackageReference Include="Microsoft.Testing.Extensions.CrashDump" />
<PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
<PackageReference Include="xunit.runner.visualstudio" />
</ItemGroup>

<Import Project="Microsoft.Testing.Platform.targets" />
</Project>
<Import Project="..\tools\VSTest.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
</Project>
35 changes: 35 additions & 0 deletions eng/testing/.runsettings
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="utf-8"?>
<RunSettings>
<RunConfiguration>
<!-- Timeout in ms, 20 minutes -->
<TestSessionTimeout>1200000</TestSessionTimeout>
<!-- Filter out failing (wrong framework, platform, runtime or activeissue) tests -->
<TestCaseFilter>category!=failing</TestCaseFilter>
</RunConfiguration>
<LoggerRunSettings>
<Loggers>
<Logger friendlyName="trx">
<Configuration>
<LogFileName>TestResults.trx</LogFileName>
</Configuration>
</Logger>
<Logger friendlyName="console">
<Configuration>
<Verbosity>normal</Verbosity>
</Configuration>
</Logger>
<Logger friendlyName="blame" enabled="True" />
</Loggers>
</LoggerRunSettings>
<DataCollectionRunSettings>
<DataCollectors>
<!-- Enables blame -->
<DataCollector friendlyName="blame" enabled="True">
<Configuration>
<CollectDump DumpType="Full" />
<CollectDumpOnTestSessionHang TestTimeout="7min" HangDumpType="Full" />
</Configuration>
</DataCollector>
</DataCollectors>
</DataCollectionRunSettings>
</RunSettings>
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@
<IsTestUtilityProject>true</IsTestUtilityProject>
<SkipTests>true</SkipTests>
<DeployOutsideOfRepoSupportFiles>false</DeployOutsideOfRepoSupportFiles>

<!-- https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-exit-codes -->
<!-- Exit code 8 is "zero tests ran" -->
<!-- Currently, none of the tests in this project run in CI. All are ignored -->
<TestingPlatformCommandLineArguments>$(TestingPlatformCommandLineArguments) --ignore-exit-code 8</TestingPlatformCommandLineArguments>
</PropertyGroup>

<ItemGroup>
Expand Down
26 changes: 26 additions & 0 deletions tests/Aspire.EndToEnd.Tests/.runsettings
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="utf-8"?>
<RunSettings>
<xUnit>
<ShowLiveOutput>true</ShowLiveOutput>
</xUnit>
<RunConfiguration>
<!-- Timeout in ms, 15 minutes -->
<TestSessionTimeout>900000</TestSessionTimeout>
<!-- Filter out failing (wrong framework, platform, runtime or activeissue) tests -->
<TestCaseFilter>category!=failing</TestCaseFilter>
</RunConfiguration>
<LoggerRunSettings>
<Loggers>
<Logger friendlyName="trx">
<Configuration>
<LogFileName>TestResults.trx</LogFileName>
</Configuration>
</Logger>
<Logger friendlyName="console">
<Configuration>
<Verbosity>normal</Verbosity>
</Configuration>
</Logger>
</Loggers>
</LoggerRunSettings>
</RunSettings>
Loading
Loading