diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 3ae84667ed..9bab109119 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -64,6 +64,7 @@ jobs: _includeBenchmarkData: true _targetFramework: netcoreapp3.0 innerLoop: true + vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -72,6 +73,7 @@ jobs: name: Windows_x64_NetCoreApp21 buildScript: build.cmd innerLoop: true + vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -91,6 +93,7 @@ jobs: _includeBenchmarkData: false _targetFramework: win-x64 innerLoop: true + vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -100,5 +103,6 @@ jobs: architecture: x86 buildScript: build.cmd innerLoop: true + vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index e760e811ee..5f029f3e40 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -9,6 +9,7 @@ parameters: innerLoop: false runSpecific: false container: '' + useVSTestTask: false jobs: - job: ${{ parameters.name }} @@ -26,6 +27,7 @@ jobs: nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework) packageUpdaterProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj versionFilePath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/latest_versions.txt + PROCDUMP_PATH: '$(Build.SourcesDirectory)/Tools/procdump/' strategy: matrix: ${{ if eq(parameters.customMatrixes, '') }}: @@ -79,12 +81,33 @@ jobs: - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData) displayName: Download Benchmark Data - ${{ if eq(parameters.innerLoop, 'false') }}: - - ${{ if eq(parameters.runSpecific, 'false') }}: + - ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}: - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }} displayName: Run All Tests. - - ${{ if eq(parameters.runSpecific, 'true') }}: + - ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}: - script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }} displayName: Run Specific Tests. + - ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}: + - task: VSTest@2 + displayName: Run Tests with VSTest + inputs: + testSelector: testAssemblies + testAssemblyVer2: | + **\*test.dll + **\*tests.dll + !**\obj\** + runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings + searchFolder: '$(System.DefaultWorkingDirectory)' + vstestLocationMethod: 'version' + vsTestVersion: 'latest' + runInParallel: False + runTestsInIsolation: True + codeCoverageEnabled: ${{ parameters.codeCoverage }} + otherConsoleOptions: ${{ parameters.vsTestConfiguration }} + dontDistribute: True + diagnosticsEnabled: False + collectDumpOn: onAbortOnly + publishRunAttachments: true - ${{ if eq(parameters.innerLoop, 'true') }}: - script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }} displayName: Run CI Tests. @@ -123,7 +146,7 @@ jobs: inputs: sourceFolder: $(Build.SourcesDirectory) contents: | - CrashDumps/*.dmp + *.dmp bin/**/*.pdb targetFolder: $(Build.ArtifactStagingDirectory) - task: PublishBuildArtifacts@1 diff --git a/docs/project-docs/developer-guide.md b/docs/project-docs/developer-guide.md index 34dc591fc5..b3531e339e 100644 --- a/docs/project-docs/developer-guide.md +++ b/docs/project-docs/developer-guide.md @@ -78,3 +78,13 @@ Steps to update `core_manifest.json` and `core_ep-list.tsv`: 3. Verify the changes to `core_manifest.json` and `core_ep-list.tsv` are correct. 4. Re-enable the skip attribute on the `RegenerateEntryPointCatalog` test. 5. Commit the updated `core_manifest.json` and `core_ep-list.tsv` files to your branch. + +### Running unit tests through VSTest Task & Collecting memory dumps + +During development, there may also arise a need to debug hanging tests. In this scenario, it can be beneficial to collect the memory dump while a given test is hanging. + +In this case, the given needs needs to be implemented according to the Microsoft test framework. Please check out the [Microsoft test framework walkthrough](https://docs.microsoft.com/en-us/visualstudio/test/walkthrough-creating-and-running-unit-tests-for-managed-code?view=vs-2019) and the VSTest [sample](https://github.com/dotnet/samples/tree/master/core/getting-started/unit-testing-using-mstest) demonstrating the "TestClass", "TestMethod", "DataTestMethod", and "DataRow" attributes. + +Once the unit test(s) are implemented according to VSTest and ready to be debugged, the `useVSTestTask` parameter in `build\ci\job-template.yml` needs to be set to `True`. Once these steps are completed and pushed in your pull request, the unit test(s) will run and produce a full memory dump. At the end of a run, the memory dump `.dmp` file will be availible for downloading and inspection in the published artifacts of the build, in the folder `TestResults`. + +Note: this is only supported on Windows builds, as [ProcDump](https://docs.microsoft.com/en-us/sysinternals/downloads/procdump) is officially only available on Windows. \ No newline at end of file diff --git a/init-tools.cmd b/init-tools.cmd index b7d6e298c1..3968696a53 100644 --- a/init-tools.cmd +++ b/init-tools.cmd @@ -67,6 +67,13 @@ if NOT [%AGENT_ID%] == [] ( reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /f /v DumpFolder /t REG_SZ /d "%~dp0CrashDumps" ) +:: install procdump.exe to take process dump when test crashes, hangs or fails +echo Installing procdump.exe +powershell -Command "Invoke-WebRequest https://download.sysinternals.com/files/Procdump.zip -UseBasicParsing -outfile procdump.zip | Out-Null" +powershell -Command "Expand-Archive -Force procdump.zip Tools" +del /f procdump.zip +echo Finish install procdump.exe + :: install the extra runtime first, so the SDK install will overwrite the root dotnet executable echo Installing dotnet runtime %DOTNET_EXTRA_RUNTIME_VERSION%... set DOTNET_EXTRA_RUNTIME_ZIP_NAME=dotnet-runtime-%DOTNET_EXTRA_RUNTIME_VERSION%-win-%ARCH%.zip diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 289aa187aa..c05bb189cc 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -28,6 +28,8 @@ + + diff --git a/tools-local/vstest.runsettings b/tools-local/vstest.runsettings new file mode 100644 index 0000000000..852698bd34 --- /dev/null +++ b/tools-local/vstest.runsettings @@ -0,0 +1,21 @@ + + + 1000 + D:\a\1\a + + + + + + + + + + + + + + + + +