Added hanging test mem dump feature

.vsts-dotnet-ci.yml

@@ -64,6 +64,7 @@ jobs:
         _includeBenchmarkData: true
         _targetFramework: netcoreapp3.0
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0"
     pool:
       name: Hosted VS2017
 
@@ -72,6 +73,7 @@ jobs:
     name: Windows_x64_NetCoreApp21
     buildScript: build.cmd
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
     pool:
       name: Hosted VS2017
 
@@ -91,6 +93,7 @@ jobs:
         _includeBenchmarkData: false
         _targetFramework: win-x64
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0"
     pool:
       name: Hosted VS2017
 
@@ -100,5 +103,6 @@ jobs:
     architecture: x86
     buildScript: build.cmd
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
     pool:
       name: Hosted VS2017

build/ci/job-template.yml

@@ -9,6 +9,7 @@ parameters:
   innerLoop: false
   runSpecific: false
   container: ''
+  useVSTestTask: false
 
 jobs:
   - job: ${{ parameters.name }}
@@ -26,6 +27,7 @@ jobs:
       nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework)
       packageUpdaterProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj
       versionFilePath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/latest_versions.txt
+      PROCDUMP_PATH: '$(Build.SourcesDirectory)/Tools/procdump/'
     strategy:
       matrix:
         ${{ if eq(parameters.customMatrixes, '') }}:
@@ -79,12 +81,33 @@ jobs:
       - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
         displayName: Download Benchmark Data
       - ${{ if eq(parameters.innerLoop, 'false') }}:
-        - ${{ if eq(parameters.runSpecific, 'false') }}:
+        - ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
           - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
             displayName: Run All Tests.
-        - ${{ if eq(parameters.runSpecific, 'true') }}:
+        - ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
           - script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }}
             displayName: Run Specific Tests.
+        - ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
+          - task: VSTest@2
+            displayName: Run Tests with VSTest
+            inputs:
+              testSelector: testAssemblies
+              testAssemblyVer2: |
+                **\*test.dll
+                **\*tests.dll
+                !**\obj\** 
+              runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings
+              searchFolder: '$(System.DefaultWorkingDirectory)'
+              vstestLocationMethod: 'version' 
+              vsTestVersion: 'latest'
+              runInParallel: False
+              runTestsInIsolation: True
+              codeCoverageEnabled: ${{ parameters.codeCoverage }}
+              otherConsoleOptions: ${{ parameters.vsTestConfiguration }}
+              dontDistribute: True
+              diagnosticsEnabled: False
+              collectDumpOn: onAbortOnly
+              publishRunAttachments: true
       - ${{ if eq(parameters.innerLoop, 'true') }}:
         - script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }}
           displayName: Run CI Tests.
@@ -123,7 +146,7 @@ jobs:
       inputs:
         sourceFolder: $(Build.SourcesDirectory)
         contents: |
-          CrashDumps/*.dmp
+          *.dmp
           bin/**/*.pdb
         targetFolder: $(Build.ArtifactStagingDirectory)
     - task: PublishBuildArtifacts@1

docs/project-docs/developer-guide.md

@@ -78,3 +78,13 @@ Steps to update `core_manifest.json` and `core_ep-list.tsv`:
 3. Verify the changes to `core_manifest.json` and `core_ep-list.tsv` are correct.
 4. Re-enable the skip attribute on the `RegenerateEntryPointCatalog` test.
 5. Commit the updated `core_manifest.json` and `core_ep-list.tsv` files to your branch.
+
+### Running unit tests through VSTest Task & Collecting memory dumps
+
+During development, there may also arise a need to debug hanging tests. In this scenario, it can be beneficial to collect the memory dump while a given test is hanging.
+
+In this case, the given needs needs to be implemented according to the Microsoft test framework. Please check out the [Microsoft test framework walkthrough](https://docs.microsoft.com/en-us/visualstudio/test/walkthrough-creating-and-running-unit-tests-for-managed-code?view=vs-2019) and the VSTest [sample](https://github.com/dotnet/samples/tree/master/core/getting-started/unit-testing-using-mstest) demonstrating the "TestClass", "TestMethod", "DataTestMethod", and "DataRow" attributes.
+
+Once the unit test(s) are implemented according to VSTest and ready to be debugged, the `useVSTestTask` parameter in `build\ci\job-template.yml` needs to be set to `True`. Once these steps are completed and pushed in your pull request, the unit test(s) will run and produce a full memory dump. At the end of a run, the memory dump `.dmp` file will be availible for downloading and inspection in the published artifacts of the build, in the folder `TestResults`.
+
+Note: this is only supported on Windows builds, as [ProcDump](https://docs.microsoft.com/en-us/sysinternals/downloads/procdump) is officially only available on Windows.

init-tools.cmd

@@ -67,6 +67,13 @@ if NOT [%AGENT_ID%] == [] (
   reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /f /v DumpFolder /t REG_SZ /d "%~dp0CrashDumps"
 )
 
+:: install procdump.exe to take process dump when test crashes, hangs or fails
+echo Installing procdump.exe
+powershell -Command "Invoke-WebRequest https://download.sysinternals.com/files/Procdump.zip -UseBasicParsing -outfile procdump.zip | Out-Null"
+powershell -Command "Expand-Archive -Force procdump.zip Tools"
+del /f procdump.zip
+echo Finish install procdump.exe
+
 :: install the extra runtime first, so the SDK install will overwrite the root dotnet executable
 echo Installing dotnet runtime %DOTNET_EXTRA_RUNTIME_VERSION%...
 set DOTNET_EXTRA_RUNTIME_ZIP_NAME=dotnet-runtime-%DOTNET_EXTRA_RUNTIME_VERSION%-win-%ARCH%.zip

test/Directory.Build.props

@@ -28,6 +28,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.8.0" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
     <PackageReference Include="xunit" Version="2.4.0" />
     <PackageReference Include="Xunit.Combinatorial" Version="$(XunitCombinatorialVersion)" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />

tools-local/vstest.runsettings

@@ -0,0 +1,21 @@
+<RunSettings>
+    <RunConfiguration>
+        <BatchSize>1000</BatchSize>
+        <ResultsDirectory>D:\a\1\a</ResultsDirectory>
+    </RunConfiguration>
+    <LoggerRunSettings>
+        <Loggers>
+            <Logger friendlyName="blame" enabled="True" />
+        </Loggers>
+    </LoggerRunSettings>
+    <DataCollectionRunSettings>
+        <DataCollectors>
+            <DataCollector friendlyName="blame" enabled="True">
+                <Configuration>
+                    <CollectDump CollectAlways="false" DumpType="full" />
+                    <CollectDumpOnTestSessionHang TestTimeout="300000" DumpType="full" />
+                </Configuration>
+            </DataCollector>
+        </DataCollectors>
+    </DataCollectionRunSettings>
+</RunSettings>