submit all helix items in single job

azure-pipelines.yml

@@ -209,37 +209,39 @@ stages:
   jobs:
   - template: eng/pipelines/test-windows-job.yml
     parameters:
-      testRunName: 'Test Windows Desktop Debug 64'
-      jobName: Test_Windows_Desktop_Debug_64
+      jobName: Submit_Windows_Debug_Helix_Tests
       testArtifactName: Transport_Artifacts_Windows_Debug
       configuration: Debug
-      testArguments: -testDesktop -testArch x64
-      helixQueueName: $(HelixWindowsVsQueueName)
       helixApiAccessToken: $(HelixApiAccessToken)
       poolParameters: ${{ parameters.windowsPool }}
-
-  - template: eng/pipelines/test-windows-job.yml
-    parameters:
-      testRunName: 'Test Windows CoreClr Debug'
-      jobName: Test_Windows_CoreClr_Debug
-      testArtifactName: Transport_Artifacts_Windows_Debug
-      configuration: Debug
-      testArguments: -testCoreClr
-      helixQueueName: $(HelixWindowsQueueName)
-      helixApiAccessToken: $(HelixApiAccessToken)
-      poolParameters: ${{ parameters.windowsPool }}
-
-  - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-    - template: eng/pipelines/test-windows-job.yml
-      parameters:
-        testRunName: 'Test Windows CoreClr Debug Spanish'
-        jobName: Test_Windows_CoreClr_Debug_Spanish
-        testArtifactName: Transport_Artifacts_Windows_Debug
-        configuration: Debug
+      testRuns:
+      - name: desktop-debug-x64
+        displayName: Test Windows Desktop Debug 64
+        testArguments: -testDesktop -testArch x64
+        helixQueueName: $(HelixWindowsVsQueueName)
+      - name: coreclr-debug
+        displayName: Test Windows CoreClr Debug
         testArguments: -testCoreClr
-        helixQueueName: $(HelixWindowsSpanishQueueName)
-        helixApiAccessToken: $(HelixApiAccessToken)
-        poolParameters: ${{ parameters.windowsPool }}
+        helixQueueName: $(HelixWindowsQueueName)
+      - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        - name: coreclr-debug-spanish
+          displayName: Test Windows CoreClr Debug Spanish
+          testArguments: -testCoreClr
+          helixQueueName: $(HelixWindowsSpanishQueueName)
+      - name: coreclr-ioperation-debug
+        displayName: Test Windows CoreCLR IOperation Debug
+        testArguments: -testCoreClr -testIOperation -testCompilerOnly
+        helixQueueName: $(HelixWindowsQueueName)
+      - name: coreclr-runtimeasync-debug
+        displayName: Test Windows CoreCLR RuntimeAsync Debug
+        testArguments: -testCoreClr -testRuntimeAsync -testCompilerOnly
+        helixQueueName: $(HelixWindowsQueueName)
+      # This submission runs almost all the compiler tests supported on CoreCLR, but
+      # with additional validation for used assemblies and GetEmitDiagnostics.
+      - name: coreclr-usedassemblies-debug
+        displayName: Test Windows CoreCLR UsedAssemblies Debug
+        testArguments: -testCoreClr -testUsedAssemblies -testCompilerOnly
+        helixQueueName: $(HelixWindowsQueueName)
 
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     - template: eng/pipelines/test-windows-job-single-machine.yml
@@ -251,41 +253,6 @@ stages:
         poolParameters: ${{ parameters.vs2026PreviewPool }}
         testArguments: -testCoreClr
 
-  - template: eng/pipelines/test-windows-job.yml
-    parameters:
-      testRunName: 'Test Windows CoreCLR IOperation Debug'
-      jobName: Test_Windows_CoreClr_IOperation_Debug
-      testArtifactName: Transport_Artifacts_Windows_Debug
-      configuration: Debug
-      testArguments: -testCoreClr -testIOperation -testCompilerOnly
-      helixQueueName: $(HelixWindowsQueueName)
-      helixApiAccessToken: $(HelixApiAccessToken)
-      poolParameters: ${{ parameters.windowsPool }}
-
-  - template: eng/pipelines/test-windows-job.yml
-    parameters:
-      testRunName: 'Test Windows CoreCLR RuntimeAsync Debug'
-      jobName: Test_Windows_CoreClr_RuntimeAsync_Debug
-      testArtifactName: Transport_Artifacts_Windows_Debug
-      configuration: Debug
-      testArguments: -testCoreClr -testRuntimeAsync -testCompilerOnly
-      helixQueueName: $(HelixWindowsQueueName)
-      helixApiAccessToken: $(HelixApiAccessToken)
-      poolParameters: ${{ parameters.windowsPool }}
-
-  # This leg runs almost all the compiler tests supported on CoreCLR, but
-  #  with additional validation for used assemblies and GetEmitDiagnostics
-  - template: eng/pipelines/test-windows-job.yml
-    parameters:
-      testRunName: 'Test Windows CoreCLR UsedAssemblies Debug'
-      jobName: Test_Windows_CoreClr_UsedAssemblies_Debug
-      testArtifactName: Transport_Artifacts_Windows_Debug
-      configuration: Debug
-      testArguments: -testCoreClr -testUsedAssemblies -testCompilerOnly
-      helixQueueName: $(HelixWindowsQueueName)
-      helixApiAccessToken: $(HelixApiAccessToken)
-      poolParameters: ${{ parameters.windowsPool }}
-
   - template: /eng/common/core-templates/job/helix-job-monitor.yml
     parameters:
       helixAccessToken: $(HelixApiAccessToken)
@@ -296,51 +263,32 @@ stages:
   - ${{ if ne(variables['System.TeamProject'], 'public') }}:
     - group: DotNet-HelixApi-Access
   jobs:
-  - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-    - template: eng/pipelines/test-windows-job.yml
-      parameters:
-        testRunName: 'Test Windows Desktop Release 32'
-        jobName: Test_Windows_Desktop_Release_32
-        testArtifactName: Transport_Artifacts_Windows_Release
-        configuration: Release
-        testArguments: -testDesktop -testArch x86
-        helixQueueName: $(HelixWindowsVsQueueName)
-        helixApiAccessToken: $(HelixApiAccessToken)
-        poolParameters: ${{ parameters.windowsPool }}
-
   - template: eng/pipelines/test-windows-job.yml
     parameters:
-      testRunName: 'Test Windows Desktop Release 64'
-      jobName: Test_Windows_Desktop_Release_64
+      jobName: Submit_Windows_Release_Helix_Tests
       testArtifactName: Transport_Artifacts_Windows_Release
       configuration: Release
-      testArguments: -testDesktop -testArch x64
-      helixQueueName: $(HelixWindowsVsQueueName)
       helixApiAccessToken: $(HelixApiAccessToken)
       poolParameters: ${{ parameters.windowsPool }}
-
-  - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-    - template: eng/pipelines/test-windows-job.yml
-      parameters:
-        testRunName: 'Test Windows Desktop Spanish Release 64'
-        jobName: Test_Windows_Desktop_Spanish_Release_64
-        testArtifactName: Transport_Artifacts_Windows_Release
-        configuration: Release
+      testRuns:
+      - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        - name: desktop-release-x86
+          displayName: Test Windows Desktop Release 32
+          testArguments: -testDesktop -testArch x86
+          helixQueueName: $(HelixWindowsVsQueueName)
+      - name: desktop-release-x64
+        displayName: Test Windows Desktop Release 64
         testArguments: -testDesktop -testArch x64
-        helixQueueName: $(HelixWindowsSpanishQueueName)
-        helixApiAccessToken: $(HelixApiAccessToken)
-        poolParameters: ${{ parameters.windowsPool }}
-
-  - template: eng/pipelines/test-windows-job.yml
-    parameters:
-      testRunName: 'Test Windows CoreClr Release'
-      jobName: Test_Windows_CoreClr_Release
-      testArtifactName: Transport_Artifacts_Windows_Release
-      configuration: Release
-      testArguments: -testCoreClr
-      helixQueueName: $(HelixWindowsQueueName)
-      helixApiAccessToken: $(HelixApiAccessToken)
-      poolParameters: ${{ parameters.windowsPool }}
+        helixQueueName: $(HelixWindowsVsQueueName)
+      - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        - name: desktop-release-spanish-x64
+          displayName: Test Windows Desktop Spanish Release 64
+          testArguments: -testDesktop -testArch x64
+          helixQueueName: $(HelixWindowsSpanishQueueName)
+      - name: coreclr-release
+        displayName: Test Windows CoreClr Release
+        testArguments: -testCoreClr
+        helixQueueName: $(HelixWindowsQueueName)
 
   - template: /eng/common/core-templates/job/helix-job-monitor.yml
     parameters:

eng/pipelines/test-windows-job.yml

@@ -1,10 +1,5 @@
 # Test on Windows Desktop using Helix
 parameters:
-- name: helixQueueName
-  type: string
-- name: testRunName
-  type: string
-  default: ''
 - name: jobName
   type: string
   default: ''
@@ -14,9 +9,9 @@ parameters:
 - name: configuration
   type: string
   default: 'Debug'
-- name: testArguments
-  type: string
-  default: ''
+- name: testRuns
+  type: object
+  default: []
 - name: helixApiAccessToken
   type: string
   default: ''
@@ -43,19 +38,67 @@ jobs:
       env:
         HELIX_CORRELATION_PAYLOAD: '$(Build.SourcesDirectory)\.duplicate'
 
+    # Attempt every submission even if an earlier one fails, then fail the aggregate job below.
+    - ${{ each testRun in parameters.testRuns }}:
+      - task: PowerShell@2
+        displayName: Submit ${{ testRun.displayName }}
+        continueOnError: true
+        inputs:
+          filePath: eng/build.ps1
+          ${{ if ne(variables['System.TeamProject'], 'public') }}:
+            arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ testRun.helixQueueName }} -helixApiAccessToken ${{ parameters.helixApiAccessToken }} ${{ testRun.testArguments }} -collectDumps
+          ${{ else }}:
+            arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ testRun.helixQueueName }} ${{ testRun.testArguments }} -collectDumps
+        env:
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+      # RunTests uses the same output paths for every Helix submission. Move each submission's
+      # outputs aside so the next sequential submission cannot overwrite or mix its diagnostics.
+      - task: PowerShell@2
+        displayName: Preserve ${{ testRun.displayName }} submission logs
+        condition: succeededOrFailed()
+        continueOnError: true
+        inputs:
+          targetType: inline
+          script: |
+            $artifactsDirectory = '$(Build.SourcesDirectory)\artifacts'
+            $logDirectory = Join-Path $artifactsDirectory 'log\${{ parameters.configuration }}'
+            $submissionLogDirectory = Join-Path $logDirectory '${{ testRun.name }}'
+            $payloadsDirectory = Join-Path $artifactsDirectory 'payloads'
+            $submissionPayloadsDirectory = Join-Path $artifactsDirectory 'payloads-${{ testRun.name }}-$(System.JobAttempt)'
+            $helixProjectPath = Join-Path $artifactsDirectory 'helix.proj'
+            $helixBinaryLogPath = Join-Path $logDirectory 'helix.binlog'
+            $submissionBinaryLogPath = Join-Path $logDirectory 'helix-${{ testRun.name }}.binlog'
+            $loggedHelixProjectPath = Join-Path $logDirectory 'helix.proj'
+
+            if (Test-Path $submissionLogDirectory) {
+              Remove-Item $submissionLogDirectory -Recurse -Force
+            }
+            New-Item -ItemType Directory -Path $submissionLogDirectory -Force | Out-Null
+            if (Test-Path $helixBinaryLogPath) {
+              Move-Item $helixBinaryLogPath $submissionBinaryLogPath -Force
+            }
+            if (Test-Path $loggedHelixProjectPath) {
+              Move-Item $loggedHelixProjectPath $submissionLogDirectory -Force
+            }
+            Get-ChildItem -Path $logDirectory -Filter 'workitem_*' -Directory |
+              Move-Item -Destination $submissionLogDirectory -Force
+
+            if (Test-Path $payloadsDirectory) {
+              Move-Item $payloadsDirectory $submissionPayloadsDirectory
+            }
+            if (Test-Path $helixProjectPath) {
+              Remove-Item $helixProjectPath -Force
+            }
+
     - task: PowerShell@2
-      displayName: Run Unit Tests
+      displayName: Fail job if any submission failed
+      condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
       inputs:
-        filePath: eng/build.ps1
-        ${{ if ne(variables['System.TeamProject'], 'public') }}:
-          arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ parameters.helixQueueName }} -helixApiAccessToken ${{ parameters.helixApiAccessToken }} ${{ parameters.testArguments }} -collectDumps        
-        ${{ else }}:
-          arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ parameters.helixQueueName }} ${{ parameters.testArguments }} -collectDumps
-      env:
-        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+        targetType: inline
+        script: throw 'One or more Helix submissions failed.'
 
     - template: publish-logs.yml
       parameters:
         configuration: ${{ parameters.configuration }}
         jobName: ${{ parameters.jobName }}
-        testRunName: ${{ parameters.testRunName }}

src/Tools/RunTests/AzdoClient.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -74,12 +74,12 @@ public static AzdoClient Create(string projectUri, string accessToken)
     }
 
     /// <summary>
-    /// Gets test runs for a build within the build's time range, matching a stage/phase name.
+    /// Gets a test run for a build within the build's time range, matching its exact name.
     /// </summary>
-    public async Task<AzdoTestRun?> GetRunForStageAsync(
+    public async Task<AzdoTestRun?> GetTestRunAsync(
         string project,
         AzdoBuild build,
-        string phaseName,
+        string testRunName,
         CancellationToken cancellationToken)
     {
         var minTime = build.QueueTime!.Value.ToString("o");
@@ -94,7 +94,7 @@ public static AzdoClient Create(string projectUri, string accessToken)
 
         // If the last successful build had multiple attempts then there are potentially multiple runs with
         // the same name. Take the last one as it will be the successful one.
-        return runs.LastOrDefault(r => r.Name.Contains(phaseName));
+        return runs.LastOrDefault(r => string.Equals(r.Name, testRunName, StringComparison.OrdinalIgnoreCase));
     }
 
     /// <summary>