diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 43658242cbbac..af388c1dcdf95 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -209,37 +209,39 @@ stages: jobs: - template: eng/pipelines/test-windows-job.yml parameters: - testRunName: 'Test Windows Desktop Debug 64' - jobName: Test_Windows_Desktop_Debug_64 + jobName: Submit_Windows_Debug_Helix_Tests testArtifactName: Transport_Artifacts_Windows_Debug configuration: Debug - testArguments: -testDesktop -testArch x64 - helixQueueName: $(HelixWindowsVsQueueName) helixApiAccessToken: $(HelixApiAccessToken) poolParameters: ${{ parameters.windowsPool }} - - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows CoreClr Debug' - jobName: Test_Windows_CoreClr_Debug - testArtifactName: Transport_Artifacts_Windows_Debug - configuration: Debug - testArguments: -testCoreClr - helixQueueName: $(HelixWindowsQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} - - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows CoreClr Debug Spanish' - jobName: Test_Windows_CoreClr_Debug_Spanish - testArtifactName: Transport_Artifacts_Windows_Debug - configuration: Debug + testRuns: + - name: desktop-debug-x64 + displayName: Test Windows Desktop Debug 64 + testArguments: -testDesktop -testArch x64 + helixQueueName: $(HelixWindowsVsQueueName) + - name: coreclr-debug + displayName: Test Windows CoreClr Debug testArguments: -testCoreClr - helixQueueName: $(HelixWindowsSpanishQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} + helixQueueName: $(HelixWindowsQueueName) + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - name: coreclr-debug-spanish + displayName: Test Windows CoreClr Debug Spanish + testArguments: -testCoreClr + helixQueueName: $(HelixWindowsSpanishQueueName) + - name: coreclr-ioperation-debug + displayName: Test Windows CoreCLR IOperation Debug + testArguments: -testCoreClr -testIOperation -testCompilerOnly + helixQueueName: $(HelixWindowsQueueName) + - name: coreclr-runtimeasync-debug + displayName: Test Windows CoreCLR RuntimeAsync Debug + testArguments: -testCoreClr -testRuntimeAsync -testCompilerOnly + helixQueueName: $(HelixWindowsQueueName) + # This submission runs almost all the compiler tests supported on CoreCLR, but + # with additional validation for used assemblies and GetEmitDiagnostics. + - name: coreclr-usedassemblies-debug + displayName: Test Windows CoreCLR UsedAssemblies Debug + testArguments: -testCoreClr -testUsedAssemblies -testCompilerOnly + helixQueueName: $(HelixWindowsQueueName) - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: eng/pipelines/test-windows-job-single-machine.yml @@ -251,41 +253,6 @@ stages: poolParameters: ${{ parameters.vs2026PreviewPool }} testArguments: -testCoreClr - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows CoreCLR IOperation Debug' - jobName: Test_Windows_CoreClr_IOperation_Debug - testArtifactName: Transport_Artifacts_Windows_Debug - configuration: Debug - testArguments: -testCoreClr -testIOperation -testCompilerOnly - helixQueueName: $(HelixWindowsQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} - - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows CoreCLR RuntimeAsync Debug' - jobName: Test_Windows_CoreClr_RuntimeAsync_Debug - testArtifactName: Transport_Artifacts_Windows_Debug - configuration: Debug - testArguments: -testCoreClr -testRuntimeAsync -testCompilerOnly - helixQueueName: $(HelixWindowsQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} - - # This leg runs almost all the compiler tests supported on CoreCLR, but - # with additional validation for used assemblies and GetEmitDiagnostics - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows CoreCLR UsedAssemblies Debug' - jobName: Test_Windows_CoreClr_UsedAssemblies_Debug - testArtifactName: Transport_Artifacts_Windows_Debug - configuration: Debug - testArguments: -testCoreClr -testUsedAssemblies -testCompilerOnly - helixQueueName: $(HelixWindowsQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} - - template: /eng/common/core-templates/job/helix-job-monitor.yml parameters: helixAccessToken: $(HelixApiAccessToken) @@ -296,51 +263,32 @@ stages: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - group: DotNet-HelixApi-Access jobs: - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows Desktop Release 32' - jobName: Test_Windows_Desktop_Release_32 - testArtifactName: Transport_Artifacts_Windows_Release - configuration: Release - testArguments: -testDesktop -testArch x86 - helixQueueName: $(HelixWindowsVsQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} - - template: eng/pipelines/test-windows-job.yml parameters: - testRunName: 'Test Windows Desktop Release 64' - jobName: Test_Windows_Desktop_Release_64 + jobName: Submit_Windows_Release_Helix_Tests testArtifactName: Transport_Artifacts_Windows_Release configuration: Release - testArguments: -testDesktop -testArch x64 - helixQueueName: $(HelixWindowsVsQueueName) helixApiAccessToken: $(HelixApiAccessToken) poolParameters: ${{ parameters.windowsPool }} - - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows Desktop Spanish Release 64' - jobName: Test_Windows_Desktop_Spanish_Release_64 - testArtifactName: Transport_Artifacts_Windows_Release - configuration: Release + testRuns: + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - name: desktop-release-x86 + displayName: Test Windows Desktop Release 32 + testArguments: -testDesktop -testArch x86 + helixQueueName: $(HelixWindowsVsQueueName) + - name: desktop-release-x64 + displayName: Test Windows Desktop Release 64 testArguments: -testDesktop -testArch x64 - helixQueueName: $(HelixWindowsSpanishQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} - - - template: eng/pipelines/test-windows-job.yml - parameters: - testRunName: 'Test Windows CoreClr Release' - jobName: Test_Windows_CoreClr_Release - testArtifactName: Transport_Artifacts_Windows_Release - configuration: Release - testArguments: -testCoreClr - helixQueueName: $(HelixWindowsQueueName) - helixApiAccessToken: $(HelixApiAccessToken) - poolParameters: ${{ parameters.windowsPool }} + helixQueueName: $(HelixWindowsVsQueueName) + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - name: desktop-release-spanish-x64 + displayName: Test Windows Desktop Spanish Release 64 + testArguments: -testDesktop -testArch x64 + helixQueueName: $(HelixWindowsSpanishQueueName) + - name: coreclr-release + displayName: Test Windows CoreClr Release + testArguments: -testCoreClr + helixQueueName: $(HelixWindowsQueueName) - template: /eng/common/core-templates/job/helix-job-monitor.yml parameters: diff --git a/eng/pipelines/test-windows-job.yml b/eng/pipelines/test-windows-job.yml index e9f1e1aa63ffb..e13532f338199 100644 --- a/eng/pipelines/test-windows-job.yml +++ b/eng/pipelines/test-windows-job.yml @@ -1,10 +1,5 @@ # Test on Windows Desktop using Helix parameters: -- name: helixQueueName - type: string -- name: testRunName - type: string - default: '' - name: jobName type: string default: '' @@ -14,9 +9,9 @@ parameters: - name: configuration type: string default: 'Debug' -- name: testArguments - type: string - default: '' +- name: testRuns + type: object + default: [] - name: helixApiAccessToken type: string default: '' @@ -43,19 +38,67 @@ jobs: env: HELIX_CORRELATION_PAYLOAD: '$(Build.SourcesDirectory)\.duplicate' + # Attempt every submission even if an earlier one fails, then fail the aggregate job below. + - ${{ each testRun in parameters.testRuns }}: + - task: PowerShell@2 + displayName: Submit ${{ testRun.displayName }} + continueOnError: true + inputs: + filePath: eng/build.ps1 + ${{ if ne(variables['System.TeamProject'], 'public') }}: + arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ testRun.helixQueueName }} -helixApiAccessToken ${{ parameters.helixApiAccessToken }} ${{ testRun.testArguments }} -collectDumps + ${{ else }}: + arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ testRun.helixQueueName }} ${{ testRun.testArguments }} -collectDumps + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + # RunTests uses the same output paths for every Helix submission. Move each submission's + # outputs aside so the next sequential submission cannot overwrite or mix its diagnostics. + - task: PowerShell@2 + displayName: Preserve ${{ testRun.displayName }} submission logs + condition: succeededOrFailed() + continueOnError: true + inputs: + targetType: inline + script: | + $artifactsDirectory = '$(Build.SourcesDirectory)\artifacts' + $logDirectory = Join-Path $artifactsDirectory 'log\${{ parameters.configuration }}' + $submissionLogDirectory = Join-Path $logDirectory '${{ testRun.name }}' + $payloadsDirectory = Join-Path $artifactsDirectory 'payloads' + $submissionPayloadsDirectory = Join-Path $artifactsDirectory 'payloads-${{ testRun.name }}-$(System.JobAttempt)' + $helixProjectPath = Join-Path $artifactsDirectory 'helix.proj' + $helixBinaryLogPath = Join-Path $logDirectory 'helix.binlog' + $submissionBinaryLogPath = Join-Path $logDirectory 'helix-${{ testRun.name }}.binlog' + $loggedHelixProjectPath = Join-Path $logDirectory 'helix.proj' + + if (Test-Path $submissionLogDirectory) { + Remove-Item $submissionLogDirectory -Recurse -Force + } + New-Item -ItemType Directory -Path $submissionLogDirectory -Force | Out-Null + if (Test-Path $helixBinaryLogPath) { + Move-Item $helixBinaryLogPath $submissionBinaryLogPath -Force + } + if (Test-Path $loggedHelixProjectPath) { + Move-Item $loggedHelixProjectPath $submissionLogDirectory -Force + } + Get-ChildItem -Path $logDirectory -Filter 'workitem_*' -Directory | + Move-Item -Destination $submissionLogDirectory -Force + + if (Test-Path $payloadsDirectory) { + Move-Item $payloadsDirectory $submissionPayloadsDirectory + } + if (Test-Path $helixProjectPath) { + Remove-Item $helixProjectPath -Force + } + - task: PowerShell@2 - displayName: Run Unit Tests + displayName: Fail job if any submission failed + condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues') inputs: - filePath: eng/build.ps1 - ${{ if ne(variables['System.TeamProject'], 'public') }}: - arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ parameters.helixQueueName }} -helixApiAccessToken ${{ parameters.helixApiAccessToken }} ${{ parameters.testArguments }} -collectDumps - ${{ else }}: - arguments: -ci -helix -configuration ${{ parameters.configuration }} -helixQueueName ${{ parameters.helixQueueName }} ${{ parameters.testArguments }} -collectDumps - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) + targetType: inline + script: throw 'One or more Helix submissions failed.' - template: publish-logs.yml parameters: configuration: ${{ parameters.configuration }} jobName: ${{ parameters.jobName }} - testRunName: ${{ parameters.testRunName }} diff --git a/src/Tools/RunTests/AzdoClient.cs b/src/Tools/RunTests/AzdoClient.cs index 70d24d7fc5de7..77430aa99d5b2 100644 --- a/src/Tools/RunTests/AzdoClient.cs +++ b/src/Tools/RunTests/AzdoClient.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -74,12 +74,12 @@ public static AzdoClient Create(string projectUri, string accessToken) } /// - /// Gets test runs for a build within the build's time range, matching a stage/phase name. + /// Gets a test run for a build within the build's time range, matching its exact name. /// - public async Task GetRunForStageAsync( + public async Task GetTestRunAsync( string project, AzdoBuild build, - string phaseName, + string testRunName, CancellationToken cancellationToken) { var minTime = build.QueueTime!.Value.ToString("o"); @@ -94,7 +94,7 @@ public static AzdoClient Create(string projectUri, string accessToken) // If the last successful build had multiple attempts then there are potentially multiple runs with // the same name. Take the last one as it will be the successful one. - return runs.LastOrDefault(r => r.Name.Contains(phaseName)); + return runs.LastOrDefault(r => string.Equals(r.Name, testRunName, StringComparison.OrdinalIgnoreCase)); } /// diff --git a/src/Tools/RunTests/HelixTestRunner.cs b/src/Tools/RunTests/HelixTestRunner.cs index 388796d06564e..54a01c23bca0e 100644 --- a/src/Tools/RunTests/HelixTestRunner.cs +++ b/src/Tools/RunTests/HelixTestRunner.cs @@ -34,6 +34,10 @@ public sealed class HelixWorkItem( internal sealed class HelixTestRunner { + private const string IOperationEnvironmentVariable = "ROSLYN_TEST_IOPERATION"; + private const string RuntimeAsyncEnvironmentVariable = "DOTNET_RuntimeAsync"; + private const string UsedAssembliesEnvironmentVariable = "ROSLYN_TEST_USEDASSEMBLIES"; + /// /// The amount of time we will allocate for each helix work item. When changing this value, consider that test execution time is only part of the /// total time in a work item: @@ -121,6 +125,7 @@ internal static async Task CreateHelixArtifactsAsync(Options options, Im : TestOS.Linux; var platform = !string.IsNullOrEmpty(options.Architecture) ? options.Architecture : "x64"; + var testRunName = GetTestRunName(options); var dotnetSdkVersion = GetDotNetSdkVersion(options.ArtifactsDirectory); // This is the directory where all of the work item payloads are stored. @@ -128,7 +133,7 @@ internal static async Task CreateHelixArtifactsAsync(Options options, Im var logsDir = Path.Combine(options.ArtifactsDirectory, "log", options.Configuration); // Retrieve test runtimes from azure devops historical data. - var testHistory = await TestHistoryManager.GetTestHistoryAsync(options, cancellationToken); + var testHistory = await TestHistoryManager.GetTestHistoryAsync(options, testRunName, cancellationToken); var helixWorkItems = AssemblyScheduler.Schedule(assemblies.Select(x => x.AssemblyPath), platform, testHistory); var timeout = testHistory is null ? WorkItemExecutionTimeout * 2 : WorkItemExecutionTimeout; var helixProjectFileContent = GetHelixProjectFileContent( @@ -136,6 +141,7 @@ internal static async Task CreateHelixArtifactsAsync(Options options, Im testOS, dotnetSdkVersion, platform, + testRunName, options.HelixQueueName, options.ArtifactsDirectory, payloadsDir, @@ -214,6 +220,7 @@ private static string GetHelixProjectFileContent( TestOS testOS, string dotnetSdkVersion, string platform, + string testRunName, string helixQueueName, string artifactsDir, string payloadsDir, @@ -237,7 +244,6 @@ private static string GetHelixProjectFileContent( // it's possible we should be using the BUILD_SOURCEVERSIONAUTHOR instead here a la https://github.com/dotnet/arcade/blob/main/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md#how-to-use // however that variable isn't documented at https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml var queuedBy = GetEnv("BUILD_QUEUEDBY", "roslyn").Replace(" ", ""); - var jobName = GetEnv("SYSTEM_JOBDISPLAYNAME", ""); var buildNumber = GetEnv("BUILD_BUILDNUMBER", "0"); var duplicateDir = Path.Combine(Path.GetDirectoryName(artifactsDir)!, ".duplicate"); @@ -245,7 +251,7 @@ private static string GetHelixProjectFileContent( builder.AppendLine($""" - {jobName}_ + {testRunName}_ test {buildNumber} {helixQueueName} @@ -354,9 +360,9 @@ static void AppendHelixWorkItemProject( string[] knownEnvironmentVariables = [ - "ROSLYN_TEST_IOPERATION", - "ROSLYN_TEST_USEDASSEMBLIES", - "DOTNET_RuntimeAsync" + IOperationEnvironmentVariable, + UsedAssembliesEnvironmentVariable, + RuntimeAsyncEnvironmentVariable ]; foreach (var knownEnvironmentVariable in knownEnvironmentVariables) @@ -469,6 +475,38 @@ static void AppendHelixWorkItemProject( } } + private static string GetTestRunName(Options options) + { + var runtime = options.TestRuntime switch + { + TestRuntime.Core => "CoreClr", + TestRuntime.Framework => "Desktop", + TestRuntime.Both => "Both", + _ => throw new ArgumentOutOfRangeException(nameof(options.TestRuntime)), + }; + + var nameParts = new List + { + options.Configuration, + runtime, + options.Architecture, + }; + + AddEnvironmentVariableToken(IOperationEnvironmentVariable, "IOperation"); + AddEnvironmentVariableToken(RuntimeAsyncEnvironmentVariable, "RuntimeAsync"); + AddEnvironmentVariableToken(UsedAssembliesEnvironmentVariable, "UsedAssemblies"); + + return string.Join("_", nameParts); + + void AddEnvironmentVariableToken(string environmentVariable, string token) + { + if (Environment.GetEnvironmentVariable(environmentVariable) is { Length: > 0 }) + { + nameParts.Add(token); + } + } + } + private static string GetEnv(string name, string defaultValue) { if (Environment.GetEnvironmentVariable(name) is { } value) diff --git a/src/Tools/RunTests/Options.cs b/src/Tools/RunTests/Options.cs index 454f7e34ae961..ef82c49d62dc9 100644 --- a/src/Tools/RunTests/Options.cs +++ b/src/Tools/RunTests/Options.cs @@ -114,8 +114,6 @@ internal class Options public string? PipelineDefinitionId { get; set; } - public string? PhaseName { get; set; } - public string? TargetBranchName { get; set; } public Options( @@ -158,7 +156,6 @@ public Options( string? accessToken = null; string? projectUri = null; string? pipelineDefinitionId = null; - string? phaseName = null; string? targetBranchName = null; var optionSet = new OptionSet() { @@ -184,7 +181,6 @@ public Options( { "accessToken=", "Pipeline access token with permissions to view test history", s => accessToken = s }, { "projectUri=", "ADO project containing the pipeline", s => projectUri = s }, { "pipelineDefinitionId=", "Pipeline definition id", s => pipelineDefinitionId = s }, - { "phaseName=", "Pipeline phase name associated with this test run", s => phaseName = s }, { "targetBranchName=", "Target branch of this pipeline run", s => targetBranchName = s }, }; @@ -254,7 +250,6 @@ public Options( AccessToken = accessToken, ProjectUri = projectUri, PipelineDefinitionId = pipelineDefinitionId, - PhaseName = phaseName, TargetBranchName = targetBranchName, }; diff --git a/src/Tools/RunTests/TestHistoryManager.cs b/src/Tools/RunTests/TestHistoryManager.cs index ab30be73f3ac0..d5d61a26b3e43 100644 --- a/src/Tools/RunTests/TestHistoryManager.cs +++ b/src/Tools/RunTests/TestHistoryManager.cs @@ -20,7 +20,7 @@ internal class TestHistoryManager private const int MaxTestsReturnedPerRequest = 10_000; /// - /// Looks up the last passing test run for the current build and stage to estimate execution times for each + /// Looks up the last passing test run for the current build to estimate execution times for each /// tests. The dictionary is indexed by test full name and contains the body duration and theory instance count. /// The theory instance count is sourced from the AzDO subResultsCount field which represents individual /// theory invocations reported under a grouped test result. @@ -29,7 +29,7 @@ internal class TestHistoryManager /// In xUnit v2, DurationInMs does NOT include IAsyncLifetime.InitializeAsync or DisposeAsync time. /// The caller is responsible for adjusting the duration based on the HasAsyncLifetime flag from test discovery. /// - public static async Task?> GetTestHistoryAsync(Options options, CancellationToken cancellationToken) + public static async Task?> GetTestHistoryAsync(Options options, string testRunNamePrefix, CancellationToken cancellationToken) { // Access token that has permissions to lookup test history. This typically comes from the pipeline. var accessToken = options.AccessToken ?? GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); @@ -40,23 +40,19 @@ internal class TestHistoryManager // Id of the pipeline to get test history from. var pipelineDefinitionIdStr = options.PipelineDefinitionId ?? GetEnvironmentVariable("SYSTEM_DEFINITIONID"); - // The phase name is used to filter the tests on the last passing build to only those that apply to the currently running phase. - // Note here that 'phaseName' corresponds to the 'jobName' defined in our pipeline yaml file and the job name env var is not correct. - // See https://developercommunity.visualstudio.com/t/systemjobname-seems-to-be-incorrectly-assigned-and/1209736 - var phaseName = options.PhaseName ?? GetEnvironmentVariable("SYSTEM_PHASENAME"); - // We use the target branch of the current build to lookup the last successful build for the same branch. // For PR builds, SYSTEM_PULLREQUEST_TARGETBRANCH gives us the target (e.g. "main"). // For CI builds, we need the full branch ref. BUILD_SOURCEBRANCH gives us the full ref // (e.g. "refs/heads/features/unions") while BUILD_SOURCEBRANCHNAME only gives the last // segment (e.g. "unions"), which breaks history lookup for nested branch names. var targetBranch = options.TargetBranchName ?? GetEnvironmentVariable("SYSTEM_PULLREQUEST_TARGETBRANCH") ?? GetEnvironmentVariable("BUILD_SOURCEBRANCH"); - if (string.IsNullOrEmpty(accessToken) || string.IsNullOrEmpty(projectUri) || string.IsNullOrEmpty(phaseName) || string.IsNullOrEmpty(targetBranch) || !int.TryParse(pipelineDefinitionIdStr, out var pipelineDefinitionId)) + if (string.IsNullOrEmpty(accessToken) || string.IsNullOrEmpty(projectUri) || string.IsNullOrEmpty(testRunNamePrefix) || string.IsNullOrEmpty(options.HelixQueueName) || string.IsNullOrEmpty(targetBranch) || !int.TryParse(pipelineDefinitionIdStr, out var pipelineDefinitionId)) { - ConsoleUtil.Warning($"Missing required options to lookup test history, projectUri={projectUri}, phaseName={phaseName}, targetBranchName={targetBranch}, pipelineDefinitionId={pipelineDefinitionIdStr}"); + ConsoleUtil.Warning($"Missing required options to lookup test history, projectUri={projectUri}, testRunName={testRunNamePrefix}, helixQueueName={options.HelixQueueName}, targetBranchName={targetBranch}, pipelineDefinitionId={pipelineDefinitionIdStr}"); return null; } + var testRunName = $"{testRunNamePrefix}_{options.HelixQueueName}"; using var azdoClient = AzdoClient.Create(projectUri, accessToken); ConsoleUtil.WriteLine($"Getting last successful build for branch {targetBranch}"); @@ -79,17 +75,16 @@ internal class TestHistoryManager return null; } - var runForThisStage = await GetRunForStageAsync(azdoClient, lastSuccessfulBuild, phaseName, cancellationToken); - if (runForThisStage == null) + var testRun = await GetTestRunAsync(azdoClient, lastSuccessfulBuild, testRunName, cancellationToken); + if (testRun == null) { - // If this is a new stage, historical runs will not have any data for it. - ConsoleUtil.Warning($"Unable to get a run with name {phaseName} from build {lastSuccessfulBuild.Url}."); + ConsoleUtil.Warning($"Unable to get a run with name {testRunName} from build {lastSuccessfulBuild.Url}."); return null; } - ConsoleUtil.WriteLine($"Looking up test execution data for build {lastSuccessfulBuild.Id} on branch {targetBranch} and stage {phaseName}"); + ConsoleUtil.WriteLine($"Looking up test execution data for build {lastSuccessfulBuild.Id} on branch {targetBranch} and test run {testRunName}"); - var totalTests = runForThisStage.TotalTests; + var totalTests = testRun.TotalTests; Dictionary testInfos = new(); var duplicateCount = 0; @@ -99,7 +94,7 @@ internal class TestHistoryManager timer.Start(); for (var i = 0; i < totalTests; i += MaxTestsReturnedPerRequest) { - var testResults = await GetTestResultsAsync(azdoClient, runForThisStage, i, MaxTestsReturnedPerRequest, cancellationToken); + var testResults = await GetTestResultsAsync(azdoClient, testRun, i, MaxTestsReturnedPerRequest, cancellationToken); foreach (var testResult in testResults) { // Helix outputs results for the whole dll work item suffixed with WorkItemExecution which we should ignore. @@ -135,12 +130,12 @@ internal class TestHistoryManager if (duplicateCount > 0) { - Logger.Log($"Found {duplicateCount} duplicate tests in run {runForThisStage.Name}."); + Logger.Log($"Found {duplicateCount} duplicate tests in run {testRun.Name}."); } if (testInfos.Count == 0) { - ConsoleUtil.Warning($"Retrieved zero test results from build {lastSuccessfulBuild.Id} and stage {phaseName}, falling back to count based scheduling"); + ConsoleUtil.Warning($"Retrieved zero test results from build {lastSuccessfulBuild.Id} and test run {testRunName}, falling back to count based scheduling"); return null; } @@ -181,11 +176,11 @@ private static string CleanTestName(string fullyQualifiedTestName) } } - private static async Task GetRunForStageAsync(AzdoClient azdoClient, AzdoBuild build, string phaseName, CancellationToken cancellationToken) + private static async Task GetTestRunAsync(AzdoClient azdoClient, AzdoBuild build, string testRunName, CancellationToken cancellationToken) { try { - return await azdoClient.GetRunForStageAsync("public", build, phaseName, cancellationToken); + return await azdoClient.GetTestRunAsync("public", build, testRunName, cancellationToken); } catch (Exception ex) {