From 0da8413d5ebae5d200decc5c5d19d78ccfb0ecff Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 22 Apr 2025 10:17:51 +0200 Subject: [PATCH] Reapply "Use new HelixAPI `/job/{job}/results` endpoint (#15230)" (#15335) This reverts commit 255d5e0c89958af276883a988108c2d616438805. --- .../Client/CSharp/generated-code/Job.cs | 87 +++++++++++++++++++ .../Models/JobCreationResult.cs | 4 - .../generated-code/Models/JobResultsUri.cs | 22 +++++ .../JobSender/ISentJob.cs | 11 --- .../JobSender/JobDefinition.cs | 4 +- .../JobSender/SentJob.cs | 6 +- .../Sdk/DownloadFromResultsContainer.cs | 7 +- .../Sdk/SendHelixJob.cs | 14 --- ...crosoft.DotNet.Helix.Sdk.MonoQueue.targets | 4 - .../DownloadFromResultsContainer.targets | 7 +- .../Languages/csharp/CSharp.cs | 11 +++ .../Languages/csharp/MethodGroup.hb | 2 +- 12 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobResultsUri.cs diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs index 49bdead3882..13b6133036a 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs @@ -20,6 +20,7 @@ public partial interface IJob Task NewAsync( Models.JobCreationRequest body, string idempotencyKey, + bool? returnSas = default, CancellationToken cancellationToken = default ); @@ -33,6 +34,11 @@ public partial interface IJob CancellationToken cancellationToken = default ); + Task ResultsAsync( + string job, + CancellationToken cancellationToken = default + ); + Task PassFailAsync( string job, CancellationToken cancellationToken = default @@ -77,6 +83,7 @@ public Job(HelixApi client) public async Task NewAsync( Models.JobCreationRequest body, string idempotencyKey, + bool? returnSas = default, CancellationToken cancellationToken = default ) { @@ -118,6 +125,11 @@ public Job(HelixApi client) _req.Headers.Add("Idempotency-Key", idempotencyKey); } + if (returnSas != default(bool?)) + { + _req.Headers.Add("return-sas", returnSas.ToString()); + } + if (body != default(Models.JobCreationRequest)) { _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); @@ -268,6 +280,81 @@ internal async Task OnListFailed(Request req, Response res) throw ex; } + partial void HandleFailedResultsRequest(RestApiException ex); + + public async Task ResultsAsync( + string job, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(job)) + { + throw new ArgumentNullException(nameof(job)); + } + + const string apiVersion = "2019-06-17"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/jobs/{job}/results".Replace("{job}", Uri.EscapeDataString(Client.Serialize(job))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnResultsFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnResultsFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnResultsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedResultsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + partial void HandleFailedPassFailRequest(RestApiException ex); public async Task PassFailAsync( diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobCreationResult.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobCreationResult.cs index 78060c7c714..f0f20522879 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobCreationResult.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobCreationResult.cs @@ -49,10 +49,6 @@ public bool IsValid { return false; } - if (string.IsNullOrEmpty(ResultsUriRSAS)) - { - return false; - } return true; } } diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobResultsUri.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobResultsUri.cs new file mode 100644 index 00000000000..cb7a0f77ec3 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobResultsUri.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Client.Models +{ + public partial class JobResultsUri + { + public JobResultsUri() + { + } + + [JsonProperty("ResultsUri")] + public string ResultsUri { get; set; } + + [JsonProperty("ResultsUriRSAS")] + public string ResultsUriRSAS { get; set; } + } +} diff --git a/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs b/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs index 3087aefa3b5..959824a777a 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs @@ -23,17 +23,6 @@ public interface ISentJob /// string HelixCancellationToken { get; } - /// - /// URI for blob storage container with the results. - /// - string ResultsContainerUri { get; } - - /// - /// Shared Access Signature for access to the container with results. - /// Used for internal builds. - /// - string ResultsContainerReadSAS { get; } - /// /// Poll for the job to actually finish inside Helix. /// diff --git a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs index 482294d1d78..1c52f9a4811 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs @@ -239,9 +239,9 @@ public async Task SendAsync(Action log, CancellationToken canc } string jobStartIdentifier = Guid.NewGuid().ToString("N"); - var newJob = await JobApi.NewAsync(creationRequest, jobStartIdentifier, cancellationToken).ConfigureAwait(false); + var newJob = await JobApi.NewAsync(creationRequest, jobStartIdentifier, cancellationToken: cancellationToken).ConfigureAwait(false); - return new SentJob(JobApi, newJob, newJob.ResultsUri, newJob.ResultsUriRSAS); + return new SentJob(JobApi, newJob); } private void WarnForImpendingRemoval(Action log, QueueInfo queueInfo) diff --git a/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs b/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs index caf3d00abc3..0e44962bdcf 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs @@ -9,20 +9,16 @@ namespace Microsoft.DotNet.Helix.Client { internal class SentJob : ISentJob { - public SentJob(IJob jobApi, JobCreationResult newJob, string resultsContainerUri, string resultsContainerReadSAS) + public SentJob(IJob jobApi, JobCreationResult newJob) { JobApi = jobApi; CorrelationId = newJob.Name; HelixCancellationToken = newJob.CancellationToken; - ResultsContainerUri = resultsContainerUri; - ResultsContainerReadSAS = resultsContainerReadSAS; } public IJob JobApi { get; } public string CorrelationId { get; } public string HelixCancellationToken { get; } - public string ResultsContainerUri { get; } - public string ResultsContainerReadSAS { get; } public Task WaitAsync(int pollingIntervalMs = 10000, CancellationToken cancellationToken = default) { diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs index beb1f0378de..13ac6caac6b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs @@ -26,8 +26,6 @@ public class DownloadFromResultsContainer : HelixTask, ICancelableTask [Required] public ITaskItem[] MetadataToWrite { get; set; } - public string ResultsContainerReadSAS { get; set; } - private const string MetadataFile = "metadata.txt"; private readonly CancellationTokenSource _cancellationSource = new CancellationTokenSource(); @@ -74,6 +72,7 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory // Use the Helix API to get the last possible iteration of the work item's execution var allAvailableFiles = await HelixApi.WorkItem.ListFilesAsync(workItemName, JobId, true, ct); + var resultsUri = await HelixApi.Job.ResultsAsync(JobId, ct); DirectoryInfo destinationDir = Directory.CreateDirectory(Path.Combine(directoryPath, workItemName)); foreach (string file in filesToDownload) @@ -109,14 +108,14 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory // If we have no read SAS token from the build, make a best-effort attempt using the URL from the Helix API. // For restricted queues, there will be no read SAS token available to use in the Helix API's result // (but hopefully the 'else' branch will be hit in this case) - if (string.IsNullOrEmpty(ResultsContainerReadSAS)) + if (string.IsNullOrEmpty(resultsUri.ResultsUriRSAS)) { blob = new BlobClient(new Uri(fileAvailableForDownload.Link), blobClientOptions); } else { var strippedFileUri = new Uri(fileAvailableForDownload.Link.Substring(0, fileAvailableForDownload.Link.LastIndexOf('?'))); - blob = new BlobClient(strippedFileUri, new AzureSasCredential(ResultsContainerReadSAS), blobClientOptions); + blob = new BlobClient(strippedFileUri, new AzureSasCredential(resultsUri.ResultsUriRSAS), blobClientOptions); } await blob.DownloadToAsync(destinationFile); } diff --git a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs index accc868956c..bb507fcaf8e 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs @@ -78,18 +78,6 @@ public static class MetadataNames [Output] public string JobCancellationToken { get; set; } - /// - /// When the task finishes, the results container uri should be available in case we want to download files. - /// - [Output] - public string ResultsContainerUri { get; set; } - - /// - /// If the job is internal, we need to give the DownloadFromResultsContainer task the Write SAS to download files. - /// - [Output] - public string ResultsContainerReadSAS { get; set; } - /// /// A collection of commands that will run for each work item before any work item commands. /// Use a semicolon to delimit these and escape semicolons by percent coding them ('%3B'). @@ -270,8 +258,6 @@ protected override async Task ExecuteCore(CancellationToken cancellationToken) ISentJob job = await def.SendAsync(msg => Log.LogMessageFromText(msg, MessageImportance.Normal), cancellationToken); JobCorrelationId = job.CorrelationId; JobCancellationToken = job.HelixCancellationToken; - ResultsContainerUri = job.ResultsContainerUri; - ResultsContainerReadSAS = job.ResultsContainerReadSAS; cancellationToken.ThrowIfCancellationRequested(); } diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets index d86d54c76bc..f1e7cd0a403 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets @@ -59,15 +59,11 @@ HelixProperties="@(HelixProperties)"> - - @(HelixWorkItem->Count()) $(HelixTargetQueue) - $(HelixResultsContainer) - $(HelixResultsContainerReadSAS) $(HelixJobCancellationToken) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets index f51a2852b4d..b7b08975f49 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets @@ -21,16 +21,13 @@ <_shouldDownloadResults Condition="'@(_workItemsWithDownloadMetadata)' != '' AND '$(HelixResultsDestinationDir)' != ''">true - - + JobId="%(SentJob.Identity)" /> diff --git a/src/Microsoft.DotNet.SwaggerGenerator/Microsoft.DotNet.SwaggerGenerator.CodeGenerator/Languages/csharp/CSharp.cs b/src/Microsoft.DotNet.SwaggerGenerator/Microsoft.DotNet.SwaggerGenerator.CodeGenerator/Languages/csharp/CSharp.cs index 1d8a8d27b10..0b63882a64e 100644 --- a/src/Microsoft.DotNet.SwaggerGenerator/Microsoft.DotNet.SwaggerGenerator.CodeGenerator/Languages/csharp/CSharp.cs +++ b/src/Microsoft.DotNet.SwaggerGenerator/Microsoft.DotNet.SwaggerGenerator.CodeGenerator/Languages/csharp/CSharp.cs @@ -193,6 +193,17 @@ public void NotNullCheck(TextWriter output, object context, Action