diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index 926fa188d10..4b2784921bb 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -86,7 +86,7 @@ public void TemporarySymbolDirectoryDoesNotExists() }; var path = TestInputs.GetFullPath("Symbol"); var buildAsset = new Dictionary>(); - var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, path); + var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, null, path); Assert.True(task.Log.HasLoggedErrors); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 0f5be881c36..ee706c1efc0 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -91,11 +91,6 @@ public class PublishArtifactsInManifest : MSBuildTaskBase [Required] public string BuildAssetRegistryToken { get; set; } - /// - /// Maximum number of parallel uploads for the upload tasks - /// - public int MaxClients { get; set; } = 16; - /// /// Directory where "nuget.exe" is installed. This will be used to publish packages. /// @@ -331,7 +326,6 @@ internal PublishArtifactsInManifestBase ConstructPublishingV2Task(BuildModel bui BARBuildId = this.BARBuildId, MaestroApiEndpoint = this.MaestroApiEndpoint, BuildAssetRegistryToken = this.BuildAssetRegistryToken, - MaxClients = this.MaxClients, NugetPath = this.NugetPath, InternalBuild = this.InternalBuild, SkipSafetyChecks = this.SkipSafetyChecks, @@ -359,7 +353,6 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui BARBuildId = this.BARBuildId, MaestroApiEndpoint = this.MaestroApiEndpoint, BuildAssetRegistryToken = this.BuildAssetRegistryToken, - MaxClients = this.MaxClients, NugetPath = this.NugetPath, InternalBuild = this.InternalBuild, SkipSafetyChecks = this.SkipSafetyChecks, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 8cf716cf8c0..41d001be794 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -69,10 +69,15 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities [Required] public string NugetPath { get; set; } + private const int StreamingPublishingMaxClients = 20; + private const int NonStreamingPublishingMaxClients = 16; + /// - /// Maximum number of parallel uploads for the upload tasks + /// Maximum number of parallel uploads for the upload tasks. + /// For streaming publishing, 20 is used as the most optimal. + /// For non-streaming publishing, 16 is used (there are multiple sets of 16-parallel uploads) /// - public int MaxClients { get; set; } = 16; + public int MaxClients { get { return UseStreamingPublishing ? StreamingPublishingMaxClients : NonStreamingPublishingMaxClients; } } /// /// Whether this build is internal or not. If true, extra checks are done to avoid accidental @@ -365,6 +370,7 @@ public void CheckForStableAssetsInNonIsolatedFeeds() /// Token to authenticate symweb /// Right now we do not add any files to this, so this is going to be null /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published + /// To avoid starting too many processes /// Task public async Task PublishSymbolsUsingStreamingAsync( string pdbArtifactsBasePath, @@ -372,7 +378,8 @@ public async Task PublishSymbolsUsingStreamingAsync( string symWebToken, string symbolPublishingExclusionsFile, bool publishSpecialClrFiles, - Dictionary> buildAssets) + Dictionary> buildAssets, + SemaphoreSlim clientThrottle) { StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); @@ -403,49 +410,62 @@ public async Task PublishSymbolsUsingStreamingAsync( { using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true)) { - foreach (var symbol in symbolsToPublish) + await Task.WhenAll(symbolsToPublish.Select(async symbol => { - string temporarySymbolsDirectory = CreateTemporaryDirectory(); - string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); - symbolLog.AppendLine($"Downloading symbol : {symbol} to {localSymbolPath}"); - - await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, symbol, localSymbolPath); - symbolLog.AppendLine($"Successfully downloaded symbol : {symbol} to {localSymbolPath}"); - List symbolFiles = new List(); - symbolFiles.Add(localSymbolPath); - symbolLog.AppendLine($"Uploading symbol file '{string.Join(",", symbolFiles)}'"); - - foreach (var server in serversToPublish) + try { - var serverPath = server.Key; - var token = server.Value; - symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:"); - - try + await clientThrottle.WaitAsync(); + string temporarySymbolsDirectory = CreateTemporaryDirectory(); + string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); + symbolLog.AppendLine($"Downloading symbol : {symbol} to {localSymbolPath}"); + + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, + symbol, + localSymbolPath); + symbolLog.AppendLine($"Successfully downloaded symbol : {symbol} to {localSymbolPath}"); + List symbolFiles = new List(); + symbolFiles.Add(localSymbolPath); + symbolLog.AppendLine($"Uploading symbol file '{string.Join(",", symbolFiles)}'"); + + foreach (var server in serversToPublish) { - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - symbolFiles, - null, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:"); + + try + { + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFiles, + null, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + catch (Exception ex) + { + Log.LogError(ex.Message); + } } - catch (Exception ex) - { - Log.LogError(ex.Message); - } - } - DeleteTemporaryDirectory(temporarySymbolsDirectory); - } + DeleteTemporaryDirectory(temporarySymbolsDirectory); + } + finally + { + clientThrottle.Release(); + } + })); symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); @@ -524,6 +544,7 @@ await PublishSymbolsHelper.PublishAsync( /// Token to authenticate symweb /// Right now we do not add any files to this, so this is going to be null /// Path to Symbol.nupkgs + /// To avoid starting too many processes /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published public async Task HandleSymbolPublishingAsync ( string pdbArtifactsBasePath, @@ -532,6 +553,7 @@ public async Task HandleSymbolPublishingAsync ( string symbolPublishingExclusionsFile, bool publishSpecialClrFiles, Dictionary> buildAssets, + SemaphoreSlim clientThrottle = null, string temporarySymbolsLocation = null) { if (UseStreamingPublishing) @@ -542,7 +564,8 @@ await PublishSymbolsUsingStreamingAsync( symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles, - buildAssets); + buildAssets, + clientThrottle); } else { @@ -669,8 +692,9 @@ public Dictionary GetTargetSymbolServers(HashSet /// Maestro API client /// Assets information about build being published. + /// To avoid starting too many processes /// Task - protected async Task HandlePackagePublishingAsync(Dictionary> buildAssets) + protected async Task HandlePackagePublishingAsync(Dictionary> buildAssets, SemaphoreSlim clientThrottle =null) { List publishTasks = new List(); @@ -704,7 +728,8 @@ protected async Task HandlePackagePublishingAsync(Dictionary> buildAssets) + protected async Task HandleBlobPublishingAsync(Dictionary> buildAssets, SemaphoreSlim clientThrottle= null) { List publishTasks = new List(); @@ -939,14 +964,16 @@ protected async Task HandleBlobPublishingAsync(Dictionary PublishBlobsToAzDoNugetFeedAsync( filteredBlobs, buildAssets, - feedConfig)); + feedConfig, + clientThrottle)); break; case FeedType.AzureStorageFeed: publishTasks.Add( PublishBlobsToAzureStorageNugetFeedAsync( filteredBlobs, buildAssets, - feedConfig)); + feedConfig, + clientThrottle)); break; default: Log.LogError( @@ -1088,7 +1115,8 @@ await PushNugetPackageAsync( private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( HashSet packagesToPublish, Dictionary> buildAssets, - TargetFeedConfig feedConfig) + TargetFeedConfig feedConfig, + SemaphoreSlim clientThrottle) { string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts); @@ -1099,62 +1127,74 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); - foreach (var package in packagesToPublish) + await Task.WhenAll(packagesToPublish.Select(async package => { - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - string localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - Log.LogMessage(MessageImportance.Low, $"Downloading package : {packageFilename} to {localPackagePath}"); + try + { + await clientThrottle.WaitAsync(); + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + string localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + Log.LogMessage(MessageImportance.Low, + $"Downloading package : {packageFilename} to {localPackagePath}"); - await DownloadFileAsync( - client, - ArtifactName.PackageArtifacts, - containerId, - packageFilename, - localPackagePath); + await DownloadFileAsync( + client, + ArtifactName.PackageArtifacts, + containerId, + packageFilename, + localPackagePath); - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - Log.LogMessage(MessageImportance.Low, $"Successfully downloaded package : {packageFilename} to {localPackagePath}"); + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } - TryAddAssetLocation( - package.Id, - package.Version, - buildAssets, - feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); + Log.LogMessage(MessageImportance.Low, + $"Successfully downloaded package : {packageFilename} to {localPackagePath}"); - using HttpClient httpClient = new HttpClient(new HttpClientHandler - { - CheckCertificateRevocationList = true - }); + TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); - httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String( - Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + using HttpClient httpClient = new HttpClient(new HttpClientHandler + { + CheckCertificateRevocationList = true + }); - await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version); + httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); - DeleteTemporaryDirectory(localPackagePath); - } + await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version); + + DeleteTemporaryDirectory(localPackagePath); + } + finally + { + clientThrottle.Release(); + } + })); } private async Task PublishPackagesToAzDoNugetFeedAsync( HashSet packagesToPublish, Dictionary> buildAssets, - TargetFeedConfig feedConfig) + TargetFeedConfig feedConfig, + SemaphoreSlim clientThrottle) { if (UseStreamingPublishing) { - await PublishPackagesUsingStreamingToAzdoNugetAsync(packagesToPublish, buildAssets, feedConfig); + await PublishPackagesUsingStreamingToAzdoNugetAsync(packagesToPublish, buildAssets, feedConfig, clientThrottle); } else { @@ -1342,7 +1382,8 @@ public async Task PushNugetPackageAsync( private async Task PublishBlobsToAzDoNugetFeedAsync( HashSet blobsToPublish, Dictionary> buildAssets, - TargetFeedConfig feedConfig) + TargetFeedConfig feedConfig, + SemaphoreSlim clientThrottle) { HashSet packagesToPublish = new HashSet(); @@ -1362,7 +1403,7 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( if (UseStreamingPublishing) { - await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); + await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig, clientThrottle); } else { @@ -1423,7 +1464,8 @@ await PushNugetPackageAsync( private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( HashSet blobsToPublish, Dictionary> buildAssets, - TargetFeedConfig feedConfig) + TargetFeedConfig feedConfig, + SemaphoreSlim clientThrottle) { string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); @@ -1432,52 +1474,62 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( return; } using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - - foreach (var blob in blobsToPublish) + + await Task.WhenAll(blobsToPublish.Select(async blob => { - if (TryAddAssetLocation( - blob.Id, - assetVersion: null, - buildAssets, - feedConfig, - AddAssetLocationToAssetAssetLocationType.Container)) + try { - string temporaryBlobDirectory = CreateTemporaryDirectory(); - string fileName = Path.GetFileName(blob.Id); - string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); - - await DownloadFileAsync( - client, - ArtifactName.BlobArtifacts, - containerId, - fileName, - localBlobPath); - - if (!File.Exists(localBlobPath)) + await clientThrottle.WaitAsync(); + if (TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - Log.LogMessage(MessageImportance.Low, $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + string temporaryBlobDirectory = CreateTemporaryDirectory(); + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); - string id; - string version; - using (var packageReader = new PackageArchiveReader(localBlobPath)) - { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, + fileName, + localBlobPath); - await PushBlobToNugetFeed( - feedConfig, - localBlobPath, - id, - version); - - DeleteTemporaryDirectory(temporaryBlobDirectory); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + + Log.LogMessage(MessageImportance.Low, + $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + + string id; + string version; + using (var packageReader = new PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } + + await PushBlobToNugetFeed( + feedConfig, + localBlobPath, + id, + version); + + DeleteTemporaryDirectory(temporaryBlobDirectory); + } } - } + finally + { + clientThrottle.Release(); + } + })); } private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id, @@ -1535,11 +1587,12 @@ public string CreateTemporaryDirectory() private async Task PublishBlobsToAzureStorageNugetFeedAsync( HashSet blobsToPublish, Dictionary> buildAssets, - TargetFeedConfig feedConfig) + TargetFeedConfig feedConfig, + SemaphoreSlim clientThrottle) { if (UseStreamingPublishing) { - await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, feedConfig); + await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, feedConfig, clientThrottle); } else { @@ -1568,7 +1621,8 @@ await LinkManager.CreateOrUpdateLatestLinksAsync( private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( HashSet blobsToPublish, Dictionary> buildAssets, - TargetFeedConfig feedConfig) + TargetFeedConfig feedConfig, + SemaphoreSlim clientThrottle) { string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); @@ -1584,42 +1638,52 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( }; using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - foreach (var blob in blobsToPublish) + await Task.WhenAll(blobsToPublish.Select(async blob => { - string temporaryBlobDirectory = CreateTemporaryDirectory(); - var fileName = Path.GetFileName(blob.Id); - var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); - - await DownloadFileAsync( - client, - ArtifactName.BlobArtifacts, - containerId, - fileName, - localBlobPath); - - if (!File.Exists(localBlobPath)) + try { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - Log.LogMessage(MessageImportance.Low, $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + await clientThrottle.WaitAsync(); + string temporaryBlobDirectory = CreateTemporaryDirectory(); + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); - var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, - new Dictionary + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, + fileName, + localBlobPath); + + if (!File.Exists(localBlobPath)) { - {"RelativeBlobPath", blob.Id} - }); - - TryAddAssetLocation( - blob.Id, - assetVersion: null, - buildAssets, - feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); - - await blobFeedAction.UploadAssetAsync(item, pushOptions, null); - DeleteTemporaryDirectory(temporaryBlobDirectory); - } + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + + Log.LogMessage(MessageImportance.Low, + $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + + TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + + await blobFeedAction.UploadAssetAsync(item, pushOptions, null); + DeleteTemporaryDirectory(temporaryBlobDirectory); + } + finally + { + clientThrottle.Release(); + } + })); } private async Task PublishBlobsToAzureStorageNugetAsync( diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 4228c42d701..2320ba8a1d0 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.Tasks.Feed.Model; @@ -85,7 +86,8 @@ public override async Task ExecuteAsync() { if (!int.TryParse(channelIdStr, out var channelId)) { - Log.LogError($"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID. To add a channel refer to https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Publishing.md#how-to-add-a-new-channel-to-use-v3-publishing."); + Log.LogError( + $"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID. To add a channel refer to https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Publishing.md#how-to-add-a-new-channel-to-use-v3-publishing."); continue; } @@ -94,7 +96,8 @@ public override async Task ExecuteAsync() if (Log.HasLoggedErrors) { - Log.LogError($"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers."); + Log.LogError( + $"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers."); return false; } @@ -135,8 +138,9 @@ public override async Task ExecuteAsync() Log.LogMessage(MessageImportance.High, $"Publishing to this target channel: {targetChannelConfig}"); - string shortLinkUrl = string.IsNullOrEmpty(targetChannelConfig.AkaMSChannelName) ? - $"dotnet/" : $"dotnet/{targetChannelConfig.AkaMSChannelName}/{BuildQuality}"; + string shortLinkUrl = string.IsNullOrEmpty(targetChannelConfig.AkaMSChannelName) + ? $"dotnet/" + : $"dotnet/{targetChannelConfig.AkaMSChannelName}/{BuildQuality}"; var targetFeedsSetup = new SetupTargetFeedConfigV3( targetChannelConfig.IsInternal, @@ -146,9 +150,9 @@ public override async Task ExecuteAsync() AzureStorageTargetFeedKey, PublishInstallersAndChecksums, GetFeed(targetChannelConfig.InstallersFeed, InstallersFeedOverride), - targetChannelConfig.IsInternal? InternalInstallersFeedKey : InstallersFeedKey, + targetChannelConfig.IsInternal ? InternalInstallersFeedKey : InstallersFeedKey, GetFeed(targetChannelConfig.ChecksumsFeed, ChecksumsFeedOverride), - targetChannelConfig.IsInternal? InternalCheckSumsFeedKey : CheckSumsFeedKey, + targetChannelConfig.IsInternal ? InternalCheckSumsFeedKey : CheckSumsFeedKey, GetFeed(targetChannelConfig.ShippingFeed, ShippingFeedOverride), GetFeed(targetChannelConfig.TransportFeed, TransportFeedOverride), GetFeed(targetChannelConfig.SymbolsFeed, SymbolsFeedOverride), @@ -203,9 +207,12 @@ public override async Task ExecuteAsync() CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); } - await Task.WhenAll(new Task[] { - HandlePackagePublishingAsync(buildAssets), - HandleBlobPublishingAsync(buildAssets), + using var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients); + + await Task.WhenAll(new Task[] + { + HandlePackagePublishingAsync(buildAssets, clientThrottle), + HandleBlobPublishingAsync(buildAssets, clientThrottle), HandleSymbolPublishingAsync( PdbArtifactsBasePath, MsdlToken, @@ -213,12 +220,13 @@ await Task.WhenAll(new Task[] { SymbolPublishingExclusionsFile, PublishSpecialClrFiles, buildAssets, + clientThrottle, temporarySymbolsLocation) }); DeleteTemporaryFiles(temporarySymbolsLocation); DeleteTemporaryDirectory(temporarySymbolsLocation); - + await PersistPendingAssetLocationAsync(client); } catch (Exception e)