From 4a0cafa130a75b77192329797ec7ec28326a2a64 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Fri, 3 Sep 2021 15:45:46 -0700 Subject: [PATCH 1/5] Add DotNetCliUseProductVersion property; if true, get the version of the inner package from productVersion.txt, NOT from the specified version. Allows "non-stable" outer container with "stable" inner contents https://github.com/dotnet/arcade/issues/7836 --- .../Sdk/FindDotNetCliPackage.cs | 38 ++++++++++++++++--- .../Sdk/tools/dotnet-cli/DotNetCli.props | 1 + .../Sdk/tools/dotnet-cli/DotNetCli.targets | 2 +- tests/UnitTests.proj | 1 + 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs index 5d82a45f509..739b062876e 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs @@ -23,6 +23,13 @@ public class FindDotNetCliPackage : BaseTask [Required] public string Version { get; set; } + /// + /// If true, checks DotNetCLI paths for a productVersion.txt file to + /// enable intentional mismatch of versions in the path + /// + [Required] + public bool UseProductVersion { get; set; } + /// /// RID of dotnet cli to get /// @@ -49,7 +56,7 @@ private async Task ExecuteAsync() NormalizeParameters(); await ResolveVersionAsync(); - string downloadUrl = GetDownloadUrl(); + string downloadUrl = await GetDownloadUrlAsync(); Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl}, testing..."); @@ -80,14 +87,35 @@ private async Task ExecuteAsync() } } - private string GetDownloadUrl() + private async Task GetDownloadUrlAsync() { string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz"; + string effectiveVersion = Version; + if (UseProductVersion) + { + var productVersionText = (PackageType switch + { + "sdk" => await _client.GetStringAsync($"{DotNetCliAzureFeed}/Sdk/{Version}/productVersion.txt"), + "aspnetcore-runtime" => await _client.GetStringAsync($"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/productVersion.txt"), + _ => await _client.GetStringAsync($"{DotNetCliAzureFeed}/Runtime/{Version}/productVersion.txt") + }).Trim(); // The file contains the version + '\r\n' + + if (!productVersionText.Equals(Version)) + { + effectiveVersion = productVersionText; + Log.LogMessage($"Switched to effective .NET Core version '{productVersionText}' from productVersion.txt"); + } + else + { + Log.LogMessage(MessageImportance.Low, $"'UseProductVersion' was set to true, but the version ('{Version}') was the same as specified"); + } + + } return PackageType switch { - "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{Version}-{Runtime}.{extension}", - "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{Version}-{Runtime}.{extension}", - _ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{Version}-{Runtime}.{extension}" + "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}", + "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}", + _ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}" }; } diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props index a18693c65f8..fcda8f34629 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props @@ -8,6 +8,7 @@ $(AspNetCoreRuntimeVersion) $(NETCoreSdkVersion) + false Current dotnet-cli <_HelixMonoQueueTargets>$(_HelixMonoQueueTargets);$(MSBuildThisFileDirectory)DotNetCli.targets diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets index 50a93f39490..551077d27ef 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets @@ -2,7 +2,7 @@ - + diff --git a/tests/UnitTests.proj b/tests/UnitTests.proj index b200bcfd4a6..d73d78edc71 100644 --- a/tests/UnitTests.proj +++ b/tests/UnitTests.proj @@ -12,6 +12,7 @@ true sdk + true $(AGENT_JOBNAME) From 57dda103de2351b05b3063b5db0e61c97d569ed1 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Fri, 3 Sep 2021 16:12:43 -0700 Subject: [PATCH 2/5] Add the property to Helix SDK readme --- src/Microsoft.DotNet.Helix/Sdk/Readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index 5b8774440e4..dfc0f26325f 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -120,6 +120,8 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r 2.1.403 Current + + false false From f6cea011bf23ac31f2a15ce1562be4612b1a5b5f Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 7 Sep 2021 16:03:45 -0700 Subject: [PATCH 3/5] PR feedbacks - Remove boolean for whether to use it, only do the check for > 3.1, and when available use the specific-productVersion.txt files (mostly seem to contain the same version) --- .../Sdk/FindDotNetCliPackage.cs | 87 +++++++++++++------ .../Sdk/Microsoft.DotNet.Helix.Sdk.csproj | 1 + .../Sdk/tools/dotnet-cli/DotNetCli.props | 1 - .../Sdk/tools/dotnet-cli/DotNetCli.targets | 2 +- 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs index 739b062876e..c2db5c29251 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs @@ -3,6 +3,7 @@ using System.Net.Http; using System.Threading.Tasks; using Microsoft.Build.Framework; +using NuGet.Versioning; namespace Microsoft.DotNet.Helix.Sdk { @@ -23,13 +24,6 @@ public class FindDotNetCliPackage : BaseTask [Required] public string Version { get; set; } - /// - /// If true, checks DotNetCLI paths for a productVersion.txt file to - /// enable intentional mismatch of versions in the path - /// - [Required] - public bool UseProductVersion { get; set; } - /// /// RID of dotnet cli to get /// @@ -37,7 +31,7 @@ public class FindDotNetCliPackage : BaseTask public string Runtime { get; set; } /// - /// 'sdk', 'runtime' or 'aspnetcore-runtime' + /// 'sdk', 'runtime' or 'aspnetcore-runtime' (default is runtime) /// [Required] public string PackageType { get; set; } @@ -90,33 +84,70 @@ private async Task ExecuteAsync() private async Task GetDownloadUrlAsync() { string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz"; - string effectiveVersion = Version; - if (UseProductVersion) + string effectiveVersion = await GetEffectiveVersion(); + + return PackageType switch { - var productVersionText = (PackageType switch - { - "sdk" => await _client.GetStringAsync($"{DotNetCliAzureFeed}/Sdk/{Version}/productVersion.txt"), - "aspnetcore-runtime" => await _client.GetStringAsync($"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/productVersion.txt"), - _ => await _client.GetStringAsync($"{DotNetCliAzureFeed}/Runtime/{Version}/productVersion.txt") - }).Trim(); // The file contains the version + '\r\n' + "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}", + "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}", + _ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}" + }; + } - if (!productVersionText.Equals(Version)) + private async Task GetEffectiveVersion() + { + if (NuGetVersion.TryParse(Version, out NuGetVersion semanticVersion)) + { + // Pared down version of the logic from https://github.com/dotnet/install-scripts/blob/main/src/dotnet-install.ps1 + // If this functionality stops working, review changes made there. + // Current strategy is to start with a runtime-specific name then fall back to 'productVersion.txt' + string effectiveVersion = Version; + + // Do nothing for older runtimes; the file won't exist + if (semanticVersion > new NuGetVersion("3.1.0")) + { + var productVersionText = PackageType switch + { + "sdk" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Sdk/{Version}", "sdk-productVersion.txt"), + "aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}", "aspnetcore-productVersion.txt"), + _ => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Runtime/{Version}", "runtime-productVersion.txt") + }; + + if (!productVersionText.Equals(Version)) + { + effectiveVersion = productVersionText; + Log.LogMessage($"Switched to effective .NET Core version '{productVersionText}' from matching productVersion.txt"); + } + } + return effectiveVersion; + } + else + { + throw new ArgumentException($"'{Version}' is not a valid semantic version."); + } + } + private async Task GetMatchingProductVersionTxtContents(string baseUri, string customVersionTextFileName) + { + using HttpResponseMessage specificResponse = await _client.GetAsync($"{baseUri}/{customVersionTextFileName}"); + if (specificResponse.StatusCode == HttpStatusCode.NotFound) + { + using HttpResponseMessage genericResponse = await _client.GetAsync($"{baseUri}/productVersion.txt"); + if (genericResponse.StatusCode != HttpStatusCode.NotFound) { - effectiveVersion = productVersionText; - Log.LogMessage($"Switched to effective .NET Core version '{productVersionText}' from productVersion.txt"); + genericResponse.EnsureSuccessStatusCode(); + return (await genericResponse.Content.ReadAsStringAsync()).Trim(); } else { - Log.LogMessage(MessageImportance.Low, $"'UseProductVersion' was set to true, but the version ('{Version}') was the same as specified"); + Log.LogMessage(MessageImportance.Low, $"No *productVersion.txt files found for {Version} under {baseUri}"); } - } - return PackageType switch + else { - "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}", - "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}", - _ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}" - }; + specificResponse.EnsureSuccessStatusCode(); + return (await specificResponse.Content.ReadAsStringAsync()).Trim(); + } + return Version; } private void NormalizeParameters() @@ -164,9 +195,9 @@ private async Task ResolveVersionAsync() Log.LogMessage(MessageImportance.Low, "Resolving latest dotnet cli version."); string latestVersionUrl = PackageType switch { - "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Channel}/latest.version", + "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Channel}/latest.version", "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Channel}/latest.version", - _ => $"{DotNetCliAzureFeed}/Runtime/{Channel}/latest.version" + _ => $"{DotNetCliAzureFeed}/Runtime/{Channel}/latest.version" }; Log.LogMessage(MessageImportance.Low, $"Resolving latest version from url {latestVersionUrl}"); diff --git a/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj b/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj index 2e510066ae4..7c4a029a946 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj @@ -12,6 +12,7 @@ + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props index fcda8f34629..a18693c65f8 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props @@ -8,7 +8,6 @@ $(AspNetCoreRuntimeVersion) $(NETCoreSdkVersion) - false Current dotnet-cli <_HelixMonoQueueTargets>$(_HelixMonoQueueTargets);$(MSBuildThisFileDirectory)DotNetCli.targets diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets index 551077d27ef..50a93f39490 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.targets @@ -2,7 +2,7 @@ - + From 3961f8cd0e4a83bb28a439814e65f80529660b6b Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 7 Sep 2021 16:55:27 -0700 Subject: [PATCH 4/5] More feedback --- src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs | 2 +- src/Microsoft.DotNet.Helix/Sdk/Readme.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs index c2db5c29251..1e00efe249b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs @@ -104,7 +104,7 @@ private async Task GetEffectiveVersion() string effectiveVersion = Version; // Do nothing for older runtimes; the file won't exist - if (semanticVersion > new NuGetVersion("3.1.0")) + if (semanticVersion >= new NuGetVersion("5.0.0")) { var productVersionText = PackageType switch { diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index dfc0f26325f..5b8774440e4 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -120,8 +120,6 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r 2.1.403 Current - - false false From 915b665ba69eb21800af4f9c262e4bd9179176c6 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 7 Sep 2021 16:56:59 -0700 Subject: [PATCH 5/5] Missed a spot. --- tests/UnitTests.proj | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/UnitTests.proj b/tests/UnitTests.proj index d73d78edc71..b200bcfd4a6 100644 --- a/tests/UnitTests.proj +++ b/tests/UnitTests.proj @@ -12,7 +12,6 @@ true sdk - true $(AGENT_JOBNAME)