Merged PR 44008: [internal/release/8.0] Merge from public

vseanreesermsft · vseanreesermsft · commit cb771e5630e5 · 2024-10-18T18:32:49.000Z
Merge from public release/8.0 to internal/release/8.0 and resolve conflicts if necessary

----
#### AI description  (iteration 1)
#### PR Classification
Merge from public branch to internal release branch.

#### PR Summary
This pull request updates dependencies, fixes a bug related to SAS token generation, and improves error handling for certificate export.
- `/eng/Version.Details.xml`, `/global.json`, `/eng/Versions.props`: Updated various dependencies to newer versions.
- `/eng/common/templates-official/steps/get-delegation-sas.yml`, `/eng/common/templates/steps/get-delegation-sas.yml`: Added workaround for SAS token generation issue.
- `/src/Tools/dotnet-dev-certs/src/Program.cs`, `/src/Shared/CertificateGeneration/CertificateManager.cs`, `/src/Shared/CertificateGeneration/EnsureCertificateResult.cs`: Improved error handling for exporting certificates to non-existent directories.
- `/NuGet.config`: Added multiple new package sources.
- `/src/Tools/FirstRunCertGenerator/test/CertificateManagerTests.cs`: Updated test to reflect new error handling for certificate export.
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -189,9 +189,9 @@
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>9cb3b725e3ad2b57ddc9fb2dd48d2d170563a8f5</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="8.0.0-alpha.1.24379.1">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="8.0.0-alpha.1.24510.2">
       <Uri>https://github.com/dotnet/source-build-externals</Uri>
-      <Sha>fb970eccb0a9cae3092464e29cbabda0d4115049</Sha>
+      <Sha>d4feb7e49067fc9bbf7dfb9fa76a326c33fa0595</Sha>
       <SourceBuild RepoName="source-build-externals" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.symreader" Version="2.0.0-beta-23228-03">
@@ -376,26 +376,26 @@
       <Uri>https://github.com/dotnet/winforms</Uri>
       <Sha>abda8e3bfa78319363526b5a5f86863ec979940e</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24504.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24516.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>103916ccdbe7f4ab2e194068a1a3cd330542601f</Sha>
+      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="8.0.0-beta.24504.2">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="8.0.0-beta.24516.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>103916ccdbe7f4ab2e194068a1a3cd330542601f</Sha>
+      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="8.0.0-beta.24504.2">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="8.0.0-beta.24516.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>103916ccdbe7f4ab2e194068a1a3cd330542601f</Sha>
+      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.24504.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.24516.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>103916ccdbe7f4ab2e194068a1a3cd330542601f</Sha>
+      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="8.0.0-beta.24504.2">
+    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="8.0.0-beta.24516.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>103916ccdbe7f4ab2e194068a1a3cd330542601f</Sha>
+      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Extensions.Diagnostics.Testing" Version="8.1.0-preview.23604.1">
       <Uri>https://github.com/dotnet/extensions</Uri>
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -162,11 +162,11 @@
     <NuGetVersioningVersion>6.2.4</NuGetVersioningVersion>
     <NuGetFrameworksVersion>6.2.4</NuGetFrameworksVersion>
     <!-- Packages from dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersVersion>8.0.0-beta.24504.2</MicrosoftDotNetBuildTasksInstallersVersion>
-    <MicrosoftDotNetBuildTasksTemplatingVersion>8.0.0-beta.24504.2</MicrosoftDotNetBuildTasksTemplatingVersion>
-    <MicrosoftDotNetRemoteExecutorVersion>8.0.0-beta.24504.2</MicrosoftDotNetRemoteExecutorVersion>
+    <MicrosoftDotNetBuildTasksInstallersVersion>8.0.0-beta.24516.1</MicrosoftDotNetBuildTasksInstallersVersion>
+    <MicrosoftDotNetBuildTasksTemplatingVersion>8.0.0-beta.24516.1</MicrosoftDotNetBuildTasksTemplatingVersion>
+    <MicrosoftDotNetRemoteExecutorVersion>8.0.0-beta.24516.1</MicrosoftDotNetRemoteExecutorVersion>
     <!-- Packages from dotnet/source-build-externals -->
-    <MicrosoftSourceBuildIntermediatesourcebuildexternalsVersion>8.0.0-alpha.1.24379.1</MicrosoftSourceBuildIntermediatesourcebuildexternalsVersion>
+    <MicrosoftSourceBuildIntermediatesourcebuildexternalsVersion>8.0.0-alpha.1.24510.2</MicrosoftSourceBuildIntermediatesourcebuildexternalsVersion>
     <!-- Packages from dotnet/source-build-reference-packages -->
     <MicrosoftSourceBuildIntermediatesourcebuildreferencepackagesVersion>8.0.0-alpha.1.24415.1</MicrosoftSourceBuildIntermediatesourcebuildreferencepackagesVersion>
     <!-- Packages from dotnet/symreader -->
diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml
@@ -28,7 +28,16 @@ steps:
       # Calculate the expiration of the SAS token and convert to UTC
       $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
 
-      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+      # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+      # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+      $sas = ""
+      do {
+        $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+        if ($LASTEXITCODE -ne 0) {
+          Write-Error "Failed to generate SAS token."
+          exit 1
+        }
+      } while($sas.IndexOf('/') -ne -1)
 
       if ($LASTEXITCODE -ne 0) {
         Write-Error "Failed to generate SAS token."
diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml
@@ -28,7 +28,16 @@ steps:
       # Calculate the expiration of the SAS token and convert to UTC
       $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
 
-      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+      # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+      # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+      $sas = ""
+      do {
+        $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+        if ($LASTEXITCODE -ne 0) {
+          Write-Error "Failed to generate SAS token."
+          exit 1
+        }
+      } while($sas.IndexOf('/') -ne -1)
 
       if ($LASTEXITCODE -ne 0) {
         Write-Error "Failed to generate SAS token."
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -892,7 +892,7 @@ function IsWindowsPlatform() {
 }
 
 function Get-Darc($version) {
-  $darcPath  = "$TempDir\darc\$(New-Guid)"
+  $darcPath  = "$TempDir\darc\$([guid]::NewGuid())"
   if ($version -ne $null) {
     & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
   } else {
diff --git a/global.json b/global.json
@@ -27,7 +27,7 @@
   },
   "msbuild-sdks": {
     "Yarn.MSBuild": "1.22.19",
-    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24504.2",
-    "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24504.2"
+    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24516.1",
+    "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24516.1"
   }
 }
diff --git a/src/Shared/CertificateGeneration/CertificateManager.cs b/src/Shared/CertificateGeneration/CertificateManager.cs
@@ -328,6 +328,7 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(
                 var exportDir = Path.GetDirectoryName(path);
                 if (!string.IsNullOrEmpty(exportDir) && !Directory.Exists(exportDir))
                 {
+                    result = EnsureCertificateResult.ErrorExportingTheCertificateToNonExistentDirectory;
                     throw new InvalidOperationException($"The directory '{exportDir}' does not exist.  Choose permissions carefully when creating it.");
                 }
 
diff --git a/src/Shared/CertificateGeneration/EnsureCertificateResult.cs b/src/Shared/CertificateGeneration/EnsureCertificateResult.cs
@@ -10,6 +10,7 @@ internal enum EnsureCertificateResult
     ErrorCreatingTheCertificate,
     ErrorSavingTheCertificateIntoTheCurrentUserPersonalStore,
     ErrorExportingTheCertificate,
+    ErrorExportingTheCertificateToNonExistentDirectory,
     FailedToTrustTheCertificate,
     PartiallyFailedToTrustTheCertificate,
     UserCancelledTrustStep,
diff --git a/src/Tools/FirstRunCertGenerator/test/CertificateManagerTests.cs b/src/Tools/FirstRunCertGenerator/test/CertificateManagerTests.cs
@@ -373,7 +373,7 @@ public void EnsureCreateHttpsCertificate_CannotExportToNonExistentDirectory()
             .EnsureAspNetCoreHttpsDevelopmentCertificate(now, now.AddYears(1), Path.Combine("NoSuchDirectory", CertificateName));
 
         // Assert
-        Assert.Equal(EnsureCertificateResult.ErrorExportingTheCertificate, result);
+        Assert.Equal(EnsureCertificateResult.ErrorExportingTheCertificateToNonExistentDirectory, result);
     }
 
     [Fact]
diff --git a/src/Tools/dotnet-dev-certs/src/Program.cs b/src/Tools/dotnet-dev-certs/src/Program.cs
@@ -425,6 +425,10 @@ private static int EnsureHttpsCertificate(CommandOption exportPath, CommandOptio
             case EnsureCertificateResult.ErrorExportingTheCertificate:
                 reporter.Warn("There was an error exporting the HTTPS developer certificate to a file.");
                 return ErrorExportingTheCertificate;
+            case EnsureCertificateResult.ErrorExportingTheCertificateToNonExistentDirectory:
+                // A distinct warning is useful, but a distinct error code is probably not.
+                reporter.Warn("There was an error exporting the HTTPS developer certificate to a file. Please create the target directory before exporting. Choose permissions carefully when creating it.");
+                return ErrorExportingTheCertificate;
             case EnsureCertificateResult.PartiallyFailedToTrustTheCertificate:
                 // A distinct warning is useful, but a distinct error code is probably not.
                 reporter.Warn("There was an error trusting the HTTPS developer certificate. It will be trusted by some clients but not by others.");

Original file line number	Diff line number	Diff line change
`@@ -892,7 +892,7 @@ function IsWindowsPlatform() {`
`892`	`892`	`}`
`893`	`893`
`894`	`894`	`function Get-Darc($version) {`
`895`		`- $darcPath = "$TempDir\darc\$(New-Guid)"`
	`895`	`+ $darcPath = "$TempDir\darc\$([guid]::NewGuid())"`
`896`	`896`	`if ($version -ne $null) {`
`897`	`897`	`& $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version \| Out-Host`
`898`	`898`	`} else {`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@`
`27`	`27`	`},`
`28`	`28`	`"msbuild-sdks": {`
`29`	`29`	`"Yarn.MSBuild": "1.22.19",`
`30`		`- "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24504.2",`
`31`		`- "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24504.2"`
	`30`	`+ "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24516.1",`
	`31`	`+ "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24516.1"`
`32`	`32`	`}`
`33`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -328,6 +328,7 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(`
`328`	`328`	`var exportDir = Path.GetDirectoryName(path);`
`329`	`329`	`if (!string.IsNullOrEmpty(exportDir) && !Directory.Exists(exportDir))`
`330`	`330`	`{`
	`331`	`+ result = EnsureCertificateResult.ErrorExportingTheCertificateToNonExistentDirectory;`
`331`	`332`	`throw new InvalidOperationException($"The directory '{exportDir}' does not exist. Choose permissions carefully when creating it.");`
`332`	`333`	`}`
`333`	`334`
Original file line number	Diff line number	Diff line change
`@@ -373,7 +373,7 @@ public void EnsureCreateHttpsCertificate_CannotExportToNonExistentDirectory()`
`373`	`373`	`.EnsureAspNetCoreHttpsDevelopmentCertificate(now, now.AddYears(1), Path.Combine("NoSuchDirectory", CertificateName));`
`374`	`374`
`375`	`375`	`// Assert`
`376`		`- Assert.Equal(EnsureCertificateResult.ErrorExportingTheCertificate, result);`
	`376`	`+ Assert.Equal(EnsureCertificateResult.ErrorExportingTheCertificateToNonExistentDirectory, result);`
`377`	`377`	`}`
`378`	`378`
`379`	`379`	`[Fact]`