[main] Update dependencies from dotnet/arcade

NuGet.config

@@ -5,6 +5,7 @@
     <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
+    <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />

eng/Version.Details.xml

@@ -15,9 +15,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.22630.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23118.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e82404fca08383513e0b0b3c5308d4a9b18b7c7a</Sha>
+      <Sha>38d3196e1596eb525367ec8d65fcc1f30a58261c</Sha>
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="8.0.0-alpha.1.23179.1">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>

eng/common/cross/build-rootfs.sh

@@ -48,12 +48,14 @@ __UbuntuPackages+=" symlinks"
 __UbuntuPackages+=" libicu-dev"
 __UbuntuPackages+=" liblttng-ust-dev"
 __UbuntuPackages+=" libunwind8-dev"
+__UbuntuPackages+=" libnuma-dev"
 
 __AlpinePackages+=" gettext-dev"
 __AlpinePackages+=" icu-dev"
 __AlpinePackages+=" libunwind-dev"
 __AlpinePackages+=" lttng-ust-dev"
 __AlpinePackages+=" compiler-rt-static"
+__AlpinePackages+=" numactl-dev"
 
 # runtime libraries' dependencies
 __UbuntuPackages+=" libcurl4-openssl-dev"
@@ -147,17 +149,17 @@ while :; do
             __BuildArch=ppc64le
             __UbuntuArch=ppc64el
             __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
+            __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp5/}"
             unset __LLDB_Package
             ;;
         riscv64)
             __BuildArch=riscv64
             __UbuntuArch=riscv64
             __UbuntuRepo="http://deb.debian.org/debian-ports"
             __CodeName=sid
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+            __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
             unset __LLDB_Package
 
             if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then
@@ -168,9 +170,9 @@ while :; do
             __BuildArch=s390x
             __UbuntuArch=s390x
             __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
+            __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp5/}"
             unset __LLDB_Package
             ;;
         x64)
@@ -310,6 +312,8 @@ done
 
 if [[ "$__BuildArch" == "armel" ]]; then
     __LLDB_Package="lldb-3.5-dev"
+elif [[ "$__BuildArch" == "arm" && "$__AlpineVersion" == "3.13" ]]; then
+    __AlpinePackages="${__AlpinePackages//numactl-dev/}"
 fi
 
 __UbuntuPackages+=" ${__LLDB_Package:-}"

eng/common/generate-locproject.ps1

@@ -34,6 +34,25 @@ $jsonTemplateFiles | ForEach-Object {
 $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
 $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
+if (-not $wxlFiles) {
+    $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } #  pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
+    if ($wxlEnFiles) {
+      $wxlFiles = @()
+      $wxlEnFiles | ForEach-Object {
+        $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+        $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+      }
+    }
+}
+
+$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files
+$macosHtmlFiles = @()
+if ($macosHtmlEnFiles) {
+    $macosHtmlEnFiles | ForEach-Object {
+        $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+        $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+    }
+}
 
 $xlfFiles = @()
 
@@ -99,8 +118,7 @@ $locJson = @{
                     $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
                     $continue = $true
                     foreach ($exclusion in $exclusions.Exclusions) {
-                        if ($_.FullName.Contains($exclusion))
-                        {
+                        if ($_.FullName.Contains($exclusion)) {
                             $continue = $false
                         }
                     }
@@ -115,6 +133,35 @@ $locJson = @{
                     }
                 }
             )
+        },
+        @{
+            LanguageSet = $LanguageSet
+            CloneLanguageSet = "VS_macOS_CloneLanguages"
+            LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" )
+            LocItems = @(
+                $macosHtmlFiles | ForEach-Object {
+                    $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($_.FullName.Contains($exclusion)) {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    $lciFile = $sourceFile + ".lci"
+                    if ($continue) {
+                        $result = @{
+                            SourceFile = $sourceFile
+                            CopyOption = "LangIDOnPath"
+                            OutputPath = $outputPath
+                        }
+                        if (Test-Path $lciFile -PathType Leaf) {
+                            $result["LciFile"] = $lciFile
+                        }
+                        return $result
+                    }
+                }
+            )
         }
     )
 }

eng/common/init-tools-native.ps1

@@ -83,7 +83,8 @@ try {
                     Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
   if ($NativeTools) {
     if ($PathPromotion -eq $True) {
-      if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build
+      $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools"
+      if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine
         $NativeTools.PSObject.Properties | ForEach-Object {
           $ToolName = $_.Name
           $ToolVersion = $_.Value
@@ -93,11 +94,6 @@ try {
             if ($ToolVersion -eq "latest") {
               $ToolVersion = ""
             }
-            $ArcadeToolsDirectory = "C:\arcade-tools"
-            if (-not (Test-Path $ArcadeToolsDirectory)) {
-              Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed."
-              exit 1
-            }
             $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)
             if ($ToolDirectories -eq $null) {
               Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
@@ -125,6 +121,7 @@ try {
 
           if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
             Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding."
+            Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image."
           }
         }
         exit 0

eng/common/native/init-compiler.sh

@@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then
     # Set default versions
     if [ -z "$majorVersion" ]; then
         # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
-        if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
+        if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
         elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi
 
         for version in $versions; do

eng/common/templates/job/job.yml

@@ -88,6 +88,16 @@ jobs:
     - ${{ if ne(variable.group, '') }}:
       - group: ${{ variable.group }}
 
+    # handle template variable syntax
+    # example:
+    # - template: path/to/template.yml
+    #   parameters:
+    #     [key]: [value]
+    - ${{ if ne(variable.template, '') }}:
+      - template: ${{ variable.template }}
+        ${{ if ne(variable.parameters, '') }}:
+          parameters: ${{ variable.parameters }}
+
     # handle key-value variable syntax.
     # example:
     # - [key]: [value]
@@ -124,7 +134,7 @@ jobs:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@0
 
-  - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+  - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}:
     - task: DownloadPipelineArtifact@2
       inputs:
         buildType: current
@@ -161,7 +171,7 @@ jobs:
           TeamName: $(_TeamName)
 
   - ${{ if ne(parameters.artifacts.publish, '') }}:
-    - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+    - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
       - task: CopyFiles@2
         displayName: Gather binaries for publish to artifacts
         inputs:
@@ -182,7 +192,7 @@ jobs:
           ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
         continueOnError: true
         condition: always()
-    - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+    - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
       - publish: artifacts/log
         artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
         displayName: Publish logs

eng/common/templates/job/source-build.yml

@@ -46,20 +46,12 @@ jobs:
     # source-build builds run in Docker, including the default managed platform.
     # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic
     pool:
-      # Main environments
-      ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}:
-        name: NetCore-Public
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
-      ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}:
-        name: NetCore1ESPool-Internal
-        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
 
-      # Servicing build environments
-      ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}:
-        name: NetCore-Svc-Public
-        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
-      ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}:
-        name: NetCore1ESPool-Svc-Internal
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
 
   ${{ if ne(parameters.platform.pool, '') }}:

eng/common/templates/job/source-index-stage1.yml

@@ -1,6 +1,6 @@
 parameters:
   runAsPublic: false
-  sourceIndexPackageVersion: 1.0.1-20220804.1
+  sourceIndexPackageVersion: 1.0.1-20221220.2
   sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
@@ -40,10 +40,10 @@ jobs:
     - ${{ preStep }}
 
   - task: UseDotNet@2
-    displayName: Use .NET Core sdk 3.1
+    displayName: Use .NET Core SDK 6
     inputs:
       packageType: sdk
-      version: 3.1.x
+      version: 6.0.x
       installationPath: $(Agent.TempDirectory)/dotnet
       workingDirectory: $(Agent.TempDirectory)
 

eng/common/templates/variables/pool-providers.yml

@@ -16,7 +16,7 @@
 #  First, import the template in an arcade-ified repo to pick up the variables, e.g.:
 #
 #  variables:
-#  - template: eng/common/templates/variables/pool-providers.yml
+#  - template: /eng/common/templates/variables/pool-providers.yml
 #
 #  ... then anywhere specifying the pool provider use the runtime variables,
 #      $(DncEngInternalBuildPool) and $  (DncEngPublicBuildPool), e.g.:
@@ -26,23 +26,32 @@
 #           demands: ImageOverride -equals windows.vs2019.amd64
 
 variables:
-# Coalesce the target and source branches so we know when a PR targets a release branch
-# If these variables are somehow missing, fall back to main (tends to have more capacity)
-- name: BranchNameForPoolSelection
-  value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }}
+  # Coalesce the target and source branches so we know when a PR targets a release branch
+  # If these variables are somehow missing, fall back to main (tends to have more capacity)
 
-# Any new -Svc alternative pools should have variables added here to allow for splitting work
-
-# Main branch pools
-- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}:
+  # Any new -Svc alternative pools should have variables added here to allow for splitting work
   - name: DncEngPublicBuildPool
-    value: NetCore-Public
-  - name: DncEngInternalBuildPool
-    value: NetCore1ESPool-Internal
+    value: $[
+        replace(
+          replace(
+            eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+            True,
+            'NetCore-Svc-Public'
+          ),
+          False,
+          'NetCore-Public'
+        )
+      ]
 
-# Release branch pools
-- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}:
-  - name: DncEngPublicBuildPool
-    value: NetCore-Svc-Public
   - name: DncEngInternalBuildPool
-    value: NetCore1ESPool-Svc-Internal
+    value: $[
+        replace(
+          replace(
+            eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+            True,
+            'NetCore1ESPool-Svc-Internal'
+          ),
+          False,
+          'NetCore1ESPool-Internal'
+        )
+      ]

eng/common/tools.ps1

@@ -581,7 +581,7 @@ function InitializeBuildTool() {
       ExitWithExitCode 1
     }
     $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
-    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net7.0' }
+    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' }
   } elseif ($msbuildEngine -eq "vs") {
     try {
       $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -743,6 +743,8 @@ function MSBuild() {
       (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
       (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
       (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
+      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
     )
     $selectedPath = $null
     foreach ($path in $possiblePaths) {

eng/common/tools.sh

@@ -312,7 +312,7 @@ function InitializeBuildTool {
   # return values
   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
   _InitializeBuildToolCommand="msbuild"
-  _InitializeBuildToolFramework="net7.0"
+  _InitializeBuildToolFramework="net8.0"
 }
 
 # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
@@ -428,6 +428,8 @@ function MSBuild {
     possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
     possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" )
     possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+    possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" )
+    possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" )
     for path in "${possiblePaths[@]}"; do
       if [[ -f $path ]]; then
         selectedPath=$path

global.json

@@ -1,8 +1,8 @@
 {
   "tools": {
-    "dotnet": "7.0.100"
+    "dotnet": "8.0.100-alpha.1.23061.8"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22630.1"
+    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23118.1"
   }
 }

src/Directory.Build.targets

@@ -5,7 +5,7 @@
 
   <ItemGroup>
     <NuspecProperty Include="DesktopTfm=net472"/>
-    <NuspecProperty Include="CoreTfm=net7.0" Condition="'$(DotNetBuildFromSource)' == 'true'"/>
+    <NuspecProperty Include="CoreTfm=$(NetCurrent)" Condition="'$(DotNetBuildFromSource)' == 'true'"/>
     <NuspecProperty Include="CoreTfm=netcoreapp3.1" Condition="'$(DotNetBuildFromSource)' != 'true'"/>
   </ItemGroup>