[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23577.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23628.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0b786af5e4da6164ac878842bd449d4d58ce865e</Sha>
+      <Sha>9e5fd26c0d1a187dfa08242d83e341634d4b1b07</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/build.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*"
+exit /b %ErrorLevel%

eng/common/build.ps1

@@ -19,6 +19,7 @@ Param(
   [switch] $pack,
   [switch] $publish,
   [switch] $clean,
+  [switch] $verticalBuild,
   [switch][Alias('bl')]$binaryLog,
   [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $ci,
@@ -58,6 +59,7 @@ function Print-Usage() {
   Write-Host "  -sign                   Sign build outputs"
   Write-Host "  -publish                Publish artifacts (e.g. symbols)"
   Write-Host "  -clean                  Clean the solution"
+  Write-Host "  -verticalBuild          Run in 'vertical build' infra mode."
   Write-Host ""
 
   Write-Host "Advanced settings:"
@@ -120,6 +122,7 @@ function Build {
     /p:Deploy=$deploy `
     /p:Test=$test `
     /p:Pack=$pack `
+    /p:ArcadeBuildVertical=$verticalBuild `
     /p:IntegrationTest=$integrationTest `
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `

eng/common/build.sh

@@ -59,6 +59,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 restore=false
 build=false
 source_build=false
+vertical_build=false
 rebuild=false
 test=false
 integration_test=false
@@ -129,6 +130,12 @@ while [[ $# > 0 ]]; do
       restore=true
       pack=true
       ;;
+    -verticalbuild|-vb)
+      build=true
+      vertical_build=true
+      restore=true
+      pack=true
+      ;;
     -test|-t)
       test=true
       ;;
@@ -220,6 +227,7 @@ function Build {
     /p:Restore=$restore \
     /p:Build=$build \
     /p:ArcadeBuildFromSource=$source_build \
+    /p:ArcadeBuildVertical=$vertical_build \
     /p:Rebuild=$rebuild \
     /p:Test=$test \
     /p:Pack=$pack \

eng/common/darc-init.ps1

@@ -1,6 +1,6 @@
 param (
     $darcVersion = $null,
-    $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16',
     $verbosity = 'minimal',
     $toolpath = $null
 )

eng/common/darc-init.sh

@@ -2,7 +2,7 @@
 
 source="${BASH_SOURCE[0]}"
 darcVersion=''
-versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do

eng/common/post-build/add-build-to-channel.ps1

@@ -2,7 +2,7 @@ param(
   [Parameter(Mandatory=$true)][int] $BuildId,
   [Parameter(Mandatory=$true)][int] $ChannelId,
   [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
 )
 

eng/common/post-build/publish-using-darc.ps1

@@ -3,7 +3,7 @@ param(
   [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
   [Parameter(Mandatory=$true)][string] $AzdoToken,
   [Parameter(Mandatory=$true)][string] $MaestroToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
   [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters

eng/common/post-build/redact-logs.ps1

@@ -0,0 +1,81 @@
+[CmdletBinding(PositionalBinding=$False)]
+param(
+  [Parameter(Mandatory=$true, Position=0)][string] $InputPath,
+  [Parameter(Mandatory=$true)][string] $BinlogToolVersion,
+  [Parameter(Mandatory=$false)][string] $DotnetPath,
+  [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json',
+  # File with strings to redact - separated by newlines.
+  #  For comments start the line with '# ' - such lines are ignored 
+  [Parameter(Mandatory=$false)][string] $TokensFilePath,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  $packageName = 'binlogtool'
+
+  $dotnet = $DotnetPath
+
+  if (!$dotnet) {
+    $dotnetRoot = InitializeDotNetCli -install:$true
+    $dotnet = "$dotnetRoot\dotnet.exe"
+  }
+
+  $toolList = & "$dotnet" tool list -g
+
+  if ($toolList -like "*$packageName*") {
+    & "$dotnet" tool uninstall $packageName -g
+  }
+
+  $toolPath  = "$PSScriptRoot\..\..\..\.tools"
+  $verbosity = 'minimal'
+
+  New-Item -ItemType Directory -Force -Path $toolPath
+
+  Push-Location -Path $toolPath
+
+  try {
+    Write-Host "Installing Binlog redactor CLI..."
+    Write-Host "'$dotnet' new tool-manifest"
+    & "$dotnet" new tool-manifest
+    Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion"
+    & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion
+
+    if (Test-Path $TokensFilePath) {
+        Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath
+        $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " }
+    }
+
+    $optionalParams = [System.Collections.ArrayList]::new()
+
+    Foreach ($p in $TokensToRedact)
+    {
+      if($p -match '^\$\(.*\)$')
+      {
+        Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable"  -f $p)
+      }          
+      elseif($p)
+      {
+        $optionalParams.Add("-p:" + $p) | Out-Null
+      }
+    }
+
+    & $dotnet binlogtool redact --input:$InputPath --recurse --in-place `
+      @optionalParams
+
+    if ($LastExitCode -ne 0) {
+      Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now."
+    }
+  }
+  finally {
+    Pop-Location
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_"
+  ExitWithExitCode 1
+}

eng/common/post-build/trigger-subscriptions.ps1

@@ -2,7 +2,7 @@ param(
   [Parameter(Mandatory=$true)][string] $SourceRepo,
   [Parameter(Mandatory=$true)][int] $ChannelId,
   [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
 )
 

eng/common/sdl/trim-assets-version.ps1

@@ -72,4 +72,4 @@ catch {
   Write-Host $_
   Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
   ExitWithExitCode 1
-}
+}

eng/common/templates/job/job.yml

@@ -136,7 +136,7 @@ jobs:
         condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
-    - task: NuGetAuthenticate@0
+    - task: NuGetAuthenticate@1
 
   - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}:
     - task: DownloadPipelineArtifact@2

eng/common/templates/job/publish-build-assets.yml

@@ -48,8 +48,8 @@ jobs:
     - group: AzureDevOps-Artifact-Feeds-Pats
     - name: runCodesignValidationInjection
       value: false
-    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-      - template: /eng/common/templates/post-build/common-variables.yml
+    # unconditional - needed for logs publishing (redactor tool version)
+    - template: /eng/common/templates/post-build/common-variables.yml
 
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
@@ -72,7 +72,7 @@ jobs:
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
 
-    - task: NuGetAuthenticate@0
+    - task: NuGetAuthenticate@1
 
     - task: PowerShell@2
       displayName: Publish Build Assets
@@ -81,7 +81,7 @@ jobs:
         arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
           /p:BuildAssetRegistryToken=$(MaestroAccessToken)
-          /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+          /p:MaestroApiEndpoint=https://maestro.dot.net
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
       condition: ${{ parameters.condition }}

eng/common/templates/job/source-index-stage1.yml

@@ -1,6 +1,6 @@
 parameters:
   runAsPublic: false
-  sourceIndexPackageVersion: 1.0.1-20230228.2
+  sourceIndexPackageVersion: 1.0.1-20231213.4
   sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
@@ -30,20 +30,20 @@ jobs:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
         name: $(DncEngPublicBuildPool)
-        demands: ImageOverride -equals windows.vs2019.amd64.open
+        demands: ImageOverride -equals windows.vs2022.amd64.open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: $(DncEngInternalBuildPool)
-        demands: ImageOverride -equals windows.vs2019.amd64
+        demands: ImageOverride -equals windows.vs2022.amd64
 
   steps:
   - ${{ each preStep in parameters.preSteps }}:
     - ${{ preStep }}
 
   - task: UseDotNet@2
-    displayName: Use .NET Core SDK 6
+    displayName: Use .NET 8 SDK
     inputs:
       packageType: sdk
-      version: 6.0.x
+      version: 8.0.x
       installationPath: $(Agent.TempDirectory)/dotnet
       workingDirectory: $(Agent.TempDirectory)
 

eng/common/templates/post-build/common-variables.yml

@@ -7,7 +7,7 @@ variables:
 
   # Default Maestro++ API Endpoint and API Version
   - name: MaestroApiEndPoint
-    value: "https://maestro-prod.westus2.cloudapp.azure.com"
+    value: "https://maestro.dot.net"
   - name: MaestroApiAccessToken
     value: $(MaestroAccessToken)
   - name: MaestroApiVersion
@@ -17,6 +17,8 @@ variables:
     value: 3.0.0
   - name: SymbolToolVersion
     value: 1.0.1
+  - name: BinlogToolVersion
+    value: 1.0.9
 
   - name: runCodesignValidationInjection
     value: false

eng/common/templates/post-build/post-build.yml

@@ -169,7 +169,7 @@ stages:
         # This is necessary whenever we want to publish/restore to an AzDO private feed
         # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
         # otherwise it'll complain about accessing a private feed.
-        - task: NuGetAuthenticate@0
+        - task: NuGetAuthenticate@1
           displayName: 'Authenticate to AzDO Feeds'
 
         # Signing validation will optionally work with the buildmanifest file which is downloaded from
@@ -187,6 +187,7 @@ stages:
           parameters:
             StageLabel: 'Validation'
             JobLabel: 'Signing'
+            BinlogToolVersion: $(BinlogToolVersion)
 
     - job:
       displayName: SourceLink Validation
@@ -266,7 +267,7 @@ stages:
             BARBuildId: ${{ parameters.BARBuildId }}
             PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-        - task: NuGetAuthenticate@0
+        - task: NuGetAuthenticate@1
 
         - task: PowerShell@2
           displayName: Publish Using Darc

eng/common/templates/steps/publish-logs.yml

@@ -1,6 +1,9 @@
 parameters:
   StageLabel: ''
   JobLabel: ''
+  CustomSensitiveDataList: ''
+  # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed
+  BinlogToolVersion: '1.0.9'
 
 steps:
 - task: Powershell@2
@@ -12,7 +15,30 @@ steps:
       Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
   continueOnError: true
   condition: always()
-
+
+- task: PowerShell@2
+  displayName: Redact Logs
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1
+    # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml
+    # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+    #  If the file exists - sensitive data for redaction will be sourced from it
+    #  (single entry per line, lines starting with '# ' are considered comments and skipped)
+    arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' 
+      -BinlogToolVersion ${{parameters.BinlogToolVersion}}
+      -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+      '$(publishing-dnceng-devdiv-code-r-build-re)'
+      '$(MaestroAccessToken)'
+      '$(dn-bot-all-orgs-artifact-feeds-rw)'
+      '$(akams-client-id)'
+      '$(akams-client-secret)'
+      '$(microsoft-symbol-server-pat)'
+      '$(symweb-symbol-server-pat)'
+      '$(dn-bot-all-orgs-build-rw-code-rw)'
+      ${{parameters.CustomSensitiveDataList}}
+  continueOnError: true
+  condition: always()
+
 - task: PublishBuildArtifacts@1
   displayName: Publish Logs
   inputs:

eng/common/templates/steps/source-build.yml

@@ -105,7 +105,7 @@ steps:
     Contents: |
       **/*.log
       **/*.binlog
-      artifacts/source-build/self/prebuilt-report/**
+      artifacts/sb/prebuilt-report/**
     TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
     CleanTargetFolder: true
   continueOnError: true
@@ -126,4 +126,4 @@ steps:
 - task: ComponentGovernanceComponentDetection@0
   displayName: Component Detection (Exclude upstream cache)
   inputs:
-    ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache'
+    ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache'