dotnet · twsouthwick · Nov 15, 2024 · Sep 2, 2024 · Sep 6, 2024 · Sep 9, 2024
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24423.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.24560.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>91599268652b51969b8d8088d4f2f2ba7b3ebb19</Sha>
+      <Sha>232061b49ae2157efbb83acde9acae79ef3d6d40</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/common/build.sh b/eng/common/build.sh
@@ -231,7 +231,6 @@ function Build {
     /p:Restore=$restore \
     /p:Build=$build \
     /p:DotNetBuildRepo=$product_build \
-    /p:ArcadeBuildFromSource=$source_build \
     /p:DotNetBuildSourceOnly=$source_build \
     /p:Rebuild=$rebuild \
     /p:Test=$test \

diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml
@@ -19,6 +19,7 @@ parameters:
   # publishing defaults
   artifacts: ''
   enableMicrobuild: false
+  enableMicrobuildForMacAndLinux: false
   enablePublishBuildArtifacts: false
   enablePublishBuildAssets: false
   enablePublishTestResults: false
@@ -127,18 +128,11 @@ jobs:
       - ${{ preStep }}
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-      - task: MicroBuildSigningPlugin@4
-        displayName: Install MicroBuild plugin
-        inputs:
-          signType: $(_SignType)
-          zipSources: false
-          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
-        env:
-          TeamName: $(_TeamName)
-          MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+    - template: /eng/common/core-templates/steps/install-microbuild.yml
+      parameters:
+        enableMicrobuild: ${{ parameters.enableMicrobuild }}
+        enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
         continueOnError: ${{ parameters.continueOnError }}
-        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@1
@@ -167,14 +161,12 @@ jobs:
   - ${{ each step in parameters.componentGovernanceSteps }}:
     - ${{ step }}
 
-  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      - task: MicroBuildCleanup@1
-        displayName: Execute Microbuild cleanup tasks
-        condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
+      parameters:
+        enableMicrobuild: ${{ parameters.enableMicrobuild }}
+        enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
         continueOnError: ${{ parameters.continueOnError }}
-        env:
-          TeamName: $(_TeamName)
 
   # Publish test results
   - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:

diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml
@@ -1,8 +1,5 @@
 parameters:
   runAsPublic: false
-  sourceIndexUploadPackageVersion: 2.0.0-20240522.1
-  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
-  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
   binlogPath: artifacts/log/Debug/Build.binlog
@@ -16,12 +13,6 @@ jobs:
   dependsOn: ${{ parameters.dependsOn }}
   condition: ${{ parameters.condition }}
   variables:
-  - name: SourceIndexUploadPackageVersion
-    value: ${{ parameters.sourceIndexUploadPackageVersion }}
-  - name: SourceIndexProcessBinlogPackageVersion
-    value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
-  - name: SourceIndexPackageSource
-    value: ${{ parameters.sourceIndexPackageSource }}
   - name: BinlogPath
     value: ${{ parameters.binlogPath }}
   - template: /eng/common/core-templates/variables/pool-providers.yml
@@ -34,48 +25,20 @@ jobs:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
         name: $(DncEngPublicBuildPool)
-        image: 1es-windows-2022-open
-        os: windows
+        image: windows.vs2022.amd64.open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: $(DncEngInternalBuildPool)
-        image: 1es-windows-2022
-        os: windows
+        image: windows.vs2022.amd64
 
   steps:
   - ${{ if eq(parameters.is1ESPipeline, '') }}:
     - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
 
   - ${{ each preStep in parameters.preSteps }}:
     - ${{ preStep }}
-
-  - task: UseDotNet@2
-    displayName: Use .NET 8 SDK
-    inputs:
-      packageType: sdk
-      version: 8.0.x
-      installationPath: $(Agent.TempDirectory)/dotnet
-      workingDirectory: $(Agent.TempDirectory)
-
-  - script: |
-      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-    displayName: Download Tools
-    # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
-    workingDirectory: $(Agent.TempDirectory)
-
   - script: ${{ parameters.sourceIndexBuildCommand }}
     displayName: Build Repository
 
-  - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
-    displayName: Process Binlog into indexable sln
-
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - task: AzureCLI@2
-      displayName: Log in to Azure and upload stage1 artifacts to source index
-      inputs:
-        azureSubscription: 'SourceDotNet Stage1 Publish'
-        addSpnToEnvironment: true
-        scriptType: 'ps'
-        scriptLocation: 'inlineScript'
-        inlineScript: |
-          $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
+  - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
+    parameters:
+      binLogPath: ${{ parameters.binLogPath }}
diff --git a/eng/common/core-templates/steps/cleanup-microbuild.yml b/eng/common/core-templates/steps/cleanup-microbuild.yml
@@ -0,0 +1,28 @@
+parameters:
+  # Enable cleanup tasks for MicroBuild
+  enableMicrobuild: false
+  # Enable cleanup tasks for MicroBuild on Mac and Linux
+  # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+  enableMicrobuildForMacAndLinux: false
+  continueOnError: false
+
+steps:
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - task: MicroBuildCleanup@1
+      displayName: Execute Microbuild cleanup tasks
+      condition: and(
+        always(),
+        or(
+          and(
+            eq(variables['Agent.Os'], 'Windows_NT'),
+            in(variables['_SignType'], 'real', 'test')
+          ),
+          and(
+            ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+            ne(variables['Agent.Os'], 'Windows_NT'),
+            eq(variables['_SignType'], 'real')
+          )
+        ))
+      continueOnError: ${{ parameters.continueOnError }}
+      env:
+        TeamName: $(_TeamName)
diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml
@@ -0,0 +1,43 @@
+parameters:
+  # Enable cleanup tasks for MicroBuild
+  enableMicrobuild: false
+  # Enable cleanup tasks for MicroBuild on Mac and Linux
+  # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+  enableMicrobuildForMacAndLinux: false
+  continueOnError: false
+
+steps:
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    # Remove Python downgrade with https://github.com/dotnet/arcade/issues/15151
+    - ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python 3.11.x'
+        inputs:
+          versionSpec: '3.11.x'
+
+    - task: MicroBuildSigningPlugin@4
+      displayName: Install MicroBuild plugin
+      inputs:
+        signType: $(_SignType)
+        zipSources: false
+        feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+        ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
+          azureSubscription: 'MicroBuild Signing Task (DevDiv)'
+      env:
+        TeamName: $(_TeamName)
+        MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      continueOnError: ${{ parameters.continueOnError }}
+      condition: and(
+        succeeded(),
+        or(
+          and(
+            eq(variables['Agent.Os'], 'Windows_NT'),
+            in(variables['_SignType'], 'real', 'test')
+          ),
+          and(
+            ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+            ne(variables['Agent.Os'], 'Windows_NT'),
+            eq(variables['_SignType'], 'real')
+          )
+        ))
diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml
@@ -34,7 +34,9 @@ steps:
       '$(akams-client-id)'
       '$(microsoft-symbol-server-pat)'
       '$(symweb-symbol-server-pat)'
+      '$(dnceng-symbol-server-pat)'
       '$(dn-bot-all-orgs-build-rw-code-rw)'
+      '$(System.AccessToken)'
       ${{parameters.CustomSensitiveDataList}}
   continueOnError: true
   condition: always()
@@ -45,6 +47,7 @@ steps:
     SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs'
     Contents: '**'
     TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
+  condition: always()
 
 - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
   parameters:

diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml
@@ -86,7 +86,6 @@ steps:
       $runtimeOsArgs \
       $baseOsArgs \
       /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
-      /p:ArcadeBuildFromSource=true \
       /p:DotNetBuildSourceOnly=true \
       /p:DotNetBuildRepo=true \
       /p:AssetManifestFileName=$assetManifestFileName

diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml
@@ -0,0 +1,35 @@
+parameters:
+  sourceIndexUploadPackageVersion: 2.0.0-20240522.1
+  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
+  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+  binlogPath: artifacts/log/Debug/Build.binlog
+
+steps:
+- task: UseDotNet@2
+  displayName: "Source Index: Use .NET 8 SDK"
+  inputs:
+    packageType: sdk
+    version: 8.0.x
+    installationPath: $(Agent.TempDirectory)/dotnet
+    workingDirectory: $(Agent.TempDirectory)
+
+- script: |
+    $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
+    $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
+  displayName: "Source Index: Download netsourceindex Tools"
+  # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
+  workingDirectory: $(Agent.TempDirectory)
+
+- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+  displayName: "Source Index: Process Binlog into indexable sln"
+
+- ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - task: AzureCLI@2
+    displayName: "Source Index: Upload Source Index stage1 artifacts to Azure"
+    inputs:
+      azureSubscription: 'SourceDotNet Stage1 Publish'
+      addSpnToEnvironment: true
+      scriptType: 'ps'
+      scriptLocation: 'inlineScript'
+      inlineScript: |
+        $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
@@ -66,6 +66,7 @@ __UbuntuPackages+=" libcurl4-openssl-dev"
 __UbuntuPackages+=" libkrb5-dev"
 __UbuntuPackages+=" libssl-dev"
 __UbuntuPackages+=" zlib1g-dev"
+__UbuntuPackages+=" libbrotli-dev"
 
 __AlpinePackages+=" curl-dev"
 __AlpinePackages+=" krb5-dev"
@@ -91,18 +92,18 @@ __HaikuPackages="gcc_syslibs"
 __HaikuPackages+=" gcc_syslibs_devel"
 __HaikuPackages+=" gmp"
 __HaikuPackages+=" gmp_devel"
-__HaikuPackages+=" icu66"
-__HaikuPackages+=" icu66_devel"
+__HaikuPackages+=" icu[0-9]+"
+__HaikuPackages+=" icu[0-9]*_devel"
 __HaikuPackages+=" krb5"
 __HaikuPackages+=" krb5_devel"
 __HaikuPackages+=" libiconv"
 __HaikuPackages+=" libiconv_devel"
-__HaikuPackages+=" llvm12_libunwind"
-__HaikuPackages+=" llvm12_libunwind_devel"
+__HaikuPackages+=" llvm[0-9]*_libunwind"
+__HaikuPackages+=" llvm[0-9]*_libunwind_devel"
 __HaikuPackages+=" mpfr"
 __HaikuPackages+=" mpfr_devel"
-__HaikuPackages+=" openssl"
-__HaikuPackages+=" openssl_devel"
+__HaikuPackages+=" openssl3"
+__HaikuPackages+=" openssl3_devel"
 __HaikuPackages+=" zlib"
 __HaikuPackages+=" zlib_devel"
 
@@ -496,7 +497,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
     arch="$(uname -m)"
 
     ensureDownloadTool
-    
+
     if [[ "$__hasWget" == 1 ]]; then
         wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
     else
@@ -681,7 +682,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then
 
     ensureDownloadTool
 
-    echo "Downloading Haiku package tool"
+    echo "Downloading Haiku package tools"
     git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
     if [[ "$__hasWget" == 1 ]]; then
         wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
@@ -691,34 +692,42 @@ elif [[ "$__CodeName" == "haiku" ]]; then
 
     unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
 
-    DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
-    HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+    HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+    HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current"
+
+    echo "Downloading HaikuPorts package repository index..."
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
+    else
+        curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
+    fi
 
-    # Download Haiku packages
     echo "Downloading Haiku packages"
     read -ra array <<<"$__HaikuPackages"
     for package in "${array[@]}"; do
         echo "Downloading $package..."
-        # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
-        # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
+        hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" |
+            grep -E "${package}-" | sort -V | tail -n 1 | xargs)"
+        if [ -z "$hpkgFilename" ]; then
+            >&2 echo "ERROR: package $package missing."
+            exit 1
+        fi
+        echo "Resolved filename: $hpkgFilename..."
+        hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename"
         if [[ "$__hasWget" == 1 ]]; then
-            hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
-                --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
             wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
         else
-            hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
-                --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
             curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
         fi
     done
     for package in haiku haiku_devel; do
         echo "Downloading $package..."
         if [[ "$__hasWget" == 1 ]]; then
-            hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
-            wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+            hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+            wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
         else
-            hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
-            curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+            hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+            curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
         fi
     done