diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 3b5a0c53b49..386a2c36bb6 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -206,17 +206,17 @@
-
+
https://github.com/dotnet/arcade
- a4c9a07d978c070ef5c19d2ec9f811d6a5b20914
+ 6666973b629b24e259162dba03486c23af464bab
-
+
https://github.com/dotnet/arcade
- a4c9a07d978c070ef5c19d2ec9f811d6a5b20914
+ 6666973b629b24e259162dba03486c23af464bab
-
+
https://github.com/dotnet/arcade
- a4c9a07d978c070ef5c19d2ec9f811d6a5b20914
+ 6666973b629b24e259162dba03486c23af464bab
diff --git a/eng/Versions.props b/eng/Versions.props
index 7b6ad58b5b6..de3b5a35479 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -86,7 +86,7 @@
9.0.9
- 10.0.0-beta.25476.2
+ 9.0.0-beta.25515.2
diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd
index ac1f72bf94e..56c2f25ac22 100644
--- a/eng/common/CIBuild.cmd
+++ b/eng/common/CIBuild.cmd
@@ -1,2 +1,2 @@
@echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
\ No newline at end of file
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index 9445c314325..792b60b49d4 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -157,7 +157,7 @@ if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
}
-$dotnetVersions = @('5','6','7','8','9','10')
+$dotnetVersions = @('5','6','7','8','9')
foreach ($dotnetVersion in $dotnetVersions) {
$feedPrefix = "dotnet" + $dotnetVersion;
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
index ddf4efc81a4..facb415ca6f 100755
--- a/eng/common/SetupNugetSources.sh
+++ b/eng/common/SetupNugetSources.sh
@@ -99,7 +99,7 @@ if [ "$?" == "0" ]; then
PackageSources+=('dotnet3.1-internal-transport')
fi
-DotNetVersions=('5' '6' '7' '8' '9' '10')
+DotNetVersions=('5' '6' '7' '8' '9')
for DotNetVersion in ${DotNetVersions[@]} ; do
FeedPrefix="dotnet${DotNetVersion}";
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
index 8cfee107e7a..438f9920c43 100644
--- a/eng/common/build.ps1
+++ b/eng/common/build.ps1
@@ -7,7 +7,6 @@ Param(
[string] $msbuildEngine = $null,
[bool] $warnAsError = $true,
[bool] $nodeReuse = $true,
- [switch] $buildCheck = $false,
[switch][Alias('r')]$restore,
[switch] $deployDeps,
[switch][Alias('b')]$build,
@@ -21,7 +20,6 @@ Param(
[switch] $publish,
[switch] $clean,
[switch][Alias('pb')]$productBuild,
- [switch]$fromVMR,
[switch][Alias('bl')]$binaryLog,
[switch][Alias('nobl')]$excludeCIBinarylog,
[switch] $ci,
@@ -73,9 +71,6 @@ function Print-Usage() {
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
- Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
- Write-Host " -buildCheck Sets /check msbuild parameter"
- Write-Host " -fromVMR Set when building from within the VMR"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -102,7 +97,6 @@ function Build {
$bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
$platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
- $check = if ($buildCheck) { '/check' } else { '' }
if ($projects) {
# Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@@ -119,7 +113,6 @@ function Build {
MSBuild $toolsetBuildProj `
$bl `
$platformArg `
- $check `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
/p:Restore=$restore `
@@ -129,13 +122,11 @@ function Build {
/p:Deploy=$deploy `
/p:Test=$test `
/p:Pack=$pack `
- /p:DotNetBuild=$productBuild `
- /p:DotNetBuildFromVMR=$fromVMR `
+ /p:DotNetBuildRepo=$productBuild `
/p:IntegrationTest=$integrationTest `
/p:PerformanceTest=$performanceTest `
/p:Sign=$sign `
/p:Publish=$publish `
- /p:RestoreStaticGraphEnableBinaryLogger=$binaryLog `
@properties
}
diff --git a/eng/common/build.sh b/eng/common/build.sh
index 9767bb411a4..ac1ee8620cd 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -42,8 +42,6 @@ usage()
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
- echo " --buildCheck Sets /check msbuild parameter"
- echo " --fromVMR Set when building from within the VMR"
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
@@ -65,7 +63,6 @@ restore=false
build=false
source_build=false
product_build=false
-from_vmr=false
rebuild=false
test=false
integration_test=false
@@ -79,7 +76,6 @@ clean=false
warn_as_error=true
node_reuse=true
-build_check=false
binary_log=false
exclude_ci_binary_log=false
pipelines_log=false
@@ -91,7 +87,7 @@ verbosity='minimal'
runtime_source_feed=''
runtime_source_feed_key=''
-properties=()
+properties=''
while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
@@ -131,22 +127,19 @@ while [[ $# > 0 ]]; do
-pack)
pack=true
;;
- -sourcebuild|-source-build|-sb)
+ -sourcebuild|-sb)
build=true
source_build=true
product_build=true
restore=true
pack=true
;;
- -productbuild|-product-build|-pb)
+ -productBuild|-pb)
build=true
product_build=true
restore=true
pack=true
;;
- -fromvmr|-from-vmr)
- from_vmr=true
- ;;
-test|-t)
test=true
;;
@@ -180,9 +173,6 @@ while [[ $# > 0 ]]; do
node_reuse=$2
shift
;;
- -buildcheck)
- build_check=true
- ;;
-runtimesourcefeed)
runtime_source_feed=$2
shift
@@ -192,7 +182,7 @@ while [[ $# > 0 ]]; do
shift
;;
*)
- properties+=("$1")
+ properties="$properties $1"
;;
esac
@@ -226,7 +216,7 @@ function Build {
InitializeCustomToolset
if [[ ! -z "$projects" ]]; then
- properties+=("/p:Projects=$projects")
+ properties="$properties /p:Projects=$projects"
fi
local bl=""
@@ -234,21 +224,15 @@ function Build {
bl="/bl:\"$log_dir/Build.binlog\""
fi
- local check=""
- if [[ "$build_check" == true ]]; then
- check="/check"
- fi
-
MSBuild $_InitializeToolset \
$bl \
- $check \
/p:Configuration=$configuration \
/p:RepoRoot="$repo_root" \
/p:Restore=$restore \
/p:Build=$build \
- /p:DotNetBuild=$product_build \
+ /p:DotNetBuildRepo=$product_build \
+ /p:ArcadeBuildFromSource=$source_build \
/p:DotNetBuildSourceOnly=$source_build \
- /p:DotNetBuildFromVMR=$from_vmr \
/p:Rebuild=$rebuild \
/p:Test=$test \
/p:Pack=$pack \
@@ -256,8 +240,7 @@ function Build {
/p:PerformanceTest=$performance_test \
/p:Sign=$sign \
/p:Publish=$publish \
- /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \
- ${properties[@]+"${properties[@]}"}
+ $properties
ExitWithExitCode 0
}
diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh
index 66e3b0ac61c..1a02c0dec8f 100755
--- a/eng/common/cibuild.sh
+++ b/eng/common/cibuild.sh
@@ -13,4 +13,4 @@ while [[ -h $source ]]; do
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
+. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
\ No newline at end of file
diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml
index 5ce51840619..8da43d3b583 100644
--- a/eng/common/core-templates/job/job.yml
+++ b/eng/common/core-templates/job/job.yml
@@ -19,11 +19,11 @@ parameters:
# publishing defaults
artifacts: ''
enableMicrobuild: false
- enableMicrobuildForMacAndLinux: false
microbuildUseESRP: true
enablePublishBuildArtifacts: false
enablePublishBuildAssets: false
enablePublishTestResults: false
+ enablePublishUsingPipelines: false
enableBuildRetry: false
mergeTestResults: false
testRunTitle: ''
@@ -74,6 +74,9 @@ jobs:
- ${{ if ne(parameters.enableTelemetry, 'false') }}:
- name: DOTNET_CLI_TELEMETRY_PROFILE
value: '$(Build.Repository.Uri)'
+ - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
+ - name: EnableRichCodeNavigation
+ value: 'true'
# Retry signature validation up to three times, waiting 2 seconds between attempts.
# See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
- name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
@@ -125,12 +128,23 @@ jobs:
- ${{ preStep }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - template: /eng/common/core-templates/steps/install-microbuild.yml
- parameters:
- enableMicrobuild: ${{ parameters.enableMicrobuild }}
- enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
- microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - task: MicroBuildSigningPlugin@4
+ displayName: Install MicroBuild plugin
+ inputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+ ${{ if eq(parameters.microbuildUseESRP, true) }}:
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea
+ ${{ else }}:
+ ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca
+ env:
+ TeamName: $(_TeamName)
+ MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
- ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
- task: NuGetAuthenticate@1
@@ -146,15 +160,27 @@ jobs:
- ${{ each step in parameters.steps }}:
- ${{ step }}
+ - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
+ - task: RichCodeNavIndexer@0
+ displayName: RichCodeNav Upload
+ inputs:
+ languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
+ environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }}
+ richNavLogOutputDirectory: $(System.DefaultWorkingDirectory)/artifacts/bin
+ uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
+ continueOnError: true
+
- ${{ each step in parameters.componentGovernanceSteps }}:
- ${{ step }}
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
- parameters:
- enableMicrobuild: ${{ parameters.enableMicrobuild }}
- enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: MicroBuildCleanup@1
+ displayName: Execute Microbuild cleanup tasks
+ condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
+ env:
+ TeamName: $(_TeamName)
# Publish test results
- ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml
index c5788829a87..edefa789d36 100644
--- a/eng/common/core-templates/job/onelocbuild.yml
+++ b/eng/common/core-templates/job/onelocbuild.yml
@@ -4,7 +4,7 @@ parameters:
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool: ''
-
+
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
@@ -27,7 +27,7 @@ parameters:
is1ESPipeline: ''
jobs:
- job: OneLocBuild${{ parameters.JobNameSuffix }}
-
+
dependsOn: ${{ parameters.dependsOn }}
displayName: OneLocBuild${{ parameters.JobNameSuffix }}
@@ -86,7 +86,8 @@ jobs:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
${{ if eq(parameters.CreatePr, true) }}:
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
- isShouldReusePrSelected: ${{ parameters.ReusePr }}
+ ${{ if eq(parameters.RepoType, 'gitHub') }}:
+ isShouldReusePrSelected: ${{ parameters.ReusePr }}
packageSourceAuth: patAuth
patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
@@ -99,20 +100,22 @@ jobs:
mirrorBranch: ${{ parameters.MirrorBranch }}
condition: ${{ parameters.condition }}
- # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact
- - task: CopyFiles@2
- displayName: Copy LocProject.json
- inputs:
- SourceFolder: '$(System.DefaultWorkingDirectory)/eng/Localize/'
- Contents: 'LocProject.json'
- TargetFolder: '$(Build.ArtifactStagingDirectory)/loc'
- condition: ${{ parameters.condition }}
-
- - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+ - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
args:
- targetPath: '$(Build.ArtifactStagingDirectory)/loc'
- artifactName: 'Loc'
- displayName: 'Publish Localization Files'
+ displayName: Publish Localization Files
+ pathToPublish: '$(Build.ArtifactStagingDirectory)/loc'
+ publishLocation: Container
+ artifactName: Loc
condition: ${{ parameters.condition }}
+
+ - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ args:
+ displayName: Publish LocProject.json
+ pathToPublish: '$(System.DefaultWorkingDirectory)/eng/Localize/'
+ publishLocation: Container
+ artifactName: Loc
+ condition: ${{ parameters.condition }}
\ No newline at end of file
diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml
index 37dff559fc1..a58c8a418e8 100644
--- a/eng/common/core-templates/job/publish-build-assets.yml
+++ b/eng/common/core-templates/job/publish-build-assets.yml
@@ -20,6 +20,9 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
+ # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+ publishUsingPipelines: false
+
# Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
publishAssetsImmediately: false
@@ -29,15 +32,6 @@ parameters:
is1ESPipeline: ''
- # Optional: 🌤️ or not the build has assets it wants to publish to BAR
- isAssetlessBuild: false
-
- # Optional, publishing version
- publishingVersion: 3
-
- # Optional: A minimatch pattern for the asset manifests to publish to BAR
- assetManifestsPattern: '*/manifests/**/*.xml'
-
repositoryAlias: self
officialBuildId: ''
@@ -90,33 +84,15 @@ jobs:
- checkout: ${{ parameters.repositoryAlias }}
fetchDepth: 3
clean: true
-
- - ${{ if eq(parameters.isAssetlessBuild, 'false') }}:
- - ${{ if eq(parameters.publishingVersion, 3) }}:
- - task: DownloadPipelineArtifact@2
- displayName: Download Asset Manifests
- inputs:
- artifactName: AssetManifests
- targetPath: '$(Build.StagingDirectory)/AssetManifests'
- condition: ${{ parameters.condition }}
- continueOnError: ${{ parameters.continueOnError }}
- - ${{ if eq(parameters.publishingVersion, 4) }}:
- - task: DownloadPipelineArtifact@2
- displayName: Download V4 asset manifests
- inputs:
- itemPattern: '*/manifests/**/*.xml'
- targetPath: '$(Build.StagingDirectory)/AllAssetManifests'
- condition: ${{ parameters.condition }}
- continueOnError: ${{ parameters.continueOnError }}
- - task: CopyFiles@2
- displayName: Copy V4 asset manifests to AssetManifests
- inputs:
- SourceFolder: '$(Build.StagingDirectory)/AllAssetManifests'
- Contents: ${{ parameters.assetManifestsPattern }}
- TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
- flattenFolders: true
- condition: ${{ parameters.condition }}
- continueOnError: ${{ parameters.continueOnError }}
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download artifact
+ inputs:
+ artifactName: AssetManifests
+ downloadPath: '$(Build.StagingDirectory)/Download'
+ checkDownloadedFiles: true
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
- task: NuGetAuthenticate@1
@@ -128,9 +104,9 @@ jobs:
scriptLocation: scriptPath
scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
- /p:ManifestsPath='$(Build.StagingDirectory)/AssetManifests'
- /p:IsAssetlessBuild=${{ parameters.isAssetlessBuild }}
+ /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
/p:MaestroApiEndpoint=https://maestro.dot.net
+ /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:OfficialBuildId=$(OfficialBuildId)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
@@ -153,17 +129,6 @@ jobs:
Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs"
}
- - ${{ if eq(parameters.publishingVersion, 4) }}:
- - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
- args:
- targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml'
- artifactName: AssetManifests
- displayName: 'Publish Merged Manifest'
- retryCountOnTaskFailure: 10 # for any logs being locked
- sbomEnabled: false # we don't need SBOM for logs
-
- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
@@ -173,7 +138,7 @@ jobs:
publishLocation: Container
artifactName: ReleaseConfigs
- - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}:
+ - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
@@ -194,7 +159,6 @@ jobs:
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
- -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}'
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- template: /eng/common/core-templates/steps/publish-logs.yml
diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml
index d805d5faeb9..5baedac1e03 100644
--- a/eng/common/core-templates/job/source-build.yml
+++ b/eng/common/core-templates/job/source-build.yml
@@ -12,10 +12,9 @@ parameters:
# The name of the job. This is included in the job ID.
# targetRID: ''
# The name of the target RID to use, instead of the one auto-detected by Arcade.
- # portableBuild: false
+ # nonPortable: false
# Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
- # linux-x64), and compiling against distro-provided packages rather than portable ones. The
- # default is portable mode.
+ # linux-x64), and compiling against distro-provided packages rather than portable ones.
# skipPublishValidation: false
# Disables publishing validation. By default, a check is performed to ensure no packages are
# published by source-build.
@@ -34,6 +33,9 @@ parameters:
# container and pool.
platform: {}
+ # Optional list of directories to ignore for component governance scans.
+ componentGovernanceIgnoreDirectories: []
+
is1ESPipeline: ''
# If set to true and running on a non-public project,
@@ -94,3 +96,4 @@ jobs:
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
platform: ${{ parameters.platform }}
+ componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml
index 30530359a5d..662b9fcce15 100644
--- a/eng/common/core-templates/job/source-index-stage1.yml
+++ b/eng/common/core-templates/job/source-index-stage1.yml
@@ -1,5 +1,8 @@
parameters:
runAsPublic: false
+ sourceIndexUploadPackageVersion: 2.0.0-20250425.2
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
@@ -13,6 +16,12 @@ jobs:
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
+ - name: SourceIndexUploadPackageVersion
+ value: ${{ parameters.sourceIndexUploadPackageVersion }}
+ - name: SourceIndexProcessBinlogPackageVersion
+ value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
+ - name: SourceIndexPackageSource
+ value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- template: /eng/common/core-templates/variables/pool-providers.yml
@@ -25,10 +34,12 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $(DncEngPublicBuildPool)
- image: windows.vs2022.amd64.open
+ image: 1es-windows-2022-open
+ os: windows
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $(DncEngInternalBuildPool)
- image: windows.vs2022.amd64
+ image: 1es-windows-2022
+ os: windows
steps:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
@@ -36,9 +47,35 @@ jobs:
- ${{ each preStep in parameters.preSteps }}:
- ${{ preStep }}
+
+ - task: UseDotNet@2
+ displayName: Use .NET 8 SDK
+ inputs:
+ packageType: sdk
+ version: 8.0.x
+ installationPath: $(Agent.TempDirectory)/dotnet
+ workingDirectory: $(Agent.TempDirectory)
+
+ - script: |
+ $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+ displayName: Download Tools
+ # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
+ workingDirectory: $(Agent.TempDirectory)
+
- script: ${{ parameters.sourceIndexBuildCommand }}
displayName: Build Repository
- - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
- parameters:
- binLogPath: ${{ parameters.binLogPath }}
\ No newline at end of file
+ - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: Process Binlog into indexable sln
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: AzureCLI@2
+ displayName: Log in to Azure and upload stage1 artifacts to source index
+ inputs:
+ azureSubscription: 'SourceDotNet Stage1 Publish'
+ addSpnToEnvironment: true
+ scriptType: 'ps'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml
index dbc14ac580a..4571a7864df 100644
--- a/eng/common/core-templates/jobs/codeql-build.yml
+++ b/eng/common/core-templates/jobs/codeql-build.yml
@@ -15,6 +15,7 @@ jobs:
enablePublishBuildArtifacts: false
enablePublishTestResults: false
enablePublishBuildAssets: false
+ enablePublishUsingPipelines: false
enableTelemetry: true
variables:
diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml
index 01ada747665..bf33cdc2cc7 100644
--- a/eng/common/core-templates/jobs/jobs.yml
+++ b/eng/common/core-templates/jobs/jobs.yml
@@ -5,6 +5,9 @@ parameters:
# Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false
+ # Optional: Enable publishing using release pipelines
+ enablePublishUsingPipelines: false
+
# Optional: Enable running the source-build jobs to build repo from source
enableSourceBuild: false
@@ -27,9 +30,6 @@ parameters:
# Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage.
publishAssetsImmediately: false
- # Optional: 🌤️ or not the build has assets it wants to publish to BAR
- isAssetlessBuild: false
-
# Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml)
artifactsPublishingAdditionalParameters: ''
signingValidationAdditionalParameters: ''
@@ -85,6 +85,7 @@ jobs:
- template: /eng/common/core-templates/jobs/source-build.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }}
@@ -97,7 +98,7 @@ jobs:
${{ parameter.key }}: ${{ parameter.value }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, ''), eq(parameters.isAssetlessBuild, true)) }}:
+ - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
@@ -109,10 +110,12 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
+ - ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - Source_Build_Complete
runAsPublic: ${{ parameters.runAsPublic }}
- publishAssetsImmediately: ${{ or(parameters.publishAssetsImmediately, parameters.isAssetlessBuild) }}
- isAssetlessBuild: ${{ parameters.isAssetlessBuild }}
+ publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+ publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }}
diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml
index d92860cba20..0b408a67bd5 100644
--- a/eng/common/core-templates/jobs/source-build.yml
+++ b/eng/common/core-templates/jobs/source-build.yml
@@ -2,19 +2,28 @@ parameters:
# This template adds arcade-powered source-build to CI. A job is created for each platform, as
# well as an optional server job that completes when all platform jobs complete.
+ # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+ # not included. Existing repo pipelines can use this job depend on all source-build jobs
+ # completing without maintaining a separate list of every single job ID: just depend on this one
+ # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+ allCompletedJobId: ''
+
# See /eng/common/core-templates/job/source-build.yml
jobNamePrefix: 'Source_Build'
# This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
- container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-10-amd64'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. If no platforms are specified,
# one job runs on 'defaultManagedPlatform'.
platforms: []
+ # Optional list of directories to ignore for component governance scans.
+ componentGovernanceIgnoreDirectories: []
+
is1ESPipeline: ''
# If set to true and running on a non-public project,
@@ -25,12 +34,23 @@ parameters:
jobs:
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+ - job: ${{ parameters.allCompletedJobId }}
+ displayName: Source-Build Complete
+ pool: server
+ dependsOn:
+ - ${{ each platform in parameters.platforms }}:
+ - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+ - ${{ if eq(length(parameters.platforms), 0) }}:
+ - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
- ${{ each platform in parameters.platforms }}:
- template: /eng/common/core-templates/job/source-build.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
+ componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
enableInternalSources: ${{ parameters.enableInternalSources }}
- ${{ if eq(length(parameters.platforms), 0) }}:
@@ -39,4 +59,5 @@ jobs:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
+ componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
enableInternalSources: ${{ parameters.enableInternalSources }}
diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml
index f6f87fe5c67..2ee8bbfff54 100644
--- a/eng/common/core-templates/post-build/post-build.yml
+++ b/eng/common/core-templates/post-build/post-build.yml
@@ -60,11 +60,6 @@ parameters:
artifactNames: ''
downloadArtifacts: true
- - name: isAssetlessBuild
- type: boolean
- displayName: Is Assetless Build
- default: false
-
# These parameters let the user customize the call to sdk-task.ps1 for publishing
# symbols & general artifacts as well as for signing validation
- name: symbolPublishingAdditionalParameters
@@ -193,6 +188,9 @@ stages:
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@@ -322,4 +320,3 @@ stages:
-RequireDefaultChannels ${{ parameters.requireDefaultChannels }}
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
- -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}'
diff --git a/eng/common/core-templates/steps/cleanup-microbuild.yml b/eng/common/core-templates/steps/cleanup-microbuild.yml
deleted file mode 100644
index c0fdcd3379d..00000000000
--- a/eng/common/core-templates/steps/cleanup-microbuild.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-parameters:
- # Enable cleanup tasks for MicroBuild
- enableMicrobuild: false
- # Enable cleanup tasks for MicroBuild on Mac and Linux
- # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
- enableMicrobuildForMacAndLinux: false
- continueOnError: false
-
-steps:
- - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- - task: MicroBuildCleanup@1
- displayName: Execute Microbuild cleanup tasks
- condition: and(
- always(),
- or(
- and(
- eq(variables['Agent.Os'], 'Windows_NT'),
- in(variables['_SignType'], 'real', 'test')
- ),
- and(
- ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
- ne(variables['Agent.Os'], 'Windows_NT'),
- eq(variables['_SignType'], 'real')
- )
- ))
- continueOnError: ${{ parameters.continueOnError }}
- env:
- TeamName: $(_TeamName)
diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml
index c05f6502797..7f5b84c4cb8 100644
--- a/eng/common/core-templates/steps/generate-sbom.yml
+++ b/eng/common/core-templates/steps/generate-sbom.yml
@@ -5,7 +5,7 @@
# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
parameters:
- PackageVersion: 10.0.0
+ PackageVersion: 9.0.0
BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts'
PackageName: '.NET'
ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
diff --git a/eng/common/core-templates/steps/get-delegation-sas.yml b/eng/common/core-templates/steps/get-delegation-sas.yml
index d2901470a7f..9db5617ea7d 100644
--- a/eng/common/core-templates/steps/get-delegation-sas.yml
+++ b/eng/common/core-templates/steps/get-delegation-sas.yml
@@ -31,7 +31,16 @@ steps:
# Calculate the expiration of the SAS token and convert to UTC
$expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
- $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+ # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+ # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+ $sas = ""
+ do {
+ $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+ } while($sas.IndexOf('/') -ne -1)
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to generate SAS token."
diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml
deleted file mode 100644
index d6b9878f54d..00000000000
--- a/eng/common/core-templates/steps/install-microbuild.yml
+++ /dev/null
@@ -1,91 +0,0 @@
-parameters:
- # Enable install tasks for MicroBuild
- enableMicrobuild: false
- # Enable install tasks for MicroBuild on Mac and Linux
- # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
- enableMicrobuildForMacAndLinux: false
- # Determines whether the ESRP service connection information should be passed to the signing plugin.
- # This overlaps with _SignType to some degree. We only need the service connection for real signing.
- # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place.
- # Doing so will cause the service connection to be authorized for the pipeline, which isn't allowed and won't work for non-prod.
- # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The
- # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough.
- microbuildUseESRP: true
- # Location of the MicroBuild output folder
- # NOTE: There's something that relies on this being in the "default" source directory for tasks such as Signing to work properly.
- microBuildOutputFolder: '$(Build.SourcesDirectory)'
-
- continueOnError: false
-
-steps:
- - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
- # Needed to download the MicroBuild plugin nupkgs on Mac and Linux when nuget.exe is unavailable
- - task: UseDotNet@2
- displayName: Install .NET 8.0 SDK for MicroBuild Plugin
- inputs:
- packageType: sdk
- version: 8.0.x
- installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet
- workingDirectory: ${{ parameters.microBuildOutputFolder }}
- condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
-
- - script: |
- REM Check if ESRP is disabled while SignType is real
- if /I "${{ parameters.microbuildUseESRP }}"=="false" if /I "$(_SignType)"=="real" (
- echo Error: ESRP must be enabled when SignType is real.
- exit /b 1
- )
- displayName: 'Validate ESRP usage (Windows)'
- condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
- - script: |
- # Check if ESRP is disabled while SignType is real
- if [ "${{ parameters.microbuildUseESRP }}" = "false" ] && [ "$(_SignType)" = "real" ]; then
- echo "Error: ESRP must be enabled when SignType is real."
- exit 1
- fi
- displayName: 'Validate ESRP usage (Non-Windows)'
- condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
-
- # Two different MB install steps. This is due to not being able to use the agent OS during
- # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However,
- # we can avoid including the MB install step if not enabled at all. This avoids a bunch of
- # extra pipeline authorizations, since most pipelines do not sign on non-Windows.
- - task: MicroBuildSigningPlugin@4
- displayName: Install MicroBuild plugin (Windows)
- inputs:
- signType: $(_SignType)
- zipSources: false
- feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
- ${{ if eq(parameters.microbuildUseESRP, true) }}:
- ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
- ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
- ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea
- ${{ else }}:
- ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca
- env:
- TeamName: $(_TeamName)
- MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- continueOnError: ${{ parameters.continueOnError }}
- condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test'))
-
- - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}:
- - task: MicroBuildSigningPlugin@4
- displayName: Install MicroBuild plugin (non-Windows)
- inputs:
- signType: $(_SignType)
- zipSources: false
- feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
- ${{ if eq(parameters.microbuildUseESRP, true) }}:
- ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
- ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
- ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39
- ${{ else }}:
- ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc
- env:
- TeamName: $(_TeamName)
- MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- continueOnError: ${{ parameters.continueOnError }}
- condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real'))
diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml
index 10f825e270a..0623ac6e112 100644
--- a/eng/common/core-templates/steps/publish-logs.yml
+++ b/eng/common/core-templates/steps/publish-logs.yml
@@ -34,9 +34,7 @@ steps:
'$(akams-client-id)'
'$(microsoft-symbol-server-pat)'
'$(symweb-symbol-server-pat)'
- '$(dnceng-symbol-server-pat)'
'$(dn-bot-all-orgs-build-rw-code-rw)'
- '$(System.AccessToken)'
${{parameters.CustomSensitiveDataList}}
continueOnError: true
condition: always()
@@ -47,7 +45,6 @@ steps:
SourceFolder: '$(System.DefaultWorkingDirectory)/PostBuildLogs'
Contents: '**'
TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
- condition: always()
- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
parameters:
diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml
index acf16ed3496..0718e4ba902 100644
--- a/eng/common/core-templates/steps/source-build.yml
+++ b/eng/common/core-templates/steps/source-build.yml
@@ -11,6 +11,10 @@ parameters:
# for details. The entire object is described in the 'job' template for simplicity, even though
# the usage of the properties on this object is split between the 'job' and 'steps' templates.
platform: {}
+
+ # Optional list of directories to ignore for component governance scans.
+ componentGovernanceIgnoreDirectories: []
+
is1ESPipeline: false
steps:
@@ -19,12 +23,25 @@ steps:
set -x
df -h
+ # If file changes are detected, set CopyWipIntoInnerSourceBuildRepo to copy the WIP changes into the inner source build repo.
+ internalRestoreArgs=
+ if ! git diff --quiet; then
+ internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
+ # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
+ # This only works if there is a username/email configured, which won't be the case in most CI runs.
+ git config --get user.email
+ if [ $? -ne 0 ]; then
+ git config user.email dn-bot@microsoft.com
+ git config user.name dn-bot
+ fi
+ fi
+
# If building on the internal project, the internal storage variable may be available (usually only if needed)
# In that case, add variables to allow the download of internal runtimes if the specified versions are not found
# in the default public locations.
internalRuntimeDownloadArgs=
if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then
- internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
+ internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
fi
buildConfig=Release
@@ -33,33 +50,88 @@ steps:
buildConfig='$(_BuildConfig)'
fi
+ officialBuildArgs=
+ if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+ officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+ fi
+
targetRidArgs=
if [ '${{ parameters.platform.targetRID }}' != '' ]; then
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi
- portableBuildArgs=
- if [ '${{ parameters.platform.portableBuild }}' != '' ]; then
- portableBuildArgs='/p:PortableBuild=${{ parameters.platform.portableBuild }}'
+ runtimeOsArgs=
+ if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then
+ runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
+ fi
+
+ baseOsArgs=
+ if [ '${{ parameters.platform.baseOS }}' != '' ]; then
+ baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}'
+ fi
+
+ publishArgs=
+ if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
+ publishArgs='--publish'
+ fi
+
+ assetManifestFileName=SourceBuild_RidSpecific.xml
+ if [ '${{ parameters.platform.name }}' != '' ]; then
+ assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml
fi
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \
- --restore --build --pack -bl \
- --source-build \
+ --restore --build --pack $publishArgs -bl \
${{ parameters.platform.buildArguments }} \
+ $officialBuildArgs \
$internalRuntimeDownloadArgs \
+ $internalRestoreArgs \
$targetRidArgs \
- $portableBuildArgs \
+ $runtimeOsArgs \
+ $baseOsArgs \
+ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+ /p:ArcadeBuildFromSource=true \
+ /p:DotNetBuildSourceOnly=true \
+ /p:DotNetBuildRepo=true \
+ /p:AssetManifestFileName=$assetManifestFileName
displayName: Build
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+ displayName: Prepare BuildLogs staging directory
+ inputs:
+ SourceFolder: '$(System.DefaultWorkingDirectory)'
+ Contents: |
+ **/*.log
+ **/*.binlog
+ artifacts/sb/prebuilt-report/**
+ TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+ CleanTargetFolder: true
+ continueOnError: true
+ condition: succeededOrFailed()
+
- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
args:
displayName: Publish BuildLogs
- targetPath: artifacts/log/${{ coalesce(variables._BuildConfig, 'Release') }}
+ targetPath: '$(Build.StagingDirectory)/BuildLogs'
artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
continueOnError: true
condition: succeededOrFailed()
sbomEnabled: false # we don't need SBOM for logs
+
+# Manually inject component detection so that we can ignore the source build upstream cache, which contains
+# a nupkg cache of input packages (a local feed).
+# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir'
+# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets
+- template: /eng/common/core-templates/steps/component-governance.yml
+ parameters:
+ displayName: Component Detection (Exclude upstream cache)
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ ${{ if eq(length(parameters.componentGovernanceIgnoreDirectories), 0) }}:
+ componentGovernanceIgnoreDirectories: '$(System.DefaultWorkingDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache'
+ ${{ else }}:
+ componentGovernanceIgnoreDirectories: ${{ join(',', parameters.componentGovernanceIgnoreDirectories) }}
+ disableComponentGovernance: ${{ eq(variables['System.TeamProject'], 'public') }}
diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml
deleted file mode 100644
index e9a694afa58..00000000000
--- a/eng/common/core-templates/steps/source-index-stage1-publish.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-parameters:
- sourceIndexUploadPackageVersion: 2.0.0-20250818.1
- sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1
- sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
- binlogPath: artifacts/log/Debug/Build.binlog
-
-steps:
-- task: UseDotNet@2
- displayName: "Source Index: Use .NET 9 SDK"
- inputs:
- packageType: sdk
- version: 9.0.x
- installationPath: $(Agent.TempDirectory)/dotnet
- workingDirectory: $(Agent.TempDirectory)
-
-- script: |
- $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
- $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
- displayName: "Source Index: Download netsourceindex Tools"
- # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
- workingDirectory: $(Agent.TempDirectory)
-
-- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
- displayName: "Source Index: Process Binlog into indexable sln"
-
-- ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - task: AzureCLI@2
- displayName: "Source Index: Upload Source Index stage1 artifacts to Azure"
- inputs:
- azureSubscription: 'SourceDotNet Stage1 Publish'
- addSpnToEnvironment: true
- scriptType: 'ps'
- scriptLocation: 'inlineScript'
- inlineScript: |
- $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch
index 2cebc547382..af7c8be0590 100644
--- a/eng/common/cross/arm64/tizen/tizen.patch
+++ b/eng/common/cross/arm64/tizen/tizen.patch
@@ -5,5 +5,5 @@ diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
Use the shared library, but some functions are only in
the static library, so try that secondarily. */
OUTPUT_FORMAT(elf64-littleaarch64)
--GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-aarch64.so.1 ) )
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) )
+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) )
diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch
new file mode 100644
index 00000000000..2d261561935
--- /dev/null
+++ b/eng/common/cross/armel/armel.jessie.patch
@@ -0,0 +1,43 @@
+diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
+--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
++++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
+@@ -69,10 +69,10 @@
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- return __sync_val_compare_and_swap_2(addr, old, _new);
++ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
+ #endif
+ case 4:
+- return __sync_val_compare_and_swap_4(addr, old, _new);
++ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_val_compare_and_swap_8(addr, old, _new);
+@@ -109,7 +109,7 @@
+ return;
+ #endif
+ case 4:
+- __sync_and_and_fetch_4(addr, val);
++ __sync_and_and_fetch_4((uint32_t*) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+@@ -148,7 +148,7 @@
+ return;
+ #endif
+ case 4:
+- __sync_or_and_fetch_4(addr, val);
++ __sync_or_and_fetch_4((uint32_t*) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+@@ -187,7 +187,7 @@
+ return __sync_add_and_fetch_2(addr, val);
+ #endif
+ case 4:
+- return __sync_add_and_fetch_4(addr, val);
++ return __sync_add_and_fetch_4((uint32_t*) addr, val);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_add_and_fetch_8(addr, val);
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index fbd8d80848a..7e9ba2b75ed 100755
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -6,11 +6,10 @@ usage()
{
echo "Creates a toolchain and sysroot used for cross-compiling for Android."
echo
- echo "Usage: $0 [BuildArch] [ApiLevel] [--ndk NDKVersion]"
+ echo "Usage: $0 [BuildArch] [ApiLevel]"
echo
echo "BuildArch is the target architecture of Android. Currently only arm64 is supported."
echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html"
- echo "NDKVersion is the version of Android NDK. The default is r21. See https://developer.android.com/ndk/downloads/revision_history"
echo
echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior"
echo "by setting the TOOLCHAIN_DIR environment variable"
@@ -26,15 +25,10 @@ __BuildArch=arm64
__AndroidArch=aarch64
__AndroidToolchain=aarch64-linux-android
-while :; do
- if [[ "$#" -le 0 ]]; then
- break
- fi
-
- i=$1
-
- lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
- case $lowerI in
+for i in "$@"
+ do
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
-?|-h|--help)
usage
exit 1
@@ -49,10 +43,6 @@ while :; do
__AndroidArch=arm
__AndroidToolchain=arm-linux-androideabi
;;
- --ndk)
- shift
- __NDK_Version=$1
- ;;
*[0-9])
__ApiLevel=$i
;;
@@ -60,17 +50,8 @@ while :; do
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $i"
;;
esac
- shift
done
-if [[ "$__NDK_Version" == "r21" ]] || [[ "$__NDK_Version" == "r22" ]]; then
- __NDK_File_Arch_Spec=-x86_64
- __SysRoot=sysroot
-else
- __NDK_File_Arch_Spec=
- __SysRoot=toolchains/llvm/prebuilt/linux-x86_64/sysroot
-fi
-
# Obtain the location of the bash script to figure out where the root of the repo is.
__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -97,7 +78,6 @@ fi
echo "Target API level: $__ApiLevel"
echo "Target architecture: $__BuildArch"
-echo "NDK version: $__NDK_Version"
echo "NDK location: $__NDK_Dir"
echo "Target Toolchain location: $__ToolchainDir"
@@ -105,8 +85,8 @@ echo "Target Toolchain location: $__ToolchainDir"
if [ ! -d $__NDK_Dir ]; then
echo Downloading the NDK into $__NDK_Dir
mkdir -p $__NDK_Dir
- wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux$__NDK_File_Arch_Spec.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux.zip
- unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux.zip -d $__CrossDir
+ wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip
+ unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir
fi
if [ ! -d $__lldb_Dir ]; then
@@ -136,11 +116,16 @@ for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/
fi
done
-cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/$__SysRoot/usr/"
+cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/"
# Generate platform file for build.sh script to assign to __DistroRid
echo "Generating platform file..."
-echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/$__SysRoot/android_platform
-
-echo "Now to build coreclr, libraries and host; run:"
-echo ROOTFS_DIR=$(realpath $__ToolchainDir/$__SysRoot) ./build.sh clr+libs+host --cross --arch $__BuildArch
+echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform
+
+echo "Now to build coreclr, libraries and installers; run:"
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+ --subsetCategory coreclr
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+ --subsetCategory libraries
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+ --subsetCategory installer
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 8abfb71f727..4b5e8d7166b 100755
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -5,7 +5,7 @@ set -e
usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]"
- echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86"
+ echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine"
echo " for alpine can be specified with version: alpineX.YY or alpineedge"
echo " for FreeBSD can be: freebsd13, freebsd14"
@@ -15,7 +15,6 @@ usage()
echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)."
- echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
echo "--jobs N - optional, restrict to N jobs."
exit 1
@@ -53,27 +52,28 @@ __UbuntuPackages+=" symlinks"
__UbuntuPackages+=" libicu-dev"
__UbuntuPackages+=" liblttng-ust-dev"
__UbuntuPackages+=" libunwind8-dev"
+__UbuntuPackages+=" libnuma-dev"
__AlpinePackages+=" gettext-dev"
__AlpinePackages+=" icu-dev"
__AlpinePackages+=" libunwind-dev"
__AlpinePackages+=" lttng-ust-dev"
__AlpinePackages+=" compiler-rt"
+__AlpinePackages+=" numactl-dev"
# runtime libraries' dependencies
__UbuntuPackages+=" libcurl4-openssl-dev"
__UbuntuPackages+=" libkrb5-dev"
__UbuntuPackages+=" libssl-dev"
__UbuntuPackages+=" zlib1g-dev"
-__UbuntuPackages+=" libbrotli-dev"
__AlpinePackages+=" curl-dev"
__AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
-__FreeBSDBase="13.4-RELEASE"
-__FreeBSDPkg="1.21.3"
+__FreeBSDBase="13.3-RELEASE"
+__FreeBSDPkg="1.17.0"
__FreeBSDABI="13"
__FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu"
@@ -91,18 +91,18 @@ __HaikuPackages="gcc_syslibs"
__HaikuPackages+=" gcc_syslibs_devel"
__HaikuPackages+=" gmp"
__HaikuPackages+=" gmp_devel"
-__HaikuPackages+=" icu[0-9]+"
-__HaikuPackages+=" icu[0-9]*_devel"
+__HaikuPackages+=" icu66"
+__HaikuPackages+=" icu66_devel"
__HaikuPackages+=" krb5"
__HaikuPackages+=" krb5_devel"
__HaikuPackages+=" libiconv"
__HaikuPackages+=" libiconv_devel"
-__HaikuPackages+=" llvm[0-9]*_libunwind"
-__HaikuPackages+=" llvm[0-9]*_libunwind_devel"
+__HaikuPackages+=" llvm12_libunwind"
+__HaikuPackages+=" llvm12_libunwind_devel"
__HaikuPackages+=" mpfr"
__HaikuPackages+=" mpfr_devel"
-__HaikuPackages+=" openssl3"
-__HaikuPackages+=" openssl3_devel"
+__HaikuPackages+=" openssl"
+__HaikuPackages+=" openssl_devel"
__HaikuPackages+=" zlib"
__HaikuPackages+=" zlib_devel"
@@ -128,12 +128,10 @@ __AlpineKeys='
616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ==
616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ==
616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ==
-66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ==
'
__Keyring=
__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
__SkipSigCheck=0
-__SkipEmulation=0
__UseMirror=0
__UnprocessedBuildArgs=
@@ -164,13 +162,9 @@ while :; do
armel)
__BuildArch=armel
__UbuntuArch=armel
- __UbuntuRepo="http://archive.debian.org/debian/"
- __CodeName=buster
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ __CodeName=jessie
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
- __LLDB_Package="liblldb-6.0-dev"
- __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
- __UbuntuPackages="${__UbuntuPackages// libomp5/}"
- __UbuntuSuites=
;;
armv6)
__BuildArch=armv6
@@ -186,18 +180,6 @@ while :; do
__Keyring="--keyring $__KeyringFile"
fi
;;
- loongarch64)
- __BuildArch=loongarch64
- __AlpineArch=loongarch64
- __QEMUArch=loongarch64
- __UbuntuArch=loong64
- __UbuntuSuites=unreleased
- __LLDB_Package="liblldb-19-dev"
-
- if [[ "$__CodeName" == "sid" ]]; then
- __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
- fi
- ;;
riscv64)
__BuildArch=riscv64
__AlpineArch=riscv64
@@ -282,21 +264,44 @@ while :; do
;;
xenial) # Ubuntu 16.04
- __CodeName=xenial
+ if [[ "$__CodeName" != "jessie" ]]; then
+ __CodeName=xenial
+ fi
+ ;;
+ zesty) # Ubuntu 17.04
+ if [[ "$__CodeName" != "jessie" ]]; then
+ __CodeName=zesty
+ fi
;;
bionic) # Ubuntu 18.04
- __CodeName=bionic
+ if [[ "$__CodeName" != "jessie" ]]; then
+ __CodeName=bionic
+ fi
;;
focal) # Ubuntu 20.04
- __CodeName=focal
+ if [[ "$__CodeName" != "jessie" ]]; then
+ __CodeName=focal
+ fi
;;
jammy) # Ubuntu 22.04
- __CodeName=jammy
+ if [[ "$__CodeName" != "jessie" ]]; then
+ __CodeName=jammy
+ fi
;;
noble) # Ubuntu 24.04
- __CodeName=noble
- if [[ -z "$__LLDB_Package" ]]; then
- __LLDB_Package="liblldb-19-dev"
+ if [[ "$__CodeName" != "jessie" ]]; then
+ __CodeName=noble
+ fi
+ if [[ -n "$__LLDB_Package" ]]; then
+ __LLDB_Package="liblldb-18-dev"
+ fi
+ ;;
+ jessie) # Debian 8
+ __CodeName=jessie
+ __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+
+ if [[ -z "$__UbuntuRepo" ]]; then
+ __UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
stretch) # Debian 9
@@ -314,7 +319,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://archive.debian.org/debian/"
+ __UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
bullseye) # Debian 11
@@ -335,28 +340,10 @@ while :; do
;;
sid) # Debian sid
__CodeName=sid
- __UbuntuSuites=
-
- # Debian-Ports architectures need different values
- case "$__UbuntuArch" in
- amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x)
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
-
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
- fi
- ;;
- *)
- __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg"
-
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
- fi
- ;;
- esac
+ __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
- if [[ -e "$__KeyringFile" ]]; then
- __Keyring="--keyring $__KeyringFile"
+ if [[ -z "$__UbuntuRepo" ]]; then
+ __UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
tizen)
@@ -383,7 +370,7 @@ while :; do
;;
freebsd14)
__CodeName=freebsd
- __FreeBSDBase="14.2-RELEASE"
+ __FreeBSDBase="14.0-RELEASE"
__FreeBSDABI="14"
__SkipUnmount=1
;;
@@ -401,9 +388,6 @@ while :; do
--skipsigcheck)
__SkipSigCheck=1
;;
- --skipemulation)
- __SkipEmulation=1
- ;;
--rootfsdir|-rootfsdir)
shift
__RootfsDir="$1"
@@ -436,15 +420,16 @@ case "$__AlpineVersion" in
elif [[ "$__AlpineArch" == "x86" ]]; then
__AlpineVersion=3.17 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm15-libs"
- elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then
- __AlpineVersion=3.21 # minimum version that supports lldb-dev
- __AlpinePackages+=" llvm19-libs"
- elif [[ -n "$__AlpineMajorVersion" ]]; then
- # use whichever alpine version is provided and select the latest toolchain libs
+ elif [[ "$__AlpineArch" == "riscv64" ]]; then
__AlpineLlvmLibsLookup=1
+ __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive)
else
__AlpineVersion=3.13 # 3.13 to maximize compatibility
__AlpinePackages+=" llvm10-libs"
+
+ if [[ "$__AlpineArch" == "armv7" ]]; then
+ __AlpinePackages="${__AlpinePackages//numactl-dev/}"
+ fi
fi
esac
@@ -454,6 +439,15 @@ if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then
__AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}"
fi
+if [[ "$__BuildArch" == "armel" ]]; then
+ __LLDB_Package="lldb-3.5-dev"
+fi
+
+if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then
+ # libnuma-dev is not available on armhf for xenial
+ __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}"
+fi
+
__UbuntuPackages+=" ${__LLDB_Package:-}"
if [[ -z "$__UbuntuRepo" ]]; then
@@ -502,7 +496,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
arch="$(uname -m)"
ensureDownloadTool
-
+
if [[ "$__hasWget" == 1 ]]; then
wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
else
@@ -518,6 +512,11 @@ if [[ "$__CodeName" == "alpine" ]]; then
echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c
chmod +x "$__ApkToolsDir/apk.static"
+ if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then
+ mkdir -p "$__RootfsDir"/usr/bin
+ cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
+ fi
+
if [[ "$__AlpineVersion" == "edge" ]]; then
version=edge
else
@@ -537,10 +536,6 @@ if [[ "$__CodeName" == "alpine" ]]; then
__ApkSignatureArg="--keys-dir $__ApkKeysDir"
fi
- if [[ "$__SkipEmulation" == "1" ]]; then
- __NoEmulationArg="--no-scripts"
- fi
-
# initialize DB
# shellcheck disable=SC2086
"$__ApkToolsDir/apk.static" \
@@ -562,7 +557,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
"$__ApkToolsDir/apk.static" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
- -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \
+ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
add $__AlpinePackages
rm -r "$__ApkToolsDir"
@@ -578,7 +573,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
fi
echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
- echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
+ echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
mkdir -p "$__RootfsDir"/tmp
# get and build package manager
if [[ "$__hasWget" == 1 ]]; then
@@ -686,7 +681,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then
ensureDownloadTool
- echo "Downloading Haiku package tools"
+ echo "Downloading Haiku package tool"
git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
if [[ "$__hasWget" == 1 ]]; then
wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
@@ -696,42 +691,34 @@ elif [[ "$__CodeName" == "haiku" ]]; then
unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
- HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
- HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current"
-
- echo "Downloading HaikuPorts package repository index..."
- if [[ "$__hasWget" == 1 ]]; then
- wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
- else
- curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
- fi
+ DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
+ HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+ # Download Haiku packages
echo "Downloading Haiku packages"
read -ra array <<<"$__HaikuPackages"
for package in "${array[@]}"; do
echo "Downloading $package..."
- hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" |
- grep -E "${package}-" | sort -V | tail -n 1 | xargs)"
- if [ -z "$hpkgFilename" ]; then
- >&2 echo "ERROR: package $package missing."
- exit 1
- fi
- echo "Resolved filename: $hpkgFilename..."
- hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename"
+ # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
+ # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
if [[ "$__hasWget" == 1 ]]; then
+ hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+ --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
else
+ hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+ --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
fi
done
for package in haiku haiku_devel; do
echo "Downloading $package..."
if [[ "$__hasWget" == 1 ]]; then
- hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
- wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+ hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+ wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
else
- hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
- curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+ hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+ curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
fi
done
@@ -757,67 +744,25 @@ elif [[ "$__CodeName" == "haiku" ]]; then
popd
rm -rf "$__RootfsDir/tmp"
elif [[ -n "$__CodeName" ]]; then
- __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)"
-
- if [[ "$__SkipEmulation" == "1" ]]; then
- if [[ -z "$AR" ]]; then
- if command -v ar &>/dev/null; then
- AR="$(command -v ar)"
- elif command -v llvm-ar &>/dev/null; then
- AR="$(command -v llvm-ar)"
- else
- echo "Unable to find ar or llvm-ar on PATH, add them to PATH or set AR environment variable pointing to the available AR tool"
- exit 1
- fi
- fi
-
- PYTHON=${PYTHON_EXECUTABLE:-python3}
-
- # shellcheck disable=SC2086,SC2046
- echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
- $(for suite in $__Suites; do echo -n "--suite $suite "; done) \
- $__UbuntuPackages
-
- # shellcheck disable=SC2086,SC2046
- "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
- $(for suite in $__Suites; do echo -n "--suite $suite "; done) \
- $__UbuntuPackages
- exit 0
- fi
-
- __UpdateOptions=
if [[ "$__SkipSigCheck" == "0" ]]; then
__Keyring="$__Keyring --force-check-gpg"
- else
- __Keyring=
- __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
fi
# shellcheck disable=SC2086
echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
+ debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
- # shellcheck disable=SC2086
- if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then
- echo "debootstrap failed! dumping debootstrap.log"
- cat "$__RootfsDir/debootstrap/debootstrap.log"
- exit 1
- fi
-
- rm -rf "$__RootfsDir"/etc/apt/*.{sources,list} "$__RootfsDir"/etc/apt/sources.list.d
mkdir -p "$__RootfsDir/etc/apt/sources.list.d/"
-
- # shellcheck disable=SC2086
cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" < token2) - (token1 < token2)
- else:
- return -1 if isinstance(token1, str) else 1
-
- return len(tokens1) - len(tokens2)
-
-def compare_debian_versions(version1, version2):
- """Compare two Debian package versions."""
- epoch1, upstream1, revision1 = parse_debian_version(version1)
- epoch2, upstream2, revision2 = parse_debian_version(version2)
-
- if epoch1 != epoch2:
- return epoch1 - epoch2
-
- result = compare_upstream_version(upstream1, upstream2)
- if result != 0:
- return result
-
- return compare_upstream_version(revision1, revision2)
-
-def resolve_dependencies(packages, aliases, desired_packages):
- """Recursively resolves dependencies for the desired packages."""
- resolved = []
- to_process = deque(desired_packages)
-
- while to_process:
- current = to_process.popleft()
- resolved_package = current if current in packages else aliases.get(current, [None])[0]
-
- if not resolved_package:
- print(f"Error: Package '{current}' was not found in the available packages.")
- sys.exit(1)
-
- if resolved_package not in resolved:
- resolved.append(resolved_package)
-
- deps = packages.get(resolved_package, {}).get("Depends", "")
- if deps:
- deps = [dep.split(' ')[0] for dep in deps.split(', ') if dep]
- for dep in deps:
- if dep not in resolved and dep not in to_process and dep in packages:
- to_process.append(dep)
-
- return resolved
-
-def parse_package_index(content):
- """Parses the Packages.gz file and returns package information."""
- packages = {}
- aliases = {}
- entries = re.split(r'\n\n+', content)
-
- for entry in entries:
- fields = dict(re.findall(r'^(\S+): (.+)$', entry, re.MULTILINE))
- if "Package" in fields:
- package_name = fields["Package"]
- version = fields.get("Version")
- filename = fields.get("Filename")
- depends = fields.get("Depends")
- provides = fields.get("Provides", None)
-
- # Only update if package_name is not in packages or if the new version is higher
- if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0:
- packages[package_name] = {
- "Version": version,
- "Filename": filename,
- "Depends": depends
- }
-
- # Update aliases if package provides any alternatives
- if provides:
- provides_list = [x.strip() for x in provides.split(",")]
- for alias in provides_list:
- # Strip version specifiers
- alias_name = re.sub(r'\s*\(=.*\)', '', alias)
- if alias_name not in aliases:
- aliases[alias_name] = []
- if package_name not in aliases[alias_name]:
- aliases[alias_name].append(package_name)
-
- return packages, aliases
-
-def install_packages(mirror, packages_info, aliases, tmp_dir, extract_dir, ar_tool, desired_packages):
- """Downloads .deb files and extracts them."""
- resolved_packages = resolve_dependencies(packages_info, aliases, desired_packages)
- print(f"Resolved packages (including dependencies): {resolved_packages}")
-
- packages_to_download = {}
-
- for pkg in resolved_packages:
- if pkg in packages_info:
- packages_to_download[pkg] = packages_info[pkg]
-
- if pkg in aliases:
- for alias in aliases[pkg]:
- if alias in packages_info:
- packages_to_download[alias] = packages_info[alias]
-
- asyncio.run(download_deb_files_parallel(mirror, packages_to_download, tmp_dir))
-
- package_to_deb_file_map = {}
- for pkg in resolved_packages:
- pkg_info = packages_info.get(pkg)
- if pkg_info:
- deb_filename = pkg_info.get("Filename")
- if deb_filename:
- deb_file_path = os.path.join(tmp_dir, os.path.basename(deb_filename))
- package_to_deb_file_map[pkg] = deb_file_path
-
- for pkg in reversed(resolved_packages):
- deb_file = package_to_deb_file_map.get(pkg)
- if deb_file and os.path.exists(deb_file):
- extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool)
-
- print("All done!")
-
-def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool):
- """Extract .deb file contents"""
-
- os.makedirs(extract_dir, exist_ok=True)
-
- with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir:
- result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
- tar_filename = None
- for line in result.stdout.decode().splitlines():
- if line.startswith("data.tar"):
- tar_filename = line.strip()
- break
-
- if not tar_filename:
- raise FileNotFoundError(f"Could not find 'data.tar.*' in {deb_file}.")
-
- tar_file_path = os.path.join(tmp_subdir, tar_filename)
- print(f"Extracting {tar_filename} from {deb_file}..")
-
- subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True)
-
- file_extension = os.path.splitext(tar_file_path)[1].lower()
-
- if file_extension == ".xz":
- mode = "r:xz"
- elif file_extension == ".gz":
- mode = "r:gz"
- elif file_extension == ".zst":
- # zstd is not supported by standard library yet
- decompressed_tar_path = tar_file_path.replace(".zst", "")
- with open(tar_file_path, "rb") as zst_file, open(decompressed_tar_path, "wb") as decompressed_file:
- dctx = zstandard.ZstdDecompressor()
- dctx.copy_stream(zst_file, decompressed_file)
-
- tar_file_path = decompressed_tar_path
- mode = "r"
- else:
- raise ValueError(f"Unsupported compression format: {file_extension}")
-
- with tarfile.open(tar_file_path, mode) as tar:
- tar.extractall(path=extract_dir, filter='fully_trusted')
-
-def finalize_setup(rootfsdir):
- lib_dir = os.path.join(rootfsdir, 'lib')
- usr_lib_dir = os.path.join(rootfsdir, 'usr', 'lib')
-
- if os.path.exists(lib_dir):
- if os.path.islink(lib_dir):
- os.remove(lib_dir)
- else:
- os.makedirs(usr_lib_dir, exist_ok=True)
-
- for item in os.listdir(lib_dir):
- src = os.path.join(lib_dir, item)
- dest = os.path.join(usr_lib_dir, item)
-
- if os.path.isdir(src):
- shutil.copytree(src, dest, dirs_exist_ok=True)
- else:
- shutil.copy2(src, dest)
-
- shutil.rmtree(lib_dir)
-
- os.symlink(usr_lib_dir, lib_dir)
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS")
- parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)")
- parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)")
- parser.add_argument("--rootfsdir", required=True, help="Destination directory.")
- parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.')
- parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)")
- parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)")
- parser.add_argument("packages", nargs="+", help="List of package names to be installed.")
-
- args = parser.parse_args()
-
- if args.mirror is None:
- if args.distro == "ubuntu":
- args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports"
- elif args.distro == "debian":
- args.mirror = "http://ftp.debian.org/debian-ports"
- else:
- raise Exception("Unsupported distro")
-
- DESIRED_PACKAGES = args.packages + [ # base packages
- "dpkg",
- "busybox",
- "libc-bin",
- "base-files",
- "base-passwd",
- "debianutils"
- ]
-
- print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}")
-
- package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite))
-
- packages_info, aliases = parse_package_index(package_index_content)
-
- with tempfile.TemporaryDirectory() as tmp_dir:
- install_packages(args.mirror, packages_info, aliases, tmp_dir, args.rootfsdir, args.artool, DESIRED_PACKAGES)
-
- finalize_setup(args.rootfsdir)
diff --git a/eng/common/cross/tizen-fetch.sh b/eng/common/cross/tizen-fetch.sh
index 37c3a61f1de..28936ceef3a 100755
--- a/eng/common/cross/tizen-fetch.sh
+++ b/eng/common/cross/tizen-fetch.sh
@@ -156,8 +156,13 @@ fetch_tizen_pkgs()
done
}
-BASE="Tizen-Base"
-UNIFIED="Tizen-Unified"
+if [ "$TIZEN_ARCH" == "riscv64" ]; then
+ BASE="Tizen-Base-RISCV"
+ UNIFIED="Tizen-Unified-RISCV"
+else
+ BASE="Tizen-Base"
+ UNIFIED="Tizen-Unified"
+fi
Inform "Initialize ${TIZEN_ARCH} base"
fetch_tizen_pkgs_init standard $BASE
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
index 0ff85cf0367..9a7ecfbd42c 100644
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@@ -67,13 +67,6 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif()
-elseif(TARGET_ARCH_NAME STREQUAL "loongarch64")
- set(CMAKE_SYSTEM_PROCESSOR "loongarch64")
- if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/loongarch64-alpine-linux-musl)
- set(TOOLCHAIN "loongarch64-alpine-linux-musl")
- else()
- set(TOOLCHAIN "loongarch64-linux-gnu")
- endif()
elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
set(CMAKE_SYSTEM_PROCESSOR ppc64le)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl)
@@ -125,7 +118,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu")
endif()
else()
- message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64 and x86 are supported!")
+ message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
endif()
if(DEFINED ENV{TOOLCHAIN})
@@ -155,25 +148,6 @@ if(TIZEN)
include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN})
endif()
-function(locate_toolchain_exec exec var)
- set(TOOLSET_PREFIX ${TOOLCHAIN}-)
- string(TOUPPER ${exec} EXEC_UPPERCASE)
- if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
- set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
- return()
- endif()
-
- find_program(EXEC_LOCATION_${exec}
- NAMES
- "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
- "${TOOLSET_PREFIX}${exec}")
-
- if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
- message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
- endif()
- set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
-endfunction()
-
if(ANDROID)
if(TARGET_ARCH_NAME STREQUAL "arm")
set(ANDROID_ABI armeabi-v7a)
@@ -204,24 +178,66 @@ elseif(FREEBSD)
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld")
elseif(ILLUMOS)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
- set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
- set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
- set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
include_directories(SYSTEM ${CROSS_ROOTFS}/include)
+ set(TOOLSET_PREFIX ${TOOLCHAIN}-)
+ function(locate_toolchain_exec exec var)
+ string(TOUPPER ${exec} EXEC_UPPERCASE)
+ if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
+ set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
+ return()
+ endif()
+
+ find_program(EXEC_LOCATION_${exec}
+ NAMES
+ "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
+ "${TOOLSET_PREFIX}${exec}")
+
+ if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
+ message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
+ endif()
+ set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
+ endfunction()
+
+ set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
+
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
+
+ set(TOOLSET_PREFIX ${TOOLCHAIN}-)
+ function(locate_toolchain_exec exec var)
+ string(TOUPPER ${exec} EXEC_UPPERCASE)
+ if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
+ set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
+ return()
+ endif()
+
+ find_program(EXEC_LOCATION_${exec}
+ NAMES
+ "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
+ "${TOOLSET_PREFIX}${exec}")
+
+ if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
+ message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
+ endif()
+ set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
+ endfunction()
+
set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
- set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
- set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
+
# let CMake set up the correct search paths
include(Platform/Haiku)
else()
@@ -291,7 +307,7 @@ endif()
# Specify compile options
-if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
+if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index e889f439b8d..36dbd45e1ce 100755
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -68,7 +68,7 @@ function InstallDarcCli {
fi
fi
- local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
+ local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
echo "Installing Darc CLI version $darcVersion..."
echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
diff --git a/eng/common/dotnet.cmd b/eng/common/dotnet.cmd
deleted file mode 100644
index 527fa4bb38f..00000000000
--- a/eng/common/dotnet.cmd
+++ /dev/null
@@ -1,7 +0,0 @@
-@echo off
-
-:: This script is used to install the .NET SDK.
-:: It will also invoke the SDK with any provided arguments.
-
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet.ps1""" %*"
-exit /b %ErrorLevel%
diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1
deleted file mode 100644
index 45e5676c9eb..00000000000
--- a/eng/common/dotnet.ps1
+++ /dev/null
@@ -1,11 +0,0 @@
-# This script is used to install the .NET SDK.
-# It will also invoke the SDK with any provided arguments.
-
-. $PSScriptRoot\tools.ps1
-$dotnetRoot = InitializeDotNetCli -install:$true
-
-# Invoke acquired SDK with args if they are provided
-if ($args.count -gt 0) {
- $env:DOTNET_NOLOGO=1
- & "$dotnetRoot\dotnet.exe" $args
-}
diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh
deleted file mode 100644
index 2ef68235675..00000000000
--- a/eng/common/dotnet.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is used to install the .NET SDK.
-# It will also invoke the SDK with any provided arguments.
-
-source="${BASH_SOURCE[0]}"
-# resolve $SOURCE until the file is no longer a symlink
-while [[ -h $source ]]; do
- scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
- source="$(readlink "$source")"
-
- # if $source was a relative symlink, we need to resolve it relative to the path where the
- # symlink file was located
- [[ $source != /* ]] && source="$scriptroot/$source"
-done
-scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-
-source $scriptroot/tools.sh
-InitializeDotNetCli true # install
-
-# Invoke acquired SDK with args if they are provided
-if [[ $# > 0 ]]; then
- __dotnetDir=${_InitializeDotNetCli}
- dotnetPath=${__dotnetDir}/dotnet
- ${dotnetPath} "$@"
-fi
diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
index fa1cdc2b300..524aaa57f2b 100644
--- a/eng/common/generate-locproject.ps1
+++ b/eng/common/generate-locproject.ps1
@@ -33,27 +33,15 @@ $jsonTemplateFiles | ForEach-Object {
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
-$wxlFilesV3 = @()
-$wxlFilesV5 = @()
$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
if (-not $wxlFiles) {
$wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
if ($wxlEnFiles) {
- $wxlFiles = @()
- $wxlEnFiles | ForEach-Object {
- $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
- $content = Get-Content $_.FullName -Raw
-
- # Split files on schema to select different parser settings in the generated project.
- if ($content -like "*http://wixtoolset.org/schemas/v4/wxl*")
- {
- $wxlFilesV5 += Copy-Item $_.FullName -Destination $destinationFile -PassThru
- }
- elseif ($content -like "*http://schemas.microsoft.com/wix/2006/localization*")
- {
- $wxlFilesV3 += Copy-Item $_.FullName -Destination $destinationFile -PassThru
- }
- }
+ $wxlFiles = @()
+ $wxlEnFiles | ForEach-Object {
+ $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+ $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+ }
}
}
@@ -126,32 +114,7 @@ $locJson = @{
CloneLanguageSet = "WiX_CloneLanguages"
LssFiles = @( "wxl_loc.lss" )
LocItems = @(
- $wxlFilesV3 | ForEach-Object {
- $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
- $continue = $true
- foreach ($exclusion in $exclusions.Exclusions) {
- if ($_.FullName.Contains($exclusion)) {
- $continue = $false
- }
- }
- $sourceFile = ($_.FullName | Resolve-Path -Relative)
- if ($continue)
- {
- return @{
- SourceFile = $sourceFile
- CopyOption = "LangIDOnPath"
- OutputPath = $outputPath
- }
- }
- }
- )
- },
- @{
- LanguageSet = $LanguageSet
- CloneLanguageSet = "WiX_CloneLanguages"
- LssFiles = @( "P210WxlSchemaV4.lss" )
- LocItems = @(
- $wxlFilesV5 | ForEach-Object {
+ $wxlFiles | ForEach-Object {
$outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh
deleted file mode 100644
index 477a44f335b..00000000000
--- a/eng/common/native/install-dependencies.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# This is a simple script primarily used for CI to install necessary dependencies
-#
-# Usage:
-#
-# ./install-dependencies.sh
-
-os="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
-
-if [ -z "$os" ]; then
- . "$(dirname "$0")"/init-os-and-arch.sh
-fi
-
-case "$os" in
- linux)
- if [ -e /etc/os-release ]; then
- . /etc/os-release
- fi
-
- if [ "$ID" = "debian" ] || [ "$ID_LIKE" = "debian" ]; then
- apt update
-
- apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
- libssl-dev libkrb5-dev pigz cpio
-
- localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
- elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then
- pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)"
- $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio
- elif [ "$ID" = "alpine" ]; then
- apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio
- else
- echo "Unsupported distro. distro: $ID"
- exit 1
- fi
- ;;
-
- osx|maccatalyst|ios|iossimulator|tvos|tvossimulator)
- echo "Installed xcode version: $(xcode-select -p)"
-
- export HOMEBREW_NO_INSTALL_CLEANUP=1
- export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
- # Skip brew update for now, see https://github.com/actions/setup-python/issues/577
- # brew update --preinstall
- brew bundle --no-upgrade --file=- < Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
- Write-Host " -excludeCIBinaryLog When running on CI, allow no binary log (short: -nobl)"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
}
@@ -37,11 +34,10 @@ function Print-Usage() {
function Build([string]$target) {
$logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog"
- $binaryLogArg = if ($binaryLog) { "/bl:$log" } else { "" }
$outputPath = Join-Path $ToolsetDir "$task\"
MSBuild $taskProject `
- $binaryLogArg `
+ /bl:$log `
/t:$target `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
@@ -68,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
- $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty
+ $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh
deleted file mode 100644
index 3270f83fa9a..00000000000
--- a/eng/common/sdk-task.sh
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-
-show_usage() {
- echo "Common settings:"
- echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)"
- echo " --restore Restore dependencies"
- echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]"
- echo " --help Print help and exit"
- echo ""
-
- echo "Advanced settings:"
- echo " --excludeCIBinarylog Don't output binary log (short: -nobl)"
- echo " --noWarnAsError Do not warn as error"
- echo ""
- echo "Command line arguments not listed above are passed thru to msbuild."
-}
-
-source="${BASH_SOURCE[0]}"
-
-# resolve $source until the file is no longer a symlink
-while [[ -h "$source" ]]; do
- scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
- source="$(readlink "$source")"
- # if $source was a relative symlink, we need to resolve it relative to the path where the
- # symlink file was located
- [[ $source != /* ]] && source="$scriptroot/$source"
-done
-scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-
-Build() {
- local target=$1
- local log_suffix=""
- [[ "$target" != "Execute" ]] && log_suffix=".$target"
- local log="$log_dir/$task$log_suffix.binlog"
- local binaryLogArg=""
- [[ $binary_log == true ]] && binaryLogArg="/bl:$log"
- local output_path="$toolset_dir/$task/"
-
- MSBuild "$taskProject" \
- $binaryLogArg \
- /t:"$target" \
- /p:Configuration="$configuration" \
- /p:RepoRoot="$repo_root" \
- /p:BaseIntermediateOutputPath="$output_path" \
- /v:"$verbosity" \
- $properties
-}
-
-binary_log=true
-configuration="Debug"
-verbosity="minimal"
-exclude_ci_binary_log=false
-restore=false
-help=false
-properties=''
-warnAsError=true
-
-while (($# > 0)); do
- lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
- case $lowerI in
- --task)
- task=$2
- shift 2
- ;;
- --restore)
- restore=true
- shift 1
- ;;
- --verbosity)
- verbosity=$2
- shift 2
- ;;
- --excludecibinarylog|--nobl)
- binary_log=false
- exclude_ci_binary_log=true
- shift 1
- ;;
- --noWarnAsError)
- warnAsError=false
- shift 1
- ;;
- --help)
- help=true
- shift 1
- ;;
- *)
- properties="$properties $1"
- shift 1
- ;;
- esac
-done
-
-ci=true
-
-if $help; then
- show_usage
- exit 0
-fi
-
-. "$scriptroot/tools.sh"
-InitializeToolset
-
-if [[ -z "$task" ]]; then
- Write-PipelineTelemetryError -Category 'Task' -Name 'MissingTask' -Message "Missing required parameter '-task '"
- ExitWithExitCode 1
-fi
-
-taskProject=$(GetSdkTaskProject "$task")
-if [[ ! -e "$taskProject" ]]; then
- Write-PipelineTelemetryError -Category 'Task' -Name 'UnknownTask' -Message "Unknown task: $task"
- ExitWithExitCode 1
-fi
-
-if $restore; then
- Build "Restore"
-fi
-
-Build "Execute"
-
-
-ExitWithExitCode 0
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index e5f543ea68c..4585cfd6bba 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml
index 92a0664f564..81ea7a261f2 100644
--- a/eng/common/templates-official/job/job.yml
+++ b/eng/common/templates-official/job/job.yml
@@ -31,7 +31,6 @@ jobs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
condition: always()
- retryCountOnTaskFailure: 10 # for any logs being locked
continueOnError: true
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- output: pipelineArtifact
@@ -40,7 +39,6 @@ jobs:
displayName: 'Publish logs'
continueOnError: true
condition: always()
- retryCountOnTaskFailure: 10 # for any logs being locked
sbomEnabled: false # we don't need SBOM for logs
- ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}:
@@ -48,7 +46,7 @@ jobs:
displayName: Publish Logs
PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
publishLocation: Container
- ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
+ ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
continueOnError: true
condition: always()
sbomEnabled: false # we don't need SBOM for logs
diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml
index fcf6637b2eb..100a3fc9849 100644
--- a/eng/common/templates-official/steps/publish-build-artifacts.yml
+++ b/eng/common/templates-official/steps/publish-build-artifacts.yml
@@ -24,10 +24,6 @@ parameters:
- name: is1ESPipeline
type: boolean
default: true
-
-- name: retryCountOnTaskFailure
- type: string
- default: 10
steps:
- ${{ if ne(parameters.is1ESPipeline, true) }}:
@@ -42,5 +38,4 @@ steps:
PathtoPublish: ${{ parameters.pathToPublish }}
${{ if parameters.artifactName }}:
ArtifactName: ${{ parameters.artifactName }}
- ${{ if parameters.retryCountOnTaskFailure }}:
- retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
+
diff --git a/eng/common/templates-official/steps/source-index-stage1-publish.yml b/eng/common/templates-official/steps/source-index-stage1-publish.yml
deleted file mode 100644
index 9b8b80942b5..00000000000
--- a/eng/common/templates-official/steps/source-index-stage1-publish.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-steps:
-- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
- parameters:
- is1ESPipeline: true
-
- ${{ each parameter in parameters }}:
- ${{ parameter.key }}: ${{ parameter.value }}
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index 238fa0818f7..5bdd3dd85fd 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -46,7 +46,6 @@ jobs:
artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
continueOnError: true
condition: always()
- retryCountOnTaskFailure: 10 # for any logs being locked
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
@@ -57,7 +56,6 @@ jobs:
displayName: 'Publish logs'
continueOnError: true
condition: always()
- retryCountOnTaskFailure: 10 # for any logs being locked
sbomEnabled: false # we don't need SBOM for logs
- ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
@@ -68,7 +66,7 @@ jobs:
displayName: Publish Logs
pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
publishLocation: Container
- artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
+ artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
continueOnError: true
condition: always()
diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml
index 605e602e94d..6428a98dfef 100644
--- a/eng/common/templates/steps/publish-build-artifacts.yml
+++ b/eng/common/templates/steps/publish-build-artifacts.yml
@@ -25,10 +25,6 @@ parameters:
type: string
default: 'Container'
-- name: retryCountOnTaskFailure
- type: string
- default: 10
-
steps:
- ${{ if eq(parameters.is1ESPipeline, true) }}:
- 'eng/common/templates cannot be referenced from a 1ES managed template': error
@@ -41,6 +37,4 @@ steps:
PublishLocation: ${{ parameters.publishLocation }}
PathtoPublish: ${{ parameters.pathToPublish }}
${{ if parameters.artifactName }}:
- ArtifactName: ${{ parameters.artifactName }}
- ${{ if parameters.retryCountOnTaskFailure }}:
- retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
+ ArtifactName: ${{ parameters.artifactName }}
\ No newline at end of file
diff --git a/eng/common/templates/steps/source-index-stage1-publish.yml b/eng/common/templates/steps/source-index-stage1-publish.yml
deleted file mode 100644
index 182cec33a7b..00000000000
--- a/eng/common/templates/steps/source-index-stage1-publish.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-steps:
-- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
- parameters:
- is1ESPipeline: false
-
- ${{ each parameter in parameters }}:
- ${{ parameter.key }}: ${{ parameter.value }}
diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml
deleted file mode 100644
index 599afb6186b..00000000000
--- a/eng/common/templates/steps/vmr-sync.yml
+++ /dev/null
@@ -1,207 +0,0 @@
-### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet).
-### They initialize the darc CLI and pull the new updates.
-### Changes are applied locally onto the already cloned VMR (located in $vmrPath).
-
-parameters:
-- name: targetRef
- displayName: Target revision in dotnet/ to synchronize
- type: string
- default: $(Build.SourceVersion)
-
-- name: vmrPath
- displayName: Path where the dotnet/dotnet is checked out to
- type: string
- default: $(Agent.BuildDirectory)/vmr
-
-- name: additionalSyncs
- displayName: Optional list of package names whose repo's source will also be synchronized in the local VMR, e.g. NuGet.Protocol
- type: object
- default: []
-
-steps:
-- checkout: vmr
- displayName: Clone dotnet/dotnet
- path: vmr
- clean: true
-
-- checkout: self
- displayName: Clone $(Build.Repository.Name)
- path: repo
- fetchDepth: 0
-
-# This step is needed so that when we get a detached HEAD / shallow clone,
-# we still pull the commit into the temporary repo clone to use it during the sync.
-# Also unshallow the clone so that forwardflow command would work.
-- script: |
- git branch repo-head
- git rev-parse HEAD
- displayName: Label PR commit
- workingDirectory: $(Agent.BuildDirectory)/repo
-
-- script: |
- vmr_sha=$(grep -oP '(?<=Sha=")[^"]*' $(Agent.BuildDirectory)/repo/eng/Version.Details.xml)
- echo "##vso[task.setvariable variable=vmr_sha]$vmr_sha"
- displayName: Obtain the vmr sha from Version.Details.xml (Unix)
- condition: ne(variables['Agent.OS'], 'Windows_NT')
- workingDirectory: $(Agent.BuildDirectory)/repo
-
-- powershell: |
- [xml]$xml = Get-Content -Path $(Agent.BuildDirectory)/repo/eng/Version.Details.xml
- $vmr_sha = $xml.SelectSingleNode("//Source").Sha
- Write-Output "##vso[task.setvariable variable=vmr_sha]$vmr_sha"
- displayName: Obtain the vmr sha from Version.Details.xml (Windows)
- condition: eq(variables['Agent.OS'], 'Windows_NT')
- workingDirectory: $(Agent.BuildDirectory)/repo
-
-- script: |
- git fetch --all
- git checkout $(vmr_sha)
- displayName: Checkout VMR at correct sha for repo flow
- workingDirectory: ${{ parameters.vmrPath }}
-
-- script: |
- git config --global user.name "dotnet-maestro[bot]"
- git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"
- displayName: Set git author to dotnet-maestro[bot]
- workingDirectory: ${{ parameters.vmrPath }}
-
-- script: |
- ./eng/common/vmr-sync.sh \
- --vmr ${{ parameters.vmrPath }} \
- --tmp $(Agent.TempDirectory) \
- --azdev-pat '$(dn-bot-all-orgs-code-r)' \
- --ci \
- --debug
-
- if [ "$?" -ne 0 ]; then
- echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
- exit 1
- fi
- displayName: Sync repo into VMR (Unix)
- condition: ne(variables['Agent.OS'], 'Windows_NT')
- workingDirectory: $(Agent.BuildDirectory)/repo
-
-- script: |
- git config --global diff.astextplain.textconv echo
- git config --system core.longpaths true
- displayName: Configure Windows git (longpaths, astextplain)
- condition: eq(variables['Agent.OS'], 'Windows_NT')
-
-- powershell: |
- ./eng/common/vmr-sync.ps1 `
- -vmr ${{ parameters.vmrPath }} `
- -tmp $(Agent.TempDirectory) `
- -azdevPat '$(dn-bot-all-orgs-code-r)' `
- -ci `
- -debugOutput
-
- if ($LASTEXITCODE -ne 0) {
- echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
- exit 1
- }
- displayName: Sync repo into VMR (Windows)
- condition: eq(variables['Agent.OS'], 'Windows_NT')
- workingDirectory: $(Agent.BuildDirectory)/repo
-
-- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- - task: CopyFiles@2
- displayName: Collect failed patches
- condition: failed()
- inputs:
- SourceFolder: '$(Agent.TempDirectory)'
- Contents: '*.patch'
- TargetFolder: '$(Build.ArtifactStagingDirectory)/FailedPatches'
-
- - publish: '$(Build.ArtifactStagingDirectory)/FailedPatches'
- artifact: $(System.JobDisplayName)_FailedPatches
- displayName: Upload failed patches
- condition: failed()
-
-- ${{ each assetName in parameters.additionalSyncs }}:
- # The vmr-sync script ends up staging files in the local VMR so we have to commit those
- - script:
- git commit --allow-empty -am "Forward-flow $(Build.Repository.Name)"
- displayName: Commit local VMR changes
- workingDirectory: ${{ parameters.vmrPath }}
-
- - script: |
- set -ex
-
- echo "Searching for details of asset ${{ assetName }}..."
-
- # Use darc to get dependencies information
- dependencies=$(./.dotnet/dotnet darc get-dependencies --name '${{ assetName }}' --ci)
-
- # Extract repository URL and commit hash
- repository=$(echo "$dependencies" | grep 'Repo:' | sed 's/Repo:[[:space:]]*//' | head -1)
-
- if [ -z "$repository" ]; then
- echo "##vso[task.logissue type=error]Asset ${{ assetName }} not found in the dependency list"
- exit 1
- fi
-
- commit=$(echo "$dependencies" | grep 'Commit:' | sed 's/Commit:[[:space:]]*//' | head -1)
-
- echo "Updating the VMR from $repository / $commit..."
- cd ..
- git clone $repository ${{ assetName }}
- cd ${{ assetName }}
- git checkout $commit
- git branch "sync/$commit"
-
- ./eng/common/vmr-sync.sh \
- --vmr ${{ parameters.vmrPath }} \
- --tmp $(Agent.TempDirectory) \
- --azdev-pat '$(dn-bot-all-orgs-code-r)' \
- --ci \
- --debug
-
- if [ "$?" -ne 0 ]; then
- echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
- exit 1
- fi
- displayName: Sync ${{ assetName }} into (Unix)
- condition: ne(variables['Agent.OS'], 'Windows_NT')
- workingDirectory: $(Agent.BuildDirectory)/repo
-
- - powershell: |
- $ErrorActionPreference = 'Stop'
-
- Write-Host "Searching for details of asset ${{ assetName }}..."
-
- $dependencies = .\.dotnet\dotnet darc get-dependencies --name '${{ assetName }}' --ci
-
- $repository = $dependencies | Select-String -Pattern 'Repo:\s+([^\s]+)' | Select-Object -First 1
- $repository -match 'Repo:\s+([^\s]+)' | Out-Null
- $repository = $matches[1]
-
- if ($repository -eq $null) {
- Write-Error "Asset ${{ assetName }} not found in the dependency list"
- exit 1
- }
-
- $commit = $dependencies | Select-String -Pattern 'Commit:\s+([^\s]+)' | Select-Object -First 1
- $commit -match 'Commit:\s+([^\s]+)' | Out-Null
- $commit = $matches[1]
-
- Write-Host "Updating the VMR from $repository / $commit..."
- cd ..
- git clone $repository ${{ assetName }}
- cd ${{ assetName }}
- git checkout $commit
- git branch "sync/$commit"
-
- .\eng\common\vmr-sync.ps1 `
- -vmr ${{ parameters.vmrPath }} `
- -tmp $(Agent.TempDirectory) `
- -azdevPat '$(dn-bot-all-orgs-code-r)' `
- -ci `
- -debugOutput
-
- if ($LASTEXITCODE -ne 0) {
- echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
- exit 1
- }
- displayName: Sync ${{ assetName }} into (Windows)
- condition: ne(variables['Agent.OS'], 'Windows_NT')
- workingDirectory: $(Agent.BuildDirectory)/repo
diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml
deleted file mode 100644
index ce3c29a62fa..00000000000
--- a/eng/common/templates/vmr-build-pr.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-# This pipeline is used for running the VMR verification of the PR changes in repo-level PRs.
-#
-# It will run a full set of verification jobs defined in:
-# https://github.com/dotnet/dotnet/blob/10060d128e3f470e77265f8490f5e4f72dae738e/eng/pipelines/templates/stages/vmr-build.yml#L27-L38
-#
-# For repos that do not need to run the full set, you would do the following:
-#
-# 1. Copy this YML file to a repo-specific location, i.e. outside of eng/common.
-#
-# 2. Add `verifications` parameter to VMR template reference
-#
-# Examples:
-# - For source-build stage 1 verification, add the following:
-# verifications: [ "source-build-stage1" ]
-#
-# - For Windows only verifications, add the following:
-# verifications: [ "unified-build-windows-x64", "unified-build-windows-x86" ]
-
-trigger: none
-pr: none
-
-variables:
-- template: /eng/common/templates/variables/pool-providers.yml@self
-
-- name: skipComponentGovernanceDetection # we run CG on internal builds only
- value: true
-
-- name: Codeql.Enabled # we run CodeQL on internal builds only
- value: false
-
-resources:
- repositories:
- - repository: vmr
- type: github
- name: dotnet/dotnet
- endpoint: dotnet
-
-stages:
-- template: /eng/pipelines/templates/stages/vmr-build.yml@vmr
- parameters:
- isBuiltFromVmr: false
- scope: lite
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 06b44de7870..9b3ad8840fd 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -65,8 +65,10 @@ $ErrorActionPreference = 'Stop'
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
-# True when the build is running within the VMR.
-[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false }
+# True if the build is a product build
+[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false }
+
+[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() }
function Create-Directory ([string[]] $path) {
New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
@@ -257,20 +259,7 @@ function Retry($downloadBlock, $maxRetries = 5) {
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
- $shouldDownload = $false
-
if (!(Test-Path $installScript)) {
- $shouldDownload = $true
- } else {
- # Check if the script is older than 30 days
- $fileAge = (Get-Date) - (Get-Item $installScript).LastWriteTime
- if ($fileAge.Days -gt 30) {
- Write-Host "Existing install script is too old, re-downloading..."
- $shouldDownload = $true
- }
- }
-
- if ($shouldDownload) {
Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
$uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
@@ -394,8 +383,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here:
- # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0
- $defaultXCopyMSBuildVersion = '17.13.0'
+ # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.12.0
+ $defaultXCopyMSBuildVersion = '17.12.0'
if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@@ -544,8 +533,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
$vswhereVersion = $GlobalJson.tools.vswhere
} else {
- # keep this in sync with the VSWhereVersion in DefaultVersions.props
- $vswhereVersion = '3.1.7'
+ $vswhereVersion = '2.5.2'
}
$vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
@@ -553,8 +541,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
- Write-Host "Downloading vswhere $vswhereVersion"
- $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+ Write-Host 'Downloading vswhere'
Retry({
Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
})
@@ -617,7 +604,14 @@ function InitializeBuildTool() {
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' }
+ # Use override if it exists - commonly set by source-build
+ if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) {
+ $initializeBuildToolFramework="net9.0"
+ } else {
+ $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
+ }
+
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -626,7 +620,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
- $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS }
+ $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1
@@ -659,6 +653,7 @@ function GetNuGetPackageCachePath() {
$env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
} else {
$env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+ $env:RESTORENOHTTPCACHE = $true
}
}
@@ -780,13 +775,26 @@ function MSBuild() {
$toolsetBuildProject = InitializeToolset
$basePath = Split-Path -parent $toolsetBuildProject
- $selectedPath = Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')
-
+ $possiblePaths = @(
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ )
+ $selectedPath = $null
+ foreach ($path in $possiblePaths) {
+ if (Test-Path $path -PathType Leaf) {
+ $selectedPath = $path
+ break
+ }
+ }
if (-not $selectedPath) {
- Write-PipelineTelemetryError -Category 'Build' -Message "Unable to find arcade sdk logger assembly: $selectedPath"
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
ExitWithExitCode 1
}
-
$args += "/logger:$selectedPath"
}
@@ -849,8 +857,8 @@ function MSBuild-Core() {
}
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
- # Skip this when the build is a child of the VMR build.
- if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) {
+ # Skip this when the build is a child of the VMR orchestrator build.
+ if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) {
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index c1841c9dfd0..01b09b65796 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -5,9 +5,6 @@
# CI mode - set to true on CI server for PR validation build or official build.
ci=${ci:-false}
-# Build mode
-source_build=${source_build:-false}
-
# Set to true to use the pipelines logger which will enable Azure logging output.
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
# This flag is meant as a temporary opt-opt for the feature while validate it across
@@ -61,8 +58,7 @@ use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
# True to use global NuGet cache instead of restoring packages to repository-local directory.
-# Keep in sync with NuGetPackageroot in Arcade SDK's RepositoryLayout.props.
-if [[ "$ci" == true || "$source_build" == true ]]; then
+if [[ "$ci" == true ]]; then
use_global_nuget_cache=${use_global_nuget_cache:-false}
else
use_global_nuget_cache=${use_global_nuget_cache:-true}
@@ -72,8 +68,8 @@ fi
runtime_source_feed=${runtime_source_feed:-''}
runtime_source_feed_key=${runtime_source_feed_key:-''}
-# True when the build is running within the VMR.
-from_vmr=${from_vmr:-false}
+# True if the build is a product build
+product_build=${product_build:-false}
# Resolve any symlinks in the given path.
function ResolvePath {
@@ -300,29 +296,8 @@ function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
- local timestamp_file="$root/.dotnet-install.timestamp"
- local should_download=false
if [[ ! -a "$install_script" ]]; then
- should_download=true
- elif [[ -f "$timestamp_file" ]]; then
- # Check if the script is older than 30 days using timestamp file
- local download_time=$(cat "$timestamp_file" 2>/dev/null || echo "0")
- local current_time=$(date +%s)
- local age_seconds=$((current_time - download_time))
-
- # 30 days = 30 * 24 * 60 * 60 = 2592000 seconds
- if [[ $age_seconds -gt 2592000 ]]; then
- echo "Existing install script is too old, re-downloading..."
- should_download=true
- fi
- else
- # No timestamp file exists, assume script is old and re-download
- echo "No timestamp found for existing install script, re-downloading..."
- should_download=true
- fi
-
- if [[ "$should_download" == true ]]; then
mkdir -p "$root"
echo "Downloading '$install_script_url'"
@@ -349,9 +324,6 @@ function GetDotNetInstallScript {
ExitWithExitCode $exit_code
}
fi
-
- # Create timestamp file to track download time in seconds from epoch
- date +%s > "$timestamp_file"
fi
# return value
_GetDotNetInstallScript="$install_script"
@@ -367,14 +339,22 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
+ # use override if it exists - commonly set by source-build
+ if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then
+ _InitializeBuildToolFramework="net9.0"
+ else
+ _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}"
+ fi
}
+# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116
function GetNuGetPackageCachePath {
if [[ -z ${NUGET_PACKAGES:-} ]]; then
if [[ "$use_global_nuget_cache" == true ]]; then
export NUGET_PACKAGES="$HOME/.nuget/packages/"
else
export NUGET_PACKAGES="$repo_root/.packages/"
+ export RESTORENOHTTPCACHE=true
fi
fi
@@ -471,13 +451,25 @@ function MSBuild {
fi
local toolset_dir="${_InitializeToolset%/*}"
- local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll"
-
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ local selectedPath=
+ local possiblePaths=()
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" )
+ for path in "${possiblePaths[@]}"; do
+ if [[ -f $path ]]; then
+ selectedPath=$path
+ break
+ fi
+ done
if [[ -z "$selectedPath" ]]; then
- Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly: $selectedPath"
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
ExitWithExitCode 1
fi
-
args+=( "-logger:$selectedPath" )
fi
@@ -514,8 +506,8 @@ function MSBuild-Core {
echo "Build failed with exit code $exit_code. Check errors above."
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
- # Skip this when the build is a child of the VMR build.
- if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then
+ # Skip this when the build is a child of the VMR orchestrator build.
+ if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then
Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@@ -538,13 +530,6 @@ function GetDarc {
fi
"$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version
- darc_tool="$darc_path/darc"
-}
-
-# Returns a full path to an Arcade SDK task project file.
-function GetSdkTaskProject {
- taskName=$1
- echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj"
}
ResolvePath "${BASH_SOURCE[0]}"
diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1
deleted file mode 100644
index 97302f3205b..00000000000
--- a/eng/common/vmr-sync.ps1
+++ /dev/null
@@ -1,138 +0,0 @@
-<#
-.SYNOPSIS
-
-This script is used for synchronizing the current repository into a local VMR.
-It pulls the current repository's code into the specified VMR directory for local testing or
-Source-Build validation.
-
-.DESCRIPTION
-
-The tooling used for synchronization will clone the VMR repository into a temporary folder if
-it does not already exist. These clones can be reused in future synchronizations, so it is
-recommended to dedicate a folder for this to speed up re-runs.
-
-.EXAMPLE
- Synchronize current repository into a local VMR:
- ./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp"
-
-.PARAMETER tmpDir
-Required. Path to the temporary folder where repositories will be cloned
-
-.PARAMETER vmrBranch
-Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
-
-.PARAMETER azdevPat
-Optional. Azure DevOps PAT to use for cloning private repositories.
-
-.PARAMETER vmrDir
-Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
-
-.PARAMETER debugOutput
-Optional. Enables debug logging in the darc vmr command.
-
-.PARAMETER ci
-Optional. Denotes that the script is running in a CI environment.
-#>
-param (
- [Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")]
- [string][Alias('t', 'tmp')]$tmpDir,
- [string][Alias('b', 'branch')]$vmrBranch,
- [string]$remote,
- [string]$azdevPat,
- [string][Alias('v', 'vmr')]$vmrDir,
- [switch]$ci,
- [switch]$debugOutput
-)
-
-function Fail {
- Write-Host "> $($args[0])" -ForegroundColor 'Red'
-}
-
-function Highlight {
- Write-Host "> $($args[0])" -ForegroundColor 'Cyan'
-}
-
-$verbosity = 'verbose'
-if ($debugOutput) {
- $verbosity = 'debug'
-}
-# Validation
-
-if (-not $tmpDir) {
- Fail "Missing -tmpDir argument. Please specify the path to the temporary folder where the repositories will be cloned"
- exit 1
-}
-
-# Sanitize the input
-
-if (-not $vmrDir) {
- $vmrDir = Join-Path $tmpDir 'dotnet'
-}
-
-if (-not (Test-Path -Path $tmpDir -PathType Container)) {
- New-Item -ItemType Directory -Path $tmpDir | Out-Null
-}
-
-# Prepare the VMR
-
-if (-not (Test-Path -Path $vmrDir -PathType Container)) {
- Highlight "Cloning 'dotnet/dotnet' into $vmrDir.."
- git clone https://github.com/dotnet/dotnet $vmrDir
-
- if ($vmrBranch) {
- git -C $vmrDir switch -c $vmrBranch
- }
-}
-else {
- if ((git -C $vmrDir diff --quiet) -eq $false) {
- Fail "There are changes in the working tree of $vmrDir. Please commit or stash your changes"
- exit 1
- }
-
- if ($vmrBranch) {
- Highlight "Preparing $vmrDir"
- git -C $vmrDir checkout $vmrBranch
- git -C $vmrDir pull
- }
-}
-
-Set-StrictMode -Version Latest
-
-# Prepare darc
-
-Highlight 'Installing .NET, preparing the tooling..'
-. .\eng\common\tools.ps1
-$dotnetRoot = InitializeDotNetCli -install:$true
-$darc = Get-Darc
-$dotnet = "$dotnetRoot\dotnet.exe"
-
-Highlight "Starting the synchronization of VMR.."
-
-# Synchronize the VMR
-$darcArgs = (
- "vmr", "forwardflow",
- "--tmp", $tmpDir,
- "--$verbosity",
- $vmrDir
-)
-
-if ($ci) {
- $darcArgs += ("--ci")
-}
-
-if ($azdevPat) {
- $darcArgs += ("--azdev-pat", $azdevPat)
-}
-
-& "$darc" $darcArgs
-
-if ($LASTEXITCODE -eq 0) {
- Highlight "Synchronization succeeded"
-}
-else {
- Fail "Synchronization of repo to VMR failed!"
- Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)."
- Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)."
- Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."
- exit 1
-}
diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh
deleted file mode 100644
index 44239e331c0..00000000000
--- a/eng/common/vmr-sync.sh
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/bin/bash
-
-### This script is used for synchronizing the current repository into a local VMR.
-### It pulls the current repository's code into the specified VMR directory for local testing or
-### Source-Build validation.
-###
-### The tooling used for synchronization will clone the VMR repository into a temporary folder if
-### it does not already exist. These clones can be reused in future synchronizations, so it is
-### recommended to dedicate a folder for this to speed up re-runs.
-###
-### USAGE:
-### Synchronize current repository into a local VMR:
-### ./vmr-sync.sh --tmp "$HOME/repos/tmp" "$HOME/repos/dotnet"
-###
-### Options:
-### -t, --tmp, --tmp-dir PATH
-### Required. Path to the temporary folder where repositories will be cloned
-###
-### -b, --branch, --vmr-branch BRANCH_NAME
-### Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
-###
-### --debug
-### Optional. Turns on the most verbose logging for the VMR tooling
-###
-### --remote name:URI
-### Optional. Additional remote to use during the synchronization
-### This can be used to synchronize to a commit from a fork of the repository
-### Example: 'runtime:https://github.com/yourfork/runtime'
-###
-### --azdev-pat
-### Optional. Azure DevOps PAT to use for cloning private repositories.
-###
-### -v, --vmr, --vmr-dir PATH
-### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
-
-source="${BASH_SOURCE[0]}"
-
-# resolve $source until the file is no longer a symlink
-while [[ -h "$source" ]]; do
- scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
- source="$(readlink "$source")"
- # if $source was a relative symlink, we need to resolve it relative to the path where the
- # symlink file was located
- [[ $source != /* ]] && source="$scriptroot/$source"
-done
-scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-
-function print_help () {
- sed -n '/^### /,/^$/p' "$source" | cut -b 5-
-}
-
-COLOR_RED=$(tput setaf 1 2>/dev/null || true)
-COLOR_CYAN=$(tput setaf 6 2>/dev/null || true)
-COLOR_CLEAR=$(tput sgr0 2>/dev/null || true)
-COLOR_RESET=uniquesearchablestring
-FAILURE_PREFIX='> '
-
-function fail () {
- echo "${COLOR_RED}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_RED}}${COLOR_CLEAR}" >&2
-}
-
-function highlight () {
- echo "${COLOR_CYAN}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_CYAN}}${COLOR_CLEAR}"
-}
-
-tmp_dir=''
-vmr_dir=''
-vmr_branch=''
-additional_remotes=''
-verbosity=verbose
-azdev_pat=''
-ci=false
-
-while [[ $# -gt 0 ]]; do
- opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
- case "$opt" in
- -t|--tmp|--tmp-dir)
- tmp_dir=$2
- shift
- ;;
- -v|--vmr|--vmr-dir)
- vmr_dir=$2
- shift
- ;;
- -b|--branch|--vmr-branch)
- vmr_branch=$2
- shift
- ;;
- --remote)
- additional_remotes="$additional_remotes $2"
- shift
- ;;
- --azdev-pat)
- azdev_pat=$2
- shift
- ;;
- --ci)
- ci=true
- ;;
- -d|--debug)
- verbosity=debug
- ;;
- -h|--help)
- print_help
- exit 0
- ;;
- *)
- fail "Invalid argument: $1"
- print_help
- exit 1
- ;;
- esac
-
- shift
-done
-
-# Validation
-
-if [[ -z "$tmp_dir" ]]; then
- fail "Missing --tmp-dir argument. Please specify the path to the temporary folder where the repositories will be cloned"
- exit 1
-fi
-
-# Sanitize the input
-
-if [[ -z "$vmr_dir" ]]; then
- vmr_dir="$tmp_dir/dotnet"
-fi
-
-if [[ ! -d "$tmp_dir" ]]; then
- mkdir -p "$tmp_dir"
-fi
-
-if [[ "$verbosity" == "debug" ]]; then
- set -x
-fi
-
-# Prepare the VMR
-
-if [[ ! -d "$vmr_dir" ]]; then
- highlight "Cloning 'dotnet/dotnet' into $vmr_dir.."
- git clone https://github.com/dotnet/dotnet "$vmr_dir"
-
- if [[ -n "$vmr_branch" ]]; then
- git -C "$vmr_dir" switch -c "$vmr_branch"
- fi
-else
- if ! git -C "$vmr_dir" diff --quiet; then
- fail "There are changes in the working tree of $vmr_dir. Please commit or stash your changes"
- exit 1
- fi
-
- if [[ -n "$vmr_branch" ]]; then
- highlight "Preparing $vmr_dir"
- git -C "$vmr_dir" checkout "$vmr_branch"
- git -C "$vmr_dir" pull
- fi
-fi
-
-set -e
-
-# Prepare darc
-
-highlight 'Installing .NET, preparing the tooling..'
-source "./eng/common/tools.sh"
-InitializeDotNetCli true
-GetDarc
-dotnetDir=$( cd ./.dotnet/; pwd -P )
-dotnet=$dotnetDir/dotnet
-
-highlight "Starting the synchronization of VMR.."
-set +e
-
-if [[ -n "$additional_remotes" ]]; then
- additional_remotes="--additional-remotes $additional_remotes"
-fi
-
-if [[ -n "$azdev_pat" ]]; then
- azdev_pat="--azdev-pat $azdev_pat"
-fi
-
-ci_arg=''
-if [[ "$ci" == "true" ]]; then
- ci_arg="--ci"
-fi
-
-# Synchronize the VMR
-
-export DOTNET_ROOT="$dotnetDir"
-
-"$darc_tool" vmr forwardflow \
- --tmp "$tmp_dir" \
- $azdev_pat \
- --$verbosity \
- $ci_arg \
- $additional_remotes \
- "$vmr_dir"
-
-if [[ $? == 0 ]]; then
- highlight "Synchronization succeeded"
-else
- fail "Synchronization of repo to VMR failed!"
- fail "'$vmr_dir' is left in its last state (re-run of this script will reset it)."
- fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)."
- fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."
- exit 1
-fi
diff --git a/global.json b/global.json
index fd61e068f3a..b0500c7f83e 100644
--- a/global.json
+++ b/global.json
@@ -18,7 +18,7 @@
"msbuild-sdks": {
"Microsoft.Build.NoTargets": "3.7.0",
"Microsoft.Build.Traversal": "3.2.0",
- "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25476.2",
- "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25476.2"
+ "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25515.2",
+ "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25515.2"
}
}