From cc7af6ceb0ad8437651213954e12ed120ca1ffff Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 15:32:06 -0700 Subject: [PATCH 01/11] Rebase the installer build jobs on the global build job template. --- eng/pipelines/installer/jobs/build-job.yml | 248 ------------------ .../jobs/steps/upload-job-artifacts.yml | 65 ----- .../{jobs => }/steps/build-linux-package.yml | 3 - .../installer/steps/upload-job-artifacts.yml | 26 ++ eng/pipelines/runtime-official.yml | 4 +- eng/pipelines/runtime.yml | 113 +++++++- 6 files changed, 127 insertions(+), 332 deletions(-) delete mode 100644 eng/pipelines/installer/jobs/build-job.yml delete mode 100644 eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml rename eng/pipelines/installer/{jobs => }/steps/build-linux-package.yml (91%) create mode 100644 eng/pipelines/installer/steps/upload-job-artifacts.yml diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml deleted file mode 100644 index 9a444b2bde170..0000000000000 --- a/eng/pipelines/installer/jobs/build-job.yml +++ /dev/null @@ -1,248 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - archType: '' - osSubgroup: '' - platform: '' - crossBuild: false - timeoutInMinutes: 120 - condition: true - shouldContinueOnError: false - container: '' - buildSteps: [] - dependsOn: [] - globalBuildSuffix: '' - variables: [] - name: '' - displayName: '' - runtimeVariant: '' - pool: '' - - # The target names here should match container names in the resources section in our pipelines, like runtime.yml - packageDistroList: - - target: debpkg - packageType: deb - packagingArgs: /p:BuildDebPackage=true - - target: rpmpkg - packageType: rpm - packagingArgs: /p:BuildRpmPackage=true - - isOfficialBuild: false - buildFullPlatformManifest: false - - liveRuntimeBuildConfig: '' - liveLibrariesBuildConfig: '' - runtimeFlavor: 'coreclr' - unifiedArtifactsName: '' - unifiedBuildNameSuffix: '' - unifiedBuildConfigOverride: '' - -### Product build -jobs: -- template: /eng/common/templates/job/job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - runtimeVariant: ${{ parameters.runtimeVariant }} - testGroup: ${{ parameters.testGroup }} - helixType: 'build/product/' - enableMicrobuild: true - pool: ${{ parameters.pool }} - - # Compute job name from template parameters - name: ${{ format('installer_{0}_{1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} - displayName: ${{ format('Installer Build and Test {0} {1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - crossBuild: ${{ parameters.crossBuild }} - - gatherAssetManifests: true - - # Component governance does not work on musl machines - ${{ if eq(parameters.osSubGroup, '_musl') }}: - disableComponentGovernance: true - - variables: - - ${{ each variable in parameters.variables }}: - - ${{ variable }} - - - name: BuildAction - value: -test - - - name: buildCommandSourcesDirectory - value: '$(Build.SourcesDirectory)/' - - - name: librariesBuildConfigArg - value: -lc ${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }} - - - name: runtimeBuildConfigArg - value: -rc ${{ coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig) }} - - ### - ### Platform-specific variable setup - ### - - - ${{ if eq(parameters.osGroup, 'windows') }}: - - - name: CommonMSBuildArgs - value: >- - /p:TargetArchitecture=${{ parameters.archType }} - /p:PortableBuild=true - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - - name: TargetArchitecture - value: ${{ parameters.archType }} - - - name: BaseJobBuildCommand - value: >- - build.cmd -subset host+packs -ci - $(BuildAction) - -configuration $(_BuildConfig) - $(librariesBuildConfigArg) - $(runtimeBuildConfigArg) - $(CommonMSBuildArgs) - - - ${{ elseif eq(parameters.osGroup, 'osx') }}: - - - name: CommonMSBuildArgs - value: >- - /p:PortableBuild=true - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - /p:TargetArchitecture=${{ parameters.archType }} - /p:CrossBuild=${{ parameters.crossBuild }} - - - name: BaseJobBuildCommand - value: >- - $(Build.SourcesDirectory)/build.sh -subset host+packs -ci - $(BuildAction) - -configuration $(_BuildConfig) - -arch ${{ parameters.archType }} - $(librariesBuildConfigArg) - $(runtimeBuildConfigArg) - $(CommonMSBuildArgs) - - - ${{ elseif in(parameters.osGroup, 'linux', 'freebsd') }}: - - name: CommonMSBuildArgs - value: >- - /p:Configuration=$(_BuildConfig) - /p:TargetOS=${{ parameters.osGroup }} - /p:TargetArchitecture=${{ parameters.archType }} - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - - - name: _PortableBuild - value: ${{ eq(parameters.osSubgroup, '') }} - - - ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'linux')) }}: - - name: _PortableBuild - value: true - - - ${{ if eq(parameters.crossBuild, true) }}: - - name: ArchArg - value: -arch ${{ parameters.archType }} - - - name: BaseJobBuildCommand - value: >- - $(Build.SourcesDirectory)/build.sh -subset host+packs -ci - $(BuildAction) - $(librariesBuildConfigArg) - $(runtimeBuildConfigArg) - $(ArchArg) - /p:CrossBuild=${{ parameters.crossBuild }} - /p:PortableBuild=$(_PortableBuild) - $(CommonMSBuildArgs) - - - name: installersSubsetArg - value: --subset packs.installers - - dependsOn: - - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - - ${{ parameters.dependsOn }} - steps: - - ${{ if eq(parameters.osGroup, 'windows') }}: - - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml - - checkout: self - clean: true - fetchDepth: $(checkoutFetchDepth) - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ else }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' - artifactName: '${{ parameters.unifiedArtifactsName }}' - displayName: 'unified artifacts' - - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} - displayName: Install Build Dependencies - - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - - script: $(BaseJobBuildCommand) - displayName: Build - continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} - - - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log - displayName: Collect vslogs on exit - condition: always() - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - # Only in glibc leg, we produce RPMs and Debs - - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'linux_x64'), eq(parameters.platform, 'linux_arm64')), eq(parameters.osSubgroup, ''))}}: - - ${{ each packageBuild in parameters.packageDistroList }}: - # This leg's RID matches the build image. Build its distro-dependent packages, as well as - # the distro-independent installers. (There's no particular reason to build the distro- - # independent installers on this leg, but we need to do it somewhere.) - # Currently, linux_arm64 supports 'rpm' type only. - - ${{ if or(not(eq(parameters.platform, 'linux_arm64')), eq(packageBuild.packageType, 'rpm')) }}: - - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml - parameters: - packageType: ${{ packageBuild.packageType }} - target: ${{ packageBuild.target }} - packageStepDescription: Runtime Deps, Runtime, Framework Packs installers - subsetArg: $(installersSubsetArg) - packagingArgs: ${{ packageBuild.packagingArgs }} - - - template: /eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml - parameters: - name: ${{ coalesce(parameters.name, parameters.platform) }} - runtimeFlavor: ${{ parameters.runtimeFlavor }} - runtimeVariant: ${{ parameters.runtimeVariant }} - - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: set -x && df -h - displayName: Check remaining storage space - condition: always() - continueOnError: true diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml deleted file mode 100644 index 8cdf6e8fb7778..0000000000000 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ /dev/null @@ -1,65 +0,0 @@ -parameters: - name: '' - -steps: -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - mergeTestResults: true - testRunTitle: Installer-${{ parameters.name }}-$(_BuildConfig) - continueOnError: true - condition: always() - -# Upload binaries and symbols on failure to allow debugging issues -- task: CopyFiles@2 - displayName: Prepare binaries to publish - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' - Contents: | - */corehost/** - */corehost_test/** - TargetFolder: '$(Build.StagingDirectory)/Binaries' - continueOnError: true - condition: failed() - -- task: ArchiveFiles@2 - displayName: Zip binaries - inputs: - rootFolderOrFile: '$(Build.StagingDirectory)/Binaries' - archiveFile: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - continueOnError: true - condition: failed() - -- task: PublishBuildArtifacts@1 - displayName: Publish binaries - inputs: - pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - artifactName: Installer-Binaries-${{ parameters.name }}-$(_BuildConfig) - continueOnError: true - condition: failed() - -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: always() - -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: Installer-Logs_Attempt$(System.JobAttempt)-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) - continueOnError: true - condition: always() diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml similarity index 91% rename from eng/pipelines/installer/jobs/steps/build-linux-package.yml rename to eng/pipelines/installer/steps/build-linux-package.yml index 7f8320b55dfe3..bcc8a9a8e9706 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/steps/build-linux-package.yml @@ -16,9 +16,6 @@ steps: --ci \ ${{ parameters.subsetArg }} \ ${{ parameters.packagingArgs }} \ - $(CommonMSBuildArgs) \ - $(librariesBuildConfigArg) \ - $(runtimeBuildConfigArg) \ /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} target: ${{ parameters.target }} diff --git a/eng/pipelines/installer/steps/upload-job-artifacts.yml b/eng/pipelines/installer/steps/upload-job-artifacts.yml new file mode 100644 index 0000000000000..076f989780bbf --- /dev/null +++ b/eng/pipelines/installer/steps/upload-job-artifacts.yml @@ -0,0 +1,26 @@ +parameters: + name: '' + +steps: +# Upload binaries and symbols on failure to allow debugging issues +- task: CopyFiles@2 + displayName: Prepare binaries to publish + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' + Contents: | + */corehost/** + */corehost_test/** + TargetFolder: '$(Build.StagingDirectory)/Binaries' + continueOnError: true + condition: failed() + +- template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: '$(Build.StagingDirectory)/Binaries' + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: 'Installer-Binaries-${{ parameters.name }}-$(_BuildConfig)' + displayName: 'Binaries' + condition: failed() diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 0cc897455e2e0..1b9c18c6be33c 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -208,7 +208,7 @@ extends: PublishLocation: Container artifactName: CoreCLRCrossDacArtifacts # Create RPMs and DEBs - - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: deb target: debpkg @@ -216,7 +216,7 @@ extends: subsetArg: -s packs.installers packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64')) - - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: rpm target: rpmpkg diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 3cce0b5a17499..d1805acf12629 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -94,12 +94,34 @@ extends: buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - osx_arm64 + jobParameters: + nameSuffix: AllSubsets_CoreCLR + buildArgs: -s clr+libs+host+packs -rc Release -c Release -lc $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: - linux_arm64 - linux_musl_x64 jobParameters: nameSuffix: AllSubsets_CoreCLR buildArgs: -s clr+libs+host+packs -rc Release -c Release -lc $(_BuildConfig) timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/installer/steps/build-linux-package.yml + parameters: + packageType: rpm + target: rpmpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -1351,16 +1373,27 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - windows_x86 jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: Release - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release - unifiedBuildConfigOverride: release - unifiedBuildNameSuffix: CoreCLR_Libraries + buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + dependsOn: + - build_$(osGroup)$(osSubgroup)_$(archType)_release_CoreCLR_Libraries + preBuildSteps: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release + artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + displayName: 'unified artifacts' + enablePublishTestResults: true + testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType) + postBuildSteps: + - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -1368,18 +1401,70 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - buildConfig: Release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - osx_x64 - - linux_x64 - windows_x64 jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedBuildNameSuffix: CoreCLR_Libraries + buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + dependsOn: + - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries + preBuildSteps: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + displayName: 'unified artifacts' + enablePublishTestResults: true + testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType) + postBuildSteps: + - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + condition: + or( + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - linux_x64 + jobParameters: + buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + dependsOn: + - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries + preBuildSteps: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + displayName: 'unified artifacts' + enablePublishTestResults: true + testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType) + postBuildSteps: + - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + - template: /eng/pipelines/installer/steps/build-linux-package.yml + parameters: + packageType: deb + target: debpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true + - template: /eng/pipelines/installer/steps/build-linux-package.yml + parameters: + packageType: rpm + target: rpmpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), From 14ca5f0320c5990a7d5966e22dac40ca6f5bdf46 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 15:39:44 -0700 Subject: [PATCH 02/11] Rebase build-linux-package on global-build-step now that it's only used in global-build-job jobs. --- .../installer/steps/build-linux-package.yml | 18 ++++++++---------- eng/pipelines/runtime-official.yml | 8 ++------ eng/pipelines/runtime.yml | 12 +++--------- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml index bcc8a9a8e9706..57baa4a74fce9 100644 --- a/eng/pipelines/installer/steps/build-linux-package.yml +++ b/eng/pipelines/installer/steps/build-linux-package.yml @@ -3,7 +3,6 @@ parameters: target: '' packageStepDescription: null packagingArgs: '' - subsetArg: '' condition: succeeded() steps: @@ -11,15 +10,14 @@ steps: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - task: NuGetAuthenticate@1 target: ${{ parameters.target }} -- script: | - $(Build.SourcesDirectory)/build.sh \ - --ci \ - ${{ parameters.subsetArg }} \ - ${{ parameters.packagingArgs }} \ - /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog - displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} - target: ${{ parameters.target }} - condition: ${{ parameters.condition }} + +- template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog + container: ${{ parameters.target }} + displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages + condition: ${{ parameters.condition }} + # Broken symbolic links break the SBOM processing # We make some symlinks during the installer generation process, # but they aren't always valid on disk afterwards. Some of our tooling, diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 1b9c18c6be33c..44dfe5781c3bd 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -212,17 +212,13 @@ extends: parameters: packageType: deb target: debpkg - packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers - subsetArg: -s packs.installers - packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true + packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64')) - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: rpm target: rpmpkg - packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers - subsetArg: -s packs.installers - packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true + packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true condition: and(succeeded(), eq(variables.osSubgroup, ''), in(variables.archType, 'x64', 'arm64')) # Upload the results. diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index d1805acf12629..a17fce1e91a96 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -119,9 +119,7 @@ extends: parameters: packageType: rpm target: rpmpkg - packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers - subsetArg: -s packs.installers - packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true + packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -1455,16 +1453,12 @@ extends: parameters: packageType: deb target: debpkg - packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers - subsetArg: -s packs.installers - packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true + packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: rpm target: rpmpkg - packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers - subsetArg: -s packs.installers - packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true + packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), From 3304e445ab6baeb1b7d07df4e6fee4f99e9d8d32 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 15:53:19 -0700 Subject: [PATCH 03/11] Template-style variables for dependsOn? --- eng/pipelines/runtime.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index a17fce1e91a96..2e0bd0c2c4696 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1378,7 +1378,7 @@ extends: jobParameters: buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - - build_$(osGroup)$(osSubgroup)_$(archType)_release_CoreCLR_Libraries + - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_release_CoreCLR_Libraries preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -1407,7 +1407,7 @@ extends: jobParameters: buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries + - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -1435,7 +1435,7 @@ extends: jobParameters: buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries + - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: From 6b499ad7f4dcaf7e0a4d1a154cd8ccf900fafcf6 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 15:56:23 -0700 Subject: [PATCH 04/11] Expand out the jobs to explicitly state dependencies --- eng/pipelines/runtime.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 2e0bd0c2c4696..d3cca40f27479 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1378,7 +1378,7 @@ extends: jobParameters: buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_release_CoreCLR_Libraries + - build_windows_x86_release_CoreCLR_Libraries preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -1403,11 +1403,38 @@ extends: buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - osx_x64 + jobParameters: + buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + dependsOn: + - build_osx_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries + preBuildSteps: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + displayName: 'unified artifacts' + enablePublishTestResults: true + testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType) + postBuildSteps: + - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + condition: + or( + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: - windows_x64 jobParameters: buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries + - build_windows_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -1435,7 +1462,7 @@ extends: jobParameters: buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries + - build_linux_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: From 359005fbc171e67137571913db02f39924bc7288 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 16:49:39 -0700 Subject: [PATCH 05/11] Installer build steps aren't cross builds --- eng/pipelines/common/templates/global-build-step.yml | 3 ++- eng/pipelines/installer/steps/build-linux-package.yml | 2 ++ eng/pipelines/runtime.yml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 6597ac4feaefc..f732a5e27a4a9 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -3,11 +3,12 @@ parameters: useContinueOnErrorDuringBuild: false shouldContinueOnError: false archParameter: $(_archParameter) + crossArg: $(crossArg) displayName: Build product container: '' steps: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml index 57baa4a74fce9..443c580e18915 100644 --- a/eng/pipelines/installer/steps/build-linux-package.yml +++ b/eng/pipelines/installer/steps/build-linux-package.yml @@ -16,6 +16,8 @@ steps: buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog container: ${{ parameters.target }} displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages + # Even for cross-build targets, our installer build steps are not cross-builds + crossArg: '' condition: ${{ parameters.condition }} # Broken symbolic links break the SBOM processing diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index d3cca40f27479..72849c2e56e41 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -120,6 +120,7 @@ extends: packageType: rpm target: rpmpkg packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, '')) condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), From 33e159c58f37778854d2cd3bd384c4eace01ecf9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 18:36:01 -0700 Subject: [PATCH 06/11] Forward condition down and fix config --- eng/pipelines/common/templates/global-build-step.yml | 2 ++ eng/pipelines/runtime.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index f732a5e27a4a9..b88207c516c19 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -6,6 +6,7 @@ parameters: crossArg: $(crossArg) displayName: Build product container: '' + condition: succeeded() steps: - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) @@ -14,3 +15,4 @@ steps: continueOnError: ${{ parameters.shouldContinueOnError }} ${{ if ne(parameters.container, '') }}: target: ${{ parameters.container }} + condition: ${{ parameters.condition }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 72849c2e56e41..4403f1a8c8557 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -119,7 +119,7 @@ extends: parameters: packageType: rpm target: rpmpkg - packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true + packagingArgs: -c Release /p:BuildRpmPackage=true condition: and(succeeded(), eq(variables.osSubgroup, '')) condition: >- or( From 521b84acab3737910a1826178c6c855b702182e9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 1 Apr 2024 15:19:00 -0700 Subject: [PATCH 07/11] Fix cross-config problems --- eng/pipelines/runtime.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 4403f1a8c8557..553e519e3a732 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1373,10 +1373,11 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + buildConfig: release platforms: - windows_x86 jobParameters: + nameSuffix: Installer_Build_And_Test buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test dependsOn: - build_windows_x86_release_CoreCLR_Libraries @@ -1401,11 +1402,12 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + buildConfig: release platforms: - osx_x64 jobParameters: - buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + nameSuffix: Installer_Build_And_Test + buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test dependsOn: - build_osx_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: @@ -1429,11 +1431,12 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + buildConfig: release platforms: - windows_x64 jobParameters: - buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + nameSuffix: Installer_Build_And_Test + buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test dependsOn: - build_windows_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: @@ -1457,11 +1460,12 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + buildConfig: release platforms: - linux_x64 jobParameters: - buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test + nameSuffix: Installer_Build_And_Test + buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test dependsOn: - build_linux_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries preBuildSteps: From 2034fe0d13090968494c04590d7e9941c9ffdc6f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 1 Apr 2024 15:35:19 -0700 Subject: [PATCH 08/11] Re-collapse the installer jobs by providing a dependency model that can calculate a global build job name when we have a platform. --- eng/pipelines/common/global-build-job.yml | 14 +++- .../installer/steps/build-linux-package.yml | 4 +- eng/pipelines/runtime.yml | 72 +++---------------- 3 files changed, 25 insertions(+), 65 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index eb83415112e7b..421aacce1a607 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -12,6 +12,13 @@ parameters: targetRid: '' timeoutInMinutes: '' dependsOn: [] + # The following parameter is used to specify dependencies on other global build for the same platform. + # We provide this mechanism to allow for global builds to depend on other global builds and use the multiplexing + # that platform-matrix.yml enables. + # Each item can have the following properties: + # - nameSuffix: The suffix of the job name to depend on. + # - buildConfig: The configuration of the job to depend on. + dependsOnGlobalBuilds: [] pool: '' platform: '' condition: true @@ -61,8 +68,11 @@ jobs: workspace: clean: all - ${{ if ne(parameters.dependsOn,'') }}: - dependsOn: ${{ parameters.dependsOn }} + ${{ if or(ne(parameters.dependsOn,''), ne(parameters.dependsOnGlobalBuilds,'')) }}: + dependsOn: + - ${{ insert }} : ${{ parameters.dependsOn }} + - ${{ each globalBuild in parameters.dependsOnGlobalBuilds }}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, coalesce(globalBuild.buildConfig, parameters.buildConfig), globalBuild.nameSuffix) }} variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml index 443c580e18915..5c7dcb57ef060 100644 --- a/eng/pipelines/installer/steps/build-linux-package.yml +++ b/eng/pipelines/installer/steps/build-linux-package.yml @@ -7,9 +7,10 @@ parameters: steps: ## Run NuGet Authentication for each of the side containers -- ${{ if ne(variables['System.TeamProject'], 'public') }}: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(parameters.target, '')) }}: - task: NuGetAuthenticate@1 target: ${{ parameters.target }} + condition: ${{ parameters.condition }} - template: /eng/pipelines/common/templates/global-build-step.yml parameters: @@ -26,3 +27,4 @@ steps: # in particular the SBOM tooling, breaks on broken symlinks. - script: find . -xtype l -delete displayName: Remove broken symbolic links + condition: ${{ parameters.condition }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 553e519e3a732..9816d8b43f81f 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1373,14 +1373,15 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - windows_x86 jobParameters: nameSuffix: Installer_Build_And_Test buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test - dependsOn: - - build_windows_x86_release_CoreCLR_Libraries + dependsOnGlobalBuilds: + - nameSuffix: CoreCLR_Libraries + buildConfig: release preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -1399,75 +1400,20 @@ extends: eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - platforms: - - osx_x64 - jobParameters: - nameSuffix: Installer_Build_And_Test - buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test - dependsOn: - - build_osx_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries - preBuildSteps: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - displayName: 'unified artifacts' - enablePublishTestResults: true - testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType) - postBuildSteps: - - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) - condition: - or( - eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 - jobParameters: - nameSuffix: Installer_Build_And_Test - buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test - dependsOn: - - build_windows_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries - preBuildSteps: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - displayName: 'unified artifacts' - enablePublishTestResults: true - testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType) - postBuildSteps: - - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) - condition: - or( - eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - platforms: + - osx_x64 - linux_x64 jobParameters: nameSuffix: Installer_Build_And_Test buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test - dependsOn: - - build_linux_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries + dependsOnGlobalBuilds: + - nameSuffix: CoreCLR_Libraries + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} preBuildSteps: - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -1486,11 +1432,13 @@ extends: packageType: deb target: debpkg packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true + condition: eq(variables['osGroup'], 'linux') - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: rpm target: rpmpkg packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true + condition: eq(variables['osGroup'], 'linux') condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), From 9e55e61bff67f8bdf04d1399df3606776d588ee9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 1 Apr 2024 15:37:48 -0700 Subject: [PATCH 09/11] Try using an each instead of insert. --- eng/pipelines/common/global-build-job.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 421aacce1a607..9b81acc80d811 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -70,7 +70,8 @@ jobs: ${{ if or(ne(parameters.dependsOn,''), ne(parameters.dependsOnGlobalBuilds,'')) }}: dependsOn: - - ${{ insert }} : ${{ parameters.dependsOn }} + - ${{ each build in parameters.dependsOn }}: + - ${{ build }} - ${{ each globalBuild in parameters.dependsOnGlobalBuilds }}: - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, coalesce(globalBuild.buildConfig, parameters.buildConfig), globalBuild.nameSuffix) }} From 2c62cc27d91b5841d23c04782b33e8ae01c338e5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 1 Apr 2024 18:14:34 -0700 Subject: [PATCH 10/11] Remove docker initialization attempts on non-Windows installer jobs --- .../installer/steps/build-linux-package.yml | 43 ++++++++++--------- eng/pipelines/runtime.yml | 2 - 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml index 5c7dcb57ef060..ef905d6c16465 100644 --- a/eng/pipelines/installer/steps/build-linux-package.yml +++ b/eng/pipelines/installer/steps/build-linux-package.yml @@ -1,4 +1,6 @@ parameters: + osGroup: '' + osSubgroup: '' packageType: null target: '' packageStepDescription: null @@ -6,25 +8,26 @@ parameters: condition: succeeded() steps: -## Run NuGet Authentication for each of the side containers -- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(parameters.target, '')) }}: - - task: NuGetAuthenticate@1 - target: ${{ parameters.target }} - condition: ${{ parameters.condition }} +- ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '')) }}: + ## Run NuGet Authentication for each of the side containers + - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(parameters.target, '')) }}: + - task: NuGetAuthenticate@1 + target: ${{ parameters.target }} + condition: ${{ parameters.condition }} -- template: /eng/pipelines/common/templates/global-build-step.yml - parameters: - buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog - container: ${{ parameters.target }} - displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages - # Even for cross-build targets, our installer build steps are not cross-builds - crossArg: '' - condition: ${{ parameters.condition }} + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog + container: ${{ parameters.target }} + displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages + # Even for cross-build targets, our installer build steps are not cross-builds + crossArg: '' + condition: ${{ parameters.condition }} -# Broken symbolic links break the SBOM processing -# We make some symlinks during the installer generation process, -# but they aren't always valid on disk afterwards. Some of our tooling, -# in particular the SBOM tooling, breaks on broken symlinks. -- script: find . -xtype l -delete - displayName: Remove broken symbolic links - condition: ${{ parameters.condition }} + # Broken symbolic links break the SBOM processing + # We make some symlinks during the installer generation process, + # but they aren't always valid on disk afterwards. Some of our tooling, + # in particular the SBOM tooling, breaks on broken symlinks. + - script: find . -xtype l -delete + displayName: Remove broken symbolic links + condition: ${{ parameters.condition }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 9816d8b43f81f..e31b72f7c5836 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1432,13 +1432,11 @@ extends: packageType: deb target: debpkg packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true - condition: eq(variables['osGroup'], 'linux') - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: rpm target: rpmpkg packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true - condition: eq(variables['osGroup'], 'linux') condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), From 2822dde9ad735e3510a0efe99672f43e97aacd73 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 2 Apr 2024 10:25:44 -0700 Subject: [PATCH 11/11] Fix unified artifacts path --- eng/pipelines/runtime.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index e31b72f7c5836..6c35d9573d8c8 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -119,7 +119,7 @@ extends: parameters: packageType: rpm target: rpmpkg - packagingArgs: -c Release /p:BuildRpmPackage=true + packagingArgs: -c Release -lc $(_BuildConfig) /p:BuildRpmPackage=true condition: and(succeeded(), eq(variables.osSubgroup, '')) condition: >- or( @@ -1386,7 +1386,7 @@ extends: - template: /eng/pipelines/common/download-artifact-step.yml parameters: artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release - artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release + artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release$(archiveExtension) unpackFolder: $(Build.SourcesDirectory)/artifacts/bin displayName: 'unified artifacts' enablePublishTestResults: true @@ -1418,7 +1418,7 @@ extends: - template: /eng/pipelines/common/download-artifact-step.yml parameters: artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)$(archiveExtension) unpackFolder: $(Build.SourcesDirectory)/artifacts/bin displayName: 'unified artifacts' enablePublishTestResults: true @@ -1431,12 +1431,12 @@ extends: parameters: packageType: deb target: debpkg - packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true + packagingArgs: -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} /p:BuildDebPackage=true - template: /eng/pipelines/installer/steps/build-linux-package.yml parameters: packageType: rpm target: rpmpkg - packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true + packagingArgs: -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} /p:BuildRpmPackage=true condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),