Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
1a217bd
Pass along PGO type and use it in artifacts names
agocke Feb 11, 2021
96746dd
Add PGO to artifact names
agocke Feb 11, 2021
44a68a4
Add parameter to template
agocke Feb 11, 2021
6f0874a
Add to template parameters
agocke Feb 12, 2021
9c9d822
Add underscore
agocke Feb 12, 2021
ee2725d
Use parameter syntax
agocke Feb 12, 2021
ae3e4a4
Change intstaller dependency type
agocke Feb 12, 2021
0c2ddbd
typo
agocke Feb 13, 2021
948dbce
Add pgoType to libraries definition
agocke Feb 13, 2021
dabdc7f
Remove suffix
agocke Feb 13, 2021
65587f3
Fix mono dependency
agocke Feb 13, 2021
72ce20b
Remove set of pgoType
agocke Feb 13, 2021
d54c60f
Remove pgoType from native test artifacts
agocke Feb 13, 2021
8991709
Remove default job parameter
agocke Feb 14, 2021
646b021
remove pgotype default
agocke Feb 14, 2021
b2f159c
Merge remote-tracking branch 'upstream/master' into fix-pgo-req
agocke Feb 14, 2021
7d83f7f
Re-enable PGO builds
agocke Feb 14, 2021
d4b488a
Don't publish native test artifacts for PGO runs
agocke Feb 14, 2021
28eafa7
Pass down pgoType param
agocke Feb 15, 2021
9da03dc
Add pgoType to installer publish logs
agocke Feb 15, 2021
e3414c6
Add to xplat-setup
agocke Feb 15, 2021
17d3774
Fix
agocke Feb 15, 2021
9dd7fc6
Also re-enable in official builds
agocke Feb 22, 2021
606deb9
Also add suffix to archives
agocke Feb 25, 2021
c03516e
Merge remote-tracking branch 'upstream/master' into fix-pgo-req
agocke Feb 25, 2021
26bee1f
Temporarily disable
agocke Feb 25, 2021
3deebfb
Fix indentation
agocke Feb 25, 2021
fc07ea9
Also fix up installer artifact names
agocke Feb 25, 2021
d6375d3
PGO name for Crossgen2 pack
agocke Feb 25, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion eng/pipelines/common/templates/runtimes/build-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ parameters:
runtimeVariant: ''
dependsOn: []
dependOnEvaluatePaths: false
pgoInstrument: false
pgoType: ''

### Build managed test components (native components are getting built as part
### of the the product build job).
Expand Down
6 changes: 3 additions & 3 deletions eng/pipelines/common/xplat-setup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
runtimeFlavorDisplayName: 'CoreCLR'
${{ if eq(parameters.jobParameters.runtimeFlavor, 'mono') }}:
runtimeFlavorDisplayName: 'Mono'

shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }}

# keep in sync with /eng/pipelines/common/variables.yml
Expand All @@ -26,7 +26,7 @@ jobs:
variables:
# Disable component governance in our CI builds. These builds are not shipping nor
# are they a service. Also the component governance jobs issue lots of inconsequential
# warnings and errors into our build timelines that make it hard to track down
# warnings and errors into our build timelines that make it hard to track down
# real errors in the build
- name: skipComponentGovernanceDetection
value: true
Expand Down Expand Up @@ -90,7 +90,7 @@ jobs:
value: Mono
${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}:
value: CoreCLR

- ${{ each variable in parameters.variables }}:
- ${{ variable }}

Expand Down
5 changes: 3 additions & 2 deletions eng/pipelines/coreclr/templates/build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
pool: ${{ parameters.pool }}
condition: ${{ parameters.condition }}
dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
pgoType: ${{ parameters.pgoType }}

# Compute job name from template parameters
${{ if and(ne(parameters.testGroup, 'clrTools'), eq(parameters.compilerName, 'gcc')) }}:
Expand Down Expand Up @@ -281,7 +282,7 @@ jobs:

- ${{ if and(ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, ''), ne(parameters.testGroup, 'clrTools')) }}:
# Publish test native components for consumption by test execution.
- ${{ if ne(parameters.isOfficialBuild, true) }}:
- ${{ if and(ne(parameters.isOfficialBuild, true), eq(parameters.pgoType, '')) }}:
- template: /eng/pipelines/common/upload-artifact-step.yml
parameters:
rootFolder: $(nativeTestArtifactRootFolderPath)
Expand Down Expand Up @@ -322,6 +323,6 @@ jobs:
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: '$(publishLogsArtifactPrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
artifactName: '$(publishLogsArtifactPrefix)${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
continueOnError: true
condition: always()
7 changes: 4 additions & 3 deletions eng/pipelines/coreclr/templates/xplat-pipeline-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ jobs:
stagedBuild: ${{ parameters.stagedBuild }}
strategy: ${{ parameters.strategy }}
pool: ${{ parameters.pool }}
pgoType: ${{ parameters.pgoType }}

# arcade-specific parameters
condition: and(succeeded(), ${{ parameters.condition }})
Expand All @@ -66,7 +67,7 @@ jobs:

# Build product defines what we are trying to build, either coreclr or mono
- name: buildProductArtifactName
value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: buildProductRootFolderPath
value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)'
Expand Down Expand Up @@ -98,13 +99,13 @@ jobs:

# We need this because both mono and coreclr build currently depends on CoreClr
- name: coreClrProductArtifactName
value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: coreClrProductRootFolderPath
value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)'

- name: corelibProductArtifactName
value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
value: 'CoreLib_${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: managedGenericTestArtifactName
value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)'
Expand Down
3 changes: 2 additions & 1 deletion eng/pipelines/installer/jobs/base-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ jobs:
/p:RuntimeArtifactsPath=$(buildCommandSourcesDirectory)$(RuntimeDownloadPath)
/p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }}
- name: RuntimeArtifactName
value: $(runtimeFlavorName)Product_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName)
value: $(runtimeFlavorName)Product_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName)

- ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
- name: liveLibrariesLegName
Expand Down Expand Up @@ -471,6 +471,7 @@ jobs:
runtimeVariant: ${{ parameters.runtimeVariant }}
skipTests: $(SkipTests)
isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }}
pgoType: ${{ parameters.pgoType }}

- ${{ if ne(parameters.osGroup, 'windows') }}:
- script: set -x && df -h
Expand Down
3 changes: 2 additions & 1 deletion eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ parameters:
runtimeFlavor: 'coreclr'
runtimeVariant: ''
isOfficialBuild: false
pgoType: ''

steps:
# Upload build artifacts (packages) to pipeline only if official, to save storage space.
Expand Down Expand Up @@ -38,6 +39,6 @@ steps:
displayName: Publish BuildLogs
inputs:
targetPath: '$(Build.StagingDirectory)/BuildLogs'
artifactName: Installer-Logs-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig)
artifactName: Installer-Logs-${{parameters.pgoType }}${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig)
continueOnError: true
condition: succeededOrFailed()
3 changes: 2 additions & 1 deletion eng/pipelines/libraries/base-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ parameters:
testScope: ''
pool: ''
runTests: false
pgoType: ''

jobs:
- template: /eng/common/templates/job/job.yml
Expand Down Expand Up @@ -96,7 +97,7 @@ jobs:
- _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}'
- _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }}
- ${{ if eq(parameters.runTests, true) }}:
- _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
- _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.pgoType }}_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
- _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
- ${{ if eq(parameters.testDisplayName, '') }}:
- _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }}
Expand Down
1 change: 1 addition & 0 deletions eng/pipelines/mono/templates/build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ parameters:
dependsOn: []
monoCrossAOTTargetOS: []
dependOnEvaluatePaths: false
pgoType: ''

### Product build
jobs:
Expand Down
4 changes: 2 additions & 2 deletions eng/pipelines/mono/templates/xplat-pipeline-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,13 @@ jobs:

variables:
- name: coreClrProductArtifactName
value: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}'
value: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}'

- name: coreClrProductRootFolderPath
value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(liveRuntimeBuildConfigUpper)'

- name: buildProductArtifactName
value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
value: 'MonoProduct__${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: binTestsPath
value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr'
Expand Down
52 changes: 26 additions & 26 deletions eng/pipelines/runtime-official.yml
Original file line number Diff line number Diff line change
Expand Up @@ -327,32 +327,32 @@ stages:
- windows_arm
- windows_arm64

# #
# # Build PGO CoreCLR release
# #
# - template: /eng/pipelines/common/platform-matrix.yml
# parameters:
# jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
# buildConfig: release
# platforms:
# - windows_x64
# jobParameters:
# testGroup: innerloop
# pgoType: 'PGO'
#
# #
# # PGO Build
# #
# - template: /eng/pipelines/installer/installer-matrix.yml
# parameters:
# buildConfig: Release
# jobParameters:
# isOfficialBuild: ${{ variables.isOfficialBuild }}
# liveRuntimeBuildConfig: release
# liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
# pgoType: 'PGO'
# platforms:
# - windows_x64
#
# Build PGO CoreCLR release
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
buildConfig: release
platforms:
- windows_x64
jobParameters:
testGroup: innerloop
pgoType: 'PGO'

#
# PGO Build
#
- template: /eng/pipelines/installer/installer-matrix.yml
parameters:
buildConfig: Release
jobParameters:
isOfficialBuild: ${{ variables.isOfficialBuild }}
liveRuntimeBuildConfig: release
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
pgoType: 'PGO'
platforms:
- windows_x64

- ${{ if eq(variables.isOfficialBuild, true) }}:
- template: /eng/pipelines/official/stages/publish.yml
Expand Down
44 changes: 22 additions & 22 deletions eng/pipelines/runtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,15 @@ jobs:
#
# Build PGO CoreCLR release
#
#- template: /eng/pipelines/common/platform-matrix.yml
# parameters:
# jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
# buildConfig: release
# platforms:
# - windows_x64
# jobParameters:
# testGroup: innerloop
# pgoType: 'PGO'
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
buildConfig: release
platforms:
- windows_x64
jobParameters:
testGroup: innerloop
pgoType: 'PGO'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we expect different PGO types or why didn't you choose a bool?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't figure out a good way in YAML to turn a bool into a string, and didn't want to repeat a conditional in the build where I change the name of jobs to include "PGO". If you have a better way I'd love to use it. I couldn't find any support for something like if(pgoInstrument, 'PGO', '') in YAML.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah I see. Unfortunately there is no way in YAML to do that. So let's keep it as is.


#
# Build CoreCLR Formatting Job
Expand Down Expand Up @@ -699,19 +699,19 @@ jobs:
liveRuntimeBuildConfig: release
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}

#
# PGO Build
#
#- template: /eng/pipelines/installer/installer-matrix.yml
# parameters:
# buildConfig: Release
# jobParameters:
# isOfficialBuild: ${{ variables.isOfficialBuild }}
# liveRuntimeBuildConfig: release
# liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
# pgoType: 'PGO'
# platforms:
# - windows_x64
#
# PGO Build
#
- template: /eng/pipelines/installer/installer-matrix.yml
parameters:
buildConfig: Release
jobParameters:
isOfficialBuild: ${{ variables.isOfficialBuild }}
liveRuntimeBuildConfig: release
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
pgoType: 'PGO'
platforms:
- windows_x64

#
# Crossgen-comparison jobs
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/tools/aot/crossgen2/crossgen2.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@
CopyToOutputDirectory="PreserveNewest"
CopyToPublishDirectory="PreserveNewest"
Link="%(FileName)%(Extension)"
Condition="'$(PgoInstrument)' != ''"
/>
</ItemGroup>

Expand Down
5 changes: 5 additions & 0 deletions src/installer/Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
<Project>

<PropertyGroup>
<InstallerName Condition="'$(PgoInstrument)' != ''">$(InstallerName)-pgo</InstallerName>
<ArchiveName Condition="'$(PgoInstrument)' != ''">$(ArchiveName)-pgo</ArchiveName>
</PropertyGroup>

<!--
Import stubs for compatibility with packaging tools, if not building a pkgproj. Ordinarily,
listing this before the ../Directory.Build.targets import would be sufficient, but the packaging
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<PropertyGroup>
<VersionProp>HostResolverVersion</VersionProp>
<InstallerName>dotnet-hostfxr</InstallerName>
<InstallerName Condition="'$(PgoInstrument)' != ''">$(InstallerName)-pgo</InstallerName>
<PackageDescription>Provides an implementation of framework resolution strategy used by Microsoft.NETCore.DotNetHost</PackageDescription>
</PropertyGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<PropertyGroup>
<SkipBuild Condition="'$(RuntimeFlavor)' == 'Mono'">true</SkipBuild>
<PlatformPackageType>RuntimePack</PlatformPackageType>
<SharedFrameworkName>Microsoft.NETCore.App.Crossgen2</SharedFrameworkName>
<OverridePackageId>Microsoft.NETCore.App.Crossgen2.$(RuntimeIdentifier)</OverridePackageId>
<SharedFrameworkName>$(SharedFrameworkName).Crossgen2</SharedFrameworkName>
<OverridePackageId>$(SharedFrameworkName).Crossgen2.$(RuntimeIdentifier)</OverridePackageId>
<ArchiveName>dotnet-crossgen2</ArchiveName>
<SharedFrameworkHostFileNameOverride>crossgen2</SharedFrameworkHostFileNameOverride>
<RuntimeIdentifiers>linux-x64;linux-musl-x64;osx-x64;osx-arm64;win-x64</RuntimeIdentifiers>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<MacOSBundleIdentifierName>com.microsoft.dotnet.Microsoft.NETCore.App.$(ProductVersion).osx.$(TargetArchitecture)</MacOSBundleIdentifierName>
<MacOSBundleResourcesPath>osx_resources</MacOSBundleResourcesPath>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Build.Tasks.Installers" Version="$(MicrosoftDotNetBuildTasksInstallersVersion)" />
<PackageReference Include="Microsoft.DotNet.Build.Tasks.Archives" Version="$(MicrosoftDotNetBuildTasksArchivesVersion)" />
Expand Down