Remove various single-scenario YAML parameters in our build scripts a…

…nd shift implementation responsibility to those scenarios (dotnet#100115)
matouskozak · Apr 30, 2024 · eb025b3 · eb025b3
1 parent 37dcca1
commit eb025b3
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 171 deletions.
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -25,20 +25,14 @@ parameters:
   useContinueOnErrorDuringBuild: false
   shouldContinueOnError: false
   isOfficialBuild: false
-  isSourceBuild: false
-  isNonPortableSourceBuild: false
   runtimeFlavor: 'coreclr'
   runtimeVariant: ''
   helixQueues: ''
   enablePublishTestResults: false
   testResultsFormat: ''
   postBuildSteps: []
   extraVariablesTemplates: []
-  isManualCodeQLBuild: false
   preBuildSteps: []
-  enableRichCodeNavigation: false
-  richCodeNavigationLanguage: 'csharp'
-  disableComponentGovernance: ''
   templatePath: 'templates'
 
 jobs:
@@ -57,9 +51,6 @@ jobs:
     enablePublishTestResults: ${{ parameters.enablePublishTestResults }}
     testResultsFormat: ${{ parameters.testResultsFormat }}
 
-    enableRichCodeNavigation: ${{ parameters.enableRichCodeNavigation }}
-    richCodeNavigationLanguage: ${{ parameters.richCodeNavigationLanguage }}
-
     artifacts:
       publish:
         logs:
@@ -154,10 +145,7 @@ jobs:
 
     - checkout: self
       clean: true
-      # If running in source build mode, a git stash will be used for the inner clone. Avoid setting a fetch depth,
-      # as a stash of a shallow cloned repo is not currently supported.
-      ${{ if ne(parameters.isSourceBuild, true) }}:
-        fetchDepth: $(checkoutFetchDepth)
+      fetchDepth: $(checkoutFetchDepth)
 
     - ${{ if and(eq(parameters.isOfficialBuild, true), notin(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator')) }}:
       - template: /eng/pipelines/common/restore-internal-tools.yml
@@ -187,16 +175,6 @@ jobs:
           artifact: Mono_Offsets_${{monoCrossAOTTargetOS}}
           path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'
 
-    - ${{ if eq(parameters.isSourceBuild, true) }}:
-      - template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml
-        parameters:
-          platform:
-            baseOS: ${{ parameters.baseOS }}
-            buildScript: $(Build.SourcesDirectory)$(dir)build$(scriptExt)
-            nonPortable: ${{ parameters.isNonPortableSourceBuild }}
-            targetRID: ${{ parameters.targetRid }}
-            name: ${{ parameters.platform }}
-
     - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}:
       - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }}
         displayName: Install Build Dependencies
@@ -228,7 +206,6 @@ jobs:
               targetRid: ${{ parameters.targetRid }}
               nameSuffix: ${{ parameters.nameSuffix }}
               platform: ${{ parameters.platform }}
-              pgoType: ${{ parameters.pgoType }}
               shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
               ${{ if ne(preBuildStep.forwardedParameters, '') }}:
                 ${{ each parameter in preBuildStep.forwardedParameters }}:
@@ -239,21 +216,11 @@ jobs:
           - ${{ preBuildStep }}
 
     # Build
-    - ${{ if eq(parameters.isSourceBuild, false) }}:
-      - ${{ if eq(parameters.isManualCodeQLBuild, true) }}:
-        - task: CodeQL3000Init@0
-          displayName: Initialize CodeQL (manually-injected)
-
-      - template: /eng/pipelines/common/templates/global-build-step.yml
-        parameters:
-          buildArgs: ${{ parameters.buildArgs }}
-          useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }}
-          shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-
-      - ${{ if eq(parameters.isManualCodeQLBuild, true) }}:
-        - task: CodeQL3000Finalize@0
-          displayName: Finalize CodeQL (manually-injected)
-    #endif isSourceBuild
+    - template: /eng/pipelines/common/templates/global-build-step.yml
+      parameters:
+        buildArgs: ${{ parameters.buildArgs }}
+        useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }}
+        shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
 
     - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos', 'android') }}:
       - script: |

diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml
@@ -294,73 +294,6 @@ jobs:
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
         ${{ insert }}: ${{ parameters.jobParameters }}
 
-# Centos 8 Stream x64 Source Build
-
-- ${{ if containsValue(parameters.platforms, 'SourceBuild_centos8_x64') }}:
-  - template: xplat-setup.yml
-    parameters:
-      jobTemplate: ${{ parameters.jobTemplate }}
-      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
-      variables: ${{ parameters.variables }}
-      osGroup: linux
-      archType: x64
-      targetRid: centos.8-x64
-      platform: centos8_linux_x64
-      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: SourceBuild_centos_x64
-      jobParameters:
-        runtimeFlavor: ${{ parameters.runtimeFlavor }}
-        buildConfig: ${{ parameters.buildConfig }}
-        helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        isSourceBuild: true
-        isNonPortableSourceBuild: true
-        ${{ insert }}: ${{ parameters.jobParameters }}
-
-# Non-existent RID Source Build
-
-- ${{ if containsValue(parameters.platforms, 'SourceBuild_banana24_x64') }}:
-  - template: xplat-setup.yml
-    parameters:
-      jobTemplate: ${{ parameters.jobTemplate }}
-      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
-      variables: ${{ parameters.variables }}
-      osGroup: linux
-      archType: x64
-      targetRid: banana.24-x64
-      platform: banana24_linux_x64
-      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: SourceBuild_centos_x64 # Run the unknown-rid build on a platform with a known RID so our RID graph tooling can automatically add it to the RID graph.
-      jobParameters:
-        runtimeFlavor: ${{ parameters.runtimeFlavor }}
-        baseOS: linux
-        buildConfig: ${{ parameters.buildConfig }}
-        helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        isSourceBuild: true
-        isNonPortableSourceBuild: true
-        ${{ insert }}: ${{ parameters.jobParameters }}
-
-# Portable Linux x64 Source Build
-
-- ${{ if containsValue(parameters.platforms, 'SourceBuild_linux_x64') }}:
-  - template: xplat-setup.yml
-    parameters:
-      jobTemplate: ${{ parameters.jobTemplate }}
-      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
-      variables: ${{ parameters.variables }}
-      osGroup: linux
-      archType: x64
-      targetRid: linux-x64
-      platform: linux_x64
-      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: SourceBuild_linux_x64
-      jobParameters:
-        runtimeFlavor: ${{ parameters.runtimeFlavor }}
-        buildConfig: ${{ parameters.buildConfig }}
-        helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        isSourceBuild: true
-        isNonPortableSourceBuild: false
-        ${{ insert }}: ${{ parameters.jobParameters }}
-
 # GCC Linux x64 Build
 
 - ${{ if containsValue(parameters.platforms, 'gcc_linux_x64') }}:

diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml
@@ -163,16 +163,11 @@ extends:
                 eq(variables['isRollingBuild'], true))
 
       #
-      # SourceBuild Build
+      # Build Portable SourceBuild
       #
-      - template: /eng/pipelines/common/platform-matrix.yml
+      - template: /eng/common/templates/jobs/source-build.yml
         parameters:
-          jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: Release
           platforms:
-          - SourceBuild_linux_x64
-          jobParameters:
-            nameSuffix: PortableSourceBuild
-            timeoutInMinutes: 95
-            condition:
-                eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true)
+          - name: Linux_x64
+            targetRid: linux-x64
+            container: SourceBuild_linux_x64
diff --git a/eng/pipelines/runtime-codeql.yml b/eng/pipelines/runtime-codeql.yml
@@ -47,7 +47,12 @@ extends:
             nameSuffix: AllSubsets_CoreCLR
             isOfficialBuild: ${{ variables.isOfficialBuild }}
             timeoutInMinutes: 360
-            isManualCodeQLBuild: true
+            preBuildSteps:
+              - task: CodeQL3000Init@0
+                displayName: Initialize CodeQL (manually-injected)
+            postBuildSteps:
+              - task: CodeQL3000Finalize@0
+                displayName: Finalize CodeQL (manually-injected)
 
       #
       # Build Mono runtime packs
@@ -65,4 +70,9 @@ extends:
             nameSuffix: AllSubsets_Mono
             isOfficialBuild: ${{ variables.isOfficialBuild }}
             timeoutInMinutes: 360
-            isManualCodeQLBuild: true
+            preBuildSteps:
+              - task: CodeQL3000Init@0
+                displayName: Initialize CodeQL (manually-injected)
+            postBuildSteps:
+              - task: CodeQL3000Finalize@0
+                displayName: Finalize CodeQL (manually-injected)
diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml
@@ -562,22 +562,12 @@ extends:
       #
       # Build SourceBuild packages
       #
-      - template: /eng/pipelines/common/platform-matrix.yml
+      - template: /eng/common/templates-official/jobs/source-build.yml
         parameters:
-          jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: Release
-          helixQueueGroup: ci
           platforms:
-          - SourceBuild_linux_x64
-          jobParameters:
-            templatePath: 'templates-official'
-            nameSuffix: PortableSourceBuild
-            isOfficialBuild: ${{ variables.isOfficialBuild }}
-            postBuildSteps:
-              - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
-                parameters:
-                  name: SourceBuildPackages
-            timeoutInMinutes: 95
+          - name: Linux_x64
+            targetRid: linux-x64
+            container: SourceBuild_linux_x64
 
       #
       # Build PGO Instrumented CoreCLR Release

diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
@@ -1878,36 +1878,29 @@ extends:
                 eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true),
                 eq(variables['isRollingBuild'], true))
 
-      #
-      # Sourcebuild legs
-      # We have 3 important legs for source-build:
-      # - Centos.8 (ensures that known non-portable RID is working)
-      # - Linux-x64 portable (used for dependency flow and downstream PR verification)
-      # - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about.
-      #
-      # Running all of these everywhere is wasteful. Run Banana.24 and CentOS.8 in rolling CI,
-      # Run Linux-x64 in PR.
-
-      - template: /eng/pipelines/common/platform-matrix.yml
-        parameters:
-          jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: Release
-          helixQueueGroup: pr
-          platforms:
-          - SourceBuild_centos8_x64
-          jobParameters:
-            nameSuffix: centos8SourceBuild
-            timeoutInMinutes: 95
-            condition: eq(variables['isRollingBuild'], true)
-
-      - template: /eng/pipelines/common/platform-matrix.yml
-        parameters:
-          jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: Release
-          helixQueueGroup: pr
-          platforms:
-          - SourceBuild_banana24_x64
-          jobParameters:
-            nameSuffix: banana24SourceBuild
-            timeoutInMinutes: 95
-            condition: eq(variables['isRollingBuild'], true)
+    - stage: SourceBuild
+      displayName: Source Build Validation
+      dependsOn: []
+      condition: eq(variables['isRollingBuild'], true)
+      jobs:
+        #
+        # Sourcebuild legs
+        # We have 3 important legs for source-build:
+        # - Centos.8 (ensures that known non-portable RID is working)
+        # - Linux-x64 portable (used for dependency flow and downstream PR verification)
+        # - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about.
+        #
+        # Running all of these everywhere is wasteful. Run Banana.24 and CentOS.8 in rolling CI,
+        # Run Linux-x64 in PR.
+        - template: /eng/common/templates/jobs/source-build.yml
+          parameters:
+            platforms:
+              - name: CentOS8
+                targetRid: centos.8-x64
+                nonPortable: true
+                container: SourceBuild_centos_x64
+              - name: NonexistentRID
+                baseOS: linux
+                targetRid: banana.24-x64
+                nonPortable: true
+                container: SourceBuild_centos_x64
diff --git a/src/installer/prepare-artifacts.proj b/src/installer/prepare-artifacts.proj
@@ -58,8 +58,8 @@
     <ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
   </ItemGroup>
 
-  <!-- 
-    Set metadata for assets that are not marked as NonShipping. 
+  <!--
+    Set metadata for assets that are not marked as NonShipping.
     This is used to determine if the asset should be shipped as part of .NET release.
   -->
   <ItemDefinitionGroup>
@@ -145,10 +145,6 @@
         <RelativeBlobPath>$(InstallersRelativePath)workloads/%(Filename)%(Extension)</RelativeBlobPath>
         <PublishFlatContainer>true</PublishFlatContainer>
       </ItemsToPush>
-
-      <!-- Source build intermediated packages will be pushed and signed by the sourcebuild leg. -->
-      <ItemsToPush Remove="@(ItemsToPush)" Condition="$([System.String]::new('%(Identity)').Contains('Microsoft.SourceBuild.Intermediate'))" />
-      <ItemsToSignPostBuild Remove="@(ItemsToSignPostBuild)" Condition="$([System.String]::new('%(Identity)').Contains('Microsoft.SourceBuild.Intermediate'))" />
     </ItemGroup>
 
     <!-- Push items to AzDO as build artifacts, generating the asset manifest as a side effect. -->