Pipeline to run Exploratory fuzzing tool to catch codegen issues

eng/pipelines/coreclr/exploratory.yml

@@ -0,0 +1,63 @@
+# This job definition automates the exploratory tool.
+
+trigger:
+  batch: false
+  branches:
+    include:
+    - main
+  paths:
+    include:
+    - src/coreclr/jit/*
+    - src/coreclr/inc/jiteeversionguid.h
+
+# This pipeline is supposed to be run only on merged changes
+# and should not be triggerable from a PR. 
+pr: none
+
+schedules:
+- cron: "0 17 * * 0"
+  displayName: Sun at 9:00 AM (UTC-8:00)
+  branches:
+    include:
+    - main
+  always: true
+
+jobs:
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
+    buildConfig: checked
+    platforms:
+    # Linux tests are built on the OSX machines.
+    # - OSX_x64
+    - Linux_arm
+    - Linux_arm64
+    - Linux_x64
+    - windows_x64
+    - windows_x86
+    - windows_arm64
+    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
+    jobParameters:
+      testGroup: outerloop
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/exploratory-job.yml
+    buildConfig: checked
+    platforms:
+    # Linux tests are built on the OSX machines.
+    # - OSX_x64
+    - Linux_arm
+    - Linux_arm64
+    - Linux_x64
+    - windows_x64
+    - windows_x86
+    - windows_arm64
+    helixQueueGroup: ci
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    jobParameters:
+      testGroup: outerloop
+      liveLibrariesBuildConfig: Release
+      collectionType: pmi
+      collectionName: libraries

eng/pipelines/coreclr/templates/exploratory-job.yml

@@ -0,0 +1,93 @@
+parameters:
+  buildConfig: ''
+  archType: ''
+  osGroup: ''
+  osSubgroup: ''
+  container: ''
+  runtimeVariant: ''
+  testGroup: ''
+  framework: net5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0)
+  liveLibrariesBuildConfig: ''
+  variables: {}
+  runtimeType: 'coreclr'
+  pool: ''
+  codeGenType: 'JIT'
+  projetFile: ''
+  runKind: ''
+  runJobTemplate: '/eng/pipelines/coreclr/templates/run-exploratory-job.yml'
+  additionalSetupParameters: ''
+  collectionType: ''
+  collectionName: ''
+
+### Exploratory job
+
+### Each exploratory job depends on a corresponding build job with the same
+### buildConfig and archType.
+
+jobs:
+- template: ${{ parameters.runJobTemplate }}
+  parameters:
+    # Compute job name from template parameters
+    jobName: ${{ format('exploratory_{0}{1}_{2}_{3}_{4}_{5}_{6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind) }}
+    displayName: ${{ format('Exploratory {0}{1} {2} {3} {4} {5} {6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind) }}
+    pool: ${{ parameters.pool }}
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: ${{ parameters.archType }}
+    osGroup: ${{ parameters.osGroup }}
+    osSubgroup: ${{ parameters.osSubgroup }}
+    runtimeVariant: ${{ parameters.runtimeVariant }}
+    liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
+    runtimeType: ${{ parameters.runtimeType }}
+    codeGenType: ${{ parameters.codeGenType }}
+    runKind: ${{ parameters.runKind }}
+    testGroup: ${{ parameters.testGroup }}
+    collectionType: ${{ parameters.collectionType }}
+    collectionName: ${{ parameters.collectionName }}
+    additionalSetupParameters: ${{ parameters.additionalSetupParameters }}
+    # Test job depends on the corresponding build job
+    dependsOn:
+     - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+     # Depend on coreclr x64 so we can download it and use mcs.exe from it while publishing non-x64 arch SPMI collection
+     - ${{ if ne(parameters.archType, 'x64') }}:
+       - ${{ format('coreclr_{0}_product_build_{1}{2}_x64_{3}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.buildConfig) }}
+     - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+       - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
+
+    variables: ${{ parameters.variables }}
+
+    frameworks:
+      - ${{ parameters.framework }}
+    steps:
+    # Extra steps that will be passed to the exploratory template and run before sending the job to helix (all of which is done in the template)
+
+    # Optionally download live-built libraries
+    - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(librariesDownloadDir)
+          cleanUnpackFolder: false
+          artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)'
+          artifactName: '$(librariesBuildArtifactName)'
+          displayName: 'live-built libraries'
+
+    # Download coreclr
+    - template: /eng/pipelines/common/download-artifact-step.yml
+      parameters:
+        unpackFolder: $(buildProductRootFolderPath)
+        artifactFileName: '$(buildProductArtifactName)$(archiveExtension)'
+        artifactName: '$(buildProductArtifactName)'
+        displayName: 'Coreclr product build'
+
+    # Download x64 coreclr if running on non-x64 configuration
+    - ${{ if ne(parameters.archType, 'x64') }}:
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)'
+          artifactFileName: 'CoreCLRProduct__${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)'
+          artifactName: 'CoreCLRProduct__${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_x64_$(buildConfig)'
+          displayName: 'Coreclr product build (x64)'
+
+    # Create Core_Root
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg)
+      displayName: Create Core_Root
+      condition: succeeded()

eng/pipelines/coreclr/templates/run-exploratory-job.yml

@@ -0,0 +1,152 @@
+parameters:
+  steps: []                       # optional -- any additional steps that need to happen before pulling down the jitutils repo and sending the jitutils to helix (ie building your repo)
+  variables: []                   # optional -- list of additional variables to send to the template
+  jobName: ''                     # required -- job name
+  displayName: ''                 # optional -- display name for the job. Will use jobName if not passed
+  pool: ''                        # required -- name of the Build pool
+  container: ''                   # required -- name of the container
+  buildConfig: ''                 # required -- build configuration
+  archType: ''                    # required -- targeting CPU architecture
+  osGroup: ''                     # required -- operating system for the job
+  osSubgroup: ''                  # optional -- operating system subgroup
+  extraSetupParameters: ''        # optional -- extra arguments to pass to the setup script
+  frameworks: ['netcoreapp3.0']   # optional -- list of frameworks to run against
+  continueOnError: 'false'        # optional -- determines whether to continue the build if the step errors
+  dependsOn: ''                   # optional -- dependencies of the job
+  timeoutInMinutes: 320           # optional -- timeout for the job
+  enableTelemetry: false          # optional -- enable for telemetry
+  liveLibrariesBuildConfig: ''    # optional -- live-live libraries configuration to use for the run
+  runtimeType: 'coreclr'          # optional -- Sets the runtime as coreclr or mono
+  codeGenType: 'JIT'              # optional -- Decides on the codegen technology if running on mono
+  runKind: ''                     # required -- test category
+  collectionType: ''
+  collectionName: ''
+  dependOnEvaluatePaths: false
+
+jobs:
+- template: xplat-pipeline-job.yml
+  parameters:
+    dependsOn: ${{ parameters.dependsOn }}
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: ${{ parameters.archType }}
+    osGroup: ${{ parameters.osGroup }}
+    osSubgroup: ${{ parameters.osSubgroup }}
+    liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
+    enableTelemetry: ${{ parameters.enableTelemetry }}
+    enablePublishBuildArtifacts: true
+    continueOnError: ${{ parameters.continueOnError }}
+    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+    ${{ if ne(parameters.displayName, '') }}:
+      displayName: '${{ parameters.displayName }}'
+    ${{ if eq(parameters.displayName, '') }}:
+      displayName: '${{ parameters.jobName }}'
+
+    variables:
+
+    - ${{ each variable in parameters.variables }}:
+      - ${{ if ne(variable.name, '') }}:
+        - name: ${{ variable.name }}
+          value: ${{ variable.value }}
+      - ${{ if ne(variable.group, '') }}:
+        - group: ${{ variable.group }}
+
+    - HelixApiAccessToken: ''
+    - HelixPreCommand: ''
+
+    - ${{ if eq(parameters.osGroup, 'windows') }}:
+      - name: PythonScript
+        value: 'py -3'
+      - name: PipScript
+        value: 'py -3 -m pip'
+      - name: Core_Root_Dir
+        value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root'
+      - name: IssuesLocation
+        value: '$(Build.SourcesDirectory)\artifacts\helixresults\'
+      - name: IssuesSummaryLocation
+        value: '$(Build.SourcesDirectory)\artifacts\issues_summary\'
+
+    - ${{ if ne(parameters.osGroup, 'windows') }}:
+      - name: PythonScript
+        value: 'python3'
+      - name: PipScript
+        value: 'pip3'
+      - name: Core_Root_Dir
+        value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root'
+      - name: IssuesLocation
+        value: '$(Build.SourcesDirectory)/artifacts/helixresults/'
+      - name: IssuesSummaryLocation
+        value: '$(Build.SourcesDirectory)/artifacts/issues_summary/'
+    workspace:
+      clean: all
+    pool:
+      ${{ parameters.pool }}
+    container: ${{ parameters.container }}
+    strategy:
+      matrix:
+        ${{ each framework in parameters.frameworks }}:
+          ${{ framework }}:
+            _Framework: ${{ framework }}
+    steps:
+    - ${{ parameters.steps }}
+
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/exploratory_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup)
+      displayName: ${{ format('Exploratory setup ({0})', parameters.osGroup) }}
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken) # secret access token to clone exploratory code
+        TOOL_REPO_PATH: $(ToolRepoPath)           # tool repo path
+        TOOL_NAME: $(ToolName)                    # tool name
+
+      # Run exploratory tool in helix
+    - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml
+      parameters:
+        HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
+        HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(_Framework)/$(Architecture)'
+        HelixAccessToken: $(HelixApiAccessToken)
+        HelixTargetQueues: $(Queue)
+        HelixPreCommands: $(HelixPreCommand)
+        Creator: $(Creator)
+        WorkItemTimeout: 4:00 # 4 hours
+        WorkItemDirectory: '$(WorkItemDirectory)'
+        CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)'
+        ProjectFile: 'exploratory.proj'
+        BuildConfig: ${{ parameters.buildConfig }}
+        osGroup: ${{ parameters.osGroup }}
+        RunConfiguration: '$(RunConfiguration)'
+        ToolName: '$(ToolName)'
+        continueOnError: true # Run the future step i.e. merge-mch step even if this step fails.
+
+    - template: /eng/pipelines/common/upload-artifact-step.yml
+      parameters:
+        rootFolder: $(IssuesLocation)
+        includeRootFolder: false
+        archiveType: $(archiveType)
+        tarCompression: $(tarCompression)
+        archiveExtension: $(archiveExtension)
+        artifactName: 'Exploratory_Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        displayName: ${{ format('Upload artifacts Exploratory issues for {0}-{1}', parameters.osGroup, parameters.archType ) }}
+
+    # Always upload the available issues-summary.txt files
+    - task: CopyFiles@2
+      displayName: Copying issues-summary.txt of all partitions
+      inputs:
+        sourceFolder: '$(IssuesLocation)'
+        contents: '**/issues-summary-*.txt'
+        targetFolder: '$(IssuesSummaryLocation)'
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish issues-summary.txt files
+      inputs:
+        targetPath: $(IssuesSummaryLocation)
+        artifactName: 'Issues_Summary_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/exploratory_post.py -issues_directory $(IssuesSummaryLocation)
+      displayName: ${{ format('Print unique issues ({0})', parameters.osGroup) }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish Exploratory build logs
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/artifacts/log
+        artifactName: 'Exploratory_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+      condition: always()