From b0b2025ea468d0623ec35d7f2119a4dd621f0cba Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Oct 2020 12:11:34 -0700 Subject: [PATCH 1/5] Add back superpmi pipeline files --- .../coreclr/templates/run-superpmi-job.yml | 2 +- .../templates/superpmi-send-to-helix.yml | 53 +++++++++ src/coreclr/scripts/superpmi.proj | 102 ++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml create mode 100644 src/coreclr/scripts/superpmi.proj diff --git a/eng/pipelines/coreclr/templates/run-superpmi-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-job.yml index 703c384108d9c6..286acbc721cfa0 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-job.yml @@ -89,7 +89,7 @@ jobs: displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Run superpmi collection in helix - - template: /eng/common/templates/steps/superpmi-send-to-helix.yml + - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml parameters: HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ HelixType: 'test/superpmi/$(Kind)/$(_Framework)/$(Architecture)' diff --git a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml new file mode 100644 index 00000000000000..77a25dc0610a17 --- /dev/null +++ b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml @@ -0,0 +1,53 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + BuildConfig: 'checked' # optional -- Mostly, superpmi will be run on checked builds + LibrariesArtifacts: '' + TestsArtifacts: '' + +steps: +- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + sendParams: $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.proj /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfig)/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + environment: + MchFileTag: $(MchFileTag) + BuildConfig: ${{ parameters.BuildConfig }} + LibrariesArtifacts: ${{ parameters.LibrariesArtifacts }} + TestsArtifacts: ${{ parameters.TestsArtifacts }} + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi.proj b/src/coreclr/scripts/superpmi.proj new file mode 100644 index 00000000000000..75797de16e86fc --- /dev/null +++ b/src/coreclr/scripts/superpmi.proj @@ -0,0 +1,102 @@ + + + + \ + + + / + + + + + %HELIX_PYTHONPATH% + $(WorkItemDirectory)\pmiAssembliesDirectory + %HELIX_WORKITEM_PAYLOAD%\binaries + %HELIX_CORRELATION_PAYLOAD%\superpmi + %HELIX_WORKITEM_UPLOAD_ROOT% + + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults + $(SuperPMIDirectory)\superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)\pmi.dll + + + $HELIX_PYTHONPATH + $(WorkItemDirectory)/pmiAssembliesDirectory + $HELIX_WORKITEM_PAYLOAD/binaries + $HELIX_CORRELATION_PAYLOAD/superpmi + $HELIX_WORKITEM_UPLOAD_ROOT + + $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults + $(SuperPMIDirectory)/superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)/pmi.dll + + + + $(Python) $(WorkItemCommand) -pmi_assemblies $(PmiAssembliesDirectory) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) + + + + false + false + 5:00 + + + + + %(Identity) + + + + + + + + + + + + + + + + %(HelixWorkItem.OutputMchName).$(MchFileTag).%(HelixWorkItem.PartitionId) + $(PmiAssembliesPayload)$(FileSeparatorChar)%(HelixWorkItem.PmiAssemblies) + $(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log + $(WorkItemTimeout) + %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log + + + + + + + \ No newline at end of file From 0b0a36dc783d43cfcdf3043171673fe904794a27 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Oct 2020 14:02:17 -0700 Subject: [PATCH 2/5] add the missing import --- src/coreclr/scripts/superpmi-setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/scripts/superpmi-setup.py b/src/coreclr/scripts/superpmi-setup.py index 73de8cff73eec0..ac1a3c5f2a3ac7 100644 --- a/src/coreclr/scripts/superpmi-setup.py +++ b/src/coreclr/scripts/superpmi-setup.py @@ -33,6 +33,7 @@ ################################################################################ +import shutil import subprocess import argparse From d174cf946dd5129a8a0cb37dec4ee2f5279ddb15 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Oct 2020 15:10:01 -0700 Subject: [PATCH 3/5] add another missing import I am not sure how this worked in the past with missing imports. --- src/coreclr/scripts/superpmi-setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi-setup.py b/src/coreclr/scripts/superpmi-setup.py index ac1a3c5f2a3ac7..db4fca239f12f9 100644 --- a/src/coreclr/scripts/superpmi-setup.py +++ b/src/coreclr/scripts/superpmi-setup.py @@ -33,9 +33,10 @@ ################################################################################ +import argparse import shutil import subprocess -import argparse +import tempfile from os import listdir, path, walk from os.path import isfile, join, getsize From 5c674210cc8f20fda3e73704c7f72d6e168c68d9 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Oct 2020 16:32:23 -0700 Subject: [PATCH 4/5] remove the workaround for a bug --- src/coreclr/scripts/superpmi.proj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi.proj b/src/coreclr/scripts/superpmi.proj index 75797de16e86fc..01992f0ab7add1 100644 --- a/src/coreclr/scripts/superpmi.proj +++ b/src/coreclr/scripts/superpmi.proj @@ -22,8 +22,6 @@ %HELIX_WORKITEM_PAYLOAD%\binaries %HELIX_CORRELATION_PAYLOAD%\superpmi %HELIX_WORKITEM_UPLOAD_ROOT% - - $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults $(SuperPMIDirectory)\superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)\pmi.dll @@ -32,8 +30,6 @@ $HELIX_WORKITEM_PAYLOAD/binaries $HELIX_CORRELATION_PAYLOAD/superpmi $HELIX_WORKITEM_UPLOAD_ROOT - - $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults $(SuperPMIDirectory)/superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)/pmi.dll From b16d3188343345e013fba415f23ee8530bbc4478 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Oct 2020 22:36:26 -0700 Subject: [PATCH 5/5] Revert "remove the workaround for a bug" This reverts commit 5c674210cc8f20fda3e73704c7f72d6e168c68d9. --- src/coreclr/scripts/superpmi.proj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/coreclr/scripts/superpmi.proj b/src/coreclr/scripts/superpmi.proj index 01992f0ab7add1..75797de16e86fc 100644 --- a/src/coreclr/scripts/superpmi.proj +++ b/src/coreclr/scripts/superpmi.proj @@ -22,6 +22,8 @@ %HELIX_WORKITEM_PAYLOAD%\binaries %HELIX_CORRELATION_PAYLOAD%\superpmi %HELIX_WORKITEM_UPLOAD_ROOT% + + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults $(SuperPMIDirectory)\superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)\pmi.dll @@ -30,6 +32,8 @@ $HELIX_WORKITEM_PAYLOAD/binaries $HELIX_CORRELATION_PAYLOAD/superpmi $HELIX_WORKITEM_UPLOAD_ROOT + + $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults $(SuperPMIDirectory)/superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)/pmi.dll