forked from dotnet/corefx
-
Notifications
You must be signed in to change notification settings - Fork 0
/
perf.groovy
169 lines (148 loc) · 9.47 KB
/
perf.groovy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
// Import the utility functionality.
import jobs.generation.*;
// The input project name (e.g. dotnet/corefx)
def project = GithubProject
// The input branch name (e.g. master)
def branch = GithubBranchName
// Folder that the project jobs reside in (project/branch)
def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
// Globals
// Map of os -> osGroup.
def osGroupMap = ['Ubuntu14.04':'Linux',
'Ubuntu16.04':'Linux',
'Debian8.4':'Linux',
'OSX':'OSX',
'Windows_NT':'Windows_NT',
'CentOS7.1': 'Linux',
'OpenSUSE13.2': 'Linux',
'RHEL7.2': 'Linux',
'LinuxARMEmulator': 'Linux']
// Map of os -> nuget runtime
def targetNugetRuntimeMap = ['OSX' : 'osx.10.10-x64',
'Ubuntu14.04' : 'ubuntu.14.04-x64',
'Ubuntu16.04' : 'ubuntu.16.04-x64',
'Debian8.4' : 'debian.8-x64',
'CentOS7.1' : 'centos.7-x64',
'OpenSUSE13.2' : 'opensuse.13.2-x64',
'RHEL7.2': 'rhel.7-x64']
def osShortName = ['Windows 10': 'win10',
'Windows 7' : 'win7',
'Windows_NT' : 'windows_nt',
'Ubuntu14.04' : 'ubuntu14.04',
'OSX' : 'osx',
'Windows Nano 2016' : 'winnano16',
'Ubuntu16.04' : 'ubuntu16.04',
'CentOS7.1' : 'centos7.1',
'Debian8.4' : 'debian8.4',
'OpenSUSE13.2' : 'opensuse13.2',
'RHEL7.2' : 'rhel7.2']
// **************************
// Define perf testing. These tests should be run daily, and will run on Jenkins not in Helix
// **************************
[true, false].each { isPR ->
['Release'].each { configurationGroup ->
['Windows_NT', 'Ubuntu16.04'].each { os ->
def osGroup = osGroupMap[os]
def newJobName = "perf_${os.toLowerCase()}_${configurationGroup.toLowerCase()}"
def newJob = job(Utilities.getFullJobName(project, newJobName, isPR)) {
if (os == 'Windows_NT') {
label('windows_server_2016_clr_perf')
}
else {
label('ubuntu_1604_clr_perf')
}
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreFX Perf BenchView Sas')
}
}
if (isPR)
{
parameters
{
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
}
}
parameters {
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
}
def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
if (os == 'Windows_NT') {
def benchViewName = isPR ? 'corefx private %BenchviewCommitName%' : 'corefx rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
steps {
//We need to specify the max cpu count to be one as we do not want to be executing performance tests in parallel
batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build.cmd -release")
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\\Tools\" -Prerelease -ExcludeVersion")
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
"py \"%WORKSPACE%\\Tools\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name " + "\"" + benchViewName + "\"" + " --user-email " + "\"[email protected]\"\n" +
"py \"%WORKSPACE%\\Tools\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
batchFile("py \"%WORKSPACE%\\Tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build-managed.cmd -release -tests -- /p:Performance=true /p:TargetOS=${osGroup} /m:1 /p:LogToBenchview=true /p:BenchviewRunType=${runType} /p:PerformanceType=Profile")
batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build-managed.cmd -release -tests -- /p:Performance=true /p:TargetOS=${osGroup} /m:1 /p:LogToBenchview=true /p:BenchviewRunType=${runType} /p:PerformanceType=Diagnostic")
}
}
else {
def benchViewName = isPR ? 'corefx private \$BenchviewCommitName' : 'corefx rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
steps {
//We need to specify the max cpu count to be one as we do not want to be executing performance tests in parallel
shell("./build.sh -release")
shell("find . -type f -name dotnet | xargs chmod u+x")
shell("curl \"http://benchviewtestfeed.azurewebsites.net/nuget/FindPackagesById()?id='Microsoft.BenchView.JSONFormat'\" | grep \"content type\" | sed \"\$ s/.*src=\\\"\\([^\\\"]*\\)\\\".*/\\1/;tx;d;:x\" | xargs curl -o benchview.zip")
shell("unzip -q -o benchview.zip -d \"\${WORKSPACE}/Tools/Microsoft.BenchView.JSONFormat\"")
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
"python3.5 \"\${WORKSPACE}/Tools/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name " + "\"" + benchViewName + "\"" + " --user-email " + "\"[email protected]\"\n" +
"python3.5 \"\${WORKSPACE}/Tools/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type " + runType)
shell("python3.5 \"\${WORKSPACE}/Tools/Microsoft.BenchView.JSONFormat/tools/machinedata.py\"")
shell("bash ./build-managed.sh -release -tests -- /p:Performance=true /p:TargetOS=${osGroup} /m:1 /p:LogToBenchview=true /p:BenchviewRunType=${runType} /p:PerformanceType=Profile")
}
}
}
// Add the unit test results
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('msbuild.log')
archiveSettings.addFiles('machinedata.json')
archiveSettings.addFiles('bin/**/Perf-*Performance.Tests.csv')
archiveSettings.addFiles('bin/**/Perf-*Performance.Tests.etl')
archiveSettings.addFiles('bin/**/Perf-*Performance.Tests.md')
archiveSettings.addFiles('bin/**/Perf-*Performance.Tests.xml')
archiveSettings.setAlwaysArchive()
// Add archival for the built data.
Utilities.addArchival(newJob, archiveSettings)
// Set up standard options.
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
newJob.with {
logRotator {
artifactDaysToKeep(14)
daysToKeep(14)
artifactNumToKeep(100)
numToKeep(100)
}
wrappers {
timeout {
absolute(360)
}
}
}
// Set up triggers
if (isPR) {
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
builder.setGithubContext("${os} Perf Tests")
builder.triggerOnlyOnComment()
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
else {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
}
}
}