Skip to content

Commit 2f67e77

Browse files
author
Spencer
authored
[6.8] Jenkins pipeline with parallel cigroups (#45285) (#45599)
* Jenkins pipeline with parallel cigroups (#45285) * Pipeline * WIP some work for parallelization with ciGroups * Fix xpack kibana install dir, and add some debugging * Attempt to quick fix a few tests * Revert "Revert "Revert "[ci] compress jobs for CI stability" (#44584)"" This reverts commit 078ac28. * Recombine test groups, and try runbld again * Mostly cleanup, and fix failed_tests reporting to hopefully work for both pipeline and non-pipeline * Fix typo in shell script * Remove some debug code * Add support for changing es transport.port during testing via TEST_ES_TRANSPORT_PORT * Fix test that uses hard-coded es transport port and add it back in to parallel groups * Disable checks reporter again for now * Set env var for TEST_ES_TRANSPORT_PORT in pipeline * Update Jenkinsfile for shorter testrunner labels * Fix another hard-coded transport port * Fix a new test with hard-coded URLs * Jenkinsfile cleanup and fix one of the groups * Fix double slash * Testing vault credentials on jenkins server * Add a non-existent credential * Revert "Add a non-existent credential" This reverts commit 0dc234c. * Try github-checks-reporter again * github-checks-reporter should only run for elastic/kibana, forks won't work * Clean up some debug code * Changing names around to try to make BlueOcean UI a little better * Add more stages * Make some changes to stage structure to mirror a nested example from CloudBees * Handle TODOs, and some cleanup in Jenkinsfile * Pass GIT_BRANCH when started without GHPRB, fix branch check * Fix mailer problem and add code that ensures all tests are in cigroups back in * Test adding worker/job name to junit report paths * Remove some duplication from ci_setup scripts * Fix unit test that uses junit path * Don't reinstall node every time setup_env is run * Fix yarn install logic * Fix another unit test that uses junit output dir * Download latest ES snapshot after kibana builds * Make sure junit reports are always processed * Add two failing tests for testing purposes * Add support to Jenkinsfile for kibana build e-mails * Remove some debug code for email sending * Change JOB env handling in junit paths and move it to a sub-directory * Revert "Add two failing tests for testing purposes" This reverts commit 5715203e26922a93483feb0ebb8bb3fdcc3daf8c. * Fix junit report path in test * Don't send kibana emails on build abort * Address PR feedback, formatting and use built-in url formatting library * Fix path formatting for functional test * Add email sending back in to Jenkinsfile * Fix another unit test with path problem (cherry picked from commit 27d23c4) * remove reference to extract_bootstrap_cache.sh * remove extra jobs from Jenkinsfile * increment leadfoot server port for parallel workers * add worker index to port number, rather than concatenating with it * well, for some reason the PR job wasn't triggered for 351eaaf... * huh, PR job wasn't triggered for 1b99c69 either...
1 parent d786bac commit 2f67e77

File tree

22 files changed

+580
-179
lines changed

22 files changed

+580
-179
lines changed

‎Jenkinsfile‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
#!/bin/groovy
2+
3+
properties([
4+
durabilityHint('PERFORMANCE_OPTIMIZED'),
5+
])
6+
7+
stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
8+
timeout(time: 180, unit: 'MINUTES') {
9+
timestamps {
10+
ansiColor('xterm') {
11+
catchError {
12+
parallel([
13+
'kibana-intake-agent': legacyJobRunner('kibana-intake'),
14+
'x-pack-intake-agent': legacyJobRunner('x-pack-intake'),
15+
'kibana-oss-agent': withWorkers('kibana-oss-tests', { buildOss() }, [
16+
'oss-ciGroup1': getOssCiGroupWorker(1),
17+
'oss-ciGroup2': getOssCiGroupWorker(2),
18+
'oss-ciGroup3': getOssCiGroupWorker(3),
19+
'oss-ciGroup4': getOssCiGroupWorker(4),
20+
'oss-ciGroup5': getOssCiGroupWorker(5),
21+
'oss-ciGroup6': getOssCiGroupWorker(6),
22+
'oss-ciGroup7': getOssCiGroupWorker(7),
23+
'oss-ciGroup8': getOssCiGroupWorker(8),
24+
'oss-ciGroup9': getOssCiGroupWorker(9),
25+
'oss-ciGroup10': getOssCiGroupWorker(10),
26+
'oss-ciGroup11': getOssCiGroupWorker(11),
27+
'oss-ciGroup12': getOssCiGroupWorker(12),
28+
]),
29+
'kibana-xpack-agent': withWorkers('kibana-xpack-tests', { buildXpack() }, [
30+
'xpack-ciGroup1': getXpackCiGroupWorker(1),
31+
'xpack-ciGroup2': getXpackCiGroupWorker(2),
32+
'xpack-ciGroup3': getXpackCiGroupWorker(3),
33+
'xpack-ciGroup4': getXpackCiGroupWorker(4),
34+
'xpack-ciGroup5': getXpackCiGroupWorker(5),
35+
'xpack-ciGroup6': getXpackCiGroupWorker(6),
36+
]),
37+
])
38+
}
39+
node('flyweight') {
40+
sendMail()
41+
}
42+
}
43+
}
44+
}
45+
}
46+
47+
def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
48+
return {
49+
jobRunner('tests-xl') {
50+
try {
51+
doSetup()
52+
preWorkerClosure()
53+
54+
def nextWorker = 1
55+
def worker = { workerClosure ->
56+
def workerNumber = nextWorker
57+
nextWorker++
58+
59+
return {
60+
workerClosure(workerNumber)
61+
}
62+
}
63+
64+
def workers = [:]
65+
workerClosures.each { workerName, workerClosure ->
66+
workers[workerName] = worker(workerClosure)
67+
}
68+
69+
parallel(workers)
70+
} finally {
71+
catchError {
72+
uploadAllGcsArtifacts(name)
73+
}
74+
75+
catchError {
76+
publishJunit()
77+
}
78+
}
79+
}
80+
}
81+
}
82+
83+
def getPostBuildWorker(name, closure) {
84+
return { workerNumber ->
85+
def kibanaPort = "61${workerNumber}1"
86+
def esPort = "61${workerNumber}2"
87+
def esTransportPort = "61${workerNumber}3"
88+
89+
withEnv([
90+
"PARALLEL_PIPELINE_WORKER_INDEX=${workerNumber}",
91+
"TEST_KIBANA_HOST=localhost",
92+
"TEST_KIBANA_PORT=${kibanaPort}",
93+
"TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}",
94+
"TEST_ES_URL=http://elastic:changeme@localhost:${esPort}",
95+
"TEST_ES_TRANSPORT_PORT=${esTransportPort}",
96+
"IS_PIPELINE_JOB=1",
97+
]) {
98+
closure()
99+
}
100+
}
101+
}
102+
103+
def getOssCiGroupWorker(ciGroup) {
104+
return getPostBuildWorker("ciGroup" + ciGroup, {
105+
withEnv([
106+
"CI_GROUP=${ciGroup}",
107+
"JOB=kibana-ciGroup${ciGroup}",
108+
]) {
109+
runbld "./test/scripts/jenkins_ci_group.sh"
110+
}
111+
})
112+
}
113+
114+
def getXpackCiGroupWorker(ciGroup) {
115+
return getPostBuildWorker("xpack-ciGroup" + ciGroup, {
116+
withEnv([
117+
"CI_GROUP=${ciGroup}",
118+
"JOB=xpack-kibana-ciGroup${ciGroup}",
119+
]) {
120+
runbld "./test/scripts/jenkins_xpack_ci_group.sh"
121+
}
122+
})
123+
}
124+
125+
def legacyJobRunner(name) {
126+
return {
127+
parallel([
128+
"${name}": {
129+
withEnv([
130+
"JOB=${name}",
131+
]) {
132+
jobRunner('linux && immutable') {
133+
try {
134+
runbld '.ci/run.sh'
135+
} finally {
136+
catchError {
137+
uploadAllGcsArtifacts(name)
138+
}
139+
catchError {
140+
publishJunit()
141+
}
142+
}
143+
}
144+
}
145+
}
146+
])
147+
}
148+
}
149+
150+
def jobRunner(label, closure) {
151+
node(label) {
152+
def scmVars = checkout scm
153+
154+
withEnv([
155+
"CI=true",
156+
"HOME=${env.JENKINS_HOME}",
157+
"PR_SOURCE_BRANCH=${env.ghprbSourceBranch}",
158+
"PR_TARGET_BRANCH=${env.ghprbTargetBranch}",
159+
"PR_AUTHOR=${env.ghprbPullAuthorLogin}",
160+
"TEST_BROWSER_HEADLESS=1",
161+
"GIT_BRANCH=${scmVars.GIT_BRANCH}",
162+
]) {
163+
withCredentials([
164+
string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'),
165+
string(credentialsId: 'vault-role-id', variable: 'VAULT_ROLE_ID'),
166+
string(credentialsId: 'vault-secret-id', variable: 'VAULT_SECRET_ID'),
167+
]) {
168+
// scm is configured to check out to the ./kibana directory
169+
dir('kibana') {
170+
closure()
171+
}
172+
}
173+
}
174+
}
175+
}
176+
177+
// TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build?
178+
179+
def uploadGcsArtifact(workerName, pattern) {
180+
def storageLocation = "gs://kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" // TODO
181+
// def storageLocation = "gs://kibana-pipeline-testing/jobs/pipeline-test/${BUILD_NUMBER}/${workerName}"
182+
183+
googleStorageUpload(
184+
credentialsId: 'kibana-ci-gcs-plugin',
185+
bucket: storageLocation,
186+
pattern: pattern,
187+
sharedPublicly: true,
188+
showInline: true,
189+
)
190+
}
191+
192+
def uploadAllGcsArtifacts(workerName) {
193+
def ARTIFACT_PATTERNS = [
194+
'target/kibana-*',
195+
'target/junit/**/*',
196+
'test/**/screenshots/**/*.png',
197+
'test/functional/failure_debug/html/*.html',
198+
'x-pack/test/**/screenshots/**/*.png',
199+
'x-pack/test/functional/failure_debug/html/*.html',
200+
'x-pack/test/functional/apps/reporting/reports/session/*.pdf',
201+
]
202+
203+
ARTIFACT_PATTERNS.each { pattern ->
204+
uploadGcsArtifact(workerName, pattern)
205+
}
206+
}
207+
208+
def publishJunit() {
209+
junit(testResults: 'target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true)
210+
}
211+
212+
def sendMail() {
213+
sendInfraMail()
214+
sendKibanaMail()
215+
}
216+
217+
def sendInfraMail() {
218+
catchError {
219+
step([
220+
$class: 'Mailer',
221+
notifyEveryUnstableBuild: true,
222+
recipients: '[email protected]',
223+
sendToIndividuals: false
224+
])
225+
}
226+
}
227+
228+
def sendKibanaMail() {
229+
catchError {
230+
if(params.NOTIFY_ON_FAILURE && currentBuild.result != 'SUCCESS' && currentBuild.result != 'ABORTED') {
231+
emailext(
232+
// to: '[email protected]',
233+
to: '[email protected]', // TODO switch this out after testing
234+
subject: "${env.PROJECT_NAME} - Build # ${env.BUILD_NUMBER} - ${currentBuild.result}",
235+
body: '${SCRIPT,template="groovy-html.template"}',
236+
mimeType: 'text/html',
237+
)
238+
}
239+
}
240+
}
241+
242+
def runbld(script) {
243+
sh '#!/usr/local/bin/runbld\n' + script
244+
}
245+
246+
def bash(script) {
247+
sh "#!/bin/bash -x\n${script}"
248+
}
249+
250+
def doSetup() {
251+
runbld "./test/scripts/jenkins_setup.sh"
252+
}
253+
254+
def buildOss() {
255+
runbld "./test/scripts/jenkins_build_kibana.sh"
256+
}
257+
258+
def buildXpack() {
259+
runbld "./test/scripts/jenkins_xpack_build_kibana.sh"
260+
}

‎packages/kbn-test/src/es/es_test_cluster.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ export function createEsTestCluster(options = {}) {
5151
license,
5252
basePath,
5353
};
54+
const transportPort = esTestConfig.getTransportPort();
5455

5556
const cluster = new Cluster(log);
5657

@@ -84,6 +85,7 @@ export function createEsTestCluster(options = {}) {
8485
`cluster.name=${clusterName}`,
8586
`http.port=${port}`,
8687
'discovery.type=single-node',
88+
`transport.port=${transportPort}`,
8789
...esArgs,
8890
],
8991
});

‎packages/kbn-test/src/es/es_test_config.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ export const esTestConfig = new class EsTestConfig {
3838
return process.env.TEST_ES_FROM || 'snapshot';
3939
}
4040

41+
getTransportPort() {
42+
return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
43+
}
44+
4145
getUrlParts() {
4246
// Allow setting one complete TEST_ES_URL for Es like https://elastic:changeme@myCloudInstance:9200
4347
if (process.env.TEST_ES_URL) {

0 commit comments

Comments
 (0)