Skip to content

Commit 14b69dc

Browse files
brianseedersspalger
brianseeders
authored and
spalger
committed
Jenkins pipeline with parallel cigroups (#45285)
* Pipeline * WIP some work for parallelization with ciGroups * Fix xpack kibana install dir, and add some debugging * Attempt to quick fix a few tests * Revert "Revert "Revert "[ci] compress jobs for CI stability" (#44584)"" This reverts commit 078ac28. * Recombine test groups, and try runbld again * Mostly cleanup, and fix failed_tests reporting to hopefully work for both pipeline and non-pipeline * Fix typo in shell script * Remove some debug code * Add support for changing es transport.port during testing via TEST_ES_TRANSPORT_PORT * Fix test that uses hard-coded es transport port and add it back in to parallel groups * Disable checks reporter again for now * Set env var for TEST_ES_TRANSPORT_PORT in pipeline * Update Jenkinsfile for shorter testrunner labels * Fix another hard-coded transport port * Fix a new test with hard-coded URLs * Jenkinsfile cleanup and fix one of the groups * Fix double slash * Testing vault credentials on jenkins server * Add a non-existent credential * Revert "Add a non-existent credential" This reverts commit 0dc234c. * Try github-checks-reporter again * github-checks-reporter should only run for elastic/kibana, forks won't work * Clean up some debug code * Changing names around to try to make BlueOcean UI a little better * Add more stages * Make some changes to stage structure to mirror a nested example from CloudBees * Handle TODOs, and some cleanup in Jenkinsfile * Pass GIT_BRANCH when started without GHPRB, fix branch check * Fix mailer problem and add code that ensures all tests are in cigroups back in * Test adding worker/job name to junit report paths * Remove some duplication from ci_setup scripts * Fix unit test that uses junit path * Don't reinstall node every time setup_env is run * Fix yarn install logic * Fix another unit test that uses junit output dir * Download latest ES snapshot after kibana builds * Make sure junit reports are always processed * Add two failing tests for testing purposes * Add support to Jenkinsfile for kibana build e-mails * Remove some debug code for email sending * Change JOB env handling in junit paths and move it to a sub-directory * Revert "Add two failing tests for testing purposes" This reverts commit 5715203e26922a93483feb0ebb8bb3fdcc3daf8c. * Fix junit report path in test * Don't send kibana emails on build abort * Address PR feedback, formatting and use built-in url formatting library * Fix path formatting for functional test * Add email sending back in to Jenkinsfile * Fix another unit test with path problem (cherry picked from commit 27d23c4) # Conflicts: # src/dev/ci_setup/setup.sh # src/dev/failed_tests/cli.js # test/plugin_functional/test_suites/core_plugins/applications.js # test/scripts/jenkins_xpack_ci_group.sh
1 parent 84683cc commit 14b69dc

File tree

28 files changed

+666
-225
lines changed

28 files changed

+666
-225
lines changed

‎Jenkinsfile‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
#!/bin/groovy
2+
3+
properties([
4+
durabilityHint('PERFORMANCE_OPTIMIZED'),
5+
])
6+
7+
stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
8+
timeout(time: 180, unit: 'MINUTES') {
9+
timestamps {
10+
ansiColor('xterm') {
11+
catchError {
12+
parallel([
13+
'kibana-intake-agent': legacyJobRunner('kibana-intake'),
14+
'x-pack-intake-agent': legacyJobRunner('x-pack-intake'),
15+
'kibana-oss-agent': withWorkers('kibana-oss-tests', { buildOss() }, [
16+
'oss-ciGroup1': getOssCiGroupWorker(1),
17+
'oss-ciGroup2': getOssCiGroupWorker(2),
18+
'oss-ciGroup3': getOssCiGroupWorker(3),
19+
'oss-ciGroup4': getOssCiGroupWorker(4),
20+
'oss-ciGroup5': getOssCiGroupWorker(5),
21+
'oss-ciGroup6': getOssCiGroupWorker(6),
22+
'oss-ciGroup7': getOssCiGroupWorker(7),
23+
'oss-ciGroup8': getOssCiGroupWorker(8),
24+
'oss-ciGroup9': getOssCiGroupWorker(9),
25+
'oss-ciGroup10': getOssCiGroupWorker(10),
26+
'oss-ciGroup11': getOssCiGroupWorker(11),
27+
'oss-ciGroup12': getOssCiGroupWorker(12),
28+
'oss-visualRegression': getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }),
29+
'oss-firefoxSmoke': getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' }),
30+
]),
31+
'kibana-xpack-agent': withWorkers('kibana-xpack-tests', { buildXpack() }, [
32+
'xpack-ciGroup1': getXpackCiGroupWorker(1),
33+
'xpack-ciGroup2': getXpackCiGroupWorker(2),
34+
'xpack-ciGroup3': getXpackCiGroupWorker(3),
35+
'xpack-ciGroup4': getXpackCiGroupWorker(4),
36+
'xpack-ciGroup5': getXpackCiGroupWorker(5),
37+
'xpack-ciGroup6': getXpackCiGroupWorker(6),
38+
'xpack-ciGroup7': getXpackCiGroupWorker(7),
39+
'xpack-ciGroup8': getXpackCiGroupWorker(8),
40+
'xpack-ciGroup9': getXpackCiGroupWorker(9),
41+
'xpack-ciGroup10': getXpackCiGroupWorker(10),
42+
'xpack-firefoxSmoke': getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' }),
43+
'xpack-visualRegression': getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }),
44+
]),
45+
])
46+
}
47+
node('flyweight') {
48+
sendMail()
49+
}
50+
}
51+
}
52+
}
53+
}
54+
55+
def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
56+
return {
57+
jobRunner('tests-xl') {
58+
try {
59+
doSetup()
60+
preWorkerClosure()
61+
62+
def nextWorker = 1
63+
def worker = { workerClosure ->
64+
def workerNumber = nextWorker
65+
nextWorker++
66+
67+
return {
68+
workerClosure(workerNumber)
69+
}
70+
}
71+
72+
def workers = [:]
73+
workerClosures.each { workerName, workerClosure ->
74+
workers[workerName] = worker(workerClosure)
75+
}
76+
77+
parallel(workers)
78+
} finally {
79+
catchError {
80+
uploadAllGcsArtifacts(name)
81+
}
82+
83+
catchError {
84+
publishJunit()
85+
}
86+
}
87+
}
88+
}
89+
}
90+
91+
def getPostBuildWorker(name, closure) {
92+
return { workerNumber ->
93+
def kibanaPort = "61${workerNumber}1"
94+
def esPort = "61${workerNumber}2"
95+
def esTransportPort = "61${workerNumber}3"
96+
97+
withEnv([
98+
"CI_WORKER_NUMBER=${workerNumber}",
99+
"TEST_KIBANA_HOST=localhost",
100+
"TEST_KIBANA_PORT=${kibanaPort}",
101+
"TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}",
102+
"TEST_ES_URL=http://elastic:changeme@localhost:${esPort}",
103+
"TEST_ES_TRANSPORT_PORT=${esTransportPort}",
104+
"IS_PIPELINE_JOB=1",
105+
]) {
106+
closure()
107+
}
108+
}
109+
}
110+
111+
def getOssCiGroupWorker(ciGroup) {
112+
return getPostBuildWorker("ciGroup" + ciGroup, {
113+
withEnv([
114+
"CI_GROUP=${ciGroup}",
115+
"JOB=kibana-ciGroup${ciGroup}",
116+
]) {
117+
runbld "./test/scripts/jenkins_ci_group.sh"
118+
}
119+
})
120+
}
121+
122+
def getXpackCiGroupWorker(ciGroup) {
123+
return getPostBuildWorker("xpack-ciGroup" + ciGroup, {
124+
withEnv([
125+
"CI_GROUP=${ciGroup}",
126+
"JOB=xpack-kibana-ciGroup${ciGroup}",
127+
]) {
128+
runbld "./test/scripts/jenkins_xpack_ci_group.sh"
129+
}
130+
})
131+
}
132+
133+
def legacyJobRunner(name) {
134+
return {
135+
parallel([
136+
"${name}": {
137+
withEnv([
138+
"JOB=${name}",
139+
]) {
140+
jobRunner('linux && immutable') {
141+
try {
142+
runbld '.ci/run.sh'
143+
} finally {
144+
catchError {
145+
uploadAllGcsArtifacts(name)
146+
}
147+
catchError {
148+
publishJunit()
149+
}
150+
}
151+
}
152+
}
153+
}
154+
])
155+
}
156+
}
157+
158+
def jobRunner(label, closure) {
159+
node(label) {
160+
def scmVars = checkout scm
161+
162+
withEnv([
163+
"CI=true",
164+
"HOME=${env.JENKINS_HOME}",
165+
"PR_SOURCE_BRANCH=${env.ghprbSourceBranch}",
166+
"PR_TARGET_BRANCH=${env.ghprbTargetBranch}",
167+
"PR_AUTHOR=${env.ghprbPullAuthorLogin}",
168+
"TEST_BROWSER_HEADLESS=1",
169+
"GIT_BRANCH=${scmVars.GIT_BRANCH}",
170+
]) {
171+
withCredentials([
172+
string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'),
173+
string(credentialsId: 'vault-role-id', variable: 'VAULT_ROLE_ID'),
174+
string(credentialsId: 'vault-secret-id', variable: 'VAULT_SECRET_ID'),
175+
]) {
176+
// scm is configured to check out to the ./kibana directory
177+
dir('kibana') {
178+
closure()
179+
}
180+
}
181+
}
182+
}
183+
}
184+
185+
// TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build?
186+
187+
def uploadGcsArtifact(workerName, pattern) {
188+
def storageLocation = "gs://kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" // TODO
189+
// def storageLocation = "gs://kibana-pipeline-testing/jobs/pipeline-test/${BUILD_NUMBER}/${workerName}"
190+
191+
googleStorageUpload(
192+
credentialsId: 'kibana-ci-gcs-plugin',
193+
bucket: storageLocation,
194+
pattern: pattern,
195+
sharedPublicly: true,
196+
showInline: true,
197+
)
198+
}
199+
200+
def uploadAllGcsArtifacts(workerName) {
201+
def ARTIFACT_PATTERNS = [
202+
'target/kibana-*',
203+
'target/junit/**/*',
204+
'test/**/screenshots/**/*.png',
205+
'test/functional/failure_debug/html/*.html',
206+
'x-pack/test/**/screenshots/**/*.png',
207+
'x-pack/test/functional/failure_debug/html/*.html',
208+
'x-pack/test/functional/apps/reporting/reports/session/*.pdf',
209+
]
210+
211+
ARTIFACT_PATTERNS.each { pattern ->
212+
uploadGcsArtifact(workerName, pattern)
213+
}
214+
}
215+
216+
def publishJunit() {
217+
junit(testResults: 'target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true)
218+
}
219+
220+
def sendMail() {
221+
sendInfraMail()
222+
sendKibanaMail()
223+
}
224+
225+
def sendInfraMail() {
226+
catchError {
227+
step([
228+
$class: 'Mailer',
229+
notifyEveryUnstableBuild: true,
230+
recipients: '[email protected]',
231+
sendToIndividuals: false
232+
])
233+
}
234+
}
235+
236+
def sendKibanaMail() {
237+
catchError {
238+
if(params.NOTIFY_ON_FAILURE && currentBuild.result != 'SUCCESS' && currentBuild.result != 'ABORTED') {
239+
emailext(
240+
// to: '[email protected]',
241+
to: '[email protected]', // TODO switch this out after testing
242+
subject: "${env.PROJECT_NAME} - Build # ${env.BUILD_NUMBER} - ${currentBuild.result}",
243+
body: '${SCRIPT,template="groovy-html.template"}',
244+
mimeType: 'text/html',
245+
)
246+
}
247+
}
248+
}
249+
250+
def runbld(script) {
251+
sh '#!/usr/local/bin/runbld\n' + script
252+
}
253+
254+
def bash(script) {
255+
sh "#!/bin/bash -x\n${script}"
256+
}
257+
258+
def doSetup() {
259+
runbld "./test/scripts/jenkins_setup.sh"
260+
}
261+
262+
def buildOss() {
263+
runbld "./test/scripts/jenkins_build_kibana.sh"
264+
}
265+
266+
def buildXpack() {
267+
runbld "./test/scripts/jenkins_xpack_build_kibana.sh"
268+
}

‎packages/kbn-test/src/es/es_test_cluster.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ export function createEsTestCluster(options = {}) {
5454
basePath,
5555
esArgs,
5656
};
57+
const transportPort = esTestConfig.getTransportPort();
5758

5859
const cluster = new Cluster({ log, ssl });
5960

@@ -88,6 +89,7 @@ export function createEsTestCluster(options = {}) {
8889
`cluster.name=${clusterName}`,
8990
`http.port=${port}`,
9091
'discovery.type=single-node',
92+
`transport.port=${transportPort}`,
9193
...esArgs,
9294
],
9395
esEnvVars,

‎packages/kbn-test/src/es/es_test_config.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ export const esTestConfig = new (class EsTestConfig {
3838
return process.env.TEST_ES_FROM || 'snapshot';
3939
}
4040

41+
getTransportPort() {
42+
return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
43+
}
44+
4145
getUrlParts() {
4246
// Allow setting one complete TEST_ES_URL for Es like https://elastic:changeme@myCloudInstance:9200
4347
if (process.env.TEST_ES_URL) {

‎src/dev/ci_setup/get_percy_env.js‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,7 @@ const pkg = require('../../../package.json');
2323
const { stdout: commit } = execa.sync('git', ['rev-parse', 'HEAD']);
2424
const shortCommit = commit.slice(0, 8);
2525

26-
if (!process.env.JOB_NAME) {
27-
throw new Error('getPercyEnv: [JOB_NAME] environment variable required');
28-
}
29-
30-
const isPr = process.env.JOB_NAME.includes('elastic+kibana+pull-request');
26+
const isPr = !!process.env.ghprbPullId;
3127
if (isPr && !(process.env.PR_TARGET_BRANCH && process.env.PR_SOURCE_BRANCH)) {
3228
throw new Error(
3329
'getPercyEnv: Unable to determine percy environment in prs without [PR_TARGET_BRANCH] and [PR_SOURCE_BRANCH] environment variables'

0 commit comments

Comments
 (0)