Skip to content

Commit 146946b

Browse files
committed
[CI] Add pipeline task queue framework and merge workers into one
1 parent fa70afb commit 146946b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+858
-129
lines changed

.ci/Dockerfile

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
ARG NODE_VERSION=10.21.0
2+
3+
FROM node:${NODE_VERSION} AS base
4+
5+
RUN apt-get update && \
6+
apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
7+
libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
8+
libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
9+
libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
10+
libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
11+
rm -rf /var/lib/apt/lists/*
12+
13+
RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
14+
&& sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
15+
&& apt-get update \
16+
&& apt-get install -y rsync jq bsdtar google-chrome-stable \
17+
--no-install-recommends \
18+
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
19+
20+
RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
21+
&& curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
22+
&& unzip vault.zip \
23+
&& rm vault.zip \
24+
&& chmod +x vault \
25+
&& mv vault /usr/local/bin/vault
26+
27+
RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
28+
29+
COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
30+
RUN chmod +x /usr/local/bin/bash_standard_lib.sh
31+
32+
COPY ./runbld /usr/local/bin/runbld
33+
RUN chmod +x /usr/local/bin/runbld
34+
35+
USER kibana

.ci/bash_standard_lib.sh

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#! /usr/bin/env bash
2+
3+
# Readonly vars
4+
readonly CACHE_DIR="${HOME}/.git-references"
5+
6+
# Global vars
7+
declare -a cloned_git_repos
8+
9+
#==============================================================================
10+
#
11+
# What: A library of standard Bash functions
12+
# Why: Simplify and improve the quality of bash scripts being produced.
13+
#
14+
#==============================================================================
15+
16+
# cache_maven_deps function
17+
# -------------------------------------
18+
# Run a `mvnw compile` in order to populate the local
19+
# m2 repository.
20+
#
21+
# Expects a single param which is a project directory to change into
22+
# -------------------------------------
23+
cache_maven_deps() {
24+
local project=$1
25+
26+
# Check that 'project' dir exists
27+
if [[ ! -d $project ]]; then
28+
printf 'Project dir %s doesn''t exist. Exiting...\n' $project >&2
29+
exit 1
30+
fi
31+
32+
# Switch directory, and run mvnw compile
33+
pushd $project
34+
./mvnw compile
35+
popd
36+
}
37+
38+
# clean_git_repo_clones function
39+
# -------------------------------------
40+
# Clean up any Git repo clones performed by the 'clone_git_repo' function.
41+
# Should be called by the 'EXIT' trap.
42+
# -------------------------------------
43+
clean_git_repo_clones() {
44+
printf 'Cleaning cloned git repos...\n'
45+
46+
# Loop over cloned repos and remove them...
47+
for cloned_repo in "${cloned_git_repos[@]}"
48+
do
49+
printf 'Removing repo %s\n' "$cloned_repo"
50+
rm -rf $cloned_repo
51+
done
52+
}
53+
54+
# clone_git_repo function
55+
# -------------------------------------
56+
# Clone a Git repo, using either a Git SSH source, or a repo short-name.
57+
# If a matching local reference repo is found, then that will be used as the `--reference` for the Git clone,
58+
# otherwise a full clone is undertaken.
59+
#
60+
# Expects a single param which is the repo name or repo SSH source
61+
# -------------------------------------
62+
clone_git_repo() {
63+
local repo=$1
64+
local repo_name
65+
local repo_url
66+
67+
# Calculate the "humanish" part of the source repo
68+
repo_name=$(echo "$repo" | sed -e 's|/$||' -e 's|:*/*\.git$||' -e 's|.*[/:]||g')
69+
local mirror_dir="${CACHE_DIR}/${repo_name}.git"
70+
71+
# If $repo is a SSH repo source, use it as the repo_url.
72+
# Otherwise, attempt to get the origin URL from the matching reference repo, failing if the reference repo is not present.
73+
if [[ $repo == git* ]]; then
74+
repo_url=$repo
75+
else
76+
if [[ -d $mirror_dir ]]; then
77+
repo_url=$(cd "$mirror_dir" && git remote get-url origin)
78+
else
79+
printf 'Attempting to clone %s using the short-name, however no matching reference repo found. Exiting...\n' "$repo" >&2
80+
exit 1
81+
fi
82+
fi
83+
84+
# Check if we have a reference repo clone for this repo,
85+
# and use it for the clone if we do.
86+
# Otherwise clone the repo directly.
87+
if [[ -d $mirror_dir ]]; then
88+
printf 'Cloning repo %s using reference repo...\n' "$repo"
89+
git clone -q --reference "$mirror_dir" "$repo_url" || exit 1
90+
else
91+
printf 'Cloning repo %s directly...' "$repo"
92+
git clone -q "$repo_url" || exit 1
93+
fi
94+
95+
# Add repo to cloned repo array
96+
cloned_git_repos+=("$repo")
97+
98+
# Clean up clone on exit
99+
trap clean_git_repo_clones EXIT
100+
}
101+
102+
# retry function
103+
# -------------------------------------
104+
# Retry a command for a specified number of times until the command exits successfully.
105+
# Retry wait period backs off exponentially after each retry.
106+
#
107+
# The first argument should be the number of retries.
108+
# Remainder is treated as the command to execute.
109+
# -------------------------------------
110+
retry() {
111+
local retries=$1
112+
shift
113+
114+
local count=0
115+
until "$@"; do
116+
exit=$?
117+
wait=$((2 ** $count))
118+
count=$(($count + 1))
119+
if [ $count -lt $retries ]; then
120+
printf "Retry %s/%s exited %s, retrying in %s seconds...\n" "$count" "$retries" "$exit" "$wait" >&2
121+
sleep $wait
122+
else
123+
printf "Retry %s/%s exited %s, no more retries left.\n" "$count" "$retries" "$exit" >&2
124+
return $exit
125+
fi
126+
done
127+
return 0
128+
}

.ci/runbld_no_junit.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
profiles:
44
- ".*": # Match any job
55
tests:
6-
junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist
6+
junit-filename-pattern: false

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ npm-debug.log*
4747
.tern-project
4848
.nyc_output
4949
.ci/pipeline-library/build/
50+
.ci/runbld
5051
.gradle
5152

5253
# apm plugin

Jenkinsfile

Lines changed: 1 addition & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -8,49 +8,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true)
88
ciStats.trackBuild {
99
catchError {
1010
retryable.enable()
11-
parallel([
12-
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
13-
'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
14-
'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
15-
'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
16-
'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
17-
'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
18-
'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
19-
'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
20-
'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
21-
'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
22-
'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
23-
'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
24-
'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
25-
'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
26-
'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
27-
'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
28-
'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
29-
// 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
30-
]),
31-
'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
32-
'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
33-
'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
34-
'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
35-
'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
36-
'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
37-
'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
38-
'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
39-
'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
40-
'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
41-
'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
42-
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
43-
'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
44-
// 'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'),
45-
'xpack-securitySolutionCypress': { processNumber ->
46-
whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) {
47-
kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)
48-
}
49-
},
50-
51-
// 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
52-
]),
53-
])
11+
kibanaPipeline.allCiTasks()
5412
}
5513
}
5614
}

src/dev/ci_setup/checkout_sibling_es.sh

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,14 @@ function checkout_sibling {
77
targetDir=$2
88
useExistingParamName=$3
99
useExisting="$(eval "echo "\$$useExistingParamName"")"
10+
repoAddress="[email protected]:"
11+
if [[ "$USE_HTTPS_CLONE" ]]; then
12+
repoAddress="https://github.com/"
13+
fi
1014

1115
if [ -z ${useExisting:+x} ]; then
1216
if [ -d "$targetDir" ]; then
13-
echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$PARENT_DIR]!"
17+
echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$WORKSPACE]!"
1418
echo
1519
echo "Either define '${useExistingParamName}' or remove the existing '${project}' sibling."
1620
exit 1
@@ -21,8 +25,9 @@ function checkout_sibling {
2125
cloneBranch=""
2226

2327
function clone_target_is_valid {
28+
2429
echo " -> checking for '${cloneBranch}' branch at ${cloneAuthor}/${project}"
25-
if [[ -n "$(git ls-remote --heads "[email protected]:${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then
30+
if [[ -n "$(git ls-remote --heads "${repoAddress}${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then
2631
return 0
2732
else
2833
return 1
@@ -71,7 +76,7 @@ function checkout_sibling {
7176
fi
7277

7378
echo " -> checking out '${cloneBranch}' branch from ${cloneAuthor}/${project}..."
74-
git clone -b "$cloneBranch" "[email protected]:${cloneAuthor}/${project}.git" "$targetDir" --depth=1
79+
git clone -b "$cloneBranch" "${repoAddress}${cloneAuthor}/${project}.git" "$targetDir" --depth=1
7580
echo " -> checked out ${project} revision: $(git -C "${targetDir}" rev-parse HEAD)"
7681
echo
7782
}
@@ -87,12 +92,12 @@ function checkout_sibling {
8792
fi
8893
}
8994

90-
checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES"
95+
checkout_sibling "elasticsearch" "${WORKSPACE}/elasticsearch" "USE_EXISTING_ES"
9196
export TEST_ES_FROM=${TEST_ES_FROM:-snapshot}
9297

9398
# Set the JAVA_HOME based on the Java property file in the ES repo
9499
# This assumes the naming convention used on CI (ex: ~/.java/java10)
95-
ES_DIR="$PARENT_DIR/elasticsearch"
100+
ES_DIR="$WORKSPACE/elasticsearch"
96101
ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
97102

98103

src/dev/ci_setup/setup_env.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ export PARENT_DIR="$parentDir"
5353
kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")"
5454
export KIBANA_PKG_BRANCH="$kbnBranch"
5555

56+
export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
57+
5658
###
5759
### download node
5860
###
@@ -161,7 +163,7 @@ export -f checks-reporter-with-killswitch
161163

162164
source "$KIBANA_DIR/src/dev/ci_setup/load_env_keys.sh"
163165

164-
ES_DIR="$PARENT_DIR/elasticsearch"
166+
ES_DIR="$WORKSPACE/elasticsearch"
165167
ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
166168

167169
if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then

src/dev/notice/generate_notice_from_source.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@ export async function generateNoticeFromSource({ productName, directory, log }:
4949
ignore: [
5050
'{node_modules,build,target,dist,data,built_assets}/**',
5151
'packages/*/{node_modules,build,target,dist}/**',
52+
'src/plugins/*/{node_modules,build,target,dist}/**',
5253
'x-pack/{node_modules,build,target,dist,data}/**',
5354
'x-pack/packages/*/{node_modules,build,target,dist}/**',
55+
'x-pack/plugins/*/{node_modules,build,target,dist}/**',
5456
],
5557
};
5658

tasks/config/karma.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ module.exports = function (grunt) {
110110
customLaunchers: {
111111
Chrome_Headless: {
112112
base: 'Chrome',
113-
flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222'],
113+
flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222', '--no-sandbox'],
114114
},
115115
},
116116

tasks/test_jest.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,18 @@ const { resolve } = require('path');
2222
module.exports = function (grunt) {
2323
grunt.registerTask('test:jest', function () {
2424
const done = this.async();
25-
runJest(resolve(__dirname, '../scripts/jest.js')).then(done, done);
25+
runJest(resolve(__dirname, '../scripts/jest.js'), ['--maxWorkers=10']).then(done, done);
2626
});
2727

2828
grunt.registerTask('test:jest_integration', function () {
2929
const done = this.async();
3030
runJest(resolve(__dirname, '../scripts/jest_integration.js')).then(done, done);
3131
});
3232

33-
function runJest(jestScript) {
33+
function runJest(jestScript, args = []) {
3434
const serverCmd = {
3535
cmd: 'node',
36-
args: [jestScript, '--ci'],
36+
args: [jestScript, '--ci', ...args],
3737
opts: { stdio: 'inherit' },
3838
};
3939

0 commit comments

Comments
 (0)