From b41b60eb3b72978b40d9dbe36ea370d695646525 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 17 Jan 2020 14:00:21 +0100 Subject: [PATCH 1/6] ci: add apm beats update pipeline multibranch job --- .ci/apm-beats-update.groovy | 102 +++++++++++++++++++++++++ .ci/scripts/apm-server-update-beats.sh | 19 +++++ 2 files changed, 121 insertions(+) create mode 100644 .ci/apm-beats-update.groovy create mode 100644 .ci/scripts/apm-server-update-beats.sh diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy new file mode 100644 index 000000000000..d60a613c6fd6 --- /dev/null +++ b/.ci/apm-beats-update.groovy @@ -0,0 +1,102 @@ +#!/usr/bin/env groovy +@Library('apm@current') _ + +pipeline { + agent { label 'linux && immutable' } + environment { + REPO = 'apm-server' + BASE_DIR = "src/github.com/elastic/${env.REPO}" + NOTIFY_TO = credentials('notify-to') + GITHUB_CHECK_ITS_NAME = 'APM Server Beats update' + PATH = "${env.PATH}:${env.WORKSPACE}/bin" + HOME = "${env.WORKSPACE}" + GOPATH = "${env.WORKSPACE}" + } + options { + timeout(time: 2, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '100', artifactNumToKeepStr: '30', daysToKeepStr: '30')) + timestamps() + ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) + quietPeriod(10) + } + triggers { + issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*') + } + stages { + /** + Checkout the code and stash it, to use it on other stages. + */ + stage('Checkout') { + options { skipDefaultCheckout() } + steps { + deleteDir() + gitCheckout(basedir: "beats") + script { + dir("beats"){ + env.GO_VERSION = readFile(".go-version").trim() + def regexps =[ + "^devtools/mage.*", + "^libbeat/scripts/Makefile", + ] + env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps) + + // Skip all the stages except docs for PR's with asciidoc changes only + env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true) + } + } + } + } + /** + updates beats updates the framework part and go parts of beats. + Then build and test. + Finally archive the results. + */ + stage('Update Beats') { + options { skipDefaultCheckout() } + when { + beforeAgent true + anyOf { + branch 'master' + branch "\\d+\\.\\d+" + branch "v\\d?" + tag "v\\d+\\.\\d+\\.\\d+*" + allOf { + expression { return env.BEATS_UPDATED != "false" } + changeRequest() + } + } + } + steps { + withGithubNotify(context: 'Check Apm Server Beats Update') { + deleteDir() + gitCheckout(basedir: "${BASE_DIR}", + repo: "git@github.com:elastic/${REPO}.git", + branch: 'master', + credentialsId: 'credentials-id', + githubNotifyFirstTimeContributor: false, + depth: 1, + reference: "/var/lib/jenkins/.git-references/${REPO}.git" + ) + dir("${BASE_DIR}"){ + sh(label: 'Update Beats script', script: './ci/script/apm-server-update-beats.sh') + } + } + } + post { + always { + catchError(buildResult: 'SUCCESS', message: 'Failed to grab test results tar files', stageResult: 'SUCCESS') { + tar(file: "update-beats-system-tests-linux-files.tgz", archive: true, dir: "system-tests", pathPrefix: "${BASE_DIR}/build") + } + } + } + } + } + post { + cleanup { + notifyBuildResult() + } + } +} diff --git a/.ci/scripts/apm-server-update-beats.sh b/.ci/scripts/apm-server-update-beats.sh new file mode 100644 index 000000000000..9053b0e9a04b --- /dev/null +++ b/.ci/scripts/apm-server-update-beats.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euox pipefail + +source ./_beats/dev-tools/common.bash + +jenkins_setup + +cleanup() { + rm -rf $TEMP_PYTHON_ENV + make stop-environment fix-permissions +} +trap cleanup EXIT + +go get github.com/elastic/apm-server/vendor/github.com/kardianos/govendor +RACE_DETECTOR=1 make update-beats + +eval "$(gvm $(cat ./_beats/.go-version))" + +RACE_DETECTOR=1 make clean check testsuite apm-server From 41da564921d3ca665eaf42328dfa93ef0bb14eeb Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 17 Jan 2020 14:10:01 +0100 Subject: [PATCH 2/6] fix: force execution on comment trigger --- .ci/apm-beats-update.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index d60a613c6fd6..b3e67ad0a16b 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -64,9 +64,10 @@ pipeline { branch "v\\d?" tag "v\\d+\\.\\d+\\.\\d+*" allOf { - expression { return env.BEATS_UPDATED != "false" } + expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() } changeRequest() } + } } steps { From 8500780075308c2efe7156b87232316bcb5dad10 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 21 Jan 2020 10:36:37 +0100 Subject: [PATCH 3/6] Update .ci/apm-beats-update.groovy --- .ci/apm-beats-update.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index b3e67ad0a16b..35bb17d2b339 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -82,7 +82,10 @@ pipeline { reference: "/var/lib/jenkins/.git-references/${REPO}.git" ) dir("${BASE_DIR}"){ - sh(label: 'Update Beats script', script: './ci/script/apm-server-update-beats.sh') + sh(label: 'Update Beats script', script: """ + export BEATS_VERSION=${env.GIT_BASE_COMMIT} + ./ci/script/apm-server-update-beats.sh + """) } } } From 606e13a6a2afa3cf3c45ea613ece9fb4a5b514dd Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 22 Jan 2020 10:56:11 +0100 Subject: [PATCH 4/6] Update .ci/apm-beats-update.groovy --- .ci/apm-beats-update.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index 35bb17d2b339..ede455a4e64c 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -84,6 +84,7 @@ pipeline { dir("${BASE_DIR}"){ sh(label: 'Update Beats script', script: """ export BEATS_VERSION=${env.GIT_BASE_COMMIT} + git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*" ./ci/script/apm-server-update-beats.sh """) } From dfc8c21c29d4b86f575ae4466ac35e81c9701a15 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 22 Jan 2020 19:03:25 +0100 Subject: [PATCH 5/6] Update .ci/apm-beats-update.groovy --- .ci/apm-beats-update.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index ede455a4e64c..39572aa8d286 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -85,7 +85,7 @@ pipeline { sh(label: 'Update Beats script', script: """ export BEATS_VERSION=${env.GIT_BASE_COMMIT} git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*" - ./ci/script/apm-server-update-beats.sh + script/jenkins/update-beats.sh """) } } From daa7dd19da10de6e608a5f794eea03258ce9188c Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 22 Jan 2020 19:04:34 +0100 Subject: [PATCH 6/6] Delete apm-server-update-beats.sh --- .ci/scripts/apm-server-update-beats.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .ci/scripts/apm-server-update-beats.sh diff --git a/.ci/scripts/apm-server-update-beats.sh b/.ci/scripts/apm-server-update-beats.sh deleted file mode 100644 index 9053b0e9a04b..000000000000 --- a/.ci/scripts/apm-server-update-beats.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -euox pipefail - -source ./_beats/dev-tools/common.bash - -jenkins_setup - -cleanup() { - rm -rf $TEMP_PYTHON_ENV - make stop-environment fix-permissions -} -trap cleanup EXIT - -go get github.com/elastic/apm-server/vendor/github.com/kardianos/govendor -RACE_DETECTOR=1 make update-beats - -eval "$(gvm $(cat ./_beats/.go-version))" - -RACE_DETECTOR=1 make clean check testsuite apm-server