From 098eb35a5d65b508045a4b88dbdf761f654df6f8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 18 Nov 2020 17:37:10 +0000 Subject: [PATCH 01/12] [CI] retry windows-7 environmental issues --- Jenkinsfile | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8236ef55cf0d..b77d2ba18ddf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -266,6 +266,23 @@ def k8sTest(Map args = [:]) { * - mage then the dir(location) is required, aka by enabling isMage: true. */ def target(Map args = [:]) { + try { + runCommand(args) + } catch (err) { + if(args.label.contains('windows-7')) { + sleep 10 + // There are some environmental issues with windows-7 + runCommand(args) + } + } +} + +/** +* This method runs the given command supporting two kind of scenarios: +* - make -C then the dir(location) is not required, aka by disaling isMage: false +* - mage then the dir(location) is required, aka by enabling isMage: true. +*/ +def runCommand(Map args = [:]) { def context = args.context def command = args.command def directory = args.get('directory', '') @@ -345,7 +362,9 @@ def withBeatsEnv(Map args = [:], Closure body) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } dir("${env.BASE_DIR}") { + def noEnvironmentalIssue = true installTools() + def noEnvironmentalIssue = false if(isUnix()) { // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. // See https://github.com/elastic/beats/issues/17787. @@ -371,7 +390,7 @@ def withBeatsEnv(Map args = [:], Closure body) { upload = true error("Error '${err.toString()}'") } finally { - if (archive) { + if (archive && noEnvironmentalIssue) { archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } // Tear down the setup for the permamnent workers. From 3a119ce4fc5583085eac9742ae58fffd33b5f0ad Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 18 Nov 2020 17:42:38 +0000 Subject: [PATCH 02/12] Use file flag to rerun stages once with sleeps --- Jenkinsfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b77d2ba18ddf..1d895ca5e543 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -269,9 +269,8 @@ def target(Map args = [:]) { try { runCommand(args) } catch (err) { - if(args.label.contains('windows-7')) { + if(fileExists('environmental-issue')) { sleep 10 - // There are some environmental issues with windows-7 runCommand(args) } } @@ -362,9 +361,9 @@ def withBeatsEnv(Map args = [:], Closure body) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } dir("${env.BASE_DIR}") { - def noEnvironmentalIssue = true + def environmentalIssue = true installTools() - def noEnvironmentalIssue = false + def environmentalIssue = false if(isUnix()) { // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. // See https://github.com/elastic/beats/issues/17787. @@ -390,14 +389,17 @@ def withBeatsEnv(Map args = [:], Closure body) { upload = true error("Error '${err.toString()}'") } finally { - if (archive && noEnvironmentalIssue) { + if (archive && !environmentalIssue) { archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } - // Tear down the setup for the permamnent workers. + // Tear down the setup for the permanent workers. catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { fixPermissions("${WORKSPACE}") deleteDir() } + if (environmentalIssue) { + writeFile file: 'environmental-issue', text: 'environmental-issue' + } } } } From 6c98a747dc04438aa17912a642503c0e46b1d1dc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 18 Nov 2020 17:44:21 +0000 Subject: [PATCH 03/12] Ensure all the prerequisites are ok --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1d895ca5e543..1c4688803249 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -363,7 +363,6 @@ def withBeatsEnv(Map args = [:], Closure body) { dir("${env.BASE_DIR}") { def environmentalIssue = true installTools() - def environmentalIssue = false if(isUnix()) { // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. // See https://github.com/elastic/beats/issues/17787. @@ -373,6 +372,8 @@ def withBeatsEnv(Map args = [:], Closure body) { git config --global user.name "beatsmachine" fi''') } + // Pre-requisites to configure the environment were ok. + environmentalIssue = false // Skip to upload the generated files by default. def upload = false try { From f0f885b53a98a0450acd487eb2a010fde178b20c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 18 Nov 2020 17:48:25 +0000 Subject: [PATCH 04/12] Update docs --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1c4688803249..81d66235e22f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -261,9 +261,9 @@ def k8sTest(Map args = [:]) { } /** -* This method runs the given command supporting two kind of scenarios: -* - make -C then the dir(location) is not required, aka by disaling isMage: false -* - mage then the dir(location) is required, aka by enabling isMage: true. +* This method is a wrapper to run the runCommand method and retry if there are +* environmental issues. Therefore it passes the arguments to the runCommand. +* For further details regarding the arguments please refers to the runCommand method. */ def target(Map args = [:]) { try { From ebb6644de21cafa72e3d168ca40f49fecfad7b1d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 19 Nov 2020 11:19:12 +0000 Subject: [PATCH 05/12] Use variable to store the stage status instead a file in the toplevel agent --- Jenkinsfile | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 81d66235e22f..8cc48014e879 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,11 @@ import groovy.transform.Field */ @Field def stashedTestReports = [:] +/** + This is required to store any environmental issues to retry if so +*/ +@Field def environmentalIssues = [:] + pipeline { agent { label 'ubuntu-18 && immutable' } environment { @@ -269,7 +274,7 @@ def target(Map args = [:]) { try { runCommand(args) } catch (err) { - if(fileExists('environmental-issue')) { + if(environmentalIssues?.get(args.id, false)) { sleep 10 runCommand(args) } @@ -390,6 +395,7 @@ def withBeatsEnv(Map args = [:], Closure body) { upload = true error("Error '${err.toString()}'") } finally { + // If there are environmental issues then let's avoid the archiving of none files. if (archive && !environmentalIssue) { archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } @@ -398,14 +404,23 @@ def withBeatsEnv(Map args = [:], Closure body) { fixPermissions("${WORKSPACE}") deleteDir() } - if (environmentalIssue) { - writeFile file: 'environmental-issue', text: 'environmental-issue' - } + analyseEnvironmentalIssues(isEnvironmentalIssue: environmentalIssue) } } } } +/** +* This method analyse if the existing stage failed with some enviornmental issues. +* So far the analysis is just purelly based on a boolean that detects if the installation of +* the required tools for building and testing in the agent were successfully installed. +* We can use this method in the future to analyse the build logs for searching specific patterns +* such as, not able to access the docker registry, or failed when pulling some docker images. +*/ +def analyseEnvironmentalIssues(args) { + environmentalIssues[args.id] = environmentalIssue +} + /** * This method fixes the filesystem permissions after the build has happenend. The reason is to * ensure any non-ephemeral workers don't have any leftovers that could cause some environmental From aa3a25e2a251b79ffad58d0c004469f4be65d4bb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 19 Nov 2020 14:49:27 +0000 Subject: [PATCH 06/12] Fix missing variables in the method call --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8cc48014e879..9c767213fca9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -404,7 +404,7 @@ def withBeatsEnv(Map args = [:], Closure body) { fixPermissions("${WORKSPACE}") deleteDir() } - analyseEnvironmentalIssues(isEnvironmentalIssue: environmentalIssue) + analyseEnvironmentalIssues(id: args.id, isEnvironmentalIssue: environmentalIssue) } } } @@ -418,7 +418,7 @@ def withBeatsEnv(Map args = [:], Closure body) { * such as, not able to access the docker registry, or failed when pulling some docker images. */ def analyseEnvironmentalIssues(args) { - environmentalIssues[args.id] = environmentalIssue + environmentalIssues[args.id] = args.environmentalIssue } /** From 4a4a15d3994267437fb70d787225fc51c5da5766 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 20 Nov 2020 09:14:48 +0000 Subject: [PATCH 07/12] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9c767213fca9..705cd28cd5df 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -404,7 +404,7 @@ def withBeatsEnv(Map args = [:], Closure body) { fixPermissions("${WORKSPACE}") deleteDir() } - analyseEnvironmentalIssues(id: args.id, isEnvironmentalIssue: environmentalIssue) + analyseEnvironmentalIssues(id: args.id, environmentalIssue: environmentalIssue) } } } From b11e00ae196940441481425dc7573d870a37e085 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 20 Nov 2020 12:01:56 +0000 Subject: [PATCH 08/12] Update Jenkinsfile Co-authored-by: cachedout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 705cd28cd5df..2fb10bbfa5bd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -412,7 +412,7 @@ def withBeatsEnv(Map args = [:], Closure body) { /** * This method analyse if the existing stage failed with some enviornmental issues. -* So far the analysis is just purelly based on a boolean that detects if the installation of +* So far the analysis is just purely based on a boolean that detects if the installation of * the required tools for building and testing in the agent were successfully installed. * We can use this method in the future to analyse the build logs for searching specific patterns * such as, not able to access the docker registry, or failed when pulling some docker images. From a7c74eb9d38cc3c7e75840637289a34eaa59bc98 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 18 Feb 2021 15:50:54 +0000 Subject: [PATCH 09/12] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 78055037948a..672c8a5e687c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -622,7 +622,7 @@ def withBeatsEnv(Map args = [:], Closure body) { } /** -* This method analyse if the existing stage failed with some enviornmental issues. +* This method analyse if the existing stage failed with some environmental issues. * So far the analysis is just purely based on a boolean that detects if the installation of * the required tools for building and testing in the agent were successfully installed. * We can use this method in the future to analyse the build logs for searching specific patterns From 446f840560d60f7134668c5fe677866a1cfd468d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 18 Feb 2021 15:56:40 +0000 Subject: [PATCH 10/12] Run within the base_dir context --- Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 672c8a5e687c..006170397b02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -610,7 +610,9 @@ def withBeatsEnv(Map args = [:], Closure body) { } finally { // If there are environmental issues then let's avoid the archiving of none files. if (archive && !environmentalIssue) { - archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: uploadGeneratedFiles) + dir("${env.BASE_DIR}") { + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: uploadGeneratedFiles) + } } // Tear down the setup for the permanent workers. catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { @@ -624,7 +626,8 @@ def withBeatsEnv(Map args = [:], Closure body) { /** * This method analyse if the existing stage failed with some environmental issues. * So far the analysis is just purely based on a boolean that detects if the installation of -* the required tools for building and testing in the agent were successfully installed. +* the required tools for building and testing in the agent were successfully installed or +* some unexpected pre-building/testing errors, such as deleting the workspace. * We can use this method in the future to analyse the build logs for searching specific patterns * such as, not able to access the docker registry, or failed when pulling some docker images. */ From d018c7df48ac7b7d94e790d0a2a2237a6c9d4615 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Feb 2021 17:04:09 +0000 Subject: [PATCH 11/12] Add post build analysis --- Jenkinsfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index f1d68f06573e..f3c2d580bb74 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -167,6 +167,9 @@ VERSION=${env.VERSION}-SNAPSHOT""") analyzeFlakey: !isTag(), flakyReportIdx: "reporter-beats-beats-${getIdSuffix()}") } } + always { + notifyEnvironmentalIssues() + } } } @@ -652,6 +655,26 @@ def analyseEnvironmentalIssues(args) { environmentalIssues[args.id] = args.environmentalIssue } +/** +* This method runs only as a post build action to notify when there are environmental issues. +* Archive the file with the environmental issues details in case there are any +* reported environmental issues. +* TODO: Create JSON file to upload those details to elasticsearch +*/ +def notifyEnvironmentalIssues(Map args = [:]) { + stage('Notify environmental issues'){ + def content = '' + environmentalIssues?.findAll { k, v -> return v }.each { k, v -> + content += "${k} failed with some environmental issues (${v})\n" + } + if (content?.trim()) { + writeFile file: 'environmental.issues.txt', text: content + archiveArtifacts artifacts: 'environmental.issues.txt' + } + } +} + + /** * This method fixes the filesystem permissions after the build has happenend. The reason is to * ensure any non-ephemeral workers don't have any leftovers that could cause some environmental From 466d10df6f2faa4884680f255fff5b935a31ea00 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Feb 2021 12:30:35 +0000 Subject: [PATCH 12/12] Update Jenkinsfile --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index f3c2d580bb74..7782d575e479 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -660,6 +660,7 @@ def analyseEnvironmentalIssues(args) { * Archive the file with the environmental issues details in case there are any * reported environmental issues. * TODO: Create JSON file to upload those details to elasticsearch +* This won't be needed as soon as we use the APM transactions */ def notifyEnvironmentalIssues(Map args = [:]) { stage('Notify environmental issues'){