chore: standardise how unit/integration tests is run on CI, adding a build command to compress the system-tests files

Jenkinsfile

@@ -671,7 +671,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
           error("Error '${err.toString()}'")
         } finally {
           if (archive) {
-            archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
+            archiveTestOutput(directory: directory, testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
           }
           tearDown()
         }
@@ -767,6 +767,8 @@ def getCommonModuleInTheChangeSet(String directory) {
 * to bypass some issues when working with big repositories.
 */
 def archiveTestOutput(Map args = [:]) {
+  def directory = args.get('directory', '')
+
   catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
     if (isUnix()) {
       fixPermissions("${WORKSPACE}")
@@ -790,13 +792,20 @@ def archiveTestOutput(Map args = [:]) {
     }
     if (args.upload) {
       catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
-        def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
-        log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
-        if (folder.trim()) {
-          // TODO: nodeOS() should support ARM
-          def os_suffix = isArm() ? 'linux' : nodeOS()
-          def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
-          tarAndUploadArtifacts(file: "${name}.tgz", location: folder)
+        withMageEnv(version: "${env.GO_VERSION}"){
+          dir(directory){
+            cmd(label: "Archive system tests files", script: 'mage packageSystemTests')
+          }
+        }
+        def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method
+        dir("${BASE_DIR}"){
+          cmd(label: "List files to upload", script: "ls -l ${BASE_DIR}/${fileName}")
+          googleStorageUploadExt(
+            bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+            credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+            pattern: "${BASE_DIR}/${fileName}",
+            sharedPublicly: true
+          )
         }
       }
     }

auditbeat/Jenkinsfile.yml

@@ -35,8 +35,8 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
         stage: mandatory
     crosscompile:
         make: "make -C auditbeat crosscompile"

dev-tools/mage/common.go

@@ -353,6 +353,71 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
+// Tar compress a directory using tar + gzip algorithms
+func Tar(src string, targetFile string) error {
+	fmt.Printf(">> creating TAR file from directory: %s, target: %s\n", src, targetFile)
+	var buf bytes.Buffer
+
+	// tar > gzip > buf
+	zr := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zr)
+
+	// walk through every file in the folder
+	filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
+		// if a symlink, skip file
+		if fi.Mode().Type() == os.ModeSymlink {
+			fmt.Printf(">> skipping symlink: %s\n", file)
+			return nil
+		}
+
+		// generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			fmt.Printf(">> error getting file info header: %s\n", err)
+			return err
+		}
+
+		// must provide real name
+		// (see https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = filepath.ToSlash(file)
+
+		// write header
+		if err := tw.WriteHeader(header); err != nil {
+			fmt.Printf(">> error writing header: %s\n", err)
+			return err
+		}
+
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				fmt.Printf(">> error opening file: %s\n", err)
+				return err
+			}
+			defer data.Close()
+			if _, err := io.Copy(tw, data); err != nil {
+				fmt.Printf(">> error compressing file: %s\n", err)
+				return err
+			}
+		}
+		return nil
+	})
+
+	// produce tar
+	if err := tw.Close(); err != nil {
+		fmt.Printf(">> error closing tar file: %s\n", err)
+		return err
+	}
+	// produce gzip
+	if err := zr.Close(); err != nil {
+		fmt.Printf(">> error closing gzip file: %s\n", err)
+		return err
+	}
+
+	// write the .tar.gzip
+	return ioutil.WriteFile(targetFile, buf.Bytes(), os.FileMode(0600))
+}
+
 func untar(sourceFile, destinationDir string) error {
 	file, err := os.Open(sourceFile)
 	if err != nil {

dev-tools/mage/target/common/package.go

@@ -0,0 +1,72 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package common
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+)
+
+// PackageSystemTests packages the python system tests results
+func PackageSystemTests() error {
+	excludeds := []string{".ci", ".git", ".github", "vendor", "dev-tools"}
+
+	// include run as it's the directory we want to compress
+	systemTestsDir := filepath.Join("build", "system-tests", "run")
+	files, err := devtools.FindFilesRecursive(func(path string, _ os.FileInfo) bool {
+		base := filepath.Base(path)
+		for _, excluded := range excludeds {
+			if strings.HasPrefix(base, excluded) {
+				return false
+			}
+		}
+
+		return strings.HasPrefix(path, systemTestsDir)
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(files) == 0 {
+		fmt.Printf(">> there are no system test files under %s", systemTestsDir)
+		return nil
+	}
+
+	// create a plain directory layout for all beats
+	beat := devtools.MustExpand("{{ repo.SubDir }}")
+	beat = strings.ReplaceAll(beat, string(os.PathSeparator), "-")
+
+	targetFile := devtools.MustExpand("{{ elastic_beats_dir }}/build/system-tests-" + beat + ".tar.gz")
+	parent := filepath.Dir(targetFile)
+	if !fileExists(parent) {
+		fmt.Printf(">> creating parent dir: %s", parent)
+		os.Mkdir(parent, 0750)
+	}
+
+	return devtools.Tar(systemTestsDir, targetFile)
+}
+
+// fileExists returns true if the specified file exists.
+func fileExists(file string) bool {
+	_, err := os.Stat(file)
+	return !os.IsNotExist(err)
+}

filebeat/Jenkinsfile.yml

@@ -35,9 +35,14 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
-        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"       ## run the ITs only if the changeset affects a specific module.
         stage: mandatory
     module-compat-7.11:
       mage: >-                 ## Run module integration tests under ES 7.11 to ensure ingest pipeline compatibility.

heartbeat/Jenkinsfile.yml

@@ -35,8 +35,16 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"
+        withModule: true
         stage: mandatory
     macos:
         mage: "mage build unitTest"

libbeat/Jenkinsfile.yml

@@ -31,8 +31,11 @@ stages:
                 - "arm"
             parameters:
                 - "armTest"
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
         stage: mandatory
     crosscompile:
         make: "make -C libbeat crosscompile"

metricbeat/Jenkinsfile.yml

@@ -69,15 +69,15 @@ stages:
         platforms:             ## override default labels in this specific stage.
             - "windows-10"
         stage: extended
-    windows-8:
+    windows-2008:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
-            - "windows-8"
+            - "windows-2008-r2"
         stage: extended
-    windows-2008:
+    windows-8:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
-            - "windows-2008-r2"
+            - "windows-8"
         stage: extended
     windows-7:
         mage: "mage build unitTest"

packetbeat/Jenkinsfile.yml

@@ -36,8 +36,8 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
         stage: mandatory
     macos:
         mage: "mage build unitTest"

x-pack/dockerlogbeat/Jenkinsfile.yml

@@ -19,8 +19,11 @@ stages:
           make -C x-pack/dockerlogbeat update;
           make check-no-changes;
         stage: lint
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
         stage: mandatory
     packaging-linux:

x-pack/elastic-agent/Jenkinsfile.yml

@@ -33,8 +33,11 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
         stage: mandatory
     macos:
         mage: "mage build unitTest"

x-pack/filebeat/Jenkinsfile.yml

@@ -35,9 +35,14 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
-        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"       ## run the ITs only if the changeset affects a specific module.
         stage: mandatory
     module-compat-7.11:
       mage: >-                 ## Run module integration tests under ES 7.11 to ensure ingest pipeline compatibility.
@@ -112,4 +117,4 @@ stages:
             enabled: false
         platforms:             ## override default label in this specific stage.
           - "arm"
-        stage: packaging
+        stage: packaging

x-pack/functionbeat/Jenkinsfile.yml

@@ -30,8 +30,11 @@ stages:
                 - "arm"
             parameters:
                 - "armTest"
-    build:
-        mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions"
+    unitTest:
+        mage: "mage build unitTest && GO_VERSION=1.13.1 mage testGCPFunctions"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
         stage: mandatory
     macos:
         mage: "mage build unitTest"

x-pack/heartbeat/Jenkinsfile.yml

@@ -21,11 +21,14 @@ stages:
           make -C heartbeat update;
           make check-no-changes;
         stage: lint
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
         stage: mandatory
     macos:
-        mage: "mage build test"
+        mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
           - "macosx&&x86_64"
         when:                  ## Override the top-level when.

x-pack/libbeat/Jenkinsfile.yml

@@ -33,6 +33,12 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"
         stage: mandatory