ci: fix warnings with wildcards and archive system-tests

.ci/scripts/pre_archive_test.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import os
+import distutils
+from distutils import dir_util
+
+# Support filtering in the distutils.dir_util.copy_tree
+#ORIG_COPY_TREE = distutils.dir_util.copy_tree
+#
+#
+# def my_copy_tree(src, *args, **kwargs):
+#    '''function my_copy_tree
+#        Override distutils.dir_util.copy_tree to filter the system-tests
+#    '''
+#    if src.endswith('system-tests'):
+#        return []
+#    return ORIG_COPY_TREE(src, *args, **kwargs)
+#
+#
+#distutils.dir_util.copy_tree = my_copy_tree
+
+if __name__ == "__main__":
+
+    if not os.path.exists('build'):
+        os.makedirs('build')
+
+    # Top level folders to be excluded
+    EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools'])
+    for root, dirs, files in os.walk('.'):
+        dirs[:] = [d for d in dirs if d not in EXCLUDE]
+        if root.endswith(('build')) and not root.startswith((".{}build".format(os.sep))):
+            dest = os.path.join('build', root.replace(".{}".format(os.sep), ''))
+            print("Copy {} into {}".format(root, dest))
+            distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1)

.ci/scripts/search_system_tests.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+import os
+
+
+if __name__ == "__main__":
+
+    for root, dirs, files in os.walk('build'):
+        if root.endswith(('system-tests')):
+            print(root.replace(".{}".format(os.sep), ''))

Jenkinsfile

@@ -847,17 +847,62 @@ def withBeatsEnv(boolean archive, Closure body) {
         }
       } finally {
         if (archive) {
-          catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-            junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml")
-            archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out')
-          }
+          archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out')
         }
         reportCoverage()
       }
     }
   }
 }
 
+/**
+  This method archives and report the tests output.
+    It searches for certain folders to bypass some issues when working with big repositories. In addition,
+
+    OUTPUT:
+      - JUnit
+      - Artifacts in Jenkins
+      - Artifacts in GCP
+*/
+def archiveTestOutput(Map args = [:]) {
+  catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+    if (isUnix()) {
+      fixPermissions("${WORKSPACE}")
+    }
+    shOrBat(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
+    dir('build') {
+      junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults)
+      archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
+    }
+    catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
+      def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py')
+      log(level: 'INFO', text: "system-tests has been found in ${folder}")
+      if (folder.trim()) {
+        def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '')
+        tar(file: "${name}.tgz", archive: true, dir: folder)
+      }
+    }
+  }
+}
+
+/**
+  This method encapsulates the OS shell.
+*/
+def shOrBat(Map args = [:]) {
+  if (isUnix()) {
+    return sh(args)
+  } else {
+    def command = args.script
+    if (args.containsKey('returnStdout') && args.returnStdout) {
+      command = """@echo off
+      ${args.script}
+      """
+      args.script = command
+    }
+    return bat(args)
+  }
+}
+
 def withBeatsEnvWin(Closure body) {
   final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin'
   final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts'
@@ -881,10 +926,7 @@ def withBeatsEnvWin(Closure body) {
           body()
         }
       } finally {
-        catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-          junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
-          archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out')
-        }
+        archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out")
       }
     }
   }