elastic · MadameSheema · Jan 16, 2020 · Jan 16, 2020 · Jan 16, 2020 · Jan 16, 2020
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -9,63 +9,62 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
       ansiColor('xterm') {
         githubPr.withDefaultPrComments {
           catchError {
-            retryable.enable()
             parallel([
-              'kibana-intake-agent': kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh'),
-              'x-pack-intake-agent': kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-              'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-                // 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', {
-                //   retryable('kibana-firefoxSmoke') {
-                //     runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke')
-                //   }
-                // }),
-                'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
-                'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
-                'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
-                'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
-                'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
-                'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
-                'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
-                'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
-                'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
-                'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
-                'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
-                'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
-                'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', {
-                  retryable('kibana-accessibility') {
-                    runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility')
-                  }
-                }),
+            //  'kibana-intake-agent': kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh'),
+            //  'x-pack-intake-agent': kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
+            //  'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+            //    'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', {
+            //      retryable('kibana-firefoxSmoke') {
+            //        runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke')
+            //      }
+            //    }),
+            //    'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
+            //    'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
+            //    'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
+            //    'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
+            //    'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
+            //    'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
+            //    'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
+            //    'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
+            //    'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
+            //    'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
+            //    'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
+            //    'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
+            //    'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', {
+            //      retryable('kibana-accessibility') {
+            //        runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility')
+            //      }
+            //    }),
                 // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }),
-              ]),
+            //  ]),
               'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-                // 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', {
-                //   retryable('xpack-firefoxSmoke') {
-                //     runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke')
-                //   }
-                // }),
-                'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
-                'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
-                'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
-                'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
-                'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
-                'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
-                'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
-                'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
-                'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
-                'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
-                'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', {
-                  retryable('xpack-accessibility') {
-                    runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility')
-                  }
-                }),
+                //  'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', {
+                //  retryable('xpack-firefoxSmoke') {
+            //    runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke')
+            //      }
+                //   }),
+                //   'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
+                //'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
+                //'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
+                //'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
+                //'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
+                //'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
+                //'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
+                //'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
+                //'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
+                //'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
+                //'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', {
+                //  retryable('xpack-accessibility') {
+            //    runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility')
+            //     }
+                //   }),
                 // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }),
+                'xpack-siemCypress': kibanaPipeline.getPostBuildWorker('xpack-siemCypress', { runbld('./test/scripts/jenkins_siem_cypress.sh', 'Execute siem-cypress') }),
               ]),
             ])
           }
         }
-
-        retryable.printFlakyFailures()
+      //  retryable.printFlakyFailures()
         kibanaPipeline.sendMail()
       }
     }

diff --git a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts
@@ -60,6 +60,14 @@ export class FunctionalTestRunner {
 
       await providers.loadAll();
 
+      const customTestRunner = config.get('testRunner');
+      if (customTestRunner) {
+        this.log.warning(
+          'custom test runner defined, ignoring all mocha/suite/filtering related options'
+        );
+        return (await providers.invokeProviderFn(customTestRunner)) || 0;
+      }
+
       const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
       await this.lifecycle.beforeTests.trigger();
       this.log.info('Starting tests');
@@ -70,6 +78,10 @@ export class FunctionalTestRunner {
 
   async getTestStats() {
     return await this._run(async (config, coreProviders) => {
+      if (config.get('testRunner')) {
+        throw new Error('Unable to get test stats for config that uses a custom test runner');
+      }
+
       // replace the function of custom service providers so that they return
       // promise-like objects which never resolve, essentially disabling them
       // allowing us to load the test files and populate the mocha suites
@@ -108,8 +120,11 @@ export class FunctionalTestRunner {
       const config = await readConfigFile(this.log, this.configFile, this.configOverrides);
       this.log.info('Config loaded');
 
-      if (config.get('testFiles').length === 0) {
-        throw new Error('No test files defined.');
+      if (
+        (!config.get('testFiles') || config.get('testFiles').length === 0) &&
+        !config.get('testRunner')
+      ) {
+        throw new Error('No tests defined.');
       }
 
       // base level services that functional_test_runner exposes

diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts
@@ -61,13 +61,8 @@ const defaultRelativeToConfigPath = (path: string) => {
 
 export const schema = Joi.object()
   .keys({
-    testFiles: Joi.array()
-      .items(Joi.string())
-      .when('$primary', {
-        is: true,
-        then: Joi.required(),
-        otherwise: Joi.any().default([]),
-      }),
+    testFiles: Joi.array().items(Joi.string()),
+    testRunner: Joi.func(),
 
     excludeTestFiles: Joi.array()
       .items(Joi.string())

diff --git a/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts b/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts
@@ -68,6 +68,15 @@ export class ProviderCollection {
     }
   }
 
+  public invokeProviderFn(provider: (args: any) => any) {
+    return provider({
+      getService: this.getService,
+      hasService: this.hasService,
+      getPageObject: this.getPageObject,
+      getPageObjects: this.getPageObjects,
+    });
+  }
+
   private findProvider(type: string, name: string) {
     return this.providers.find(p => p.type === type && p.name === name);
   }
@@ -89,12 +98,7 @@ export class ProviderCollection {
       }
 
       if (!instances.has(provider)) {
-        let instance = provider({
-          getService: this.getService,
-          hasService: this.hasService,
-          getPageObject: this.getPageObject,
-          getPageObjects: this.getPageObjects,
-        });
+        let instance = this.invokeProviderFn(provider);
 
         if (instance && typeof instance.then === 'function') {
           instance = createAsyncInstance(type, name, instance);

diff --git a/packages/kbn-test/src/functional_tests/cli/index.js b/packages/kbn-test/src/functional_tests/cli/index.js
@@ -18,4 +18,6 @@
  */
 
 export { runTestsCli } from './run_tests/cli';
+export { processOptions as processRunTestsCliOptions } from './run_tests/args';
 export { startServersCli } from './start_servers/cli';
+export { processOptions as processStartServersCliOptions } from './start_servers/args';
diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js
@@ -26,24 +26,32 @@ async function createFtr({
 }) {
   const config = await readConfigFile(log, configPath);
 
-  return new FunctionalTestRunner(log, configPath, {
-    mochaOpts: {
-      bail: !!bail,
-      grep,
-    },
-    kbnTestServer: {
-      installDir,
-    },
-    updateBaselines,
-    suiteTags: {
-      include: [...suiteTags.include, ...config.get('suiteTags.include')],
-      exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],
-    },
-  });
+  return {
+    config,
+    ftr: new FunctionalTestRunner(log, configPath, {
+      mochaOpts: {
+        bail: !!bail,
+        grep,
+      },
+      kbnTestServer: {
+        installDir,
+      },
+      updateBaselines,
+      suiteTags: {
+        include: [...suiteTags.include, ...config.get('suiteTags.include')],
+        exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],
+      },
+    }),
+  };
 }
 
 export async function assertNoneExcluded({ configPath, options }) {
-  const ftr = await createFtr({ configPath, options });
+  const { config, ftr } = await createFtr({ configPath, options });
+
+  if (config.get('testRunner')) {
+    // tests with custom test runners are not included in this check
+    return;
+  }
 
   const stats = await ftr.getTestStats();
   if (stats.excludedTests.length > 0) {
@@ -60,7 +68,7 @@ export async function assertNoneExcluded({ configPath, options }) {
 }
 
 export async function runFtr({ configPath, options }) {
-  const ftr = await createFtr({ configPath, options });
+  const { ftr } = await createFtr({ configPath, options });
 
   const failureCount = await ftr.run();
   if (failureCount > 0) {
@@ -71,7 +79,12 @@ export async function runFtr({ configPath, options }) {
 }
 
 export async function hasTests({ configPath, options }) {
-  const ftr = await createFtr({ configPath, options });
+  const { ftr, config } = await createFtr({ configPath, options });
+
+  if (config.get('testRunner')) {
+    // configs with custom test runners are assumed to always have tests
+    return true;
+  }
   const stats = await ftr.getTestStats();
   return stats.testCount > 0;
 }
diff --git a/packages/kbn-test/src/index.ts b/packages/kbn-test/src/index.ts
@@ -17,8 +17,15 @@
  * under the License.
  */
 
-// @ts-ignore not typed yet
-export { runTestsCli, startServersCli } from './functional_tests/cli';
+import {
+  runTestsCli,
+  processRunTestsCliOptions,
+  startServersCli,
+  processStartServersCliOptions,
+  // @ts-ignore not typed yet
+} from './functional_tests/cli';
+
+export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions };
 
 // @ts-ignore not typed yet
 export { runTests, startServers } from './functional_tests/tasks';

diff --git a/test/scripts/jenkins_siem_cypress.sh b/test/scripts/jenkins_siem_cypress.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source test/scripts/jenkins_test_setup.sh
+
+installDir="$PARENT_DIR/install/kibana"
+destDir="${installDir}-${CI_WORKER_NUMBER}"
+cp -R "$installDir" "$destDir"
+
+export KIBANA_INSTALL_DIR="$destDir"
+
+echo " -> Running SIEM cypress tests"
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "SIEM Cypress Tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/siem_cypress/config.ts
+
+echo ""
+echo ""
diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh
@@ -1,35 +1,21 @@
 #!/usr/bin/env bash
 
-source test/scripts/jenkins_test_setup_xpack.sh
+source test/scripts/jenkins_test_setup.sh
 
-if [[ -z "$CODE_COVERAGE" ]]; then
-  echo " -> Running functional and api tests"
+installDir="$PARENT_DIR/install/kibana"
+destDir="${installDir}-${CI_WORKER_NUMBER}"
+cp -R "$installDir" "$destDir"
 
-  checks-reporter-with-killswitch "X-Pack Chrome Functional tests / Group ${CI_GROUP}" \
+export KIBANA_INSTALL_DIR="$destDir"
+
+echo " -> Running SIEM cypress tests"
+cd "$XPACK_DIR"
+
+  checks-reporter-with-killswitch "SIEM Cypress Tests" \
     node scripts/functional_tests \
       --debug --bail \
       --kibana-install-dir "$KIBANA_INSTALL_DIR" \
-      --include-tag "ciGroup$CI_GROUP"
+      --config test/siem_cypress/config.ts
 
   echo ""
   echo ""
-else
-  echo " -> Running X-Pack functional tests with code coverage"
-  export NODE_OPTIONS=--max_old_space_size=8192
-
-  echo " -> making hard link clones"
-  cd ..
-  cp -RlP kibana "kibana${CI_GROUP}"
-  cd "kibana${CI_GROUP}/x-pack"
-
-  echo " -> running tests from the clone folder"
-  node scripts/functional_tests --debug --include-tag "ciGroup$CI_GROUP"
-
-  if [[ -d ../target/kibana-coverage/functional ]]; then
-    echo " -> replacing kibana${CI_GROUP} with kibana in json files"
-    sed -i "s|kibana${CI_GROUP}|kibana|g" ../target/kibana-coverage/functional/*.json
-    echo " -> copying coverage to the original folder"
-    mkdir -p ../../kibana/target/kibana-coverage/functional
-    cp -R ../target/kibana-coverage/functional/. ../../kibana/target/kibana-coverage/functional/
-  fi
-fi