[6.x] [ci][ftr][kbn/test] split up CI jobs (#25838)

.ci/jobs.yml

@@ -1,7 +1,26 @@
 JOB:
-  - selenium
-  - intake
-  - x-pack
+  - kibana-intake
+  - x-pack-intake
+  # make sure all kibana-ciGRoups are listed in tasks/function_test_groups.js
+  - kibana-ciGroup1
+  - kibana-ciGroup2
+  - kibana-ciGroup3
+  - kibana-ciGroup4
+  - kibana-ciGroup5
+  - kibana-ciGroup6
+  - kibana-ciGroup7
+  - kibana-ciGroup8
+  - kibana-ciGroup9
+  - kibana-ciGroup10
+  - kibana-ciGroup11
+  - kibana-ciGroup12
+  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
+  - x-pack-ciGroup1
+  - x-pack-ciGroup2
+  - x-pack-ciGroup3
+  - x-pack-ciGroup4
+  - x-pack-ciGroup5
+  - x-pack-ciGroup6
 
 # `~` is yaml for `null`
 exclude: ~

.ci/run.sh

@@ -6,15 +6,20 @@ set -e
 cd "$(dirname "$0")/.."
 
 case "$JOB" in
-"selenium")
-  ./test/scripts/jenkins_selenium.sh
-  ;;
-"intake")
+kibana-intake)
   ./test/scripts/jenkins_unit.sh
   ;;
-"x-pack")
+kibana-ciGroup*)
+  export CI_GROUP="${JOB##kibana-ciGroup}"
+  ./test/scripts/jenkins_ci_group.sh
+  ;;
+x-pack-intake)
   ./test/scripts/jenkins_xpack.sh
   ;;
+x-pack-ciGroup*)
+  export CI_GROUP="${JOB##x-pack-ciGroup}"
+  ./test/scripts/jenkins_xpack_ci_group.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1

packages/kbn-test/src/functional_tests/lib/index.js

@@ -19,6 +19,6 @@
 
 export { runKibanaServer } from './run_kibana_server';
 export { runElasticsearch } from './run_elasticsearch';
-export { runFtr, hasTests, assertNoneExcluded } from './fun_ftr';
+export { runFtr, hasTests, assertNoneExcluded } from './run_ftr';
 export { KIBANA_ROOT, KIBANA_FTR_SCRIPT, FUNCTIONAL_CONFIG_PATH, API_CONFIG_PATH } from './paths';
 export { runCli } from './run_cli';

packages/kbn-test/src/functional_tests/lib/fun_ftr.js → packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -39,14 +39,15 @@ export async function assertNoneExcluded({ configPath, options }) {
   const ftr = createFtr({ configPath, options });
 
   const stats = await ftr.getTestStats();
-  if (stats.excludedTests > 0) {
+  if (stats.excludedTests.length > 0) {
     throw new CliError(`
-      ${stats.excludedTests} tests in the ${configPath} config
+      ${stats.excludedTests.length} tests in the ${configPath} config
       are excluded when filtering by the tags run on CI. Make sure that all suites are
       tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh
 
-      ${JSON.stringify(options.suiteTags)}
+      tags: ${JSON.stringify(options.suiteTags)}
 
+      - ${stats.excludedTests.join('\n      - ')}
     `);
   }
 }
@@ -65,5 +66,5 @@ export async function runFtr({ configPath, options }) {
 export async function hasTests({ configPath, options }) {
   const ftr = createFtr({ configPath, options });
   const stats = await ftr.getTestStats();
-  return stats.tests > 0;
+  return stats.testCount > 0;
 }

src/dev/jest/junit_reporter.js

@@ -49,7 +49,7 @@ export default class JestJUnitReporter {
    * @return {undefined}
    */
   onRunComplete(contexts, results) {
-    if (!process.env.CI) {
+    if (!process.env.CI || !results.testResults.length) {
       return;
     }
 

src/dev/mocha/junit_report_generation.js

@@ -27,6 +27,8 @@ import xmlBuilder from 'xmlbuilder';
 import { getSnapshotOfRunnableLogs } from './log_cache';
 import { escapeCdata } from '../xml';
 
+const dateNow = Date.now.bind(Date);
+
 export function setupJUnitReportGeneration(runner, options = {}) {
   const {
     reportName = 'Unnamed Mocha Tests',
@@ -47,11 +49,11 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   );
 
   const setStartTime = (node) => {
-    node.startTime = Date.now();
+    node.startTime = dateNow();
   };
 
   const setEndTime = node => {
-    node.endTime = Date.now();
+    node.endTime = dateNow();
   };
 
   const getFullTitle = node => {
@@ -85,6 +87,9 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   runner.on('end', () => {
     // crawl the test graph to collect all defined tests
     const allTests = findAllTests(runner.suite);
+    if (!allTests.length) {
+      return;
+    }
 
     // filter out just the failures
     const failures = results.filter(result => result.failed);

src/functional_test_runner/functional_test_runner.js

@@ -89,8 +89,8 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
         );
 
         return {
-          tests: countTests(mocha.suite),
-          excludedTests: mocha.excludedTests.length
+          testCount: countTests(mocha.suite),
+          excludedTests: mocha.excludedTests.map(t => t.fullTitle())
         };
       });
     }

src/functional_test_runner/lib/config/read_config_file.js

@@ -22,18 +22,17 @@ import { defaultsDeep } from 'lodash';
 import { Config } from './config';
 import { transformDeprecations } from './transform_deprecations';
 
-async function getSettingsFromFile(log, path, settingOverrides) {
-  log.debug('Loading config file from %j', path);
+const cache = new WeakMap();
 
+async function getSettingsFromFile(log, path, settingOverrides) {
   const configModule = require(path);
   const configProvider = configModule.__esModule
     ? configModule.default
     : configModule;
 
-  const settingsWithDefaults = defaultsDeep(
-    {},
-    settingOverrides,
-    await configProvider({
+  if (!cache.has(configProvider)) {
+    log.debug('Loading config file from %j', path);
+    cache.set(configProvider, configProvider({
       log,
       async readConfigFile(...args) {
         return new Config({
@@ -42,7 +41,13 @@ async function getSettingsFromFile(log, path, settingOverrides) {
           path,
         });
       }
-    })
+    }));
+  }
+
+  const settingsWithDefaults = defaultsDeep(
+    {},
+    settingOverrides,
+    await cache.get(configProvider)
   );
 
   const logDeprecation = (...args) => log.error(...args);

tasks/config/run.js

@@ -18,8 +18,10 @@
  */
 
 import { resolve } from 'path';
+import { getFunctionalTestGroupRunConfigs } from '../function_test_groups';
 
-const PKG_VERSION = require('../../package.json').version;
+const { version } = require('../../package.json');
+const KIBANA_INSTALL_DIR = `./build/oss/kibana-${version}-SNAPSHOT-${process.platform}-x86_64`;
 const KIBANA_BIN_PATH = process.platform.startsWith('win')
   ? '.\\bin\\kibana.bat'
   : './bin/kibana';
@@ -191,7 +193,7 @@ module.exports = function (grunt) {
         '--esFrom', esFrom,
         '--bail',
         '--debug',
-        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
+        '--kibana-install-dir', KIBANA_INSTALL_DIR,
       ],
     },
 
@@ -203,7 +205,7 @@ module.exports = function (grunt) {
         '--esFrom', esFrom,
         '--bail',
         '--debug',
-        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
+        '--kibana-install-dir', KIBANA_INSTALL_DIR,
         '--',
         '--server.maxPayloadBytes=1648576',
       ],
@@ -222,18 +224,9 @@ module.exports = function (grunt) {
       ],
     },
 
-    functionalTestsRelease: {
-      cmd: process.execPath,
-      args: [
-        'scripts/functional_tests',
-        '--config', 'test/functional/config.js',
-        '--esFrom', esFrom,
-        '--bail',
-        '--debug',
-        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
-        '--',
-        '--server.maxPayloadBytes=1648576',
-      ],
-    },
+    ...getFunctionalTestGroupRunConfigs({
+      esFrom,
+      kibanaInstallDir: KIBANA_INSTALL_DIR
+    })
   };
 };

tasks/function_test_groups.js

@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import execa from 'execa';
+import grunt from 'grunt';
+
+/**
+ * The list of tags that we use in the functional tests, if we add a new group we need to add it to this list
+ * and to the list of jobs in .ci/jobs.yml
+ */
+const TEST_TAGS = [
+  'ciGroup1',
+  'ciGroup2',
+  'ciGroup3',
+  'ciGroup4',
+  'ciGroup5',
+  'ciGroup6',
+  'ciGroup7',
+  'ciGroup8',
+  'ciGroup9',
+  'ciGroup10',
+  'ciGroup11',
+  'ciGroup12'
+];
+
+export function getFunctionalTestGroupRunConfigs({ esFrom, kibanaInstallDir } = {}) {
+  return {
+    // include a run task for each test group
+    ...TEST_TAGS.reduce((acc, tag) => ({
+      ...acc,
+      [`functionalTests_${tag}`]: {
+        cmd: process.execPath,
+        args: [
+          'scripts/functional_tests',
+          '--include-tag', tag,
+          '--config', 'test/functional/config.js',
+          '--esFrom', esFrom,
+          '--bail',
+          '--debug',
+          '--kibana-install-dir', kibanaInstallDir,
+          '--',
+          '--server.maxPayloadBytes=1648576',
+        ],
+      }
+    }), {}),
+  };
+}
+
+grunt.registerTask(
+  'functionalTests:ensureAllTestsInCiGroup',
+  'Check that all of the functional tests are in a CI group',
+  async function () {
+    const done = this.async();
+
+    try {
+      const stats = JSON.parse(await execa.stderr(process.execPath, [
+        'scripts/functional_test_runner',
+        ...TEST_TAGS.map(tag => `--include-tag=${tag}`),
+        '--config', 'test/functional/config.js',
+        '--test-stats'
+      ]));
+
+      if (stats.excludedTests.length > 0) {
+        grunt.fail.fatal(`
+          ${stats.excludedTests.length} tests are excluded by the ciGroup tags, make sure that
+          all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js"
+          knows about the tag that you are using.
+
+          tags: ${JSON.stringify({ include: TEST_TAGS })}
+
+          - ${stats.excludedTests.join('\n          - ')}
+        `);
+        return;
+      }
+
+      done();
+    } catch (error) {
+      grunt.fail.fatal(error.stack);
+    }
+  }
+);

tasks/jenkins.js

@@ -41,12 +41,6 @@ module.exports = function (grunt) {
     'run:apiIntegrationTests',
   ]);
 
-  grunt.registerTask('jenkins:selenium', [
-    'checkPlugins',
-    'run:functionalTestsRelease',
-    'run:pluginFunctionalTestsRelease',
-  ]);
-
   grunt.registerTask(
     'jenkins:report',
     'Reports failed tests found in junit xml files to Github issues',

test/functional/apps/console/index.js

@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('console app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       await remote.setWindowSize(1300, 1100);
     });

test/functional/apps/context/index.js

@@ -24,6 +24,8 @@ export default function ({ getService, getPageObjects, loadTestFile }) {
   const kibanaServer = getService('kibanaServer');
 
   describe('context app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       await remote.setWindowSize(1200, 800);
       await esArchiver.loadIfNeeded('logstash_functional');