Split Kibana ci job into much smaller chunks, more Jenkins workers, shorter elapsed time

.ci/jobs.yml

@@ -1,7 +1,24 @@
 JOB:
-  - selenium
-  - intake
-  - x-pack
+  - kibana-intake
+  - kibana-ciGroup1
+  - kibana-ciGroup2
+  - kibana-ciGroup3
+  - kibana-ciGroup4
+  - kibana-ciGroup5
+  - kibana-ciGroup6
+  - kibana-ciGroup7
+  - kibana-ciGroup8
+  - kibana-ciGroup9
+  - kibana-ciGroup10
+  - kibana-ciGroup11
+  - kibana-ciGroup12
+  - x-pack-intake
+  - x-pack-ciGroup1
+  - x-pack-ciGroup2
+  - x-pack-ciGroup3
+  - x-pack-ciGroup4
+  - x-pack-ciGroup5
+  - x-pack-ciGroup6
 
 # `~` is yaml for `null`
 exclude: ~

.ci/run.sh

@@ -6,15 +6,20 @@ set -e
 cd "$(dirname "$0")/.."
 
 case "$JOB" in
-"selenium")
-  ./test/scripts/jenkins_selenium.sh
-  ;;
-"intake")
+kibana-intake)
   ./test/scripts/jenkins_unit.sh
   ;;
-"x-pack")
+kibana-ciGroup*)
+  export CI_GROUP="${JOB##kibana-ciGroup}"
+  ./test/scripts/jenkins_ci_group.sh
+  ;;
+x-pack-intake)
   ./test/scripts/jenkins_xpack.sh
   ;;
+x-pack-ciGroup*)
+  export CI_GROUP="${JOB##x-pack-ciGroup}"
+  ./test/scripts/jenkins_xpack_ci_group.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -18,6 +18,7 @@ Options:
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
   --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
   --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
+  --assert-none-excluded         Exit with 1/0 based on if any test is excluded with the current set of tags. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 
@@ -26,11 +27,12 @@ Options:
 
 exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -41,12 +43,13 @@ Object {
 
 exports[`process options for run tests CLI accepts debug option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "debug": true,
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -56,12 +59,13 @@ Object {
 
 exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "config": "",
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -74,13 +78,14 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": Object {
     "server.foo": "bar",
   },
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -90,11 +95,12 @@ Object {
 
 exports[`process options for run tests CLI accepts quiet option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "quiet": true,
   "suiteTags": Object {
     "exclude": Array [],
@@ -105,11 +111,12 @@ Object {
 
 exports[`process options for run tests CLI accepts silent option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "silent": true,
   "suiteTags": Object {
     "exclude": Array [],
@@ -120,12 +127,13 @@ Object {
 
 exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "esFrom": "source",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -135,12 +143,13 @@ Object {
 
 exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
   "installDir": "foo",
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -150,12 +159,13 @@ Object {
 
 exports[`process options for run tests CLI accepts value for grep 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
   "grep": "management",
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -165,11 +175,12 @@ Object {
 
 exports[`process options for run tests CLI accepts verbose option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -18,6 +18,7 @@ Options:
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
   --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
   --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
+  --assert-none-excluded         Exit with 1/0 based on if any test is excluded with the current set of tags. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import dedent from 'dedent';
 import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -52,6 +54,9 @@ const options = {
     arg: '<tag>',
     desc: 'Tags that suites must NOT include to be run, can be included multiple times.',
   },
+  'assert-none-excluded': {
+    desc: 'Exit with 1/0 based on if any test is excluded with the current set of tags.',
+  },
   verbose: { desc: 'Log everything.' },
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
@@ -113,17 +118,16 @@ export function processOptions(userOptions, defaultConfigPaths) {
   delete userOptions['include-tag'];
   delete userOptions['exclude-tag'];
 
-  function createLogger() {
-    return new ToolingLog({
-      level: pickLevelFromFlags(userOptions),
-      writeTo: process.stdout,
-    });
-  }
+  userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded'];
+  delete userOptions['assert-none-excluded'];
 
   return {
     ...userOptions,
-    configs,
-    createLogger,
+    configs: configs.map(c => resolve(c)),
+    log: new ToolingLog({
+      level: pickLevelFromFlags(userOptions),
+      writeTo: process.stdout,
+    }),
     extraKbnOpts: userOptions._,
   };
 }

packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js

@@ -18,6 +18,17 @@
  */
 
 import { displayHelp, processOptions } from './args';
+import { ToolingLog } from '@kbn/dev-utils';
+
+expect.addSnapshotSerializer({
+  print: () => '[object ToolingLog]',
+  test: value => value instanceof ToolingLog,
+});
+
+expect.addSnapshotSerializer({
+  print: value => value.replace(process.cwd(), '<absolute path>').replace(/\\/g, '/'),
+  test: value => typeof value === 'string' && value.startsWith(process.cwd()),
+});
 
 describe('display help for run tests CLI', () => {
   it('displays as expected', () => {

packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js

@@ -36,7 +36,7 @@ describe('run tests CLI', () => {
     const processMock = {
       exit: exitMock,
       argv: argvMock,
-      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn(), write: jest.fn() },
       cwd: jest.fn(),
     };