diff --git a/.ci/jobs.yml b/.ci/jobs.yml
index 1740e1db33f2d..975d0a734450c 100644
--- a/.ci/jobs.yml
+++ b/.ci/jobs.yml
@@ -1,7 +1,24 @@
 JOB:
-  - selenium
-  - intake
-  - x-pack
+  - kibana-intake
+  - kibana-ciGroup1
+  - kibana-ciGroup2
+  - kibana-ciGroup3
+  - kibana-ciGroup4
+  - kibana-ciGroup5
+  - kibana-ciGroup6
+  - kibana-ciGroup7
+  - kibana-ciGroup8
+  - kibana-ciGroup9
+  - kibana-ciGroup10
+  - kibana-ciGroup11
+  - kibana-ciGroup12
+  - x-pack-intake
+  - x-pack-ciGroup1
+  - x-pack-ciGroup2
+  - x-pack-ciGroup3
+  - x-pack-ciGroup4
+  - x-pack-ciGroup5
+  - x-pack-ciGroup6
 
 # `~` is yaml for `null`
 exclude: ~
diff --git a/.ci/run.sh b/.ci/run.sh
index 32c138bd2f450..57cc4afd6a38a 100755
--- a/.ci/run.sh
+++ b/.ci/run.sh
@@ -6,15 +6,20 @@ set -e
 cd "$(dirname "$0")/.."
 
 case "$JOB" in
-"selenium")
-  ./test/scripts/jenkins_selenium.sh
-  ;;
-"intake")
+kibana-intake)
   ./test/scripts/jenkins_unit.sh
   ;;
-"x-pack")
+kibana-ciGroup*)
+  export CI_GROUP="${JOB##kibana-ciGroup}"
+  ./test/scripts/jenkins_ci_group.sh
+  ;;
+x-pack-intake)
   ./test/scripts/jenkins_xpack.sh
   ;;
+x-pack-ciGroup*)
+  export CI_GROUP="${JOB##x-pack-ciGroup}"
+  ./test/scripts/jenkins_xpack_ci_group.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
index 93d129213ff02..bf3bd92b66e47 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
@@ -18,6 +18,7 @@ Options:
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
   --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
   --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
+  --assert-none-excluded         Exit with 1/0 based on if any test is excluded with the current set of tags. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 
@@ -26,11 +27,12 @@ Options:
 
 exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -41,12 +43,13 @@ Object {
 
 exports[`process options for run tests CLI accepts debug option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "debug": true,
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -56,12 +59,13 @@ Object {
 
 exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "config": "",
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -74,13 +78,14 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": Object {
     "server.foo": "bar",
   },
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -90,11 +95,12 @@ Object {
 
 exports[`process options for run tests CLI accepts quiet option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "quiet": true,
   "suiteTags": Object {
     "exclude": Array [],
@@ -105,11 +111,12 @@ Object {
 
 exports[`process options for run tests CLI accepts silent option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "silent": true,
   "suiteTags": Object {
     "exclude": Array [],
@@ -120,12 +127,13 @@ Object {
 
 exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "esFrom": "source",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -135,12 +143,13 @@ Object {
 
 exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
   "installDir": "foo",
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -150,12 +159,13 @@ Object {
 
 exports[`process options for run tests CLI accepts value for grep 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
   "grep": "management",
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
@@ -165,11 +175,12 @@ Object {
 
 exports[`process options for run tests CLI accepts verbose option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
-  "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "suiteTags": Object {
     "exclude": Array [],
     "include": Array [],
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
index 6d4b35f754b42..e9f3dfac56746 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
@@ -18,6 +18,7 @@ Options:
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
   --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
   --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
+  --assert-none-excluded         Exit with 1/0 based on if any test is excluded with the current set of tags. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index 6aa1e1aa5c31b..df4f4b4705e61 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import dedent from 'dedent';
 import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -52,6 +54,9 @@ const options = {
     arg: '<tag>',
     desc: 'Tags that suites must NOT include to be run, can be included multiple times.',
   },
+  'assert-none-excluded': {
+    desc: 'Exit with 1/0 based on if any test is excluded with the current set of tags.',
+  },
   verbose: { desc: 'Log everything.' },
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
@@ -113,17 +118,16 @@ export function processOptions(userOptions, defaultConfigPaths) {
   delete userOptions['include-tag'];
   delete userOptions['exclude-tag'];
 
-  function createLogger() {
-    return new ToolingLog({
-      level: pickLevelFromFlags(userOptions),
-      writeTo: process.stdout,
-    });
-  }
+  userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded'];
+  delete userOptions['assert-none-excluded'];
 
   return {
     ...userOptions,
-    configs,
-    createLogger,
+    configs: configs.map(c => resolve(c)),
+    log: new ToolingLog({
+      level: pickLevelFromFlags(userOptions),
+      writeTo: process.stdout,
+    }),
     extraKbnOpts: userOptions._,
   };
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
index 2e0c2fb44022d..c06726264cb1c 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
@@ -18,6 +18,17 @@
  */
 
 import { displayHelp, processOptions } from './args';
+import { ToolingLog } from '@kbn/dev-utils';
+
+expect.addSnapshotSerializer({
+  print: () => '[object ToolingLog]',
+  test: value => value instanceof ToolingLog,
+});
+
+expect.addSnapshotSerializer({
+  print: value => value.replace(process.cwd(), '<absolute path>').replace(/\\/g, '/'),
+  test: value => typeof value === 'string' && value.startsWith(process.cwd()),
+});
 
 describe('display help for run tests CLI', () => {
   it('displays as expected', () => {
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
index 7e2e6c5a40e78..6f4458e357c16 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
@@ -36,7 +36,7 @@ describe('run tests CLI', () => {
     const processMock = {
       exit: exitMock,
       argv: argvMock,
-      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn(), write: jest.fn() },
       cwd: jest.fn(),
     };
 
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
index 50da9f6836883..64deaa825d360 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
@@ -21,24 +21,20 @@ Options:
 
 exports[`process options for start servers CLI accepts debug option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "debug": true,
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
 }
 `;
 
 exports[`process options for start servers CLI accepts empty config value if default passed 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
 }
 `;
 
@@ -47,72 +43,60 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "snapshot",
   "extraKbnOpts": Object {
     "server.foo": "bar",
   },
+  "log": [object ToolingLog],
 }
 `;
 
 exports[`process options for start servers CLI accepts quiet option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "quiet": true,
 }
 `;
 
 exports[`process options for start servers CLI accepts silent option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "silent": true,
 }
 `;
 
 exports[`process options for start servers CLI accepts source value for esFrom 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "source",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
 }
 `;
 
 exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
   "installDir": "foo",
+  "log": [object ToolingLog],
 }
 `;
 
 exports[`process options for start servers CLI accepts verbose option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
-  "createLogger": [Function],
+  "config": <absolute path>/foo,
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
+  "log": [object ToolingLog],
   "verbose": true,
 }
 `;
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index 170de1942232b..b56064dc145d9 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import dedent from 'dedent';
 import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -88,17 +90,13 @@ export function processOptions(userOptions, defaultConfigPath) {
     delete userOptions['kibana-install-dir'];
   }
 
-  function createLogger() {
-    return new ToolingLog({
-      level: pickLevelFromFlags(userOptions),
-      writeTo: process.stdout,
-    });
-  }
-
   return {
     ...userOptions,
-    config,
-    createLogger,
+    config: resolve(config),
+    log: new ToolingLog({
+      level: pickLevelFromFlags(userOptions),
+      writeTo: process.stdout,
+    }),
     extraKbnOpts: userOptions._,
   };
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
index 22c06b6ae3231..6de09f89e18b3 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
@@ -18,6 +18,17 @@
  */
 
 import { displayHelp, processOptions } from './args';
+import { ToolingLog } from '@kbn/dev-utils';
+
+expect.addSnapshotSerializer({
+  print: () => '[object ToolingLog]',
+  test: value => value instanceof ToolingLog,
+});
+
+expect.addSnapshotSerializer({
+  print: value => value.replace(process.cwd(), '<absolute path>').replace(/\\/g, '/'),
+  test: value => typeof value === 'string' && value.startsWith(process.cwd()),
+});
 
 describe('display help for start servers CLI', () => {
   it('displays as expected', () => {
@@ -39,60 +50,60 @@ describe('process options for start servers CLI', () => {
   });
 
   it('accepts empty config value if default passed', () => {
-    const options = processOptions({ config: '' }, ['foo']);
+    const options = processOptions({ config: '' }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid option', () => {
     expect(() => {
-      processOptions({ bail: true }, ['foo']);
+      processOptions({ bail: true }, 'foo');
     }).toThrow('functional_tests_server: invalid option [bail]');
   });
 
   it('accepts string value for kibana-install-dir', () => {
-    const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    const options = processOptions({ 'kibana-install-dir': 'foo' }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('rejects boolean value for kibana-install-dir', () => {
     expect(() => {
-      processOptions({ 'kibana-install-dir': true }, ['foo']);
+      processOptions({ 'kibana-install-dir': true }, 'foo');
     }).toThrow('functional_tests_server: invalid argument [true] to option [kibana-install-dir]');
   });
 
   it('accepts source value for esFrom', () => {
-    const options = processOptions({ esFrom: 'source' }, ['foo']);
+    const options = processOptions({ esFrom: 'source' }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts debug option', () => {
-    const options = processOptions({ debug: true }, ['foo']);
+    const options = processOptions({ debug: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts silent option', () => {
-    const options = processOptions({ silent: true }, ['foo']);
+    const options = processOptions({ silent: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts quiet option', () => {
-    const options = processOptions({ quiet: true }, ['foo']);
+    const options = processOptions({ quiet: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts verbose option', () => {
-    const options = processOptions({ verbose: true }, ['foo']);
+    const options = processOptions({ verbose: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts extra server options', () => {
-    const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    const options = processOptions({ _: { 'server.foo': 'bar' } }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid options even if valid options exist', () => {
     expect(() => {
-      processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+      processOptions({ debug: true, aintnothang: true, bail: true }, 'foo');
     }).toThrow('functional_tests_server: invalid option [aintnothang]');
   });
 });
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
index 43bab5a4afe89..b6bc3f6fcd30d 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
@@ -36,7 +36,7 @@ describe('start servers CLI', () => {
     const processMock = {
       exit: exitMock,
       argv: argvMock,
-      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn(), write: jest.fn() },
       cwd: jest.fn(),
     };
 
diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/ftr.js
similarity index 53%
rename from packages/kbn-test/src/functional_tests/lib/run_ftr.js
rename to packages/kbn-test/src/functional_tests/lib/ftr.js
index 779286212f5c7..e3dbe6cef6d8a 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js
+++ b/packages/kbn-test/src/functional_tests/lib/ftr.js
@@ -17,14 +17,11 @@
  * under the License.
  */
 
-import { createFunctionalTestRunner } from '../../../../../src/functional_test_runner';
+import * as FunctionalTestRunner from '../../../../../src/functional_test_runner';
 import { CliError } from './run_cli';
 
-export async function runFtr({
-  configPath,
-  options: { log, bail, grep, updateBaselines, suiteTags },
-}) {
-  const ftr = createFunctionalTestRunner({
+function createFtr({ configPath, options: { log, bail, grep, updateBaselines, suiteTags } }) {
+  return FunctionalTestRunner.createFunctionalTestRunner({
     log,
     configFile: configPath,
     configOverrides: {
@@ -36,6 +33,26 @@ export async function runFtr({
       suiteTags,
     },
   });
+}
+
+export async function assertNoneExcluded({ configPath, options }) {
+  const ftr = createFtr({ configPath, options });
+
+  const stats = await ftr.getTestStats();
+  if (stats.excludedTests > 0) {
+    throw new CliError(`
+      ${stats.excludedTests} tests in the ${configPath} config
+      are excluded when filtering by the tags run on CI. Make sure that all suites are
+      tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh
+
+      ${JSON.stringify(options.suiteTags)}
+
+    `);
+  }
+}
+
+export async function runFtr({ configPath, options }) {
+  const ftr = createFtr({ configPath, options });
 
   const failureCount = await ftr.run();
   if (failureCount > 0) {
@@ -44,3 +61,9 @@ export async function runFtr({
     );
   }
 }
+
+export async function hasTests({ configPath, options }) {
+  const ftr = createFtr({ configPath, options });
+  const stats = await ftr.getTestStats();
+  return stats.tests > 0;
+}
diff --git a/packages/kbn-test/src/functional_tests/lib/index.js b/packages/kbn-test/src/functional_tests/lib/index.js
index d66f641c10f68..89457a22bd2c2 100644
--- a/packages/kbn-test/src/functional_tests/lib/index.js
+++ b/packages/kbn-test/src/functional_tests/lib/index.js
@@ -19,6 +19,6 @@
 
 export { runKibanaServer } from './run_kibana_server';
 export { runElasticsearch } from './run_elasticsearch';
-export { runFtr } from './run_ftr';
+export { runFtr, hasTests, assertNoneExcluded } from './ftr';
 export { KIBANA_ROOT, KIBANA_FTR_SCRIPT, FUNCTIONAL_CONFIG_PATH, API_CONFIG_PATH } from './paths';
 export { runCli } from './run_cli';
diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index 0cdcc77161a60..49ae554e1f8a2 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -17,12 +17,19 @@
  * under the License.
  */
 
-import { relative, resolve } from 'path';
+import { relative } from 'path';
 import * as Rx from 'rxjs';
 import { startWith, switchMap, take } from 'rxjs/operators';
 import { withProcRunner } from '@kbn/dev-utils';
 
-import { runElasticsearch, runKibanaServer, runFtr, KIBANA_FTR_SCRIPT } from './lib';
+import {
+  runElasticsearch,
+  runKibanaServer,
+  runFtr,
+  assertNoneExcluded,
+  hasTests,
+  KIBANA_FTR_SCRIPT,
+} from './lib';
 
 import { readConfigFile } from '../../../../src/functional_test_runner/lib';
 
@@ -38,44 +45,66 @@ in another terminal session by running this command from this directory:
 /**
  * Run servers and tests for each config
  * @param {object} options                   Optional
- * @property {string[]} configPaths          Array of paths to configs
- * @property {function} options.createLogger Optional logger creation function
+ * @property {string[]} options.configs      Array of paths to configs
+ * @property {function} options.log          An instance of the ToolingLog
  * @property {string} options.installDir     Optional installation dir from which to run Kibana
  * @property {boolean} options.bail          Whether to exit test run at the first failure
  * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function runTests(options) {
+  const { log } = options;
+
   for (const configPath of options.configs) {
-    await runSingleConfig(resolve(process.cwd(), configPath), options);
+    try {
+      log.info('Running', configPath);
+      log.indent(2);
+
+      if (options.assertNoneExcluded) {
+        await assertNoneExcluded({ configPath, options });
+        continue;
+      }
+
+      if (!(await hasTests({ configPath, options }))) {
+        log.info('Skipping', configPath, 'since all tests are excluded');
+        continue;
+      }
+
+      await withProcRunner(log, async procs => {
+        const config = await readConfigFile(log, configPath);
+
+        const es = await runElasticsearch({ config, options });
+        await runKibanaServer({ procs, config, options });
+        await runFtr({ configPath, options });
+
+        await procs.stop('kibana');
+        await es.cleanup();
+      });
+    } finally {
+      log.indent(-2);
+    }
   }
 }
 
 /**
  * Start only servers using single config
  * @param {object} options                   Optional
- * @property {string} options.configPath     Path to a config file
- * @property {function} options.createLogger Optional logger creation function
+ * @property {string} options.config         Path to a config file
+ * @property {function} options.log          An instance of the ToolingLog
  * @property {string} options.installDir     Optional installation dir from which to run Kibana
  * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function startServers(options) {
-  const { config: configOption, createLogger } = options;
-  const configPath = resolve(process.cwd(), configOption);
-  const log = createLogger();
-  const opts = {
-    ...options,
-    log,
-  };
+  const { log } = options;
 
   await withProcRunner(log, async procs => {
-    const config = await readConfigFile(log, configPath);
+    const config = await readConfigFile(log, options.config);
 
-    const es = await runElasticsearch({ config, options: opts });
+    const es = await runElasticsearch({ config, options });
     await runKibanaServer({
       procs,
       config,
       options: {
-        ...opts,
+        ...options,
         extraKbnOpts: [...options.extraKbnOpts, ...(options.installDir ? [] : ['--dev'])],
       },
     });
@@ -100,25 +129,3 @@ async function silence(milliseconds, { log }) {
     )
     .toPromise();
 }
-
-/*
- * Start servers and run tests for single config
- */
-async function runSingleConfig(configPath, options) {
-  const log = options.createLogger();
-  const opts = {
-    ...options,
-    log,
-  };
-
-  await withProcRunner(log, async procs => {
-    const config = await readConfigFile(log, configPath);
-
-    const es = await runElasticsearch({ config, options: opts });
-    await runKibanaServer({ procs, config, options: opts });
-    await runFtr({ configPath, options: opts });
-
-    await procs.stop('kibana');
-    await es.cleanup();
-  });
-}
diff --git a/src/dev/jest/junit_reporter.js b/src/dev/jest/junit_reporter.js
index 9f8f042b2ddb7..9d05e1dabf59c 100644
--- a/src/dev/jest/junit_reporter.js
+++ b/src/dev/jest/junit_reporter.js
@@ -49,7 +49,7 @@ export default class JestJUnitReporter {
    * @return {undefined}
    */
   onRunComplete(contexts, results) {
-    if (!process.env.CI) {
+    if (!process.env.CI || !results.testResults.length) {
       return;
     }
 
diff --git a/src/dev/mocha/junit_report_generation.js b/src/dev/mocha/junit_report_generation.js
index e5c7aec947b8f..74b271e82f52e 100644
--- a/src/dev/mocha/junit_report_generation.js
+++ b/src/dev/mocha/junit_report_generation.js
@@ -27,6 +27,8 @@ import xmlBuilder from 'xmlbuilder';
 import { getSnapshotOfRunnableLogs } from './log_cache';
 import { escapeCdata } from '../xml';
 
+const dateNow = Date.now.bind(Date);
+
 export function setupJUnitReportGeneration(runner, options = {}) {
   const {
     reportName = 'Unnamed Mocha Tests',
@@ -47,11 +49,11 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   );
 
   const setStartTime = (node) => {
-    node.startTime = Date.now();
+    node.startTime = dateNow();
   };
 
   const setEndTime = node => {
-    node.endTime = Date.now();
+    node.endTime = dateNow();
   };
 
   const getFullTitle = node => {
@@ -85,6 +87,9 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   runner.on('end', () => {
     // crawl the test graph to collect all defined tests
     const allTests = findAllTests(runner.suite);
+    if (!allTests.length) {
+      return;
+    }
 
     // filter out just the failures
     const failures = results.filter(result => result.failed);
diff --git a/src/functional_test_runner/cli.js b/src/functional_test_runner/cli.js
index 6998a1b8bfb7a..9414dcfe1cfd9 100644
--- a/src/functional_test_runner/cli.js
+++ b/src/functional_test_runner/cli.js
@@ -48,6 +48,7 @@ cmd
   .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), [])
   .option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), [])
   .option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), [])
+  .option('--test-stats', 'Print the number of tests (included and excluded) to STDERR', false)
   .option('--verbose', 'Log everything', false)
   .option('--quiet', 'Only log errors', false)
   .option('--silent', 'Log nothing', false)
@@ -86,8 +87,16 @@ const functionalTestRunner = createFunctionalTestRunner({
 
 async function run() {
   try {
-    const failureCount = await functionalTestRunner.run();
-    process.exitCode = failureCount ? 1 : 0;
+    if (cmd.testStats) {
+      process.stderr.write(JSON.stringify(
+        await functionalTestRunner.getTestStats(),
+        null,
+        2
+      ) + '\n');
+    } else {
+      const failureCount = await functionalTestRunner.run();
+      process.exitCode = failureCount ? 1 : 0;
+    }
   } catch (err) {
     await teardown(err);
   } finally {
diff --git a/src/functional_test_runner/functional_test_runner.js b/src/functional_test_runner/functional_test_runner.js
index 65d347fbee450..1b8c9cc05e5ae 100644
--- a/src/functional_test_runner/functional_test_runner.js
+++ b/src/functional_test_runner/functional_test_runner.js
@@ -20,11 +20,24 @@
 import {
   createLifecycle,
   readConfigFile,
-  createProviderCollection,
+  ProviderCollection,
+  readProviderSpec,
   setupMocha,
   runTests,
 } from './lib';
 
+// provider that returns a promise-like object which never "resolves" and is
+// used to disable all the services while still allowing us to load the test
+// files (and therefore define the tests) for --test-stats
+const STUB_PROVIDER =  () => ({
+  then: () => {}
+});
+
+const replaceValues = (object, value) => Object.keys(object).reduce((acc, key) => ({
+  ...acc,
+  [key]: value
+}), {});
+
 export function createFunctionalTestRunner({ log, configFile, configOverrides }) {
   const lifecycle = createLifecycle();
 
@@ -36,8 +49,70 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
     log.verbose('ending %j lifecycle phase', name);
   });
 
+
   class FunctionalTestRunner {
     async run() {
+      return await this._run(async (config) => {
+        const providers = new ProviderCollection(log, [
+          // base level services that functional_test_runner exposes
+          ...readProviderSpec('Service', {
+            lifecycle: () => lifecycle,
+            log: () => log,
+            config: () => config,
+          }),
+
+          ...readProviderSpec('Service', config.get('services')),
+          ...readProviderSpec('PageObject', config.get('pageObjects'))
+        ]);
+
+        await providers.loadAll();
+
+        const mocha = await setupMocha(lifecycle, log, config, providers);
+        await lifecycle.trigger('beforeTests');
+        log.info('Starting tests');
+
+        return await runTests(lifecycle, log, mocha);
+      });
+    }
+
+    async getTestStats() {
+      return await this._run(async (config) => {
+        const providers = new ProviderCollection(log, [
+          // base level services that functional_test_runner exposes
+          ...readProviderSpec('Service', {
+            lifecycle: () => lifecycle,
+            log: () => log,
+            config: () => config,
+          }),
+
+          ...readProviderSpec('Service', replaceValues(config.get('services'), STUB_PROVIDER)),
+          ...readProviderSpec('PageObject', replaceValues(config.get('pageObjects'), STUB_PROVIDER))
+        ]);
+
+        const countTests = suite =>
+          suite.suites.reduce((sum, suite) => (
+            sum + countTests(suite)
+          ), suite.tests.length);
+
+        const mocha = await setupMocha(lifecycle, log, config, providers);
+
+        return {
+          tests: countTests(mocha.suite),
+          excludedTests: mocha.excludedTests.length
+        };
+      });
+    }
+
+    async close() {
+      if (this._closed) {
+        return;
+      }
+
+      this._closed = true;
+      await lifecycle.trigger('cleanup');
+    }
+
+    async _run(handler) {
       let runErrorOccurred = false;
 
       try {
@@ -49,14 +124,7 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
           return;
         }
 
-        const providers = createProviderCollection(lifecycle, log, config);
-        await providers.loadAll();
-
-        const mocha = await setupMocha(lifecycle, log, config, providers);
-        await lifecycle.trigger('beforeTests');
-        log.info('Starting tests');
-        return await runTests(lifecycle, log, mocha);
-
+        return await handler(config);
       } catch (runError) {
         runErrorOccurred = true;
         throw runError;
@@ -75,13 +143,6 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
         }
       }
     }
-
-    async close() {
-      if (this._closed) return;
-
-      this._closed = true;
-      await lifecycle.trigger('cleanup');
-    }
   }
 
   return new FunctionalTestRunner();
diff --git a/src/functional_test_runner/lib/config/read_config_file.js b/src/functional_test_runner/lib/config/read_config_file.js
index bd3cddcf8e25a..e733d2b989ff3 100644
--- a/src/functional_test_runner/lib/config/read_config_file.js
+++ b/src/functional_test_runner/lib/config/read_config_file.js
@@ -22,18 +22,17 @@ import { defaultsDeep } from 'lodash';
 import { Config } from './config';
 import { transformDeprecations } from './transform_deprecations';
 
-async function getSettingsFromFile(log, path, settingOverrides) {
-  log.debug('Loading config file from %j', path);
+const cache = new WeakMap();
 
+async function getSettingsFromFile(log, path, settingOverrides) {
   const configModule = require(path);
   const configProvider = configModule.__esModule
     ? configModule.default
     : configModule;
 
-  const settingsWithDefaults = defaultsDeep(
-    {},
-    settingOverrides,
-    await configProvider({
+  if (!cache.has(configProvider)) {
+    log.debug('Loading config file from %j', path);
+    cache.set(configProvider, configProvider({
       log,
       async readConfigFile(...args) {
         return new Config({
@@ -42,7 +41,13 @@ async function getSettingsFromFile(log, path, settingOverrides) {
           path,
         });
       }
-    })
+    }));
+  }
+
+  const settingsWithDefaults = defaultsDeep(
+    {},
+    settingOverrides,
+    await cache.get(configProvider)
   );
 
   const logDeprecation = (...args) => log.error(...args);
diff --git a/src/functional_test_runner/lib/create_provider_collection.js b/src/functional_test_runner/lib/create_provider_collection.js
deleted file mode 100644
index e91f473290ec3..0000000000000
--- a/src/functional_test_runner/lib/create_provider_collection.js
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import {
-  ProviderCollection,
-  readProviderSpec
-} from './providers';
-
-/**
- *  Create a ProviderCollection that includes the Service
- *  providers and PageObject providers from config, as well
- *  providers for the default services, lifecycle, log, and
- *  config
- *
- *  @param  {Lifecycle} lifecycle
- *  @param  {ToolingLog} log
- *  @param  {Config} config    [description]
- *  @return {ProviderCollection}
- */
-export function createProviderCollection(lifecycle, log, config) {
-  return new ProviderCollection(log, [
-    ...readProviderSpec('Service', {
-      // base level services that functional_test_runner exposes
-      lifecycle: () => lifecycle,
-      log: () => log,
-      config: () => config,
-
-      ...config.get('services'),
-    }),
-    ...readProviderSpec('PageObject', {
-      ...config.get('pageObjects')
-    })
-  ]);
-}
diff --git a/src/functional_test_runner/lib/index.js b/src/functional_test_runner/lib/index.js
index 1a933447bb0fb..e1f93d8ac627c 100644
--- a/src/functional_test_runner/lib/index.js
+++ b/src/functional_test_runner/lib/index.js
@@ -19,5 +19,5 @@
 
 export { createLifecycle } from './lifecycle';
 export { readConfigFile } from './config';
-export { createProviderCollection } from './create_provider_collection';
 export { setupMocha, runTests } from './mocha';
+export { readProviderSpec, ProviderCollection } from './providers';
diff --git a/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js
index fd15f936c7625..597be9bab32a2 100644
--- a/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js
+++ b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js
@@ -28,6 +28,14 @@
  * @param options.exclude an array of tags that will be used to exclude suites from the run
  */
 export function filterSuitesByTags({ log, mocha, include, exclude }) {
+  mocha.excludedTests = [];
+  // collect all the tests from some suite, including it's children
+  const collectTests = (suite) =>
+    suite.suites.reduce(
+      (acc, s) => acc.concat(collectTests(s)),
+      suite.tests
+    );
+
   // if include tags were provided, filter the tree once to
   // only include branches that are included at some point
   if (include.length) {
@@ -47,14 +55,18 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
           continue;
         }
 
+
         // this suite has an included child but is not included
         // itself, so strip out its tests and recurse to filter
         // out child suites which are not included
         if (isChildIncluded(child)) {
+          mocha.excludedTests = mocha.excludedTests.concat(child.tests);
           child.tests = [];
           parentSuite.suites.push(child);
           recurse(child);
           continue;
+        } else {
+          mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
         }
       }
     }(mocha.suite));
@@ -78,6 +90,8 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
         if (isNotExcluded(child)) {
           parentSuite.suites.push(child);
           recurse(child);
+        } else {
+          mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
         }
       }
     }(mocha.suite));
diff --git a/tasks/config/run.js b/tasks/config/run.js
index d950218ceb2e4..dc21d8cfd2897 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -18,6 +18,7 @@
  */
 
 import { resolve } from 'path';
+import { getFunctionalTestGroupRunConfigs } from '../function_test_groups';
 
 const { version } = require('../../package.json');
 const KIBANA_INSTALL_DIR = `./build/oss/kibana-${version}-SNAPSHOT-${process.platform}-x86_64`;
@@ -220,18 +221,9 @@ module.exports = function (grunt) {
       ],
     },
 
-    functionalTestsRelease: {
-      cmd: process.execPath,
-      args: [
-        'scripts/functional_tests',
-        '--config', 'test/functional/config.js',
-        '--esFrom', esFrom,
-        '--bail',
-        '--debug',
-        '--kibana-install-dir', KIBANA_INSTALL_DIR,
-        '--',
-        '--server.maxPayloadBytes=1648576',
-      ],
-    },
+    ...getFunctionalTestGroupRunConfigs({
+      esFrom,
+      kibanaInstallDir: KIBANA_INSTALL_DIR
+    })
   };
 };
diff --git a/tasks/function_test_groups.js b/tasks/function_test_groups.js
new file mode 100644
index 0000000000000..606c8efd4ab7c
--- /dev/null
+++ b/tasks/function_test_groups.js
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import execa from 'execa';
+import grunt from 'grunt';
+
+/**
+ * The list of tags that we use in the functional tests, if we add a new group we need to add it to this list
+ * and to the list of jobs in .ci/jobs.yml
+ */
+const TEST_TAGS = [
+  'ciGroup1',
+  'ciGroup2',
+  'ciGroup3',
+  'ciGroup4',
+  'ciGroup5',
+  'ciGroup6',
+  'ciGroup7',
+  'ciGroup8',
+  'ciGroup9',
+  'ciGroup10',
+  'ciGroup11',
+  'ciGroup12'
+];
+
+export function getFunctionalTestGroupRunConfigs({ esFrom, kibanaInstallDir } = {}) {
+  return {
+    // include a run task for each test group
+    ...TEST_TAGS.reduce((acc, tag) => ({
+      ...acc,
+      [`functionalTests_${tag}`]: {
+        cmd: process.execPath,
+        args: [
+          'scripts/functional_tests',
+          '--include-tag', tag,
+          '--config', 'test/functional/config.js',
+          '--esFrom', esFrom,
+          '--bail',
+          '--debug',
+          '--kibana-install-dir', kibanaInstallDir,
+          '--',
+          '--server.maxPayloadBytes=1648576',
+        ],
+      }
+    }), {}),
+  };
+}
+
+grunt.registerTask(
+  'functionalTests:ensureAllTestsInCiGroup',
+  'Check that all of the functional tests are in a CI group',
+  async function () {
+    const done = this.async();
+
+    try {
+      const stats = JSON.parse(await execa.stderr(process.execPath, [
+        'scripts/functional_test_runner',
+        ...TEST_TAGS.map(tag => `--include-tag=${tag}`),
+        '--config', 'test/functional/config.js',
+        '--test-stats'
+      ]));
+
+      if (stats.excludedTests > 0) {
+        grunt.fail.fatal(`
+          ${stats.excludedTests} tests are excluded by the ciGroup tags, make sure that
+          all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js"
+          knows about the tag that you are using.
+        `);
+        return;
+      }
+
+      done();
+    } catch (error) {
+      grunt.fail.fatal(error.stack);
+    }
+  }
+);
diff --git a/tasks/jenkins.js b/tasks/jenkins.js
index 69f3bc8205934..283e93e782b24 100644
--- a/tasks/jenkins.js
+++ b/tasks/jenkins.js
@@ -41,12 +41,6 @@ module.exports = function (grunt) {
     'run:apiIntegrationTests',
   ]);
 
-  grunt.registerTask('jenkins:selenium', [
-    'checkPlugins',
-    'run:functionalTestsRelease',
-    'run:pluginFunctionalTestsRelease',
-  ]);
-
   grunt.registerTask(
     'jenkins:report',
     'Reports failed tests found in junit xml files to Github issues',
diff --git a/test/functional/apps/console/index.js b/test/functional/apps/console/index.js
index ff7c200d3c779..abf758b40d4ff 100644
--- a/test/functional/apps/console/index.js
+++ b/test/functional/apps/console/index.js
@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('console app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       await remote.setWindowSize(1300, 1100);
     });
diff --git a/test/functional/apps/context/index.js b/test/functional/apps/context/index.js
index e912bafa68f22..0b699f2e404ca 100644
--- a/test/functional/apps/context/index.js
+++ b/test/functional/apps/context/index.js
@@ -24,6 +24,8 @@ export default function ({ getService, getPageObjects, loadTestFile }) {
   const kibanaServer = getService('kibanaServer');
 
   describe('context app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       await remote.setWindowSize(1200, 800);
       await esArchiver.loadIfNeeded('logstash_functional');
diff --git a/test/functional/apps/dashboard/index.js b/test/functional/apps/dashboard/index.js
index d53364ecdd186..53ba2a77fd0a7 100644
--- a/test/functional/apps/dashboard/index.js
+++ b/test/functional/apps/dashboard/index.js
@@ -22,26 +22,30 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
   const esArchiver = getService('esArchiver');
   const PageObjects = getPageObjects(['dashboard']);
 
+  async function loadCurrentData() {
+    await remote.setWindowSize(1300, 900);
+    await PageObjects.dashboard.initTests({
+      kibanaIndex: 'dashboard/current/kibana',
+      dataIndex: 'dashboard/current/data',
+      defaultIndex: 'logstash-*',
+    });
+    await PageObjects.dashboard.preserveCrossAppState();
+  }
+
+  async function unloadCurrentData() {
+    await PageObjects.dashboard.clearSavedObjectsFromAppLinks();
+    await esArchiver.unload('dashboard/current/kibana');
+    await esArchiver.unload('dashboard/current/data');
+  }
+
   describe('dashboard app', function () {
+    // This has to be first since the other tests create some embeddables as side affects and our counting assumes
+    // a fresh index.
     describe('using current data', function () {
-      before(async () => {
-        await remote.setWindowSize(1300, 900);
-        await PageObjects.dashboard.initTests({
-          kibanaIndex: 'dashboard/current/kibana',
-          dataIndex: 'dashboard/current/data',
-          defaultIndex: 'logstash-*',
-        });
-        await PageObjects.dashboard.preserveCrossAppState();
-      });
-
-      after(async function () {
-        await PageObjects.dashboard.clearSavedObjectsFromAppLinks();
-        await esArchiver.unload('dashboard/current/kibana');
-        await esArchiver.unload('dashboard/current/data');
-      });
+      this.tags('ciGroup2');
+      before(loadCurrentData);
+      after(unloadCurrentData);
 
-      // This has to be first since the other tests create some embeddables as side affects and our counting assumes
-      // a fresh index.
       loadTestFile(require.resolve('./_empty_dashboard'));
       loadTestFile(require.resolve('./_dark_theme'));
       loadTestFile(require.resolve('./_embeddable_rendering'));
@@ -50,6 +54,13 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
       loadTestFile(require.resolve('./_dashboard_options'));
       loadTestFile(require.resolve('./_data_shared_attributes'));
       loadTestFile(require.resolve('./_embed_mode'));
+    });
+
+    describe('using current data', function () {
+      this.tags('ciGroup3');
+      before(loadCurrentData);
+      after(unloadCurrentData);
+
       loadTestFile(require.resolve('./_full_screen_mode'));
       loadTestFile(require.resolve('./_dashboard_filter_bar'));
       loadTestFile(require.resolve('./_dashboard_filtering'));
@@ -63,12 +74,19 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
     // the data once to save on time. Eventually, all of these tests should just use current data and we can reserve
     // legacy data only for specifically testing BWC situations.
     describe('using legacy data', function () {
+      this.tags('ciGroup4');
       before(() => remote.setWindowSize(1200, 900));
 
       loadTestFile(require.resolve('./_dashboard_time_picker'));
       loadTestFile(require.resolve('./_bwc_shared_urls'));
       loadTestFile(require.resolve('./_panel_controls'));
       loadTestFile(require.resolve('./_dashboard_state'));
+    });
+
+    describe('using legacy data', function () {
+      this.tags('ciGroup5');
+      before(() => remote.setWindowSize(1200, 900));
+
       loadTestFile(require.resolve('./_dashboard_save'));
       loadTestFile(require.resolve('./_dashboard_time'));
       loadTestFile(require.resolve('./_dashboard_listing'));
diff --git a/test/functional/apps/discover/index.js b/test/functional/apps/discover/index.js
index 4c8441d5e40de..37e1853eea793 100644
--- a/test/functional/apps/discover/index.js
+++ b/test/functional/apps/discover/index.js
@@ -22,6 +22,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('discover app', function () {
+    this.tags('ciGroup6');
+
     before(function () {
       return remote.setWindowSize(1200, 800);
     });
diff --git a/test/functional/apps/getting_started/index.js b/test/functional/apps/getting_started/index.js
index 2683372b9df16..3a18bff81b81a 100644
--- a/test/functional/apps/getting_started/index.js
+++ b/test/functional/apps/getting_started/index.js
@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('Getting Started ', function () {
+    this.tags('ciGroup6');
+
     before(async function () {
       await remote.setWindowSize(1200, 800);
     });
diff --git a/test/functional/apps/home/index.js b/test/functional/apps/home/index.js
index b4ac690c37e0a..496eaf9fd6af8 100644
--- a/test/functional/apps/home/index.js
+++ b/test/functional/apps/home/index.js
@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('homepage app', function () {
+    this.tags('ciGroup6');
+
     before(function () {
       return remote.setWindowSize(1200, 800);
     });
diff --git a/test/functional/apps/management/index.js b/test/functional/apps/management/index.js
index 29ff8ddb9ad78..4d4031b4e489b 100644
--- a/test/functional/apps/management/index.js
+++ b/test/functional/apps/management/index.js
@@ -21,33 +21,38 @@ export default function ({ getService, loadTestFile }) {
   const esArchiver = getService('esArchiver');
 
   describe('management', function () {
-    // on setup, we create an settingsPage instance
-    // that we will use for all the tests
-    before(async function () {
+    before(async () => {
       await esArchiver.unload('logstash_functional');
       await esArchiver.load('empty_kibana');
       await esArchiver.loadIfNeeded('makelogs');
     });
 
-    after(async function () {
+    after(async () => {
       await esArchiver.unload('makelogs');
       await esArchiver.unload('empty_kibana');
     });
 
-    loadTestFile(require.resolve('./_create_index_pattern_wizard'));
-    loadTestFile(require.resolve('./_index_pattern_create_delete'));
-    loadTestFile(require.resolve('./_index_pattern_results_sort'));
-    loadTestFile(require.resolve('./_index_pattern_popularity'));
-    loadTestFile(require.resolve('./_kibana_settings'));
-    loadTestFile(require.resolve('./_scripted_fields'));
-    loadTestFile(require.resolve('./_scripted_fields_preview'));
-    loadTestFile(require.resolve('./_index_pattern_filter'));
-    loadTestFile(require.resolve('./_scripted_fields_filter'));
-    loadTestFile(require.resolve('./_import_objects'));
-    loadTestFile(require.resolve('./_mgmt_import_saved_objects'));
-    loadTestFile(require.resolve('./_test_huge_fields'));
-    loadTestFile(require.resolve('./_handle_alias'));
-    loadTestFile(require.resolve('./_handle_version_conflict'));
-  });
+    describe('', function () {
+      this.tags('ciGroup7');
+
+      loadTestFile(require.resolve('./_create_index_pattern_wizard'));
+      loadTestFile(require.resolve('./_index_pattern_create_delete'));
+      loadTestFile(require.resolve('./_index_pattern_results_sort'));
+      loadTestFile(require.resolve('./_index_pattern_popularity'));
+      loadTestFile(require.resolve('./_kibana_settings'));
+      loadTestFile(require.resolve('./_scripted_fields'));
+      loadTestFile(require.resolve('./_scripted_fields_preview'));
+    });
 
+    describe('', function () {
+      this.tags('ciGroup8');
+
+      loadTestFile(require.resolve('./_index_pattern_filter'));
+      loadTestFile(require.resolve('./_scripted_fields_filter'));
+      loadTestFile(require.resolve('./_import_objects'));
+      loadTestFile(require.resolve('./_test_huge_fields'));
+      loadTestFile(require.resolve('./_handle_alias'));
+      loadTestFile(require.resolve('./_handle_version_conflict'));
+    });
+  });
 }
diff --git a/test/functional/apps/status_page/index.js b/test/functional/apps/status_page/index.js
index 2d25e20042111..52d26cd9f3a45 100644
--- a/test/functional/apps/status_page/index.js
+++ b/test/functional/apps/status_page/index.js
@@ -25,6 +25,8 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common']);
 
   describe('status page', function () {
+    this.tags('ciGroup1');
+
     beforeEach(async () => {
       await PageObjects.common.navigateToApp('status_page');
     });
diff --git a/test/functional/apps/timelion/index.js b/test/functional/apps/timelion/index.js
index 73c9375460b34..286f14f0fbcdd 100644
--- a/test/functional/apps/timelion/index.js
+++ b/test/functional/apps/timelion/index.js
@@ -24,6 +24,8 @@ export default function ({ getService, loadTestFile }) {
   const kibanaServer = getService('kibanaServer');
 
   describe('timelion app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       log.debug('Starting timelion before method');
       remote.setWindowSize(1280, 800);
diff --git a/test/functional/apps/visualize/index.js b/test/functional/apps/visualize/index.js
index 808d7fdfe5bf9..c8a7fafed0ad6 100644
--- a/test/functional/apps/visualize/index.js
+++ b/test/functional/apps/visualize/index.js
@@ -23,8 +23,9 @@ export default function ({ getService, loadTestFile }) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
+
   describe('visualize app', function () {
-    before(async function () {
+    before(async ()=> {
       log.debug('Starting visualize before method');
       remote.setWindowSize(1280, 800);
       await esArchiver.loadIfNeeded('logstash_functional');
@@ -32,30 +33,49 @@ export default function ({ getService, loadTestFile }) {
       await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'UTC', 'defaultIndex': 'logstash-*' });
     });
 
-    loadTestFile(require.resolve('./_embedding_chart'));
-    loadTestFile(require.resolve('./_inspector'));
-    loadTestFile(require.resolve('./_chart_types'));
-    loadTestFile(require.resolve('./_experimental_vis'));
-    loadTestFile(require.resolve('./_gauge_chart'));
-    loadTestFile(require.resolve('./_area_chart'));
-    loadTestFile(require.resolve('./_line_chart'));
-    loadTestFile(require.resolve('./_data_table'));
-    loadTestFile(require.resolve('./_data_table_nontimeindex'));
-    loadTestFile(require.resolve('./_pie_chart'));
-    loadTestFile(require.resolve('./_tag_cloud'));
-    loadTestFile(require.resolve('./_tile_map'));
-    loadTestFile(require.resolve('./_region_map'));
-    loadTestFile(require.resolve('./_vertical_bar_chart'));
-    loadTestFile(require.resolve('./_vertical_bar_chart_nontimeindex'));
-    loadTestFile(require.resolve('./_heatmap_chart'));
-    loadTestFile(require.resolve('./_point_series_options'));
-    loadTestFile(require.resolve('./_markdown_vis'));
-    loadTestFile(require.resolve('./_tsvb_chart'));
-    loadTestFile(require.resolve('./_shared_item'));
-    loadTestFile(require.resolve('./_input_control_vis'));
-    loadTestFile(require.resolve('./_histogram_request_start'));
-    loadTestFile(require.resolve('./_vega_chart'));
-    loadTestFile(require.resolve('./_lab_mode'));
-    loadTestFile(require.resolve('./_linked_saved_searches.js'));
+    describe('', function () {
+      this.tags('ciGroup9');
+
+      loadTestFile(require.resolve('./_embedding_chart'));
+      loadTestFile(require.resolve('./_chart_types'));
+      loadTestFile(require.resolve('./_area_chart'));
+      loadTestFile(require.resolve('./_data_table'));
+      loadTestFile(require.resolve('./_data_table_nontimeindex'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup10');
+
+      loadTestFile(require.resolve('./_inspector'));
+      loadTestFile(require.resolve('./_experimental_vis'));
+      loadTestFile(require.resolve('./_gauge_chart'));
+      loadTestFile(require.resolve('./_heatmap_chart'));
+      loadTestFile(require.resolve('./_input_control_vis'));
+      loadTestFile(require.resolve('./_histogram_request_start'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup11');
+
+      loadTestFile(require.resolve('./_line_chart'));
+      loadTestFile(require.resolve('./_pie_chart'));
+      loadTestFile(require.resolve('./_region_map'));
+      loadTestFile(require.resolve('./_point_series_options'));
+      loadTestFile(require.resolve('./_markdown_vis'));
+      loadTestFile(require.resolve('./_shared_item'));
+      loadTestFile(require.resolve('./_lab_mode'));
+      loadTestFile(require.resolve('./_linked_saved_searches'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup12');
+
+      loadTestFile(require.resolve('./_tag_cloud'));
+      loadTestFile(require.resolve('./_tile_map'));
+      loadTestFile(require.resolve('./_vertical_bar_chart'));
+      loadTestFile(require.resolve('./_vertical_bar_chart_nontimeindex'));
+      loadTestFile(require.resolve('./_tsvb_chart'));
+      loadTestFile(require.resolve('./_vega_chart'));
+    });
   });
 }
diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh
new file mode 100755
index 0000000000000..304ebd3cba1f3
--- /dev/null
+++ b/test/scripts/jenkins_ci_group.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh"
+
+node scripts/build --debug --oss;
+
+export TEST_BROWSER_HEADLESS=1
+export TEST_ES_FROM=${TEST_ES_FROM:-source}
+
+"$(FORCE_COLOR=0 yarn bin)/grunt" functionalTests:ensureAllTestsInCiGroup;
+
+"$(FORCE_COLOR=0 yarn bin)/grunt" "run:functionalTests_ciGroup${CI_GROUP}" --from=source;
+
+if [ "$CI_GROUP" == "1" ]; then
+  "$(FORCE_COLOR=0 yarn bin)/grunt" run:pluginFunctionalTestsRelease --from=source;
+fi
diff --git a/test/scripts/jenkins_selenium.sh b/test/scripts/jenkins_selenium.sh
deleted file mode 100755
index c90da64da7375..0000000000000
--- a/test/scripts/jenkins_selenium.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-source "$(dirname $0)/../../src/dev/ci_setup/setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/git_setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/java_setup.sh"
-
-node scripts/build --debug --oss;
-
-export TEST_BROWSER_HEADLESS=1
-export TEST_ES_FROM=${TEST_ES_FROM:-source}
-"$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:selenium --from=source;
diff --git a/test/scripts/jenkins_xpack.sh b/test/scripts/jenkins_xpack.sh
index 25fe08088cbbd..6acde537477a8 100755
--- a/test/scripts/jenkins_xpack.sh
+++ b/test/scripts/jenkins_xpack.sh
@@ -25,18 +25,12 @@ node scripts/jest --ci --no-cache --verbose
 echo ""
 echo ""
 
-
-echo " -> building and extracting default Kibana distributable for use in functional tests"
-cd "$KIBANA_DIR"
-node scripts/build --debug --no-oss
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
-
-export TEST_ES_FROM=${TEST_ES_FROM:-source}
-echo " -> Running functional and api tests"
+echo " -> Ensuring all functional tests are in a ciGroup"
 cd "$XPACK_DIR"
-node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir"
-echo ""
-echo ""
+node scripts/functional_tests --assert-none-excluded \
+  --include-tag ciGroup1 \
+  --include-tag ciGroup2 \
+  --include-tag ciGroup3 \
+  --include-tag ciGroup4 \
+  --include-tag ciGroup5 \
+  --include-tag ciGroup6
diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh
new file mode 100755
index 0000000000000..525ef6f395f02
--- /dev/null
+++ b/test/scripts/jenkins_xpack_ci_group.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh"
+
+export TEST_BROWSER_HEADLESS=1
+export XPACK_DIR="$(cd "$(dirname "$0")/../../x-pack"; pwd)"
+echo "-> XPACK_DIR ${XPACK_DIR}"
+
+echo " -> building and extracting default Kibana distributable for use in functional tests"
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+export TEST_ES_FROM=${TEST_ES_FROM:-source}
+echo " -> Running functional and api tests"
+cd "$XPACK_DIR"
+node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir" --include-tag "ciGroup$CI_GROUP"
+echo ""
+echo ""
diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js
index aa2273ce64212..41d4cabc3b63f 100644
--- a/x-pack/test/api_integration/apis/index.js
+++ b/x-pack/test/api_integration/apis/index.js
@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('apis', () => {
+  describe('apis', function () {
+    this.tags('ciGroup5');
+
     loadTestFile(require.resolve('./es'));
     loadTestFile(require.resolve('./security'));
     loadTestFile(require.resolve('./monitoring'));
diff --git a/x-pack/test/functional/apps/dashboard_mode/index.js b/x-pack/test/functional/apps/dashboard_mode/index.js
index e7907f4fa6508..5953dd4924c5d 100644
--- a/x-pack/test/functional/apps/dashboard_mode/index.js
+++ b/x-pack/test/functional/apps/dashboard_mode/index.js
@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('dashboard mode', function () {
+    this.tags('ciGroup3');
+
     loadTestFile(require.resolve('./dashboard_view_mode'));
   });
 }
diff --git a/x-pack/test/functional/apps/graph/index.js b/x-pack/test/functional/apps/graph/index.js
index d77ec07969e63..98b360320c2c7 100644
--- a/x-pack/test/functional/apps/graph/index.js
+++ b/x-pack/test/functional/apps/graph/index.js
@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('graph app', function () {
+    this.tags('ciGroup1');
+
     loadTestFile(require.resolve('./graph'));
   });
 }
diff --git a/x-pack/test/functional/apps/grok_debugger/index.js b/x-pack/test/functional/apps/grok_debugger/index.js
index 3dc15acbe5e30..75c05f35abd28 100644
--- a/x-pack/test/functional/apps/grok_debugger/index.js
+++ b/x-pack/test/functional/apps/grok_debugger/index.js
@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('logstash', () => {
+  describe('logstash', function () {
+    this.tags('ciGroup2');
+
     loadTestFile(require.resolve('./grok_debugger'));
   });
 }
diff --git a/x-pack/test/functional/apps/infra/home_page.ts b/x-pack/test/functional/apps/infra/home_page.ts
index d7efdb85838df..04382e4964cf0 100644
--- a/x-pack/test/functional/apps/infra/home_page.ts
+++ b/x-pack/test/functional/apps/infra/home_page.ts
@@ -15,6 +15,10 @@ export default ({ getPageObjects, getService }: KibanaFunctionalTestDefaultProvi
   const pageObjects = getPageObjects(['common', 'infraHome']);
 
   describe('Home page', () => {
+    before(async () => {
+      await esArchiver.load('empty_kibana');
+    });
+
     describe('without metrics present', () => {
       before(async () => await esArchiver.unload('infra'));
 
diff --git a/x-pack/test/functional/apps/infra/index.ts b/x-pack/test/functional/apps/infra/index.ts
index efb3cc9f276bc..87abb2584b6c2 100644
--- a/x-pack/test/functional/apps/infra/index.ts
+++ b/x-pack/test/functional/apps/infra/index.ts
@@ -8,7 +8,9 @@ import { KibanaFunctionalTestDefaultProviders } from '../../../types/providers';
 
 // tslint:disable-next-line:no-default-export
 export default ({ loadTestFile }: KibanaFunctionalTestDefaultProviders) => {
-  describe('InfraOps app', () => {
+  describe('InfraOps app', function() {
+    (this as any).tags('ciGroup4');
+
     loadTestFile(require.resolve('./home_page'));
   });
 };
diff --git a/x-pack/test/functional/apps/logstash/index.js b/x-pack/test/functional/apps/logstash/index.js
index bd7577e2b88f7..bc7d2941b7198 100644
--- a/x-pack/test/functional/apps/logstash/index.js
+++ b/x-pack/test/functional/apps/logstash/index.js
@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('logstash', () => {
+  describe('logstash', function () {
+    this.tags('ciGroup2');
+
     loadTestFile(require.resolve('./pipeline_list'));
     loadTestFile(require.resolve('./pipeline_create'));
   });
diff --git a/x-pack/test/functional/apps/monitoring/index.js b/x-pack/test/functional/apps/monitoring/index.js
index 817467f847ddc..a1551f2829a98 100644
--- a/x-pack/test/functional/apps/monitoring/index.js
+++ b/x-pack/test/functional/apps/monitoring/index.js
@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('Monitoring app', () => {
+  describe('Monitoring app', function () {
+    this.tags('ciGroup1');
+
     loadTestFile(require.resolve('./cluster/list'));
     loadTestFile(require.resolve('./cluster/overview'));
     loadTestFile(require.resolve('./cluster/alerts'));
diff --git a/x-pack/test/functional/apps/security/index.js b/x-pack/test/functional/apps/security/index.js
index 0bccef48960c3..2de59ddae3d3c 100644
--- a/x-pack/test/functional/apps/security/index.js
+++ b/x-pack/test/functional/apps/security/index.js
@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('security app', function () {
+    this.tags('ciGroup4');
+
     loadTestFile(require.resolve('./security'));
     loadTestFile(require.resolve('./doc_level_security_roles'));
     loadTestFile(require.resolve('./management'));
diff --git a/x-pack/test/functional/apps/spaces/index.ts b/x-pack/test/functional/apps/spaces/index.ts
index 455692b86fc81..76eb7d911b17b 100644
--- a/x-pack/test/functional/apps/spaces/index.ts
+++ b/x-pack/test/functional/apps/spaces/index.ts
@@ -8,6 +8,8 @@ import { TestInvoker } from './lib/types';
 // tslint:disable:no-default-export
 export default function spacesApp({ loadTestFile }: TestInvoker) {
   describe('Spaces app', function spacesAppTestSuite() {
+    (this as any).tags('ciGroup4');
+
     loadTestFile(require.resolve('./spaces_selection'));
   });
 }
diff --git a/x-pack/test/functional/apps/status_page/index.ts b/x-pack/test/functional/apps/status_page/index.ts
index 15e6711090e09..6b6d5c1d97600 100644
--- a/x-pack/test/functional/apps/status_page/index.ts
+++ b/x-pack/test/functional/apps/status_page/index.ts
@@ -8,6 +8,8 @@ import { TestInvoker } from './lib/types';
 // tslint:disable:no-default-export
 export default function statusPage({ loadTestFile }: TestInvoker) {
   describe('Status page', function statusPageTestSuite() {
+    (this as any).tags('ciGroup4');
+
     loadTestFile(require.resolve('./status_page'));
   });
 }
diff --git a/x-pack/test/functional/apps/watcher/index.js b/x-pack/test/functional/apps/watcher/index.js
index 80e60c213d211..5542f5cb8f0b7 100644
--- a/x-pack/test/functional/apps/watcher/index.js
+++ b/x-pack/test/functional/apps/watcher/index.js
@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('watcher app', function () {
+    this.tags('ciGroup1');
+
     //loadTestFile(require.resolve('./management'));
     loadTestFile(require.resolve('./watcher_test'));
   });
diff --git a/x-pack/test/reporting/api/chromium_tests.js b/x-pack/test/reporting/api/chromium_tests.js
index d9da4ac802cbc..c9282eda4f0cc 100644
--- a/x-pack/test/reporting/api/chromium_tests.js
+++ b/x-pack/test/reporting/api/chromium_tests.js
@@ -10,7 +10,9 @@ export default function ({ loadTestFile, getService }) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
-  describe('chromium', () => {
+  describe('chromium', function () {
+    this.tags('ciGroup6');
+
     before(async () => {
       await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
       await esArchiver.load(OSS_DATA_ARCHIVE_PATH);
diff --git a/x-pack/test/reporting/api/phantom_tests.js b/x-pack/test/reporting/api/phantom_tests.js
index 4a3212bff46ff..e0c8c11b326bd 100644
--- a/x-pack/test/reporting/api/phantom_tests.js
+++ b/x-pack/test/reporting/api/phantom_tests.js
@@ -10,7 +10,9 @@ export default function ({ loadTestFile, getService }) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
-  describe('phantom', () => {
+  describe('phantom', function () {
+    this.tags('ciGroup6');
+
     before(async () => {
       await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
       await esArchiver.load(OSS_DATA_ARCHIVE_PATH);
diff --git a/x-pack/test/reporting/configs/chromium_functional.js b/x-pack/test/reporting/configs/chromium_functional.js
index f14d943746645..80a9024c06ab8 100644
--- a/x-pack/test/reporting/configs/chromium_functional.js
+++ b/x-pack/test/reporting/configs/chromium_functional.js
@@ -13,7 +13,7 @@ export default async function ({ readConfigFile }) {
   return {
     ...functionalConfig,
     junit: {
-      reportName: 'X-Pack Chromium API Reporting Tests',
+      reportName: 'X-Pack Chromium Functional Reporting Tests',
     },
     testFiles: [require.resolve('../functional')],
     kbnTestServer: {
diff --git a/x-pack/test/reporting/configs/phantom_functional.js b/x-pack/test/reporting/configs/phantom_functional.js
index 48d2559479186..e9a61ee813cf2 100644
--- a/x-pack/test/reporting/configs/phantom_functional.js
+++ b/x-pack/test/reporting/configs/phantom_functional.js
@@ -13,7 +13,7 @@ export default async function ({ readConfigFile }) {
   return {
     ...functionalConfig,
     junit: {
-      reportName: 'X-Pack Phantom API Reporting Tests',
+      reportName: 'X-Pack Phantom Functional Reporting Tests',
     },
     testFiles: [require.resolve('../functional')],
     kbnTestServer: {
diff --git a/x-pack/test/reporting/functional/index.js b/x-pack/test/reporting/functional/index.js
index d99cf9a900153..fa473f454a925 100644
--- a/x-pack/test/reporting/functional/index.js
+++ b/x-pack/test/reporting/functional/index.js
@@ -6,6 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('reporting app', function () {
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./reporting'));
   });
 }
diff --git a/x-pack/test/saml_api_integration/apis/index.js b/x-pack/test/saml_api_integration/apis/index.js
index ce7c48029e66a..ac08d2e078abf 100644
--- a/x-pack/test/saml_api_integration/apis/index.js
+++ b/x-pack/test/saml_api_integration/apis/index.js
@@ -5,7 +5,8 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('apis SAML', () => {
+  describe('apis SAML', function () {
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./security'));
   });
 }
diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts
index d25a9b852b789..b2b62b501e765 100644
--- a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts
+++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts
@@ -12,7 +12,9 @@ export default function({ getService, loadTestFile }: TestInvoker) {
   const es = getService('es');
   const supertest = getService('supertest');
 
-  describe('saved objects security and spaces enabled', () => {
+  describe('saved objects security and spaces enabled', function() {
+    (this as any).tags('ciGroup5');
+
     before(async () => {
       await createUsersAndRoles(es, supertest);
     });
diff --git a/x-pack/test/saved_object_api_integration/security_only/apis/index.ts b/x-pack/test/saved_object_api_integration/security_only/apis/index.ts
index c9be7152f96ea..a0797300ebf2b 100644
--- a/x-pack/test/saved_object_api_integration/security_only/apis/index.ts
+++ b/x-pack/test/saved_object_api_integration/security_only/apis/index.ts
@@ -12,7 +12,9 @@ export default function({ getService, loadTestFile }: TestInvoker) {
   const es = getService('es');
   const supertest = getService('supertest');
 
-  describe('saved objects security only enabled', () => {
+  describe('saved objects security only enabled', function() {
+    (this as any).tags('ciGroup5');
+
     before(async () => {
       await createUsersAndRoles(es, supertest);
     });
diff --git a/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts b/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts
index 113cf86454d5f..b1028fa6c74c9 100644
--- a/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts
+++ b/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts
@@ -8,7 +8,9 @@ import { TestInvoker } from '../../common/lib/types';
 
 // tslint:disable:no-default-export
 export default function({ loadTestFile }: TestInvoker) {
-  describe('saved objects spaces only enabled', () => {
+  describe('saved objects spaces only enabled', function() {
+    (this as any).tags('ciGroup5');
+
     loadTestFile(require.resolve('./bulk_create'));
     loadTestFile(require.resolve('./bulk_get'));
     loadTestFile(require.resolve('./create'));
diff --git a/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts b/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts
index 65ab34e7ae8f8..6d8402fbb9244 100644
--- a/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts
+++ b/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts
@@ -12,7 +12,9 @@ export default function({ loadTestFile, getService }: TestInvoker) {
   const es = getService('es');
   const supertest = getService('supertest');
 
-  describe('spaces api with security', () => {
+  describe('spaces api with security', function() {
+    (this as any).tags('ciGroup5');
+
     before(async () => {
       await createUsersAndRoles(es, supertest);
     });
diff --git a/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts b/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts
index 6864ee7fbda94..75b546dd16022 100644
--- a/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts
+++ b/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts
@@ -8,7 +8,9 @@ import { TestInvoker } from '../../common/lib/types';
 
 // tslint:disable:no-default-export
 export default function spacesOnlyTestSuite({ loadTestFile }: TestInvoker) {
-  describe('spaces api without security', () => {
+  describe('spaces api without security', function() {
+    (this as any).tags('ciGroup5');
+
     loadTestFile(require.resolve('./create'));
     loadTestFile(require.resolve('./delete'));
     loadTestFile(require.resolve('./get_all'));