From b05c2d3e7719006ac6109b74cf203830c507c5e4 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 9 May 2018 19:02:00 -0500
Subject: [PATCH 01/19] Replace test:api with @kbn/test runTests

---
 tasks/config/run.js                    | 40 ++++++--------------------
 tasks/jenkins.js                       |  2 +-
 tasks/test.js                          | 19 +-----------
 test/api_integration/jenkins_config.js | 11 +++++++
 4 files changed, 22 insertions(+), 50 deletions(-)
 create mode 100644 test/api_integration/jenkins_config.js

diff --git a/tasks/config/run.js b/tasks/config/run.js
index 69bf726f6fb3d..8f112aa1c7b06 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -20,9 +20,6 @@
 import { esTestConfig, kbnTestConfig } from '@kbn/test';
 import { resolve } from 'path';
 
-const SECOND = 1000;
-const MINUTE = 60 * SECOND;
-const HOUR = 60 * MINUTE;
 const PKG_VERSION = require('../../package.json').version;
 
 module.exports = function (grunt) {
@@ -58,14 +55,6 @@ module.exports = function (grunt) {
     };
   }
 
-  const apiTestServerFlags = [
-    '--optimize.enabled=false',
-    '--elasticsearch.url=' + esTestConfig.getUrl(),
-    '--elasticsearch.healthCheck.delay=' + HOUR,
-    '--server.port=' + kbnTestConfig.getPort(),
-    '--server.xsrf.disableProtection=true',
-  ];
-
   const funcTestServerFlags = [
     '--server.maxPayloadBytes=1648576', //default is 1048576
     '--elasticsearch.url=' + esTestConfig.getUrl(),
@@ -117,25 +106,6 @@ module.exports = function (grunt) {
       ]
     },
 
-    // used by the test:api task
-    //    runs the kibana server prepared for the api_integration tests
-    apiTestServer: createKbnServerTask({
-      flags: [
-        ...apiTestServerFlags
-      ]
-    }),
-
-    // used by the test:api:server task
-    //    runs the kibana server in --dev mode, prepared for developing api_integration tests
-    //    and watching for changes so the server will restart when necessary
-    devApiTestServer: createKbnServerTask({
-      flags: [
-        '--dev',
-        '--no-base-path',
-        ...apiTestServerFlags,
-      ]
-    }),
-
     // used by test:ui task
     //    runs the kibana server prepared for the functional tests
     funcTestServer: createKbnServerTask({
@@ -224,6 +194,14 @@ module.exports = function (grunt) {
         'scripts/notice',
         '--validate'
       ]
-    }
+    },
+
+    api_integration_tests: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config ../../test/api_integration/jenkins_config.js',
+      ],
+    },
   };
 };
diff --git a/tasks/jenkins.js b/tasks/jenkins.js
index daece61ea4c78..22c36f17d3cf5 100644
--- a/tasks/jenkins.js
+++ b/tasks/jenkins.js
@@ -34,7 +34,7 @@ module.exports = function (grunt) {
     'test:jest_integration',
     'test:projects',
     'test:browser-ci',
-    'test:api',
+    'run:api_integration_tests',
     'verifyTranslations',
   ]);
 
diff --git a/tasks/test.js b/tasks/test.js
index af6ac2e7779e8..04f21e5a54bed 100644
--- a/tasks/test.js
+++ b/tasks/test.js
@@ -64,7 +64,7 @@ module.exports = function (grunt) {
     'test:jest_integration',
     'test:projects',
     'test:browser',
-    'test:api'
+    'run:api_integration_tests'
   ]);
 
   grunt.registerTask('test:dev', [
@@ -97,23 +97,6 @@ module.exports = function (grunt) {
     'run:devFuncTestServer:keepalive'
   ]);
 
-  grunt.registerTask('test:api', [
-    'run:testEsServer',
-    'run:apiTestServer',
-    'functional_test_runner:apiIntegration',
-    'stop:testEsServer',
-    'stop:apiTestServer'
-  ]);
-
-  grunt.registerTask('test:api:server', [
-    'run:testEsServer',
-    'run:devApiTestServer:keepalive'
-  ]);
-
-  grunt.registerTask('test:api:runner', () => {
-    grunt.fail.fatal('test:api:runner has moved, use: `node scripts/functional_test_runner --config test/api_integration/config.js`');
-  });
-
   grunt.registerTask('test', subTask => {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 
diff --git a/test/api_integration/jenkins_config.js b/test/api_integration/jenkins_config.js
new file mode 100644
index 0000000000000..118fcf5f1f2a8
--- /dev/null
+++ b/test/api_integration/jenkins_config.js
@@ -0,0 +1,11 @@
+export default async function ({ readConfigFile }) {
+  const apiConfig = await readConfigFile(require.resolve('./config'));
+
+  return {
+    ...apiConfig,
+    esTestCluster: {
+      ...apiConfig.esTestCluster,
+      from: 'source',
+    },
+  };
+}

From b6181ff2f6d9b1fba4a212d75b6c03193f04ec28 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 9 May 2018 20:23:33 -0500
Subject: [PATCH 02/19] Improve CLI help menu :sos:

---
 .../src/functional_tests/cli/run_tests_cli.js | 28 +++++++++----------
 .../functional_tests/cli/start_servers_cli.js | 25 +++++++++--------
 2 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
index 0f366b6278dfd..4d42f9ef1b9a1 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
@@ -81,23 +81,23 @@ function displayHelp() {
     dedent(`
     Run Functional Tests
 
-    Usage:  node scripts/functional_tests [options]
+    Usage:
+      node scripts/functional_tests [options]
+      node scripts/functional_tests --help
+      node scripts/functional_tests [--config <file1> [--config <file2> ...]]
 
-    --config      Option to pass in a config
-                  Can pass in multiple configs with
-                  --config file1 --config file2 --config file3
-    --kibana-install-dir
-                  Run Kibana from an existing install directory
-                  Default: run from source
-    --bail        Stop the test run at the first failure
-    --help        Display this menu and exit
+    Options:
+      --help                         Display this menu and exit.
+      --config=<file>                Option to pass in a config. Can pass in multiple configs.
+      --bail                         Stop the test run at the first failure.
+      --kibana-install-dir           Run Kibana from an existing install directory
+                                     Default: run from source
 
     Log level options:
-
-    --verbose
-    --debug
-    --quiet       Log errors
-    --silent
+      --verbose                      Log everything.
+      --debug                        Run in debug mode.
+      --quiet                        Only log errors.
+      --silent                       Log nothing.
     `)
   );
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
index 31266b8e7d0dc..9c102fb7f1856 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
@@ -80,20 +80,23 @@ function displayHelp() {
     dedent(`
     Start Functional Test Servers
 
-    Usage:  node scripts/functional_tests_server [options]
+    Usage:
+      node scripts/functional_tests_server [options]
+      node scripts/functional_tests_server --help
+      node scripts/functional_tests_server [--config <file>]
 
-    --config      Option to pass in a config
-    --kibana-install-dir
-                  Run Kibana from an existing install directory
-                  Default: run from source
-    --help        Display this menu and exit
+    Options:
+      --help                         Display this menu and exit.
+      --config=<file>                Option to pass in a config.
+      --bail                         Stop the test run at the first failure.
+      --kibana-install-dir           Run Kibana from an existing install directory
+                                     Default: run from source
 
     Log level options:
-
-    --verbose
-    --debug
-    --quiet       Log errors
-    --silent
+      --verbose                      Log everything.
+      --debug                        Run in debug mode.
+      --quiet                        Only log errors.
+      --silent                       Log nothing.
     `)
   );
 }

From 0374e6c633300df41ed19d0bb5d8003a813d5685 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 9 May 2018 20:25:29 -0500
Subject: [PATCH 03/19] Use --es-from

---
 packages/kbn-test/src/es/es_test_cluster.js    | 17 ++++++++---------
 .../src/functional_tests/cli/run_tests_cli.js  | 11 +++++++++--
 .../functional_tests/cli/start_servers_cli.js  | 14 +++++++++++---
 .../functional_tests/lib/run_elasticsearch.js  |  5 +++--
 .../kbn-test/src/functional_tests/tasks.js     | 18 ++++++++++--------
 tasks/config/run.js                            |  4 +++-
 test/api_integration/jenkins_config.js         | 11 -----------
 7 files changed, 44 insertions(+), 36 deletions(-)
 delete mode 100644 test/api_integration/jenkins_config.js

diff --git a/packages/kbn-test/src/es/es_test_cluster.js b/packages/kbn-test/src/es/es_test_cluster.js
index 8ff7a0efefaf6..dbbe7421a6cd8 100644
--- a/packages/kbn-test/src/es/es_test_cluster.js
+++ b/packages/kbn-test/src/es/es_test_cluster.js
@@ -29,13 +29,12 @@ import elasticsearch from 'elasticsearch';
 
 export function createEsTestCluster(options = {}) {
   const {
-    port = esTestConfig.getPort(),
-    password = 'changeme',
-    license = 'oss',
-    log,
-    basePath = resolve(KIBANA_ROOT, '.es'),
-    // Use source when running on CI
-    from = esTestConfig.getBuildFrom(),
+    port = options.port || esTestConfig.getPort(),
+    password = options.password || 'changeme',
+    license = options.license || 'oss',
+    log = options.log,
+    basePath = options.basePath || resolve(KIBANA_ROOT, '.es'),
+    esFrom = options.esFrom || esTestConfig.getBuildFrom(),
   } = options;
 
   const randomHash = Math.random()
@@ -58,12 +57,12 @@ export function createEsTestCluster(options = {}) {
       const second = 1000;
       const minute = second * 60;
 
-      return from === 'snapshot' ? minute : minute * 6;
+      return esFrom === 'snapshot' ? minute : minute * 6;
     }
 
     async start(esArgs = []) {
       const { installPath } =
-        from === 'source'
+        esFrom === 'source'
           ? await cluster.installSource(config)
           : await cluster.installSnapshot(config);
 
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
index 4d42f9ef1b9a1..d8cdb53cf7cc7 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
@@ -31,7 +31,7 @@ import { runTests } from '../../';
  *                                       if no config option is passed
  */
 export async function runTestsCli(defaultConfigPaths) {
-  const { configs, help, bail, log, installDir } = processArgs(
+  const { configs, help, bail, log, installDir, esFrom } = processArgs(
     defaultConfigPaths
   );
 
@@ -45,7 +45,7 @@ export async function runTestsCli(defaultConfigPaths) {
   }
 
   try {
-    await runTests(configs, { bail, log, installDir });
+    await runTests(configs, { bail, log, installDir, esFrom });
   } catch (err) {
     log.error('FATAL ERROR');
     log.error(err);
@@ -66,9 +66,13 @@ function processArgs(defaultConfigPaths) {
   const log = createToolingLog(pickLevelFromFlags(options));
   log.pipe(process.stdout);
 
+  // Default is 'snapshot', so unless option passed is 'source', use 'snapshot'.
+  const esFrom = options['es-from'] === 'source' ? 'source' : 'snapshot';
+
   return {
     configs,
     log,
+    esFrom,
     help: options.help,
     bail: options.bail,
     installDir: options['kibana-install-dir'],
@@ -89,6 +93,9 @@ function displayHelp() {
     Options:
       --help                         Display this menu and exit.
       --config=<file>                Option to pass in a config. Can pass in multiple configs.
+      --es-from=<snapshot|source>    Either build Elasticsearch from source
+                                     or run from snapshot
+                                     default: snapshot
       --bail                         Stop the test run at the first failure.
       --kibana-install-dir           Run Kibana from an existing install directory
                                      Default: run from source
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
index 9c102fb7f1856..3490b6e35e581 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
@@ -28,7 +28,9 @@ import { startServers } from '../../';
  * @param {string} configPath path to config
  */
 export async function startServersCli(defaultConfigPath) {
-  const { config, log, help, installDir } = processArgv(defaultConfigPath);
+  const { config, log, help, installDir, esFrom } = processArgv(
+    defaultConfigPath
+  );
 
   if (help) return displayHelp();
 
@@ -40,7 +42,7 @@ export async function startServersCli(defaultConfigPath) {
   }
 
   try {
-    await startServers(config, { log, installDir });
+    await startServers(config, { log, installDir, esFrom });
   } catch (err) {
     log.error('FATAL ERROR');
     log.error(err);
@@ -66,10 +68,14 @@ function processArgv(defaultConfigPath) {
   const log = createToolingLog(pickLevelFromFlags(options));
   log.pipe(process.stdout);
 
+  // Default is 'snapshot', so unless it is 'source', use 'snapshot'.
+  const esFrom = options['es-from'] === 'source' ? 'source' : 'snapshot';
+
   return {
     config,
     log,
     installDir: options.kibanaInstallDir,
+    esFrom,
     help: options.help,
     rest: options._,
   };
@@ -88,7 +94,9 @@ function displayHelp() {
     Options:
       --help                         Display this menu and exit.
       --config=<file>                Option to pass in a config.
-      --bail                         Stop the test run at the first failure.
+      --es-from=<snapshot|source>    Either build Elasticsearch from source
+                                     or run from snapshot
+                                     default: snapshot
       --kibana-install-dir           Run Kibana from an existing install directory
                                      Default: run from source
 
diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js
index 35b6d862a299c..efcc464faf059 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js
+++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js
@@ -23,7 +23,8 @@ import { createEsTestCluster } from '../../es';
 
 import { setupUsers, DEFAULT_SUPERUSER_PASS } from './auth';
 
-export async function runElasticsearch({ config, log }) {
+export async function runElasticsearch({ config, options }) {
+  const { log, esFrom } = options;
   const isOss = config.get('esTestCluster.license') === 'oss';
 
   const cluster = createEsTestCluster({
@@ -34,7 +35,7 @@ export async function runElasticsearch({ config, log }) {
     license: config.get('esTestCluster.license'),
     log,
     basePath: resolve(KIBANA_ROOT, '.es'),
-    from: config.get('esTestCluster.from'),
+    esFrom: esFrom || config.get('esTestCluster.from'),
   });
 
   const esArgs = config.get('esTestCluster.serverArgs');
diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index 6c4ceaf4891e4..edd6b78e7ce23 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -44,9 +44,10 @@ in another terminal session by running this command from this directory:
  * @param {string[]} configPaths         Array of paths to configs
  * @param {object}   options             Optional
  * @param {Log}      options.log         Optional logger
- * @param {string}   options.installDir  Optional installation dir
- *                                       from which to run Kibana
- * @param {boolean}  options.bail  Whether to exit test run at the first failure
+ * @property {string}   options.installDir  Optional installation dir
+ *                                          from which to run Kibana
+ * @property {boolean}  options.bail        Whether to exit test run at the first failure
+ * @property {string} options.esFrom        Optionally run from source instead of snapshot
  */
 export async function runTests(configPaths, options) {
   for (const configPath of configPaths) {
@@ -58,9 +59,10 @@ export async function runTests(configPaths, options) {
  * Start only servers using single config
  * @param {string}  configPath          Path to a config file
  * @param {object}  options             Optional
- * @param {Log}     options.log         Optional logger
- * @param {string}  options.installDir  Optional installation dir
- *                                      from which to run Kibana
+ * @property {Log}    options.log        Optional logger
+ * @property {string} options.installDir Optional installation dir
+ *                                       from which to run Kibana
+ * @property {string} options.esFrom     Optionally run from source instead of snapshot
  */
 export async function startServers(configPath, options) {
   const { log } = options;
@@ -69,7 +71,7 @@ export async function startServers(configPath, options) {
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
-    const es = await runElasticsearch({ config, log });
+    const es = await runElasticsearch({ config, options });
     await runKibanaServer({
       procs,
       config,
@@ -103,7 +105,7 @@ async function runSingleConfig(configPath, options) {
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
-    const es = await runElasticsearch({ config, log });
+    const es = await runElasticsearch({ config, options });
     await runKibanaServer({ procs, config, options });
 
     // Note: When solving how to incorporate functional_test_runner
diff --git a/tasks/config/run.js b/tasks/config/run.js
index 8f112aa1c7b06..6435947b9eb40 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -200,7 +200,9 @@ module.exports = function (grunt) {
       cmd: process.execPath,
       args: [
         'scripts/functional_tests',
-        '--config ../../test/api_integration/jenkins_config.js',
+        '--config', 'test/api_integration/config.js',
+        '--es-from', 'source',
+        '--debug',
       ],
     },
   };
diff --git a/test/api_integration/jenkins_config.js b/test/api_integration/jenkins_config.js
deleted file mode 100644
index 118fcf5f1f2a8..0000000000000
--- a/test/api_integration/jenkins_config.js
+++ /dev/null
@@ -1,11 +0,0 @@
-export default async function ({ readConfigFile }) {
-  const apiConfig = await readConfigFile(require.resolve('./config'));
-
-  return {
-    ...apiConfig,
-    esTestCluster: {
-      ...apiConfig.esTestCluster,
-      from: 'source',
-    },
-  };
-}

From b0221bcd4ff29a5cda29c4fd7fa4d55f0928be19 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Mon, 21 May 2018 14:54:05 -0500
Subject: [PATCH 04/19] Replace jenkins:selenium with kbn-test

---
 .../src/functional_tests/cli/run_tests_cli.js |  4 ++--
 .../functional_tests/lib/run_kibana_server.js | 23 +++++++++++++++----
 tasks/config/run.js                           | 14 ++++++++++-
 tasks/test.js                                 | 11 +++++----
 4 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
index d8cdb53cf7cc7..baac271f441cc 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
@@ -31,7 +31,7 @@ import { runTests } from '../../';
  *                                       if no config option is passed
  */
 export async function runTestsCli(defaultConfigPaths) {
-  const { configs, help, bail, log, installDir, esFrom } = processArgs(
+  const { configs, help, bail, log, installDir, esFrom, rest } = processArgs(
     defaultConfigPaths
   );
 
@@ -45,7 +45,7 @@ export async function runTestsCli(defaultConfigPaths) {
   }
 
   try {
-    await runTests(configs, { bail, log, installDir, esFrom });
+    await runTests(configs, { bail, log, installDir, esFrom, rest });
   } catch (err) {
     log.error('FATAL ERROR');
     log.error(err);
diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
index ae92fe9e34c97..207d84a259c5f 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
+++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
@@ -25,7 +25,7 @@ export async function runKibanaServer({ procs, config, options }) {
 
   await procs.run('kibana', {
     cmd: getKibanaCmd(installDir),
-    args: getCliArgs(config, options),
+    args: collectCliArgs(config, options),
     env: {
       FORCE_COLOR: 1,
       ...process.env,
@@ -45,14 +45,29 @@ function getKibanaCmd(installDir) {
   return KIBANA_EXEC;
 }
 
-function getCliArgs(config, { devMode, installDir }) {
+function collectCliArgs(config, { devMode, installDir }) {
   const buildArgs = config.get('kbnTestServer.buildArgs') || [];
   const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
-  const serverArgs = config.get('kbnTestServer.serverArgs') || [];
+  const baseServerArgs = config.get('kbnTestServer.serverArgs') || [];
 
-  if (devMode) serverArgs.push('--dev');
+  const serverArgs = pipe(
+    baseServerArgs,
+    args => (devMode ? [...args.push('--dev')] : args),
+    args => (installDir ? args.filter(a => a !== '--oss') : args)
+  );
 
   return installDir
     ? [...serverArgs, ...buildArgs]
     : [KIBANA_EXEC_PATH, ...serverArgs, ...sourceArgs];
 }
+
+/*
+ * Apply each function in fns to the result of the
+ * previous function. The first function's input
+ * is the arr array.
+ */
+function pipe(arr, ...fns) {
+  return fns.reduce((acc, fn) => {
+    return fn(arr);
+  }, []);
+}
diff --git a/tasks/config/run.js b/tasks/config/run.js
index 6435947b9eb40..f16aeab10c03e 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -202,7 +202,19 @@ module.exports = function (grunt) {
         'scripts/functional_tests',
         '--config', 'test/api_integration/config.js',
         '--es-from', 'source',
-        '--debug',
+      ],
+    },
+
+    functional_tests: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/functional/config.js',
+        '--es-from', 'source',
+        '--verbose',
+        '--server.maxPayloadBytes=1648576', //default is 1048576
+        '--env.name=development',
+        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
       ],
     },
   };
diff --git a/tasks/test.js b/tasks/test.js
index 04f21e5a54bed..88c7becf838b7 100644
--- a/tasks/test.js
+++ b/tasks/test.js
@@ -84,11 +84,12 @@ module.exports = function (grunt) {
 
   grunt.registerTask('test:uiRelease', [
     'checkPlugins',
-    'run:testEsServer',
-    'run:ossDistFuncTestServer',
-    'functional_test_runner:functional',
-    'stop:testEsServer',
-    'stop:ossDistFuncTestServer'
+    'run:functional_tests',
+    // 'run:testEsServer',
+    // 'run:ossDistFuncTestServer',
+    // 'functional_test_runner:functional',
+    // 'stop:testEsServer',
+    // 'stop:ossDistFuncTestServer'
   ]);
 
   grunt.registerTask('test:ui:server', [

From 9e26b1e3fc97a84105664e2860b2d2638fde92f1 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Tue, 22 May 2018 15:12:52 -0500
Subject: [PATCH 05/19] Validate cli args, fixing test in the process

---
 packages/kbn-test/src/es/es_test_cluster.js   |  12 +-
 .../src/functional_tests/cli/index.js         |   4 +-
 .../functional_tests/cli/run_tests/args.js    | 151 ++++++++++++++++++
 .../src/functional_tests/cli/run_tests/cli.js |  46 ++++++
 .../cli/start_servers/args.js                 | 130 +++++++++++++++
 .../functional_tests/cli/start_servers/cli.js |  42 +++++
 .../src/functional_tests/lib/run_ftr.js       |   9 +-
 .../functional_tests/lib/run_kibana_server.js |  25 +--
 .../kbn-test/src/functional_tests/tasks.js    |   7 +-
 tasks/config/run.js                           |   7 +-
 tasks/test.js                                 |   5 -
 11 files changed, 403 insertions(+), 35 deletions(-)
 create mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests/args.js
 create mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
 create mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers/args.js
 create mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers/cli.js

diff --git a/packages/kbn-test/src/es/es_test_cluster.js b/packages/kbn-test/src/es/es_test_cluster.js
index dbbe7421a6cd8..d969f7d6ec313 100644
--- a/packages/kbn-test/src/es/es_test_cluster.js
+++ b/packages/kbn-test/src/es/es_test_cluster.js
@@ -29,12 +29,12 @@ import elasticsearch from 'elasticsearch';
 
 export function createEsTestCluster(options = {}) {
   const {
-    port = options.port || esTestConfig.getPort(),
-    password = options.password || 'changeme',
-    license = options.license || 'oss',
-    log = options.log,
-    basePath = options.basePath || resolve(KIBANA_ROOT, '.es'),
-    esFrom = options.esFrom || esTestConfig.getBuildFrom(),
+    port = esTestConfig.getPort(),
+    password = 'changeme',
+    license = 'oss',
+    log,
+    basePath = resolve(KIBANA_ROOT, '.es'),
+    esFrom = esTestConfig.getBuildFrom(),
   } = options;
 
   const randomHash = Math.random()
diff --git a/packages/kbn-test/src/functional_tests/cli/index.js b/packages/kbn-test/src/functional_tests/cli/index.js
index a423672005950..3c6280d91f582 100644
--- a/packages/kbn-test/src/functional_tests/cli/index.js
+++ b/packages/kbn-test/src/functional_tests/cli/index.js
@@ -17,5 +17,5 @@
  * under the License.
  */
 
-export { runTestsCli } from './run_tests_cli';
-export { startServersCli } from './start_servers_cli';
+export { runTestsCli } from './run_tests/cli';
+export { startServersCli } from './start_servers/cli';
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
new file mode 100644
index 0000000000000..84b571e570858
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import dedent from 'dedent';
+import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+
+export const options = {
+  help: { desc: 'Display this menu and exit.' },
+  config: {
+    arg: '<file>',
+    desc: 'Pass in a config. Can pass in multiple configs.',
+  },
+  'es-from': {
+    arg: '<snapshot|source>',
+    choices: ['snapshot', 'source'],
+    desc: 'Build Elasticsearch from source or run from snapshot.',
+    default: 'snapshot',
+  },
+  'kibana-install-dir': {
+    arg: '<dir>',
+    desc: 'Run Kibana from existing install directory instead of from source.',
+  },
+  bail: { desc: 'Stop the test run at the first failure.' },
+  grep: {
+    arg: '<pattern>',
+    desc: 'Pattern to select which tests to run.',
+  },
+  updateBaselines: {
+    desc:
+      'Replace baseline screenshots with whatever is generated from the test.',
+  },
+  verbose: { desc: 'Log everything.' },
+  debug: { desc: 'Run in debug mode.' },
+  quiet: { desc: 'Only log errors.' },
+  silent: { desc: 'Log nothing.' },
+};
+
+export function displayHelp() {
+  const helpOptions = Object.keys(options)
+    .filter(opt => opt !== '_')
+    .map(name => {
+      return {
+        ...options[name],
+        usage: `${name} ${options[name].arg || ''}`,
+        default: options[name].default
+          ? `Default: ${options[name].default}`
+          : '',
+      };
+    })
+    .map(option => {
+      return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
+    })
+    .join(`\n      `);
+
+  console.log(
+    dedent(`
+    Run Functional Tests
+
+    Usage:
+      node scripts/functional_tests --help
+      node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+      node scripts/functional_tests [options] [-- --<other args>]
+
+    Options:
+      ${helpOptions}
+    `)
+  );
+}
+
+export function processOptions(userOptions, defaultConfigPaths) {
+  // Don't validate if help passed
+  if (userOptions.help) return userOptions;
+
+  validateOptions(userOptions);
+
+  let configs;
+  if (userOptions.config) {
+    configs = [].concat(userOptions.config);
+  } else {
+    if (!defaultConfigPaths || defaultConfigPaths.length === 0) {
+      console.log(chalk.red(`functional_tests: config is required`));
+      process.exit(9);
+    } else {
+      configs = defaultConfigPaths;
+    }
+  }
+
+  const log = createToolingLog(pickLevelFromFlags(userOptions));
+  log.pipe(process.stdout);
+
+  return {
+    ...userOptions,
+    configs,
+    log,
+    extraKbnOpts: userOptions._,
+  };
+}
+
+function validateOptions(userOptions) {
+  Object.entries(userOptions).forEach(([key, val]) => {
+    if (key === '_') return;
+
+    // Validate flags passed
+    if (options[key] === undefined) {
+      console.log(chalk.red(`functional_tests: invalid option [${key}]`));
+      process.exit(9);
+    }
+
+    if (
+      // Validate boolean flags
+      (!options[key].arg && typeof val !== 'boolean') ||
+      // Validate string/array flags
+      (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
+      // Validate enum flags
+      (options[key].choices && !options[key].choices.includes(val))
+    ) {
+      console.log(
+        chalk.red(
+          `functional_tests: invalid argument [${val}] to option [${key}]`
+        )
+      );
+      process.exit(9);
+    }
+  });
+}
+
+// test cases:
+// node scripts/functional_tests --config test/functional/config.js --es-from snapshot --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
+// node scripts/functional_tests --config test/functional/config.js --es-from butter --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
+// node scripts/functional_tests --config --es-from source --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
+// node scripts/functional_tests --config test/functional/config.js --es-from source --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
+// node scripts/functional_tests --config test/functional/config.js --es-from source --debug --server.maxPayloadBytes=1648576 --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
+// node scripts/functional_tests --config test/functional/config.js --config test/api_integration/config.js --es-from snapshot --debug --grep management --bail -- --server.maxPayloadBytes=1648576
+// node scripts/functional_tests --es-from snapshot --debug --grep management --bail -- --server.maxPayloadBytes=1648576
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
new file mode 100644
index 0000000000000..e011ebb42a480
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import getopts from 'getopts';
+import { runTests } from '../../../';
+import { processOptions, displayHelp } from './args';
+
+/**
+ * Run servers and tests for each config
+ * Only cares about --config option. Other options
+ * are passed directly to functional_test_runner, such as
+ * --bail, --verbose, etc.
+ * @param {string[]} defaultConfigPaths  Array of paths to configs to use
+ *                                       if no config option is passed
+ */
+export async function runTestsCli(defaultConfigPaths) {
+  const userOptions = getopts(process.argv.slice(2)) || {};
+  const options = processOptions(userOptions, defaultConfigPaths);
+  const { help, configs } = options;
+
+  if (help) return displayHelp();
+
+  try {
+    await runTests(configs, options);
+  } catch (err) {
+    options.log.error('FATAL ERROR');
+    options.log.error(err);
+    process.exit(1);
+  }
+}
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
new file mode 100644
index 0000000000000..bd5b1611a2ff8
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import dedent from 'dedent';
+import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+
+export const options = {
+  help: { desc: 'Display this menu and exit.' },
+  config: {
+    arg: '<file>',
+    desc: 'Pass in a config',
+  },
+  'es-from': {
+    arg: '<snapshot|source>',
+    choices: ['snapshot', 'source'],
+    desc: 'Build Elasticsearch from source or run from snapshot.',
+    default: 'snapshot',
+  },
+  'kibana-install-dir': {
+    arg: '<dir>',
+    desc: 'Run Kibana from existing install directory instead of from source.',
+  },
+  verbose: { desc: 'Log everything.' },
+  debug: { desc: 'Run in debug mode.' },
+  quiet: { desc: 'Only log errors.' },
+  silent: { desc: 'Log nothing.' },
+};
+
+export function displayHelp() {
+  const helpOptions = Object.keys(options)
+    .filter(opt => opt !== '_')
+    .map(name => {
+      return {
+        ...options[name],
+        usage: `${name} ${options[name].arg || ''}`,
+        default: options[name].default
+          ? `Default: ${options[name].default}`
+          : '',
+      };
+    })
+    .map(option => {
+      return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
+    })
+    .join(`\n      `);
+
+  console.log(
+    dedent(`
+    Run Functional Tests
+
+    Usage:
+      node scripts/functional_tests_server --help
+      node scripts/functional_tests_server [--config <file>]
+      node scripts/functional_tests_server [options] [-- --<other args>]
+
+    Options:
+      ${helpOptions}
+    `)
+  );
+}
+
+export function processOptions(userOptions, defaultConfigPath) {
+  // Don't validate if help passed
+  if (userOptions.help) return userOptions;
+
+  validateOptions(userOptions);
+
+  const config = userOptions.config || defaultConfigPath;
+
+  if (!config) {
+    console.log(chalk.red(`functional_tests_server: config is required`));
+    process.exit(9);
+  }
+
+  const log = createToolingLog(pickLevelFromFlags(userOptions));
+  log.pipe(process.stdout);
+
+  return {
+    ...userOptions,
+    config,
+    log,
+    extraKbnOpts: userOptions._,
+  };
+}
+
+function validateOptions(userOptions) {
+  Object.entries(userOptions).forEach(([key, val]) => {
+    if (key === '_') return;
+
+    // Validate flags passed
+    if (options[key] === undefined) {
+      console.log(
+        chalk.red(`functional_tests_server: invalid option [${key}]`)
+      );
+      process.exit(9);
+    }
+
+    if (
+      // Validate boolean flags
+      (!options[key].arg && typeof val !== 'boolean') ||
+      // Validate string/array flags
+      (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
+      // Validate enum flags
+      (options[key].choices && !options[key].choices.includes(val))
+    ) {
+      console.log(
+        chalk.red(
+          `functional_tests_server: invalid argument [${val}] to option [${key}]`
+        )
+      );
+      process.exit(9);
+    }
+  });
+}
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
new file mode 100644
index 0000000000000..11f5f00280785
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import getopts from 'getopts';
+import { startServers } from '../../../';
+import { processOptions, displayHelp } from './args';
+
+/**
+ * Start servers
+ * @param {string} configPath path to config
+ */
+export async function startServersCli(defaultConfigPath) {
+  const userOptions = getopts(process.argv.slice(2)) || {};
+  const options = processOptions(userOptions, defaultConfigPath);
+  const { config, help } = options;
+
+  if (help) return displayHelp();
+
+  try {
+    await startServers(config, options);
+  } catch (err) {
+    options.log.error('FATAL ERROR');
+    options.log.error(err);
+    process.exit(1);
+  }
+}
diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js
index 968ec3e1f678d..2031a0d97f518 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js
+++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js
@@ -22,15 +22,16 @@ import { KIBANA_FTR_SCRIPT, PROJECT_ROOT } from './paths';
 export async function runFtr({
   procs,
   configPath,
-  bail,
-  log,
   cwd = PROJECT_ROOT,
+  options,
 }) {
   const args = [KIBANA_FTR_SCRIPT];
 
-  if (getLogFlag(log)) args.push(`--${getLogFlag(log)}`);
-  if (bail) args.push('--bail');
+  if (getLogFlag(options.log)) args.push(`--${getLogFlag(options.log)}`);
+  if (options.bail) args.push('--bail');
   if (configPath) args.push('--config', configPath);
+  if (options.grep) args.push('--grep', options.grep);
+  if (options.updateBaselines) args.push('--updateBaselines');
 
   await procs.run('ftr', {
     cmd: 'node',
diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
index 207d84a259c5f..847d34274688f 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
+++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
@@ -45,20 +45,21 @@ function getKibanaCmd(installDir) {
   return KIBANA_EXEC;
 }
 
-function collectCliArgs(config, { devMode, installDir }) {
+function collectCliArgs(config, { installDir, extraKbnOpts }) {
   const buildArgs = config.get('kbnTestServer.buildArgs') || [];
   const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
-  const baseServerArgs = config.get('kbnTestServer.serverArgs') || [];
+  const serverArgs = config.get('kbnTestServer.serverArgs') || [];
 
-  const serverArgs = pipe(
-    baseServerArgs,
-    args => (devMode ? [...args.push('--dev')] : args),
-    args => (installDir ? args.filter(a => a !== '--oss') : args)
+  return pipe(
+    serverArgs,
+    args => (installDir ? args.filter(a => a !== '--oss') : args),
+    args => {
+      return installDir
+        ? args.concat(buildArgs)
+        : [KIBANA_EXEC_PATH, ...args, ...sourceArgs];
+    },
+    args => args.concat(extraKbnOpts || [])
   );
-
-  return installDir
-    ? [...serverArgs, ...buildArgs]
-    : [KIBANA_EXEC_PATH, ...serverArgs, ...sourceArgs];
 }
 
 /*
@@ -68,6 +69,6 @@ function collectCliArgs(config, { devMode, installDir }) {
  */
 function pipe(arr, ...fns) {
   return fns.reduce((acc, fn) => {
-    return fn(arr);
-  }, []);
+    return fn(acc);
+  }, arr);
 }
diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index edd6b78e7ce23..e24ab27489e78 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -75,7 +75,7 @@ export async function startServers(configPath, options) {
     await runKibanaServer({
       procs,
       config,
-      options: { ...options, devMode: true },
+      options: { ...options, extraKbnOpts: [...options.extraKbnOpts, '--dev'] },
     });
 
     // wait for 5 seconds of silence before logging the
@@ -100,7 +100,7 @@ async function silence(milliseconds, { log }) {
  * Start servers and run tests for single config
  */
 async function runSingleConfig(configPath, options) {
-  const { bail, log } = options;
+  const { log } = options;
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
@@ -113,9 +113,8 @@ async function runSingleConfig(configPath, options) {
     await runFtr({
       procs,
       configPath,
-      bail,
-      log,
       cwd: process.cwd(),
+      options,
     });
 
     await procs.stop('kibana');
diff --git a/tasks/config/run.js b/tasks/config/run.js
index f16aeab10c03e..7631ea9d6ddb2 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -202,6 +202,7 @@ module.exports = function (grunt) {
         'scripts/functional_tests',
         '--config', 'test/api_integration/config.js',
         '--es-from', 'source',
+        '--bail',
       ],
     },
 
@@ -212,9 +213,11 @@ module.exports = function (grunt) {
         '--config', 'test/functional/config.js',
         '--es-from', 'source',
         '--verbose',
-        '--server.maxPayloadBytes=1648576', //default is 1048576
-        '--env.name=development',
+        '--bail',
         '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
+        '--',
+        '--server.maxPayloadBytes=1648576',
+        '--env.name=development',
       ],
     },
   };
diff --git a/tasks/test.js b/tasks/test.js
index 88c7becf838b7..ba1359236cbdb 100644
--- a/tasks/test.js
+++ b/tasks/test.js
@@ -85,11 +85,6 @@ module.exports = function (grunt) {
   grunt.registerTask('test:uiRelease', [
     'checkPlugins',
     'run:functional_tests',
-    // 'run:testEsServer',
-    // 'run:ossDistFuncTestServer',
-    // 'functional_test_runner:functional',
-    // 'stop:testEsServer',
-    // 'stop:ossDistFuncTestServer'
   ]);
 
   grunt.registerTask('test:ui:server', [

From 70bc0fb7d8f24d9ec48a4a1a7af3de6f95cb3a95 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 23 May 2018 07:13:19 -0500
Subject: [PATCH 06/19] Clean up some stuff

---
 .../functional_tests/cli/run_tests/args.js    |  9 ------
 tasks/config/run.js                           | 29 ++++++++++++++++---
 tasks/jenkins.js                              |  2 +-
 tasks/test.js                                 | 13 +++------
 test/common/config.js                         |  2 +-
 5 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index 84b571e570858..c1db3a7749ffc 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -140,12 +140,3 @@ function validateOptions(userOptions) {
     }
   });
 }
-
-// test cases:
-// node scripts/functional_tests --config test/functional/config.js --es-from snapshot --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
-// node scripts/functional_tests --config test/functional/config.js --es-from butter --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
-// node scripts/functional_tests --config --es-from source --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
-// node scripts/functional_tests --config test/functional/config.js --es-from source --debug --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
-// node scripts/functional_tests --config test/functional/config.js --es-from source --debug --server.maxPayloadBytes=1648576 --grep management --bail=peanut -- --server.maxPayloadBytes=1648576
-// node scripts/functional_tests --config test/functional/config.js --config test/api_integration/config.js --es-from snapshot --debug --grep management --bail -- --server.maxPayloadBytes=1648576
-// node scripts/functional_tests --es-from snapshot --debug --grep management --bail -- --server.maxPayloadBytes=1648576
diff --git a/tasks/config/run.js b/tasks/config/run.js
index 7631ea9d6ddb2..1ecbbb42524b5 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -196,7 +196,7 @@ module.exports = function (grunt) {
       ]
     },
 
-    api_integration_tests: {
+    apiIntegrationTests: {
       cmd: process.execPath,
       args: [
         'scripts/functional_tests',
@@ -206,18 +206,39 @@ module.exports = function (grunt) {
       ],
     },
 
-    functional_tests: {
+    functionalTests: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/functional/config.js',
+        '--es-from', 'source',
+        '--bail',
+        '--',
+        '--server.maxPayloadBytes=1648576',
+      ],
+    },
+
+    functionalTestsRelease: {
       cmd: process.execPath,
       args: [
         'scripts/functional_tests',
         '--config', 'test/functional/config.js',
         '--es-from', 'source',
-        '--verbose',
         '--bail',
         '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
         '--',
         '--server.maxPayloadBytes=1648576',
-        '--env.name=development',
+      ],
+    },
+
+    functionalTestsDevServer: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests_server',
+        '--config', 'test/functional/config.js',
+        '--es-from', 'source',
+        '--',
+        '--server.maxPayloadBytes=1648576',
       ],
     },
   };
diff --git a/tasks/jenkins.js b/tasks/jenkins.js
index 22c36f17d3cf5..b766e1273b20a 100644
--- a/tasks/jenkins.js
+++ b/tasks/jenkins.js
@@ -34,7 +34,7 @@ module.exports = function (grunt) {
     'test:jest_integration',
     'test:projects',
     'test:browser-ci',
-    'run:api_integration_tests',
+    'run:apiIntegrationTests',
     'verifyTranslations',
   ]);
 
diff --git a/tasks/test.js b/tasks/test.js
index ba1359236cbdb..f83207aeef15b 100644
--- a/tasks/test.js
+++ b/tasks/test.js
@@ -64,7 +64,7 @@ module.exports = function (grunt) {
     'test:jest_integration',
     'test:projects',
     'test:browser',
-    'run:api_integration_tests'
+    'run:apiIntegrationTests'
   ]);
 
   grunt.registerTask('test:dev', [
@@ -75,22 +75,17 @@ module.exports = function (grunt) {
 
   grunt.registerTask('test:ui', [
     'checkPlugins',
-    'run:testEsServer',
-    'run:funcTestServer',
-    'functional_test_runner:functional',
-    'stop:testEsServer',
-    'stop:funcTestServer'
+    'run:functionalTests',
   ]);
 
   grunt.registerTask('test:uiRelease', [
     'checkPlugins',
-    'run:functional_tests',
+    'run:functionalTestsRelease',
   ]);
 
   grunt.registerTask('test:ui:server', [
     'checkPlugins',
-    'run:testEsServer',
-    'run:devFuncTestServer:keepalive'
+    'run:functionalTestsDevServer',
   ]);
 
   grunt.registerTask('test', subTask => {
diff --git a/test/common/config.js b/test/common/config.js
index 2f2f7d8920a50..2d677e4b57f78 100644
--- a/test/common/config.js
+++ b/test/common/config.js
@@ -49,7 +49,7 @@ export default function () {
         `--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
       ],
       serverArgs: [
-        '--env=development',
+        '--env.name=development',
         '--logging.json=false',
         `--server.port=${kbnTestConfig.getPort()}`,
         `--optimize.watchPort=${kbnTestConfig.getPort() + 10}`,

From 1adc41f79102706be1545d2d9545c0b18ff83f9d Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Tue, 5 Jun 2018 17:21:24 -0500
Subject: [PATCH 07/19] Code review fixes

---
 .../src/functional_tests/cli/run_tests/cli.js |  6 ++---
 .../functional_tests/cli/start_servers/cli.js |  9 ++++----
 .../src/functional_tests/lib/run_ftr.js       | 10 ++++-----
 .../kbn-test/src/functional_tests/tasks.js    | 22 +++++++++----------
 4 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
index e011ebb42a480..3ec915e31842c 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -31,10 +31,10 @@ import { processOptions, displayHelp } from './args';
  */
 export async function runTestsCli(defaultConfigPaths) {
   const userOptions = getopts(process.argv.slice(2)) || {};
-  const options = processOptions(userOptions, defaultConfigPaths);
-  const { help, configs } = options;
+  if (userOptions.help) return displayHelp();
 
-  if (help) return displayHelp();
+  const options = processOptions(userOptions, defaultConfigPaths);
+  const { configs } = options;
 
   try {
     await runTests(configs, options);
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
index 11f5f00280785..393c1df3023f9 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -23,14 +23,15 @@ import { processOptions, displayHelp } from './args';
 
 /**
  * Start servers
- * @param {string} configPath path to config
+ * @param {string} defaultConfigPath Path to config to use
+ *                                   if no config option is passed
  */
 export async function startServersCli(defaultConfigPath) {
   const userOptions = getopts(process.argv.slice(2)) || {};
-  const options = processOptions(userOptions, defaultConfigPath);
-  const { config, help } = options;
+  if (userOptions.help) return displayHelp();
 
-  if (help) return displayHelp();
+  const options = processOptions(userOptions, defaultConfigPath);
+  const { config } = options;
 
   try {
     await startServers(config, options);
diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js
index 2031a0d97f518..90a3289f72028 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js
+++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js
@@ -23,15 +23,15 @@ export async function runFtr({
   procs,
   configPath,
   cwd = PROJECT_ROOT,
-  options,
+  options: { log, bail, grep, updateBaselines },
 }) {
   const args = [KIBANA_FTR_SCRIPT];
 
-  if (getLogFlag(options.log)) args.push(`--${getLogFlag(options.log)}`);
-  if (options.bail) args.push('--bail');
+  if (getLogFlag(log)) args.push(`--${getLogFlag(log)}`);
+  if (bail) args.push('--bail');
   if (configPath) args.push('--config', configPath);
-  if (options.grep) args.push('--grep', options.grep);
-  if (options.updateBaselines) args.push('--updateBaselines');
+  if (grep) args.push('--grep', grep);
+  if (updateBaselines) args.push('--updateBaselines');
 
   await procs.run('ftr', {
     cmd: 'node',
diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index e24ab27489e78..a8668ced6053f 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -41,13 +41,12 @@ in another terminal session by running this command from this directory:
 
 /**
  * Run servers and tests for each config
- * @param {string[]} configPaths         Array of paths to configs
- * @param {object}   options             Optional
- * @param {Log}      options.log         Optional logger
- * @property {string}   options.installDir  Optional installation dir
- *                                          from which to run Kibana
- * @property {boolean}  options.bail        Whether to exit test run at the first failure
- * @property {string} options.esFrom        Optionally run from source instead of snapshot
+ * @param {string[]} configPaths          Array of paths to configs
+ * @param {object} options                Optional
+ * @property {Log} options.log            Optional logger
+ * @property {string} options.installDir  Optional installation dir from which to run Kibana
+ * @property {boolean} options.bail       Whether to exit test run at the first failure
+ * @property {string} options.esFrom      Optionally run from source instead of snapshot
  */
 export async function runTests(configPaths, options) {
   for (const configPath of configPaths) {
@@ -57,11 +56,10 @@ export async function runTests(configPaths, options) {
 
 /**
  * Start only servers using single config
- * @param {string}  configPath          Path to a config file
- * @param {object}  options             Optional
- * @property {Log}    options.log        Optional logger
- * @property {string} options.installDir Optional installation dir
- *                                       from which to run Kibana
+ * @param {string} configPath            Path to a config file
+ * @param {object} options               Optional
+ * @property {Log} options.log           Optional logger
+ * @property {string} options.installDir Optional installation dir from which to run Kibana
  * @property {string} options.esFrom     Optionally run from source instead of snapshot
  */
 export async function startServers(configPath, options) {

From abb0c5e2f401bcc7ae06c4b6a79ab7ca27e90a33 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Tue, 5 Jun 2018 17:30:39 -0500
Subject: [PATCH 08/19] Explanation for collectCliArgs

---
 .../kbn-test/src/functional_tests/lib/run_kibana_server.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
index 847d34274688f..730f816f9c727 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
+++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js
@@ -45,6 +45,11 @@ function getKibanaCmd(installDir) {
   return KIBANA_EXEC;
 }
 
+/* When installDir is passed, we run from a built version of Kibana,
+ * which uses different command line arguments. If installDir is not
+ * passed, we run from source code. We also allow passing in extra
+ * Kibana server options, so we tack those on here.
+ */
 function collectCliArgs(config, { installDir, extraKbnOpts }) {
   const buildArgs = config.get('kbnTestServer.buildArgs') || [];
   const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
@@ -55,7 +60,7 @@ function collectCliArgs(config, { installDir, extraKbnOpts }) {
     args => (installDir ? args.filter(a => a !== '--oss') : args),
     args => {
       return installDir
-        ? args.concat(buildArgs)
+        ? [...args, ...buildArgs]
         : [KIBANA_EXEC_PATH, ...args, ...sourceArgs];
     },
     args => args.concat(extraKbnOpts || [])

From 12e037ea549e4978286d83b933570aa772f2ce62 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Tue, 5 Jun 2018 17:41:12 -0500
Subject: [PATCH 09/19] Remove exit codes, they're useless anyway.

---
 .../src/functional_tests/cli/run_tests/args.js    |  9 +++------
 .../src/functional_tests/cli/run_tests/cli.js     | 15 +++++++--------
 .../functional_tests/cli/start_servers/args.js    |  9 +++------
 .../src/functional_tests/cli/start_servers/cli.js | 15 +++++++--------
 4 files changed, 20 insertions(+), 28 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index c1db3a7749ffc..e46eebe539936 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -85,9 +85,6 @@ export function displayHelp() {
 }
 
 export function processOptions(userOptions, defaultConfigPaths) {
-  // Don't validate if help passed
-  if (userOptions.help) return userOptions;
-
   validateOptions(userOptions);
 
   let configs;
@@ -96,7 +93,7 @@ export function processOptions(userOptions, defaultConfigPaths) {
   } else {
     if (!defaultConfigPaths || defaultConfigPaths.length === 0) {
       console.log(chalk.red(`functional_tests: config is required`));
-      process.exit(9);
+      throw new Error();
     } else {
       configs = defaultConfigPaths;
     }
@@ -120,7 +117,7 @@ function validateOptions(userOptions) {
     // Validate flags passed
     if (options[key] === undefined) {
       console.log(chalk.red(`functional_tests: invalid option [${key}]`));
-      process.exit(9);
+      throw new Error();
     }
 
     if (
@@ -136,7 +133,7 @@ function validateOptions(userOptions) {
           `functional_tests: invalid argument [${val}] to option [${key}]`
         )
       );
-      process.exit(9);
+      throw new Error();
     }
   });
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
index 3ec915e31842c..afb196e292a3c 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -30,17 +30,16 @@ import { processOptions, displayHelp } from './args';
  *                                       if no config option is passed
  */
 export async function runTestsCli(defaultConfigPaths) {
-  const userOptions = getopts(process.argv.slice(2)) || {};
-  if (userOptions.help) return displayHelp();
-
-  const options = processOptions(userOptions, defaultConfigPaths);
-  const { configs } = options;
-
   try {
+    const userOptions = getopts(process.argv.slice(2)) || {};
+    if (userOptions.help) return displayHelp();
+
+    const options = processOptions(userOptions, defaultConfigPaths);
+    const { configs } = options;
     await runTests(configs, options);
   } catch (err) {
-    options.log.error('FATAL ERROR');
-    options.log.error(err);
+    console.error('FATAL ERROR');
+    console.error(err);
     process.exit(1);
   }
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index bd5b1611a2ff8..52883ad2b076f 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -76,16 +76,13 @@ export function displayHelp() {
 }
 
 export function processOptions(userOptions, defaultConfigPath) {
-  // Don't validate if help passed
-  if (userOptions.help) return userOptions;
-
   validateOptions(userOptions);
 
   const config = userOptions.config || defaultConfigPath;
 
   if (!config) {
     console.log(chalk.red(`functional_tests_server: config is required`));
-    process.exit(9);
+    throw new Error();
   }
 
   const log = createToolingLog(pickLevelFromFlags(userOptions));
@@ -108,7 +105,7 @@ function validateOptions(userOptions) {
       console.log(
         chalk.red(`functional_tests_server: invalid option [${key}]`)
       );
-      process.exit(9);
+      throw new Error();
     }
 
     if (
@@ -124,7 +121,7 @@ function validateOptions(userOptions) {
           `functional_tests_server: invalid argument [${val}] to option [${key}]`
         )
       );
-      process.exit(9);
+      throw new Error();
     }
   });
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
index 393c1df3023f9..60f849ea17136 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -27,17 +27,16 @@ import { processOptions, displayHelp } from './args';
  *                                   if no config option is passed
  */
 export async function startServersCli(defaultConfigPath) {
-  const userOptions = getopts(process.argv.slice(2)) || {};
-  if (userOptions.help) return displayHelp();
-
-  const options = processOptions(userOptions, defaultConfigPath);
-  const { config } = options;
-
   try {
+    const userOptions = getopts(process.argv.slice(2)) || {};
+    if (userOptions.help) return displayHelp();
+
+    const options = processOptions(userOptions, defaultConfigPath);
+    const { config } = options;
     await startServers(config, options);
   } catch (err) {
-    options.log.error('FATAL ERROR');
-    options.log.error(err);
+    console.error('FATAL ERROR');
+    console.error(err);
     process.exit(1);
   }
 }

From 7c44f5e23538f65adb5d1f846ec0b437b2c9efea Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 6 Jun 2018 07:48:47 -0500
Subject: [PATCH 10/19] Make markdown vis test pass with dev_mode setting

---
 tasks/config/run.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tasks/config/run.js b/tasks/config/run.js
index 1ecbbb42524b5..70f8a72866856 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -215,6 +215,7 @@ module.exports = function (grunt) {
         '--bail',
         '--',
         '--server.maxPayloadBytes=1648576',
+        '--dev_mode.enabled=false',
       ],
     },
 
@@ -228,6 +229,7 @@ module.exports = function (grunt) {
         '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
         '--',
         '--server.maxPayloadBytes=1648576',
+        '--dev_mode.enabled=false',
       ],
     },
 
@@ -239,6 +241,7 @@ module.exports = function (grunt) {
         '--es-from', 'source',
         '--',
         '--server.maxPayloadBytes=1648576',
+        '--dev_mode.enabled=false',
       ],
     },
   };

From bd193d9c76ddb36c3945792bd1f90b49e162ef8c Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Thu, 7 Jun 2018 14:24:32 -0500
Subject: [PATCH 11/19] Tests

---
 .../cli/run_tests/args.test.js                | 161 +++++++++++++++
 .../src/functional_tests/cli/run_tests/cli.js |   4 +-
 .../cli/run_tests/cli.test.js                 | 195 ++++++++++++++++++
 .../src/functional_tests/cli/run_tests_cli.js | 110 ----------
 .../cli/start_servers/args.test.js            | 149 +++++++++++++
 .../functional_tests/cli/start_servers/cli.js |   4 +-
 .../cli/start_servers/cli.test.js             | 187 +++++++++++++++++
 .../functional_tests/cli/start_servers_cli.js | 110 ----------
 8 files changed, 696 insertions(+), 224 deletions(-)
 create mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
 create mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
 delete mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
 create mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
 create mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
 delete mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers_cli.js

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
new file mode 100644
index 0000000000000..68b62701d0f7a
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
@@ -0,0 +1,161 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { processOptions } from './args';
+
+describe('process options for run tests CLI', () => {
+  describe('options', () => {
+    const originalObjects = {};
+    // mocking instead of spying so that no output is sent to the test results
+    const logMock = jest.fn();
+
+    beforeAll(() => {
+      originalObjects.console = console;
+      global.console = { log: logMock };
+    });
+
+    afterAll(() => {
+      global.console = originalObjects.console;
+    });
+
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it('rejects boolean config value', () => {
+      expect(() => {
+        processOptions({ config: true });
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests: invalid argument [true] to option [config]'
+      );
+    });
+
+    it('rejects empty config value if no default passed', () => {
+      expect(() => {
+        processOptions({});
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests: config is required'
+      );
+    });
+
+    it('accepts empty config value if default passed', () => {
+      expect(() => {
+        processOptions({ config: '' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects non-boolean value for bail', () => {
+      expect(() => {
+        processOptions({ bail: 'peanut' }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests: invalid argument [peanut] to option [bail]'
+      );
+    });
+
+    it('accepts string value for kibana-install-dir', () => {
+      expect(() => {
+        processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects boolean value for kibana-install-dir', () => {
+      expect(() => {
+        processOptions({ 'kibana-install-dir': true }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests: invalid argument [true] to option [kibana-install-dir]'
+      );
+    });
+
+    it('accepts boolean value for updateBaselines', () => {
+      expect(() => {
+        processOptions({ updateBaselines: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts source value for es-from', () => {
+      expect(() => {
+        processOptions({ 'es-from': 'source' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects non-enum value for es-from', () => {
+      expect(() => {
+        processOptions({ 'es-from': 'butter' }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests: invalid argument [butter] to option [es-from]'
+      );
+    });
+
+    it('accepts value for grep', () => {
+      expect(() => {
+        processOptions({ grep: 'management' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts debug option', () => {
+      expect(() => {
+        processOptions({ debug: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts silent option', () => {
+      expect(() => {
+        processOptions({ silent: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts quiet option', () => {
+      expect(() => {
+        processOptions({ quiet: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts verbose option', () => {
+      expect(() => {
+        processOptions({ verbose: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts extra server options', () => {
+      expect(() => {
+        processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects invalid options even if valid options exist', () => {
+      expect(() => {
+        processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests: invalid option [aintnothang]'
+      );
+    });
+  });
+});
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
index afb196e292a3c..92f2a4d8fff36 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -38,8 +38,8 @@ export async function runTestsCli(defaultConfigPaths) {
     const { configs } = options;
     await runTests(configs, options);
   } catch (err) {
-    console.error('FATAL ERROR');
-    console.error(err);
+    console.log('FATAL ERROR');
+    console.log(err);
     process.exit(1);
   }
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
new file mode 100644
index 0000000000000..8d289cb61fc8b
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
@@ -0,0 +1,195 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { runTestsCli } from './cli';
+
+// Note: Stub the runTests function to keep testing only around the cli
+// method and arguments.
+jest.mock('../../../', () => ({
+  runTests: jest.fn(),
+}));
+
+describe('run tests CLI', () => {
+  describe('options', () => {
+    const originalObjects = {};
+    const exitMock = jest.fn();
+    const logMock = jest.fn(); // mock logging so we don't send output to the test results
+    const argvMock = ['foo', 'foo'];
+
+    const processMock = {
+      exit: exitMock,
+      argv: argvMock,
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      cwd: jest.fn(),
+    };
+
+    beforeAll(() => {
+      originalObjects.process = process;
+      originalObjects.console = console;
+      global.process = processMock;
+      global.console = { log: logMock };
+    });
+
+    afterAll(() => {
+      global.process = originalObjects.process;
+      global.console = originalObjects.console;
+    });
+
+    beforeEach(() => {
+      global.process.argv = [...argvMock];
+      jest.resetAllMocks();
+    });
+
+    it('rejects boolean config value', async () => {
+      global.process.argv.push('--config');
+
+      await runTestsCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('rejects empty config value if no default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await runTestsCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts empty config value if default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-boolean value for bail', async () => {
+      global.process.argv.push('--bail', 'peanut');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts string value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir', 'foo');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects boolean value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts boolean value for updateBaselines', async () => {
+      global.process.argv.push('--updateBaselines');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith();
+    });
+
+    it('accepts source value for es-from', async () => {
+      global.process.argv.push('--es-from', 'source');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-enum value for es-from', async () => {
+      global.process.argv.push('--es-from', 'butter');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts value for grep', async () => {
+      global.process.argv.push('--grep', 'management');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts debug option', async () => {
+      global.process.argv.push('--debug');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts silent option', async () => {
+      global.process.argv.push('--silent');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts quiet option', async () => {
+      global.process.argv.push('--quiet');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts verbose option', async () => {
+      global.process.argv.push('--verbose');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts extra server options', async () => {
+      global.process.argv.push('--', '--server.foo=bar');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts help option even if invalid options passed', async () => {
+      global.process.argv.push('--debug', '--aintnothang', '--help');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith(1);
+    });
+
+    it('rejects invalid options even if valid options exist', async () => {
+      global.process.argv.push('--debug', '--aintnothang', '--bail');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+  });
+});
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
deleted file mode 100644
index baac271f441cc..0000000000000
--- a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import dedent from 'dedent';
-import getopts from 'getopts';
-import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
-import { runTests } from '../../';
-
-/**
- * Run servers and tests for each config
- * Only cares about --config option. Other options
- * are passed directly to functional_test_runner, such as
- * --bail, --verbose, etc.
- * @param {string[]} defaultConfigPaths  Array of paths to configs to use
- *                                       if no config option is passed
- */
-export async function runTestsCli(defaultConfigPaths) {
-  const { configs, help, bail, log, installDir, esFrom, rest } = processArgs(
-    defaultConfigPaths
-  );
-
-  if (help) return displayHelp();
-
-  if (!configs || configs.length === 0) {
-    log.error(
-      `Run Tests requires at least one path to a config. Leave blank to use defaults.`
-    );
-    process.exit(9);
-  }
-
-  try {
-    await runTests(configs, { bail, log, installDir, esFrom, rest });
-  } catch (err) {
-    log.error('FATAL ERROR');
-    log.error(err);
-    process.exit(1);
-  }
-}
-
-function processArgs(defaultConfigPaths) {
-  // If no args are passed, use {}
-  const options = getopts(process.argv.slice(2)) || {};
-
-  // If --config is passed without paths, it's "true", so use default
-  const configs =
-    typeof options.config === 'string' || Array.isArray(options.config)
-      ? [].concat(options.config)
-      : defaultConfigPaths;
-
-  const log = createToolingLog(pickLevelFromFlags(options));
-  log.pipe(process.stdout);
-
-  // Default is 'snapshot', so unless option passed is 'source', use 'snapshot'.
-  const esFrom = options['es-from'] === 'source' ? 'source' : 'snapshot';
-
-  return {
-    configs,
-    log,
-    esFrom,
-    help: options.help,
-    bail: options.bail,
-    installDir: options['kibana-install-dir'],
-    rest: options._,
-  };
-}
-
-function displayHelp() {
-  console.log(
-    dedent(`
-    Run Functional Tests
-
-    Usage:
-      node scripts/functional_tests [options]
-      node scripts/functional_tests --help
-      node scripts/functional_tests [--config <file1> [--config <file2> ...]]
-
-    Options:
-      --help                         Display this menu and exit.
-      --config=<file>                Option to pass in a config. Can pass in multiple configs.
-      --es-from=<snapshot|source>    Either build Elasticsearch from source
-                                     or run from snapshot
-                                     default: snapshot
-      --bail                         Stop the test run at the first failure.
-      --kibana-install-dir           Run Kibana from an existing install directory
-                                     Default: run from source
-
-    Log level options:
-      --verbose                      Log everything.
-      --debug                        Run in debug mode.
-      --quiet                        Only log errors.
-      --silent                       Log nothing.
-    `)
-  );
-}
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
new file mode 100644
index 0000000000000..4de33c1949519
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { processOptions } from './args';
+
+describe('process options for run tests CLI', () => {
+  describe('options', () => {
+    const originalObjects = {};
+    // mocking instead of spying so that no output is sent to the test results
+    const logMock = jest.fn();
+
+    beforeAll(() => {
+      originalObjects.console = console;
+      global.console = { log: logMock };
+    });
+
+    afterAll(() => {
+      global.console = originalObjects.console;
+    });
+
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it('rejects boolean config value', () => {
+      expect(() => {
+        processOptions({ config: true });
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests_server: invalid argument [true] to option [config]'
+      );
+    });
+
+    it('rejects empty config value if no default passed', () => {
+      expect(() => {
+        processOptions({});
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests_server: config is required'
+      );
+    });
+
+    it('accepts empty config value if default passed', () => {
+      expect(() => {
+        processOptions({ config: '' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects bail', () => {
+      expect(() => {
+        processOptions({ bail: 'peanut' }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests_server: invalid option [bail]'
+      );
+    });
+
+    it('accepts string value for kibana-install-dir', () => {
+      expect(() => {
+        processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects boolean value for kibana-install-dir', () => {
+      expect(() => {
+        processOptions({ 'kibana-install-dir': true }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests_server: invalid argument [true] to option [kibana-install-dir]'
+      );
+    });
+
+    it('accepts source value for es-from', () => {
+      expect(() => {
+        processOptions({ 'es-from': 'source' }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects non-enum value for es-from', () => {
+      expect(() => {
+        processOptions({ 'es-from': 'butter' }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests_server: invalid argument [butter] to option [es-from]'
+      );
+    });
+
+    it('accepts debug option', () => {
+      expect(() => {
+        processOptions({ debug: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts silent option', () => {
+      expect(() => {
+        processOptions({ silent: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts quiet option', () => {
+      expect(() => {
+        processOptions({ quiet: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts verbose option', () => {
+      expect(() => {
+        processOptions({ verbose: true }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('accepts extra server options', () => {
+      expect(() => {
+        processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+      }).not.toThrow();
+    });
+
+    it('rejects invalid options even if valid options exist', () => {
+      expect(() => {
+        processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+      }).toThrow();
+
+      expect(logMock.mock.calls[0][0]).toContain(
+        'functional_tests_server: invalid option [aintnothang]'
+      );
+    });
+  });
+});
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
index 60f849ea17136..28779b5e5df9a 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -35,8 +35,8 @@ export async function startServersCli(defaultConfigPath) {
     const { config } = options;
     await startServers(config, options);
   } catch (err) {
-    console.error('FATAL ERROR');
-    console.error(err);
+    console.log('FATAL ERROR');
+    console.log(err);
     process.exit(1);
   }
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
new file mode 100644
index 0000000000000..8a10afd5e4e07
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
@@ -0,0 +1,187 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { startServersCli } from './cli';
+
+// Note: Stub the startServers function to keep testing only around the cli
+// method and arguments.
+jest.mock('../../../', () => ({
+  startServers: jest.fn(),
+}));
+
+describe('run tests CLI', () => {
+  describe('options', () => {
+    const originalObjects = {};
+    const exitMock = jest.fn();
+    const logMock = jest.fn(); // mock logging so we don't send output to the test results
+    const argvMock = ['foo', 'foo'];
+
+    const processMock = {
+      exit: exitMock,
+      argv: argvMock,
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      cwd: jest.fn(),
+    };
+
+    beforeAll(() => {
+      originalObjects.process = process;
+      originalObjects.console = console;
+      global.process = processMock;
+      global.console = { log: logMock };
+    });
+
+    afterAll(() => {
+      global.process = originalObjects.process;
+      global.console = originalObjects.console;
+    });
+
+    beforeEach(() => {
+      global.process.argv = [...argvMock];
+      jest.resetAllMocks();
+    });
+
+    it('rejects boolean config value', async () => {
+      global.process.argv.push('--config');
+
+      await startServersCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('rejects empty config value if no default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await startServersCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts empty config value if default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-boolean value for bail', async () => {
+      global.process.argv.push('--bail', 'peanut');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts string value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir', 'foo');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects boolean value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts boolean value for updateBaselines', async () => {
+      global.process.argv.push('--updateBaselines');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith();
+    });
+
+    it('accepts source value for es-from', async () => {
+      global.process.argv.push('--es-from', 'source');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-enum value for es-from', async () => {
+      global.process.argv.push('--es-from', 'butter');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+
+    it('accepts debug option', async () => {
+      global.process.argv.push('--debug');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts silent option', async () => {
+      global.process.argv.push('--silent');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts quiet option', async () => {
+      global.process.argv.push('--quiet');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts verbose option', async () => {
+      global.process.argv.push('--verbose');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts extra server options', async () => {
+      global.process.argv.push('--', '--server.foo=bar');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts help option even if invalid options passed', async () => {
+      global.process.argv.push('--debug', '--grep', '--help');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith(1);
+    });
+
+    it('rejects invalid options even if valid options exist', async () => {
+      global.process.argv.push('--debug', '--grep', '--bail');
+
+      await startServersCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+    });
+  });
+});
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
deleted file mode 100644
index 3490b6e35e581..0000000000000
--- a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import chalk from 'chalk';
-import dedent from 'dedent';
-import getopts from 'getopts';
-import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
-import { startServers } from '../../';
-
-/**
- * Start servers
- * @param {string} configPath path to config
- */
-export async function startServersCli(defaultConfigPath) {
-  const { config, log, help, installDir, esFrom } = processArgv(
-    defaultConfigPath
-  );
-
-  if (help) return displayHelp();
-
-  if (!config) {
-    log.error(
-      `Start Servers requires one path to a config. Leave blank to use default.`
-    );
-    process.exit(1);
-  }
-
-  try {
-    await startServers(config, { log, installDir, esFrom });
-  } catch (err) {
-    log.error('FATAL ERROR');
-    log.error(err);
-    process.exit(1);
-  }
-}
-
-function processArgv(defaultConfigPath) {
-  const options = getopts(process.argv.slice(2)) || {};
-
-  if (Array.isArray(options.config)) {
-    console.log(
-      chalk.red(
-        `Starting servers requires a single config path. Multiple were passed.`
-      )
-    );
-    process.exit(9);
-  }
-
-  const config =
-    typeof options.config === 'string' ? options.config : defaultConfigPath;
-
-  const log = createToolingLog(pickLevelFromFlags(options));
-  log.pipe(process.stdout);
-
-  // Default is 'snapshot', so unless it is 'source', use 'snapshot'.
-  const esFrom = options['es-from'] === 'source' ? 'source' : 'snapshot';
-
-  return {
-    config,
-    log,
-    installDir: options.kibanaInstallDir,
-    esFrom,
-    help: options.help,
-    rest: options._,
-  };
-}
-
-function displayHelp() {
-  console.log(
-    dedent(`
-    Start Functional Test Servers
-
-    Usage:
-      node scripts/functional_tests_server [options]
-      node scripts/functional_tests_server --help
-      node scripts/functional_tests_server [--config <file>]
-
-    Options:
-      --help                         Display this menu and exit.
-      --config=<file>                Option to pass in a config.
-      --es-from=<snapshot|source>    Either build Elasticsearch from source
-                                     or run from snapshot
-                                     default: snapshot
-      --kibana-install-dir           Run Kibana from an existing install directory
-                                     Default: run from source
-
-    Log level options:
-      --verbose                      Log everything.
-      --debug                        Run in debug mode.
-      --quiet                        Only log errors.
-      --silent                       Log nothing.
-    `)
-  );
-}

From 16e74858befe899b46b1deee55a5d9c46d4f1ada Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Thu, 7 Jun 2018 22:08:40 -0500
Subject: [PATCH 12/19] Remove unneeded export

---
 packages/kbn-test/src/functional_tests/cli/run_tests/args.js    | 2 +-
 .../kbn-test/src/functional_tests/cli/start_servers/args.js     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index e46eebe539936..f0a47dcbad64f 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -21,7 +21,7 @@ import chalk from 'chalk';
 import dedent from 'dedent';
 import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
-export const options = {
+const options = {
   help: { desc: 'Display this menu and exit.' },
   config: {
     arg: '<file>',
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index 52883ad2b076f..f9073d9dd3bd6 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -21,7 +21,7 @@ import chalk from 'chalk';
 import dedent from 'dedent';
 import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
-export const options = {
+const options = {
   help: { desc: 'Display this menu and exit.' },
   config: {
     arg: '<file>',

From 45ea76b73b1edea973c2b82605eefc1f333b0088 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Thu, 7 Jun 2018 22:29:13 -0500
Subject: [PATCH 13/19] Code review: move console logging up to cli.js

---
 .../functional_tests/cli/run_tests/args.js    |  14 +-
 .../cli/run_tests/args.test.js                | 235 ++++++++----------
 .../src/functional_tests/cli/run_tests/cli.js |   4 +-
 .../cli/start_servers/args.js                 |  18 +-
 .../cli/start_servers/args.test.js            |  47 +---
 .../functional_tests/cli/start_servers/cli.js |   4 +-
 6 files changed, 117 insertions(+), 205 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index f0a47dcbad64f..4c7d172ca3001 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -17,7 +17,6 @@
  * under the License.
  */
 
-import chalk from 'chalk';
 import dedent from 'dedent';
 import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -92,8 +91,7 @@ export function processOptions(userOptions, defaultConfigPaths) {
     configs = [].concat(userOptions.config);
   } else {
     if (!defaultConfigPaths || defaultConfigPaths.length === 0) {
-      console.log(chalk.red(`functional_tests: config is required`));
-      throw new Error();
+      throw new Error(`functional_tests: config is required`);
     } else {
       configs = defaultConfigPaths;
     }
@@ -116,8 +114,7 @@ function validateOptions(userOptions) {
 
     // Validate flags passed
     if (options[key] === undefined) {
-      console.log(chalk.red(`functional_tests: invalid option [${key}]`));
-      throw new Error();
+      throw new Error(`functional_tests: invalid option [${key}]`);
     }
 
     if (
@@ -128,12 +125,9 @@ function validateOptions(userOptions) {
       // Validate enum flags
       (options[key].choices && !options[key].choices.includes(val))
     ) {
-      console.log(
-        chalk.red(
-          `functional_tests: invalid argument [${val}] to option [${key}]`
-        )
+      throw new Error(
+        `functional_tests: invalid argument [${val}] to option [${key}]`
       );
-      throw new Error();
     }
   });
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
index 68b62701d0f7a..59ea32ea1c998 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
@@ -20,142 +20,103 @@
 import { processOptions } from './args';
 
 describe('process options for run tests CLI', () => {
-  describe('options', () => {
-    const originalObjects = {};
-    // mocking instead of spying so that no output is sent to the test results
-    const logMock = jest.fn();
-
-    beforeAll(() => {
-      originalObjects.console = console;
-      global.console = { log: logMock };
-    });
-
-    afterAll(() => {
-      global.console = originalObjects.console;
-    });
-
-    beforeEach(() => {
-      jest.resetAllMocks();
-    });
-
-    it('rejects boolean config value', () => {
-      expect(() => {
-        processOptions({ config: true });
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests: invalid argument [true] to option [config]'
-      );
-    });
-
-    it('rejects empty config value if no default passed', () => {
-      expect(() => {
-        processOptions({});
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests: config is required'
-      );
-    });
-
-    it('accepts empty config value if default passed', () => {
-      expect(() => {
-        processOptions({ config: '' }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('rejects non-boolean value for bail', () => {
-      expect(() => {
-        processOptions({ bail: 'peanut' }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests: invalid argument [peanut] to option [bail]'
-      );
-    });
-
-    it('accepts string value for kibana-install-dir', () => {
-      expect(() => {
-        processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('rejects boolean value for kibana-install-dir', () => {
-      expect(() => {
-        processOptions({ 'kibana-install-dir': true }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests: invalid argument [true] to option [kibana-install-dir]'
-      );
-    });
-
-    it('accepts boolean value for updateBaselines', () => {
-      expect(() => {
-        processOptions({ updateBaselines: true }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('accepts source value for es-from', () => {
-      expect(() => {
-        processOptions({ 'es-from': 'source' }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('rejects non-enum value for es-from', () => {
-      expect(() => {
-        processOptions({ 'es-from': 'butter' }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests: invalid argument [butter] to option [es-from]'
-      );
-    });
-
-    it('accepts value for grep', () => {
-      expect(() => {
-        processOptions({ grep: 'management' }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('accepts debug option', () => {
-      expect(() => {
-        processOptions({ debug: true }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('accepts silent option', () => {
-      expect(() => {
-        processOptions({ silent: true }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('accepts quiet option', () => {
-      expect(() => {
-        processOptions({ quiet: true }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('accepts verbose option', () => {
-      expect(() => {
-        processOptions({ verbose: true }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('accepts extra server options', () => {
-      expect(() => {
-        processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
-      }).not.toThrow();
-    });
-
-    it('rejects invalid options even if valid options exist', () => {
-      expect(() => {
-        processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests: invalid option [aintnothang]'
-      );
-    });
+  it('rejects boolean config value', () => {
+    expect(() => {
+      processOptions({ config: true });
+    }).toThrow('functional_tests: invalid argument [true] to option [config]');
+  });
+
+  it('rejects empty config value if no default passed', () => {
+    expect(() => {
+      processOptions({});
+    }).toThrow('functional_tests: config is required');
+  });
+
+  it('accepts empty config value if default passed', () => {
+    expect(() => {
+      processOptions({ config: '' }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('rejects non-boolean value for bail', () => {
+    expect(() => {
+      processOptions({ bail: 'peanut' }, ['foo']);
+    }).toThrow('functional_tests: invalid argument [peanut] to option [bail]');
+  });
+
+  it('accepts string value for kibana-install-dir', () => {
+    expect(() => {
+      processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('rejects boolean value for kibana-install-dir', () => {
+    expect(() => {
+      processOptions({ 'kibana-install-dir': true }, ['foo']);
+    }).toThrow(
+      'functional_tests: invalid argument [true] to option [kibana-install-dir]'
+    );
+  });
+
+  it('accepts boolean value for updateBaselines', () => {
+    expect(() => {
+      processOptions({ updateBaselines: true }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('accepts source value for es-from', () => {
+    expect(() => {
+      processOptions({ 'es-from': 'source' }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('rejects non-enum value for es-from', () => {
+    expect(() => {
+      processOptions({ 'es-from': 'butter' }, ['foo']);
+    }).toThrow(
+      'functional_tests: invalid argument [butter] to option [es-from]'
+    );
+  });
+
+  it('accepts value for grep', () => {
+    expect(() => {
+      processOptions({ grep: 'management' }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('accepts debug option', () => {
+    expect(() => {
+      processOptions({ debug: true }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('accepts silent option', () => {
+    expect(() => {
+      processOptions({ silent: true }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('accepts quiet option', () => {
+    expect(() => {
+      processOptions({ quiet: true }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('accepts verbose option', () => {
+    expect(() => {
+      processOptions({ verbose: true }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('accepts extra server options', () => {
+    expect(() => {
+      processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    }).not.toThrow();
+  });
+
+  it('rejects invalid options even if valid options exist', () => {
+    expect(() => {
+      processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+    }).toThrow('functional_tests: invalid option [aintnothang]');
   });
 });
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
index 92f2a4d8fff36..1257ede734546 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import chalk from 'chalk';
 import getopts from 'getopts';
 import { runTests } from '../../../';
 import { processOptions, displayHelp } from './args';
@@ -38,8 +39,7 @@ export async function runTestsCli(defaultConfigPaths) {
     const { configs } = options;
     await runTests(configs, options);
   } catch (err) {
-    console.log('FATAL ERROR');
-    console.log(err);
+    console.log(chalk.red(err));
     process.exit(1);
   }
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index f9073d9dd3bd6..56e2e09b7c01b 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -17,7 +17,6 @@
  * under the License.
  */
 
-import chalk from 'chalk';
 import dedent from 'dedent';
 import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -62,7 +61,7 @@ export function displayHelp() {
 
   console.log(
     dedent(`
-    Run Functional Tests
+    Start Functional Test Servers
 
     Usage:
       node scripts/functional_tests_server --help
@@ -81,8 +80,7 @@ export function processOptions(userOptions, defaultConfigPath) {
   const config = userOptions.config || defaultConfigPath;
 
   if (!config) {
-    console.log(chalk.red(`functional_tests_server: config is required`));
-    throw new Error();
+    throw new Error(`functional_tests_server: config is required`);
   }
 
   const log = createToolingLog(pickLevelFromFlags(userOptions));
@@ -102,10 +100,7 @@ function validateOptions(userOptions) {
 
     // Validate flags passed
     if (options[key] === undefined) {
-      console.log(
-        chalk.red(`functional_tests_server: invalid option [${key}]`)
-      );
-      throw new Error();
+      throw new Error(`functional_tests_server: invalid option [${key}]`);
     }
 
     if (
@@ -116,12 +111,9 @@ function validateOptions(userOptions) {
       // Validate enum flags
       (options[key].choices && !options[key].choices.includes(val))
     ) {
-      console.log(
-        chalk.red(
-          `functional_tests_server: invalid argument [${val}] to option [${key}]`
-        )
+      throw new Error(
+        `functional_tests_server: invalid argument [${val}] to option [${key}]`
       );
-      throw new Error();
     }
   });
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
index 4de33c1949519..c77c5fb815443 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
@@ -21,29 +21,10 @@ import { processOptions } from './args';
 
 describe('process options for run tests CLI', () => {
   describe('options', () => {
-    const originalObjects = {};
-    // mocking instead of spying so that no output is sent to the test results
-    const logMock = jest.fn();
-
-    beforeAll(() => {
-      originalObjects.console = console;
-      global.console = { log: logMock };
-    });
-
-    afterAll(() => {
-      global.console = originalObjects.console;
-    });
-
-    beforeEach(() => {
-      jest.resetAllMocks();
-    });
-
     it('rejects boolean config value', () => {
       expect(() => {
         processOptions({ config: true });
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
+      }).toThrow(
         'functional_tests_server: invalid argument [true] to option [config]'
       );
     });
@@ -51,11 +32,7 @@ describe('process options for run tests CLI', () => {
     it('rejects empty config value if no default passed', () => {
       expect(() => {
         processOptions({});
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests_server: config is required'
-      );
+      }).toThrow('functional_tests_server: config is required');
     });
 
     it('accepts empty config value if default passed', () => {
@@ -67,11 +44,7 @@ describe('process options for run tests CLI', () => {
     it('rejects bail', () => {
       expect(() => {
         processOptions({ bail: 'peanut' }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests_server: invalid option [bail]'
-      );
+      }).toThrow('functional_tests_server: invalid option [bail]');
     });
 
     it('accepts string value for kibana-install-dir', () => {
@@ -83,9 +56,7 @@ describe('process options for run tests CLI', () => {
     it('rejects boolean value for kibana-install-dir', () => {
       expect(() => {
         processOptions({ 'kibana-install-dir': true }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
+      }).toThrow(
         'functional_tests_server: invalid argument [true] to option [kibana-install-dir]'
       );
     });
@@ -99,9 +70,7 @@ describe('process options for run tests CLI', () => {
     it('rejects non-enum value for es-from', () => {
       expect(() => {
         processOptions({ 'es-from': 'butter' }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
+      }).toThrow(
         'functional_tests_server: invalid argument [butter] to option [es-from]'
       );
     });
@@ -139,11 +108,7 @@ describe('process options for run tests CLI', () => {
     it('rejects invalid options even if valid options exist', () => {
       expect(() => {
         processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
-      }).toThrow();
-
-      expect(logMock.mock.calls[0][0]).toContain(
-        'functional_tests_server: invalid option [aintnothang]'
-      );
+      }).toThrow('functional_tests_server: invalid option [aintnothang]');
     });
   });
 });
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
index 28779b5e5df9a..c6b9b7c30f925 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import chalk from 'chalk';
 import getopts from 'getopts';
 import { startServers } from '../../../';
 import { processOptions, displayHelp } from './args';
@@ -35,8 +36,7 @@ export async function startServersCli(defaultConfigPath) {
     const { config } = options;
     await startServers(config, options);
   } catch (err) {
-    console.log('FATAL ERROR');
-    console.log(err);
+    console.log(chalk.red(err));
     process.exit(1);
   }
 }

From f830ffbf2932cd84de245578953b456c86e37610 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Thu, 7 Jun 2018 23:03:12 -0500
Subject: [PATCH 14/19] Code review: refactor startServers and runTests to take
 single options collection

---
 .../src/functional_tests/cli/run_tests/cli.js      |  5 ++---
 .../src/functional_tests/cli/run_tests/cli.test.js |  2 +-
 .../src/functional_tests/cli/start_servers/cli.js  |  5 ++---
 packages/kbn-test/src/functional_tests/tasks.js    | 14 +++++++-------
 4 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
index 1257ede734546..01a6db8294d61 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -19,7 +19,7 @@
 
 import chalk from 'chalk';
 import getopts from 'getopts';
-import { runTests } from '../../../';
+import { runTests } from '../../tasks';
 import { processOptions, displayHelp } from './args';
 
 /**
@@ -36,8 +36,7 @@ export async function runTestsCli(defaultConfigPaths) {
     if (userOptions.help) return displayHelp();
 
     const options = processOptions(userOptions, defaultConfigPaths);
-    const { configs } = options;
-    await runTests(configs, options);
+    await runTests(options);
   } catch (err) {
     console.log(chalk.red(err));
     process.exit(1);
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
index 8d289cb61fc8b..517527c3f27a7 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
@@ -21,7 +21,7 @@ import { runTestsCli } from './cli';
 
 // Note: Stub the runTests function to keep testing only around the cli
 // method and arguments.
-jest.mock('../../../', () => ({
+jest.mock('../../tasks', () => ({
   runTests: jest.fn(),
 }));
 
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
index c6b9b7c30f925..4eb67bbfff93d 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -19,7 +19,7 @@
 
 import chalk from 'chalk';
 import getopts from 'getopts';
-import { startServers } from '../../../';
+import { startServers } from '../../tasks';
 import { processOptions, displayHelp } from './args';
 
 /**
@@ -33,8 +33,7 @@ export async function startServersCli(defaultConfigPath) {
     if (userOptions.help) return displayHelp();
 
     const options = processOptions(userOptions, defaultConfigPath);
-    const { config } = options;
-    await startServers(config, options);
+    await startServers(options);
   } catch (err) {
     console.log(chalk.red(err));
     process.exit(1);
diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index a8668ced6053f..0f3a50aa26e02 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -41,30 +41,30 @@ in another terminal session by running this command from this directory:
 
 /**
  * Run servers and tests for each config
- * @param {string[]} configPaths          Array of paths to configs
  * @param {object} options                Optional
+ * @property {string[]} configPaths       Array of paths to configs
  * @property {Log} options.log            Optional logger
  * @property {string} options.installDir  Optional installation dir from which to run Kibana
  * @property {boolean} options.bail       Whether to exit test run at the first failure
  * @property {string} options.esFrom      Optionally run from source instead of snapshot
  */
-export async function runTests(configPaths, options) {
-  for (const configPath of configPaths) {
+export async function runTests(options) {
+  for (const configPath of options.configs) {
     await runSingleConfig(resolve(process.cwd(), configPath), options);
   }
 }
 
 /**
  * Start only servers using single config
- * @param {string} configPath            Path to a config file
  * @param {object} options               Optional
+ * @property {string} options.configPath Path to a config file
  * @property {Log} options.log           Optional logger
  * @property {string} options.installDir Optional installation dir from which to run Kibana
  * @property {string} options.esFrom     Optionally run from source instead of snapshot
  */
-export async function startServers(configPath, options) {
-  const { log } = options;
-  configPath = resolve(process.cwd(), configPath);
+export async function startServers(options) {
+  const { config: configOption, log } = options;
+  const configPath = resolve(process.cwd(), configOption);
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);

From 904efa0a0723aa9bcbe28b58ff84a65ccd5f482c Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Thu, 7 Jun 2018 23:17:47 -0500
Subject: [PATCH 15/19] Code review: Remove all things I am sure we do not use

---
 tasks/config/functional_test_runner.js | 32 ---------------
 tasks/config/run.js                    | 54 --------------------------
 tasks/jenkins.js                       |  1 -
 3 files changed, 87 deletions(-)
 delete mode 100644 tasks/config/functional_test_runner.js

diff --git a/tasks/config/functional_test_runner.js b/tasks/config/functional_test_runner.js
deleted file mode 100644
index e57d823f1d4d5..0000000000000
--- a/tasks/config/functional_test_runner.js
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-export const functional = {
-  options: {
-    logLevel: 'debug',
-    configFile: require.resolve('../../test/functional/config.js')
-  }
-};
-
-export const apiIntegration = {
-  options: {
-    logLevel: 'debug',
-    configFile: require.resolve('../../test/api_integration/config.js')
-  }
-};
diff --git a/tasks/config/run.js b/tasks/config/run.js
index 70f8a72866856..1c1ae840e4eb8 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -17,7 +17,6 @@
  * under the License.
  */
 
-import { esTestConfig, kbnTestConfig } from '@kbn/test';
 import { resolve } from 'path';
 
 const PKG_VERSION = require('../../package.json').version;
@@ -55,12 +54,6 @@ module.exports = function (grunt) {
     };
   }
 
-  const funcTestServerFlags = [
-    '--server.maxPayloadBytes=1648576', //default is 1048576
-    '--elasticsearch.url=' + esTestConfig.getUrl(),
-    '--server.port=' + kbnTestConfig.getPort(),
-  ];
-
   const browserTestServerFlags = [
     '--plugins.initialize=false',
     '--optimize.bundleFilter=tests',
@@ -106,37 +99,6 @@ module.exports = function (grunt) {
       ]
     },
 
-    // used by test:ui task
-    //    runs the kibana server prepared for the functional tests
-    funcTestServer: createKbnServerTask({
-      flags: [
-        ...funcTestServerFlags,
-      ]
-    }),
-
-    // used by the test:ui:server task
-    //    runs the kibana server in dev mode, prepared for the functional tests
-    devFuncTestServer: createKbnServerTask({
-      flags: [
-        ...funcTestServerFlags,
-        '--dev',
-        '--dev_mode.enabled=false',
-        '--no-base-path',
-        '--optimize.watchPort=5611',
-        '--optimize.watchPrebuild=true',
-        '--optimize.bundleDir=' + resolve(__dirname, '../../optimize/testUiServer'),
-      ]
-    }),
-
-    // used by test:uiRelease task
-    //    runs the kibana server from the oss distributable prepared for the functional tests
-    ossDistFuncTestServer: createKbnServerTask({
-      runBuild: `oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
-      flags: [
-        ...funcTestServerFlags,
-      ]
-    }),
-
     // used by the test:browser task
     //    runs the kibana server to serve the browser test bundle
     browserTestServer: createKbnServerTask({
@@ -169,22 +131,6 @@ module.exports = function (grunt) {
       ]
     }),
 
-    testEsServer: {
-      options: {
-        wait: false,
-        ready: /started/,
-        quiet: false,
-      },
-      cmd: process.execPath,
-      args: [
-        'scripts/es',
-        grunt.option('from') || 'snapshot',
-        '--license', 'oss',
-        '-E', `http.port=${esTestConfig.getPort()}`,
-        '-E', `discovery.zen.ping.unicast.hosts=localhost:${esTestConfig.getPort()}`,
-      ],
-    },
-
     verifyNotice: {
       options: {
         wait: true,
diff --git a/tasks/jenkins.js b/tasks/jenkins.js
index b766e1273b20a..2f7f1d5b92ae2 100644
--- a/tasks/jenkins.js
+++ b/tasks/jenkins.js
@@ -38,7 +38,6 @@ module.exports = function (grunt) {
     'verifyTranslations',
   ]);
 
-  grunt.config.set('functional_test_runner.functional.options.configOverrides.mochaOpts.bail', true);
   grunt.registerTask('jenkins:selenium', [
     'test:uiRelease'
   ]);

From a33f81e5aa0d27c81aec2c1d46cddeaba537af18 Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 20 Jun 2018 13:51:40 -0500
Subject: [PATCH 16/19] Improve tests

---
 packages/kbn-es/yarn.lock                     |  10 ++
 .../functional_tests/cli/run_tests/args.js    |  12 +-
 .../cli/run_tests/args.test.js                |  59 ++++---
 .../cli/start_servers/args.js                 |  12 +-
 .../cli/start_servers/args.test.js            | 151 +++++++++---------
 5 files changed, 123 insertions(+), 121 deletions(-)

diff --git a/packages/kbn-es/yarn.lock b/packages/kbn-es/yarn.lock
index 442537a1fac8f..b7d41d26c48e3 100644
--- a/packages/kbn-es/yarn.lock
+++ b/packages/kbn-es/yarn.lock
@@ -246,6 +246,12 @@ readable-stream@^2.0.0, readable-stream@^2.0.5:
     string_decoder "~1.0.3"
     util-deprecate "~1.0.1"
 
+rxjs@^6.1.0:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.2.1.tgz#246cebec189a6cbc143a3ef9f62d6f4c91813ca1"
+  dependencies:
+    tslib "^1.9.0"
+
 safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
@@ -322,6 +328,10 @@ tree-kill@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36"
 
+tslib@^1.9.0:
+  version "1.9.2"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.2.tgz#8be0cc9a1f6dc7727c38deb16c2ebd1a2892988e"
+
 util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index 4c7d172ca3001..8d8663d1a1faf 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -53,14 +53,12 @@ const options = {
 
 export function displayHelp() {
   const helpOptions = Object.keys(options)
-    .filter(opt => opt !== '_')
-    .map(name => {
+    .filter(name => name !== '_')
+    .map(([name, option]) => {
       return {
-        ...options[name],
-        usage: `${name} ${options[name].arg || ''}`,
-        default: options[name].default
-          ? `Default: ${options[name].default}`
-          : '',
+        ...option,
+        usage: `${name} ${option.arg || ''}`,
+        default: option.default ? `Default: ${option.default}` : '',
       };
     })
     .map(option => {
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
index 59ea32ea1c998..8559ddfc0534a 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
@@ -33,9 +33,8 @@ describe('process options for run tests CLI', () => {
   });
 
   it('accepts empty config value if default passed', () => {
-    expect(() => {
-      processOptions({ config: '' }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ config: '' }, ['foo']);
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('rejects non-boolean value for bail', () => {
@@ -45,9 +44,9 @@ describe('process options for run tests CLI', () => {
   });
 
   it('accepts string value for kibana-install-dir', () => {
-    expect(() => {
-      processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    expect(options['kibana-install-dir']).toEqual('foo');
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('rejects boolean value for kibana-install-dir', () => {
@@ -59,15 +58,15 @@ describe('process options for run tests CLI', () => {
   });
 
   it('accepts boolean value for updateBaselines', () => {
-    expect(() => {
-      processOptions({ updateBaselines: true }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ updateBaselines: true }, ['foo']);
+    expect(options.updateBaselines).toEqual(true);
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('accepts source value for es-from', () => {
-    expect(() => {
-      processOptions({ 'es-from': 'source' }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ 'es-from': 'source' }, ['foo']);
+    expect(options['es-from']).toEqual('source');
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('rejects non-enum value for es-from', () => {
@@ -79,39 +78,39 @@ describe('process options for run tests CLI', () => {
   });
 
   it('accepts value for grep', () => {
-    expect(() => {
-      processOptions({ grep: 'management' }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ grep: 'management' }, ['foo']);
+    expect(options.grep).toEqual('management');
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('accepts debug option', () => {
-    expect(() => {
-      processOptions({ debug: true }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ debug: true }, ['foo']);
+    expect(options.debug).toEqual(true);
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('accepts silent option', () => {
-    expect(() => {
-      processOptions({ silent: true }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ silent: true }, ['foo']);
+    expect(options.silent).toEqual(true);
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('accepts quiet option', () => {
-    expect(() => {
-      processOptions({ quiet: true }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ quiet: true }, ['foo']);
+    expect(options.quiet).toEqual(true);
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('accepts verbose option', () => {
-    expect(() => {
-      processOptions({ verbose: true }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ verbose: true }, ['foo']);
+    expect(options.verbose).toEqual(true);
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('accepts extra server options', () => {
-    expect(() => {
-      processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
-    }).not.toThrow();
+    const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    expect(options._['server.foo']).toEqual('bar');
+    expect(options.configs).toEqual(['foo']);
   });
 
   it('rejects invalid options even if valid options exist', () => {
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index 56e2e09b7c01b..6ecff62f900e2 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -44,14 +44,12 @@ const options = {
 
 export function displayHelp() {
   const helpOptions = Object.keys(options)
-    .filter(opt => opt !== '_')
-    .map(name => {
+    .filter(name => name !== '_')
+    .map(([name, option]) => {
       return {
-        ...options[name],
-        usage: `${name} ${options[name].arg || ''}`,
-        default: options[name].default
-          ? `Default: ${options[name].default}`
-          : '',
+        ...option,
+        usage: `${name} ${option.arg || ''}`,
+        default: option.default ? `Default: ${option.default}` : '',
       };
     })
     .map(option => {
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
index c77c5fb815443..3b783dedcf76d 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
@@ -20,95 +20,92 @@
 import { processOptions } from './args';
 
 describe('process options for run tests CLI', () => {
-  describe('options', () => {
-    it('rejects boolean config value', () => {
-      expect(() => {
-        processOptions({ config: true });
-      }).toThrow(
-        'functional_tests_server: invalid argument [true] to option [config]'
-      );
-    });
+  it('rejects boolean config value', () => {
+    expect(() => {
+      processOptions({ config: true });
+    }).toThrow(
+      'functional_tests_server: invalid argument [true] to option [config]'
+    );
+  });
 
-    it('rejects empty config value if no default passed', () => {
-      expect(() => {
-        processOptions({});
-      }).toThrow('functional_tests_server: config is required');
-    });
+  it('rejects empty config value if no default passed', () => {
+    expect(() => {
+      processOptions({});
+    }).toThrow('functional_tests_server: config is required');
+  });
 
-    it('accepts empty config value if default passed', () => {
-      expect(() => {
-        processOptions({ config: '' }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts empty config value if default passed', () => {
+    const options = processOptions({ config: '' }, ['foo']);
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('rejects bail', () => {
-      expect(() => {
-        processOptions({ bail: 'peanut' }, ['foo']);
-      }).toThrow('functional_tests_server: invalid option [bail]');
-    });
+  it('rejects invalid option', () => {
+    expect(() => {
+      processOptions({ bail: true }, ['foo']);
+    }).toThrow('functional_tests_server: invalid option [bail]');
+  });
 
-    it('accepts string value for kibana-install-dir', () => {
-      expect(() => {
-        processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts string value for kibana-install-dir', () => {
+    const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    expect(options['kibana-install-dir']).toEqual('foo');
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('rejects boolean value for kibana-install-dir', () => {
-      expect(() => {
-        processOptions({ 'kibana-install-dir': true }, ['foo']);
-      }).toThrow(
-        'functional_tests_server: invalid argument [true] to option [kibana-install-dir]'
-      );
-    });
+  it('rejects boolean value for kibana-install-dir', () => {
+    expect(() => {
+      processOptions({ 'kibana-install-dir': true }, ['foo']);
+    }).toThrow(
+      'functional_tests_server: invalid argument [true] to option [kibana-install-dir]'
+    );
+  });
 
-    it('accepts source value for es-from', () => {
-      expect(() => {
-        processOptions({ 'es-from': 'source' }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts source value for es-from', () => {
+    const options = processOptions({ 'es-from': 'source' }, ['foo']);
+    expect(options['es-from']).toEqual('source');
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('rejects non-enum value for es-from', () => {
-      expect(() => {
-        processOptions({ 'es-from': 'butter' }, ['foo']);
-      }).toThrow(
-        'functional_tests_server: invalid argument [butter] to option [es-from]'
-      );
-    });
+  it('rejects non-enum value for es-from', () => {
+    expect(() => {
+      processOptions({ 'es-from': 'butter' }, ['foo']);
+    }).toThrow(
+      'functional_tests_server: invalid argument [butter] to option [es-from]'
+    );
+  });
 
-    it('accepts debug option', () => {
-      expect(() => {
-        processOptions({ debug: true }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts debug option', () => {
+    const options = processOptions({ debug: true }, ['foo']);
+    expect(options.debug).toEqual(true);
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('accepts silent option', () => {
-      expect(() => {
-        processOptions({ silent: true }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts silent option', () => {
+    const options = processOptions({ silent: true }, ['foo']);
+    expect(options.silent).toEqual(true);
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('accepts quiet option', () => {
-      expect(() => {
-        processOptions({ quiet: true }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts quiet option', () => {
+    const options = processOptions({ quiet: true }, ['foo']);
+    expect(options.quiet).toEqual(true);
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('accepts verbose option', () => {
-      expect(() => {
-        processOptions({ verbose: true }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts verbose option', () => {
+    const options = processOptions({ verbose: true }, ['foo']);
+    expect(options.verbose).toEqual(true);
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('accepts extra server options', () => {
-      expect(() => {
-        processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
-      }).not.toThrow();
-    });
+  it('accepts extra server options', () => {
+    const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    expect(options._['server.foo']).toEqual('bar');
+    expect(options.config).toEqual(['foo']);
+  });
 
-    it('rejects invalid options even if valid options exist', () => {
-      expect(() => {
-        processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
-      }).toThrow('functional_tests_server: invalid option [aintnothang]');
-    });
+  it('rejects invalid options even if valid options exist', () => {
+    expect(() => {
+      processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+    }).toThrow('functional_tests_server: invalid option [aintnothang]');
   });
 });

From e16024e4cc0b546bdfeab4740895f729524fbbad Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Wed, 20 Jun 2018 15:14:42 -0500
Subject: [PATCH 17/19] Code review fixes

---
 .../run_tests/__snapshots__/args.test.js.snap | 137 ++++++++++++++++++
 .../run_tests/__snapshots__/cli.test.js.snap  |  75 ++++++++++
 .../functional_tests/cli/run_tests/args.js    |  18 ++-
 .../cli/run_tests/args.test.js                |  37 +++--
 .../src/functional_tests/cli/run_tests/cli.js |   9 +-
 .../cli/run_tests/cli.test.js                 |   7 +
 .../__snapshots__/args.test.js.snap           | 111 ++++++++++++++
 .../__snapshots__/cli.test.js.snap            |  57 ++++++++
 .../cli/start_servers/args.js                 |  18 ++-
 .../cli/start_servers/args.test.js            |  33 ++---
 .../functional_tests/cli/start_servers/cli.js |   7 +-
 .../cli/start_servers/cli.test.js             |  13 +-
 .../kbn-test/src/functional_tests/tasks.js    |  27 ++--
 tasks/config/run.js                           |   4 +
 14 files changed, 479 insertions(+), 74 deletions(-)
 create mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
 create mode 100644 packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
 create mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
 create mode 100644 packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
new file mode 100644
index 0000000000000..381fd6c3e1e9e
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
@@ -0,0 +1,137 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`display help for run tests CLI displays as expected 1`] = `
+"Run Functional Tests
+
+Usage:
+  node scripts/functional_tests --help
+  node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+  node scripts/functional_tests [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config. Can pass in multiple configs. 
+  --es-from <snapshot|source>    Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --bail                         Stop the test run at the first failure. 
+  --grep <pattern>               Pattern to select which tests to run. 
+  --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing."
+`;
+
+exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "updateBaselines": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts debug option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "debug": true,
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
+Object {
+  "config": "",
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts extra server options 1`] = `
+Object {
+  "_": Object {
+    "server.foo": "bar",
+  },
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": Object {
+    "server.foo": "bar",
+  },
+}
+`;
+
+exports[`process options for run tests CLI accepts quiet option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "quiet": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts silent option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "silent": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts source value for es-from 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "es-from": "source",
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "kibana-install-dir": "foo",
+}
+`;
+
+exports[`process options for run tests CLI accepts value for grep 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "grep": "management",
+}
+`;
+
+exports[`process options for run tests CLI accepts verbose option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "verbose": true,
+}
+`;
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
new file mode 100644
index 0000000000000..e185c8c6b872d
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
@@ -0,0 +1,75 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`run tests CLI options accepts help option even if invalid options passed 1`] = `
+Array [
+  Array [
+    "Run Functional Tests
+
+Usage:
+  node scripts/functional_tests --help
+  node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+  node scripts/functional_tests [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config. Can pass in multiple configs. 
+  --es-from <snapshot|source>    Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --bail                         Stop the test run at the first failure. 
+  --grep <pattern>               Pattern to select which tests to run. 
+  --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing.",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects boolean config value 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [true] to option [config]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects boolean value for kibana-install-dir 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [true] to option [kibana-install-dir]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects empty config value if no default passed 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: config is required",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects invalid options even if valid options exist 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid option [aintnothang]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects non-boolean value for bail 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [peanut] to option [bail]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects non-enum value for es-from 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [butter] to option [es-from]",
+  ],
+]
+`;
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index 8d8663d1a1faf..ff2fcaed53617 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -54,7 +54,8 @@ const options = {
 export function displayHelp() {
   const helpOptions = Object.keys(options)
     .filter(name => name !== '_')
-    .map(([name, option]) => {
+    .map(name => {
+      const option = options[name];
       return {
         ...option,
         usage: `${name} ${option.arg || ''}`,
@@ -66,8 +67,7 @@ export function displayHelp() {
     })
     .join(`\n      `);
 
-  console.log(
-    dedent(`
+  return dedent(`
     Run Functional Tests
 
     Usage:
@@ -77,8 +77,7 @@ export function displayHelp() {
 
     Options:
       ${helpOptions}
-    `)
-  );
+    `);
 }
 
 export function processOptions(userOptions, defaultConfigPaths) {
@@ -95,13 +94,16 @@ export function processOptions(userOptions, defaultConfigPaths) {
     }
   }
 
-  const log = createToolingLog(pickLevelFromFlags(userOptions));
-  log.pipe(process.stdout);
+  function createLogger() {
+    const log = createToolingLog(pickLevelFromFlags(userOptions));
+    log.pipe(process.stdout);
+    return log;
+  }
 
   return {
     ...userOptions,
     configs,
-    log,
+    createLogger,
     extraKbnOpts: userOptions._,
   };
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
index 8559ddfc0534a..0e205d8e0c36f 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
@@ -17,7 +17,13 @@
  * under the License.
  */
 
-import { processOptions } from './args';
+import { displayHelp, processOptions } from './args';
+
+describe('display help for run tests CLI', () => {
+  it('displays as expected', () => {
+    expect(displayHelp()).toMatchSnapshot();
+  });
+});
 
 describe('process options for run tests CLI', () => {
   it('rejects boolean config value', () => {
@@ -34,7 +40,7 @@ describe('process options for run tests CLI', () => {
 
   it('accepts empty config value if default passed', () => {
     const options = processOptions({ config: '' }, ['foo']);
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects non-boolean value for bail', () => {
@@ -45,8 +51,7 @@ describe('process options for run tests CLI', () => {
 
   it('accepts string value for kibana-install-dir', () => {
     const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
-    expect(options['kibana-install-dir']).toEqual('foo');
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects boolean value for kibana-install-dir', () => {
@@ -59,14 +64,12 @@ describe('process options for run tests CLI', () => {
 
   it('accepts boolean value for updateBaselines', () => {
     const options = processOptions({ updateBaselines: true }, ['foo']);
-    expect(options.updateBaselines).toEqual(true);
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts source value for es-from', () => {
     const options = processOptions({ 'es-from': 'source' }, ['foo']);
-    expect(options['es-from']).toEqual('source');
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects non-enum value for es-from', () => {
@@ -79,38 +82,32 @@ describe('process options for run tests CLI', () => {
 
   it('accepts value for grep', () => {
     const options = processOptions({ grep: 'management' }, ['foo']);
-    expect(options.grep).toEqual('management');
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts debug option', () => {
     const options = processOptions({ debug: true }, ['foo']);
-    expect(options.debug).toEqual(true);
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts silent option', () => {
     const options = processOptions({ silent: true }, ['foo']);
-    expect(options.silent).toEqual(true);
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts quiet option', () => {
     const options = processOptions({ quiet: true }, ['foo']);
-    expect(options.quiet).toEqual(true);
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts verbose option', () => {
     const options = processOptions({ verbose: true }, ['foo']);
-    expect(options.verbose).toEqual(true);
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts extra server options', () => {
     const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
-    expect(options._['server.foo']).toEqual('bar');
-    expect(options.configs).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid options even if valid options exist', () => {
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
index 01a6db8294d61..2345dffbb2376 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
@@ -27,13 +27,16 @@ import { processOptions, displayHelp } from './args';
  * Only cares about --config option. Other options
  * are passed directly to functional_test_runner, such as
  * --bail, --verbose, etc.
- * @param {string[]} defaultConfigPaths  Array of paths to configs to use
- *                                       if no config option is passed
+ * @param {string[]} defaultConfigPaths Optional paths to configs
+ *                                      if no config option is passed
  */
 export async function runTestsCli(defaultConfigPaths) {
   try {
     const userOptions = getopts(process.argv.slice(2)) || {};
-    if (userOptions.help) return displayHelp();
+    if (userOptions.help) {
+      console.log(displayHelp());
+      return undefined;
+    }
 
     const options = processOptions(userOptions, defaultConfigPaths);
     await runTests(options);
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
index 517527c3f27a7..e9bad10f8a0b8 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
@@ -62,6 +62,7 @@ describe('run tests CLI', () => {
       await runTestsCli();
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('rejects empty config value if no default passed', async () => {
@@ -70,6 +71,7 @@ describe('run tests CLI', () => {
       await runTestsCli();
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts empty config value if default passed', async () => {
@@ -86,6 +88,7 @@ describe('run tests CLI', () => {
       await runTestsCli(['foo']);
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts string value for kibana-install-dir', async () => {
@@ -102,6 +105,7 @@ describe('run tests CLI', () => {
       await runTestsCli(['foo']);
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts boolean value for updateBaselines', async () => {
@@ -126,6 +130,7 @@ describe('run tests CLI', () => {
       await runTestsCli(['foo']);
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts value for grep', async () => {
@@ -182,6 +187,7 @@ describe('run tests CLI', () => {
       await runTestsCli(['foo']);
 
       expect(exitMock).not.toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('rejects invalid options even if valid options exist', async () => {
@@ -190,6 +196,7 @@ describe('run tests CLI', () => {
       await runTestsCli(['foo']);
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
   });
 });
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
new file mode 100644
index 0000000000000..a099f459f998c
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
@@ -0,0 +1,111 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`display help for start servers CLI displays as expected 1`] = `
+"Start Functional Test Servers
+
+Usage:
+  node scripts/functional_tests_server --help
+  node scripts/functional_tests_server [--config <file>]
+  node scripts/functional_tests_server [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config 
+  --es-from <snapshot|source>    Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing."
+`;
+
+exports[`process options for start servers CLI accepts debug option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "debug": true,
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for start servers CLI accepts empty config value if default passed 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for start servers CLI accepts extra server options 1`] = `
+Object {
+  "_": Object {
+    "server.foo": "bar",
+  },
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": Object {
+    "server.foo": "bar",
+  },
+}
+`;
+
+exports[`process options for start servers CLI accepts quiet option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "quiet": true,
+}
+`;
+
+exports[`process options for start servers CLI accepts silent option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "silent": true,
+}
+`;
+
+exports[`process options for start servers CLI accepts source value for es-from 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "es-from": "source",
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "kibana-install-dir": "foo",
+}
+`;
+
+exports[`process options for start servers CLI accepts verbose option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "verbose": true,
+}
+`;
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap
new file mode 100644
index 0000000000000..3a770dad08893
--- /dev/null
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap
@@ -0,0 +1,57 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`start servers CLI options accepts boolean value for updateBaselines 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid option [updateBaselines]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects bail 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid option [bail]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects boolean config value 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid argument [true] to option [config]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects boolean value for kibana-install-dir 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid argument [true] to option [kibana-install-dir]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects empty config value if no default passed 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: config is required",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects invalid options even if valid options exist 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid option [grep]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects non-enum value for es-from 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid argument [butter] to option [es-from]",
+  ],
+]
+`;
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index 6ecff62f900e2..df17d0aa6754d 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -45,7 +45,8 @@ const options = {
 export function displayHelp() {
   const helpOptions = Object.keys(options)
     .filter(name => name !== '_')
-    .map(([name, option]) => {
+    .map(name => {
+      const option = options[name];
       return {
         ...option,
         usage: `${name} ${option.arg || ''}`,
@@ -57,8 +58,7 @@ export function displayHelp() {
     })
     .join(`\n      `);
 
-  console.log(
-    dedent(`
+  return dedent(`
     Start Functional Test Servers
 
     Usage:
@@ -68,8 +68,7 @@ export function displayHelp() {
 
     Options:
       ${helpOptions}
-    `)
-  );
+    `);
 }
 
 export function processOptions(userOptions, defaultConfigPath) {
@@ -81,13 +80,16 @@ export function processOptions(userOptions, defaultConfigPath) {
     throw new Error(`functional_tests_server: config is required`);
   }
 
-  const log = createToolingLog(pickLevelFromFlags(userOptions));
-  log.pipe(process.stdout);
+  function createLogger() {
+    const log = createToolingLog(pickLevelFromFlags(userOptions));
+    log.pipe(process.stdout);
+    return log;
+  }
 
   return {
     ...userOptions,
     config,
-    log,
+    createLogger,
     extraKbnOpts: userOptions._,
   };
 }
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
index 3b783dedcf76d..3cd893debf975 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
@@ -17,9 +17,15 @@
  * under the License.
  */
 
-import { processOptions } from './args';
+import { displayHelp, processOptions } from './args';
 
-describe('process options for run tests CLI', () => {
+describe('display help for start servers CLI', () => {
+  it('displays as expected', () => {
+    expect(displayHelp()).toMatchSnapshot();
+  });
+});
+
+describe('process options for start servers CLI', () => {
   it('rejects boolean config value', () => {
     expect(() => {
       processOptions({ config: true });
@@ -36,7 +42,7 @@ describe('process options for run tests CLI', () => {
 
   it('accepts empty config value if default passed', () => {
     const options = processOptions({ config: '' }, ['foo']);
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid option', () => {
@@ -47,8 +53,7 @@ describe('process options for run tests CLI', () => {
 
   it('accepts string value for kibana-install-dir', () => {
     const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
-    expect(options['kibana-install-dir']).toEqual('foo');
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects boolean value for kibana-install-dir', () => {
@@ -61,8 +66,7 @@ describe('process options for run tests CLI', () => {
 
   it('accepts source value for es-from', () => {
     const options = processOptions({ 'es-from': 'source' }, ['foo']);
-    expect(options['es-from']).toEqual('source');
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects non-enum value for es-from', () => {
@@ -75,32 +79,27 @@ describe('process options for run tests CLI', () => {
 
   it('accepts debug option', () => {
     const options = processOptions({ debug: true }, ['foo']);
-    expect(options.debug).toEqual(true);
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts silent option', () => {
     const options = processOptions({ silent: true }, ['foo']);
-    expect(options.silent).toEqual(true);
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts quiet option', () => {
     const options = processOptions({ quiet: true }, ['foo']);
-    expect(options.quiet).toEqual(true);
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts verbose option', () => {
     const options = processOptions({ verbose: true }, ['foo']);
-    expect(options.verbose).toEqual(true);
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('accepts extra server options', () => {
     const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
-    expect(options._['server.foo']).toEqual('bar');
-    expect(options.config).toEqual(['foo']);
+    expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid options even if valid options exist', () => {
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
index 4eb67bbfff93d..33f266194eee9 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js
@@ -24,13 +24,16 @@ import { processOptions, displayHelp } from './args';
 
 /**
  * Start servers
- * @param {string} defaultConfigPath Path to config to use
+ * @param {string} defaultConfigPath Optional path to config
  *                                   if no config option is passed
  */
 export async function startServersCli(defaultConfigPath) {
   try {
     const userOptions = getopts(process.argv.slice(2)) || {};
-    if (userOptions.help) return displayHelp();
+    if (userOptions.help) {
+      console.log(displayHelp());
+      return undefined;
+    }
 
     const options = processOptions(userOptions, defaultConfigPath);
     await startServers(options);
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
index bfb276f2e0d57..ffc5dd0587657 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
@@ -62,6 +62,7 @@ describe('start servers CLI', () => {
       await startServersCli();
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('rejects empty config value if no default passed', async () => {
@@ -70,6 +71,7 @@ describe('start servers CLI', () => {
       await startServersCli();
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts empty config value if default passed', async () => {
@@ -80,12 +82,13 @@ describe('start servers CLI', () => {
       expect(exitMock).not.toHaveBeenCalled();
     });
 
-    it('rejects non-boolean value for bail', async () => {
-      global.process.argv.push('--bail', 'peanut');
+    it('rejects bail', async () => {
+      global.process.argv.push('--bail', true);
 
       await startServersCli('foo');
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts string value for kibana-install-dir', async () => {
@@ -102,6 +105,7 @@ describe('start servers CLI', () => {
       await startServersCli('foo');
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts boolean value for updateBaselines', async () => {
@@ -109,7 +113,8 @@ describe('start servers CLI', () => {
 
       await startServersCli('foo');
 
-      expect(exitMock).not.toHaveBeenCalledWith();
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts source value for es-from', async () => {
@@ -126,6 +131,7 @@ describe('start servers CLI', () => {
       await startServersCli('foo');
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
 
     it('accepts debug option', async () => {
@@ -182,6 +188,7 @@ describe('start servers CLI', () => {
       await startServersCli('foo');
 
       expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
     });
   });
 });
diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index b711d679e52a4..573edb2cbd24c 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -42,12 +42,12 @@ in another terminal session by running this command from this directory:
 
 /**
  * Run servers and tests for each config
- * @param {object} options                Optional
- * @property {string[]} configPaths       Array of paths to configs
- * @property {Log} options.log            Optional logger
- * @property {string} options.installDir  Optional installation dir from which to run Kibana
- * @property {boolean} options.bail       Whether to exit test run at the first failure
- * @property {string} options.esFrom      Optionally run from source instead of snapshot
+ * @param {object} options                   Optional
+ * @property {string[]} configPaths          Array of paths to configs
+ * @property {function} options.createLogger Optional logger creation function
+ * @property {string} options.installDir     Optional installation dir from which to run Kibana
+ * @property {boolean} options.bail          Whether to exit test run at the first failure
+ * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function runTests(options) {
   for (const configPath of options.configs) {
@@ -57,15 +57,16 @@ export async function runTests(options) {
 
 /**
  * Start only servers using single config
- * @param {object} options               Optional
- * @property {string} options.configPath Path to a config file
- * @property {Log} options.log           Optional logger
- * @property {string} options.installDir Optional installation dir from which to run Kibana
- * @property {string} options.esFrom     Optionally run from source instead of snapshot
+ * @param {object} options                   Optional
+ * @property {string} options.configPath     Path to a config file
+ * @property {function} options.createLogger Optional logger creation function
+ * @property {string} options.installDir     Optional installation dir from which to run Kibana
+ * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function startServers(options) {
-  const { config: configOption, log } = options;
+  const { config: configOption, createLogger } = options;
   const configPath = resolve(process.cwd(), configOption);
+  const log = createLogger();
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
@@ -97,7 +98,7 @@ async function silence(milliseconds, { log }) {
  * Start servers and run tests for single config
  */
 async function runSingleConfig(configPath, options) {
-  const { log } = options;
+  const log = options.createLogger();
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
diff --git a/tasks/config/run.js b/tasks/config/run.js
index 3dfe6c973ce35..57a6fe89ae524 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -149,6 +149,7 @@ module.exports = function (grunt) {
         '--config', 'test/api_integration/config.js',
         '--es-from', 'source',
         '--bail',
+        '--debug',
       ],
     },
 
@@ -159,6 +160,7 @@ module.exports = function (grunt) {
         '--config', 'test/functional/config.js',
         '--es-from', 'source',
         '--bail',
+        '--debug',
         '--',
         '--server.maxPayloadBytes=1648576',
       ],
@@ -171,6 +173,7 @@ module.exports = function (grunt) {
         '--config', 'test/functional/config.js',
         '--es-from', 'source',
         '--bail',
+        '--debug',
         '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
         '--',
         '--server.maxPayloadBytes=1648576',
@@ -183,6 +186,7 @@ module.exports = function (grunt) {
         'scripts/functional_tests_server',
         '--config', 'test/functional/config.js',
         '--es-from', 'source',
+        '--debug',
         '--',
         '--server.maxPayloadBytes=1648576',
       ],

From f12aa46794b9c882cfc3a81050bfe3176be8b02e Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Mon, 25 Jun 2018 09:23:57 -0500
Subject: [PATCH 18/19] Pass created log to runFtr, runElasticsearch,
 runKibanaServer

---
 .../kbn-test/src/functional_tests/tasks.js    | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js
index 573edb2cbd24c..6f10dc7760c14 100644
--- a/packages/kbn-test/src/functional_tests/tasks.js
+++ b/packages/kbn-test/src/functional_tests/tasks.js
@@ -67,15 +67,22 @@ export async function startServers(options) {
   const { config: configOption, createLogger } = options;
   const configPath = resolve(process.cwd(), configOption);
   const log = createLogger();
+  const opts = {
+    ...options,
+    log,
+  };
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
-    const es = await runElasticsearch({ config, options });
+    const es = await runElasticsearch({ config, options: opts });
     await runKibanaServer({
       procs,
       config,
-      options: { ...options, extraKbnOpts: [...options.extraKbnOpts, '--dev'] },
+      options: {
+        ...opts,
+        extraKbnOpts: [...options.extraKbnOpts, '--dev'],
+      },
     });
 
     // wait for 5 seconds of silence before logging the
@@ -99,12 +106,16 @@ async function silence(milliseconds, { log }) {
  */
 async function runSingleConfig(configPath, options) {
   const log = options.createLogger();
+  const opts = {
+    ...options,
+    log,
+  };
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
-    const es = await runElasticsearch({ config, options });
-    await runKibanaServer({ procs, config, options });
+    const es = await runElasticsearch({ config, options: opts });
+    await runKibanaServer({ procs, config, options: opts });
 
     // Note: When solving how to incorporate functional_test_runner
     // clean this up
@@ -112,7 +123,7 @@ async function runSingleConfig(configPath, options) {
       procs,
       configPath,
       cwd: process.cwd(),
-      options,
+      options: opts,
     });
 
     await procs.stop('kibana');

From 47dcc07e7e38925a1d5031015bf138a52a16433d Mon Sep 17 00:00:00 2001
From: archanid <artistic.archana@gmail.com>
Date: Mon, 25 Jun 2018 14:17:05 -0500
Subject: [PATCH 19/19] Update --es-from option to --esFrom

---
 .../cli/run_tests/__snapshots__/args.test.js.snap      |  6 +++---
 .../cli/run_tests/__snapshots__/cli.test.js.snap       |  6 +++---
 .../src/functional_tests/cli/run_tests/args.js         |  2 +-
 .../src/functional_tests/cli/run_tests/args.test.js    | 10 +++++-----
 .../src/functional_tests/cli/run_tests/cli.test.js     |  8 ++++----
 .../cli/start_servers/__snapshots__/args.test.js.snap  |  6 +++---
 .../cli/start_servers/__snapshots__/cli.test.js.snap   |  4 ++--
 .../src/functional_tests/cli/start_servers/args.js     |  2 +-
 .../functional_tests/cli/start_servers/args.test.js    | 10 +++++-----
 .../src/functional_tests/cli/start_servers/cli.test.js |  8 ++++----
 tasks/config/run.js                                    |  8 ++++----
 test/scripts/jenkins_xpack.sh                          |  2 +-
 12 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
index 381fd6c3e1e9e..dd79687ea02c3 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap
@@ -11,7 +11,7 @@ Usage:
 Options:
   --help                         Display this menu and exit. 
   --config <file>                Pass in a config. Can pass in multiple configs. 
-  --es-from <snapshot|source>    Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
   --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
   --bail                         Stop the test run at the first failure. 
   --grep <pattern>               Pattern to select which tests to run. 
@@ -92,13 +92,13 @@ Object {
 }
 `;
 
-exports[`process options for run tests CLI accepts source value for es-from 1`] = `
+exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
 Object {
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "es-from": "source",
+  "esFrom": "source",
   "extraKbnOpts": undefined,
 }
 `;
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
index e185c8c6b872d..5ae52e54e23f5 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap
@@ -13,7 +13,7 @@ Usage:
 Options:
   --help                         Display this menu and exit. 
   --config <file>                Pass in a config. Can pass in multiple configs. 
-  --es-from <snapshot|source>    Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
   --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
   --bail                         Stop the test run at the first failure. 
   --grep <pattern>               Pattern to select which tests to run. 
@@ -66,10 +66,10 @@ Array [
 ]
 `;
 
-exports[`run tests CLI options rejects non-enum value for es-from 1`] = `
+exports[`run tests CLI options rejects non-enum value for esFrom 1`] = `
 Array [
   Array [
-    "Error: functional_tests: invalid argument [butter] to option [es-from]",
+    "Error: functional_tests: invalid argument [butter] to option [esFrom]",
   ],
 ]
 `;
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
index ff2fcaed53617..b6490171f5d16 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js
@@ -26,7 +26,7 @@ const options = {
     arg: '<file>',
     desc: 'Pass in a config. Can pass in multiple configs.',
   },
-  'es-from': {
+  esFrom: {
     arg: '<snapshot|source>',
     choices: ['snapshot', 'source'],
     desc: 'Build Elasticsearch from source or run from snapshot.',
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
index 0e205d8e0c36f..0cd119a2d8eb8 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js
@@ -67,16 +67,16 @@ describe('process options for run tests CLI', () => {
     expect(options).toMatchSnapshot();
   });
 
-  it('accepts source value for es-from', () => {
-    const options = processOptions({ 'es-from': 'source' }, ['foo']);
+  it('accepts source value for esFrom', () => {
+    const options = processOptions({ esFrom: 'source' }, ['foo']);
     expect(options).toMatchSnapshot();
   });
 
-  it('rejects non-enum value for es-from', () => {
+  it('rejects non-enum value for esFrom', () => {
     expect(() => {
-      processOptions({ 'es-from': 'butter' }, ['foo']);
+      processOptions({ esFrom: 'butter' }, ['foo']);
     }).toThrow(
-      'functional_tests: invalid argument [butter] to option [es-from]'
+      'functional_tests: invalid argument [butter] to option [esFrom]'
     );
   });
 
diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
index e9bad10f8a0b8..ffaa223a64ff9 100644
--- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
@@ -116,16 +116,16 @@ describe('run tests CLI', () => {
       expect(exitMock).not.toHaveBeenCalledWith();
     });
 
-    it('accepts source value for es-from', async () => {
-      global.process.argv.push('--es-from', 'source');
+    it('accepts source value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'source');
 
       await runTestsCli(['foo']);
 
       expect(exitMock).not.toHaveBeenCalled();
     });
 
-    it('rejects non-enum value for es-from', async () => {
-      global.process.argv.push('--es-from', 'butter');
+    it('rejects non-enum value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'butter');
 
       await runTestsCli(['foo']);
 
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
index a099f459f998c..5c9f5a9a8d7fc 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap
@@ -11,7 +11,7 @@ Usage:
 Options:
   --help                         Display this menu and exit. 
   --config <file>                Pass in a config 
-  --es-from <snapshot|source>    Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
   --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
@@ -77,13 +77,13 @@ Object {
 }
 `;
 
-exports[`process options for start servers CLI accepts source value for es-from 1`] = `
+exports[`process options for start servers CLI accepts source value for esFrom 1`] = `
 Object {
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "es-from": "source",
+  "esFrom": "source",
   "extraKbnOpts": undefined,
 }
 `;
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap
index 3a770dad08893..f3fee1d0942bb 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap
@@ -48,10 +48,10 @@ Array [
 ]
 `;
 
-exports[`start servers CLI options rejects non-enum value for es-from 1`] = `
+exports[`start servers CLI options rejects non-enum value for esFrom 1`] = `
 Array [
   Array [
-    "Error: functional_tests_server: invalid argument [butter] to option [es-from]",
+    "Error: functional_tests_server: invalid argument [butter] to option [esFrom]",
   ],
 ]
 `;
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
index df17d0aa6754d..467ab098918ed 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js
@@ -26,7 +26,7 @@ const options = {
     arg: '<file>',
     desc: 'Pass in a config',
   },
-  'es-from': {
+  esFrom: {
     arg: '<snapshot|source>',
     choices: ['snapshot', 'source'],
     desc: 'Build Elasticsearch from source or run from snapshot.',
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
index 3cd893debf975..e4785a26b21b9 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js
@@ -64,16 +64,16 @@ describe('process options for start servers CLI', () => {
     );
   });
 
-  it('accepts source value for es-from', () => {
-    const options = processOptions({ 'es-from': 'source' }, ['foo']);
+  it('accepts source value for esFrom', () => {
+    const options = processOptions({ esFrom: 'source' }, ['foo']);
     expect(options).toMatchSnapshot();
   });
 
-  it('rejects non-enum value for es-from', () => {
+  it('rejects non-enum value for esFrom', () => {
     expect(() => {
-      processOptions({ 'es-from': 'butter' }, ['foo']);
+      processOptions({ esFrom: 'butter' }, ['foo']);
     }).toThrow(
-      'functional_tests_server: invalid argument [butter] to option [es-from]'
+      'functional_tests_server: invalid argument [butter] to option [esFrom]'
     );
   });
 
diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
index ffc5dd0587657..f5f29d6d4c8cb 100644
--- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
+++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js
@@ -117,16 +117,16 @@ describe('start servers CLI', () => {
       expect(logMock.mock.calls).toMatchSnapshot();
     });
 
-    it('accepts source value for es-from', async () => {
-      global.process.argv.push('--es-from', 'source');
+    it('accepts source value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'source');
 
       await startServersCli('foo');
 
       expect(exitMock).not.toHaveBeenCalled();
     });
 
-    it('rejects non-enum value for es-from', async () => {
-      global.process.argv.push('--es-from', 'butter');
+    it('rejects non-enum value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'butter');
 
       await startServersCli('foo');
 
diff --git a/tasks/config/run.js b/tasks/config/run.js
index 57a6fe89ae524..8fe7a49627236 100644
--- a/tasks/config/run.js
+++ b/tasks/config/run.js
@@ -147,7 +147,7 @@ module.exports = function (grunt) {
       args: [
         'scripts/functional_tests',
         '--config', 'test/api_integration/config.js',
-        '--es-from', 'source',
+        '--esFrom', 'source',
         '--bail',
         '--debug',
       ],
@@ -158,7 +158,7 @@ module.exports = function (grunt) {
       args: [
         'scripts/functional_tests',
         '--config', 'test/functional/config.js',
-        '--es-from', 'source',
+        '--esFrom', 'source',
         '--bail',
         '--debug',
         '--',
@@ -171,7 +171,7 @@ module.exports = function (grunt) {
       args: [
         'scripts/functional_tests',
         '--config', 'test/functional/config.js',
-        '--es-from', 'source',
+        '--esFrom', 'source',
         '--bail',
         '--debug',
         '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
@@ -185,7 +185,7 @@ module.exports = function (grunt) {
       args: [
         'scripts/functional_tests_server',
         '--config', 'test/functional/config.js',
-        '--es-from', 'source',
+        '--esFrom', 'source',
         '--debug',
         '--',
         '--server.maxPayloadBytes=1648576',
diff --git a/test/scripts/jenkins_xpack.sh b/test/scripts/jenkins_xpack.sh
index 6207a9fc78fb8..8f5d75e2cdc2a 100755
--- a/test/scripts/jenkins_xpack.sh
+++ b/test/scripts/jenkins_xpack.sh
@@ -37,6 +37,6 @@ tar -xzf "$linuxBuild" -C "$installDir" --strip=1
 
 echo " -> Running functional and api tests"
 cd "$XPACK_DIR"
-xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --es-from=source
+xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --esFrom=source
 echo ""
 echo ""