[Functional Tests] Use @kbn/test on Kibana CI

packages/kbn-test/src/es/es_test_cluster.js

@@ -34,8 +34,7 @@ export function createEsTestCluster(options = {}) {
     license = 'oss',
     log,
     basePath = resolve(KIBANA_ROOT, '.es'),
-    // Use source when running on CI
-    from = esTestConfig.getBuildFrom(),
+    esFrom = esTestConfig.getBuildFrom(),
   } = options;
 
   const randomHash = Math.random()
@@ -58,12 +57,12 @@ export function createEsTestCluster(options = {}) {
       const second = 1000;
       const minute = second * 60;
 
-      return from === 'snapshot' ? minute : minute * 6;
+      return esFrom === 'snapshot' ? minute : minute * 6;
     }
 
     async start(esArgs = []) {
       const { installPath } =
-        from === 'source'
+        esFrom === 'source'
           ? await cluster.installSource(config)
           : await cluster.installSnapshot(config);
 

packages/kbn-test/src/functional_tests/cli/index.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-export { runTestsCli } from './run_tests_cli';
-export { startServersCli } from './start_servers_cli';
+export { runTestsCli } from './run_tests/cli';
+export { startServersCli } from './start_servers/cli';

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -0,0 +1,137 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`display help for run tests CLI displays as expected 1`] = `
+"Run Functional Tests
+
+Usage:
+  node scripts/functional_tests --help
+  node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+  node scripts/functional_tests [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config. Can pass in multiple configs. 
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --bail                         Stop the test run at the first failure. 
+  --grep <pattern>               Pattern to select which tests to run. 
+  --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing."
+`;
+
+exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "updateBaselines": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts debug option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "debug": true,
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
+Object {
+  "config": "",
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts extra server options 1`] = `
+Object {
+  "_": Object {
+    "server.foo": "bar",
+  },
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": Object {
+    "server.foo": "bar",
+  },
+}
+`;
+
+exports[`process options for run tests CLI accepts quiet option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "quiet": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts silent option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "silent": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "esFrom": "source",
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "kibana-install-dir": "foo",
+}
+`;
+
+exports[`process options for run tests CLI accepts value for grep 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "grep": "management",
+}
+`;
+
+exports[`process options for run tests CLI accepts verbose option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "verbose": true,
+}
+`;

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -0,0 +1,75 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`run tests CLI options accepts help option even if invalid options passed 1`] = `
+Array [
+  Array [
+    "Run Functional Tests
+
+Usage:
+  node scripts/functional_tests --help
+  node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+  node scripts/functional_tests [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config. Can pass in multiple configs. 
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --bail                         Stop the test run at the first failure. 
+  --grep <pattern>               Pattern to select which tests to run. 
+  --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing.",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects boolean config value 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [true] to option [config]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects boolean value for kibana-install-dir 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [true] to option [kibana-install-dir]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects empty config value if no default passed 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: config is required",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects invalid options even if valid options exist 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid option [aintnothang]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects non-boolean value for bail 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [peanut] to option [bail]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects non-enum value for esFrom 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [butter] to option [esFrom]",
+  ],
+]
+`;

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import dedent from 'dedent';
+import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+
+const options = {
+  help: { desc: 'Display this menu and exit.' },
+  config: {
+    arg: '<file>',
+    desc: 'Pass in a config. Can pass in multiple configs.',
+  },
+  esFrom: {
+    arg: '<snapshot|source>',
+    choices: ['snapshot', 'source'],
+    desc: 'Build Elasticsearch from source or run from snapshot.',
+    default: 'snapshot',
+  },
+  'kibana-install-dir': {
+    arg: '<dir>',
+    desc: 'Run Kibana from existing install directory instead of from source.',
+  },
+  bail: { desc: 'Stop the test run at the first failure.' },
+  grep: {
+    arg: '<pattern>',
+    desc: 'Pattern to select which tests to run.',
+  },
+  updateBaselines: {
+    desc:
+      'Replace baseline screenshots with whatever is generated from the test.',
+  },
+  verbose: { desc: 'Log everything.' },
+  debug: { desc: 'Run in debug mode.' },
+  quiet: { desc: 'Only log errors.' },
+  silent: { desc: 'Log nothing.' },
+};
+
+export function displayHelp() {
+  const helpOptions = Object.keys(options)
+    .filter(name => name !== '_')
+    .map(name => {
+      const option = options[name];
+      return {
+        ...option,
+        usage: `${name} ${option.arg || ''}`,
+        default: option.default ? `Default: ${option.default}` : '',
+      };
+    })
+    .map(option => {
+      return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
+    })
+    .join(`\n      `);
+
+  return dedent(`
+    Run Functional Tests
+
+    Usage:
+      node scripts/functional_tests --help
+      node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+      node scripts/functional_tests [options] [-- --<other args>]
+
+    Options:
+      ${helpOptions}
+    `);
+}
+
+export function processOptions(userOptions, defaultConfigPaths) {
+  validateOptions(userOptions);
+
+  let configs;
+  if (userOptions.config) {
+    configs = [].concat(userOptions.config);
+  } else {
+    if (!defaultConfigPaths || defaultConfigPaths.length === 0) {
+      throw new Error(`functional_tests: config is required`);
+    } else {
+      configs = defaultConfigPaths;
+    }
+  }
+
+  function createLogger() {
+    const log = createToolingLog(pickLevelFromFlags(userOptions));
+    log.pipe(process.stdout);
+    return log;
+  }
+
+  return {
+    ...userOptions,
+    configs,
+    createLogger,
+    extraKbnOpts: userOptions._,
+  };
+}
+
+function validateOptions(userOptions) {
+  Object.entries(userOptions).forEach(([key, val]) => {
+    if (key === '_') return;
+
+    // Validate flags passed
+    if (options[key] === undefined) {
+      throw new Error(`functional_tests: invalid option [${key}]`);
+    }
+
+    if (
+      // Validate boolean flags
+      (!options[key].arg && typeof val !== 'boolean') ||
+      // Validate string/array flags
+      (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
+      // Validate enum flags
+      (options[key].choices && !options[key].choices.includes(val))
+    ) {
+      throw new Error(
+        `functional_tests: invalid argument [${val}] to option [${key}]`
+      );
+    }
+  });
+}