diff --git a/packages/kbn-test/src/es/es_test_cluster.js b/packages/kbn-test/src/es/es_test_cluster.js index 8ff7a0efefaf6..d969f7d6ec313 100644 --- a/packages/kbn-test/src/es/es_test_cluster.js +++ b/packages/kbn-test/src/es/es_test_cluster.js @@ -34,8 +34,7 @@ export function createEsTestCluster(options = {}) { license = 'oss', log, basePath = resolve(KIBANA_ROOT, '.es'), - // Use source when running on CI - from = esTestConfig.getBuildFrom(), + esFrom = esTestConfig.getBuildFrom(), } = options; const randomHash = Math.random() @@ -58,12 +57,12 @@ export function createEsTestCluster(options = {}) { const second = 1000; const minute = second * 60; - return from === 'snapshot' ? minute : minute * 6; + return esFrom === 'snapshot' ? minute : minute * 6; } async start(esArgs = []) { const { installPath } = - from === 'source' + esFrom === 'source' ? await cluster.installSource(config) : await cluster.installSnapshot(config); diff --git a/packages/kbn-test/src/functional_tests/cli/index.js b/packages/kbn-test/src/functional_tests/cli/index.js index a423672005950..3c6280d91f582 100644 --- a/packages/kbn-test/src/functional_tests/cli/index.js +++ b/packages/kbn-test/src/functional_tests/cli/index.js @@ -17,5 +17,5 @@ * under the License. */ -export { runTestsCli } from './run_tests_cli'; -export { startServersCli } from './start_servers_cli'; +export { runTestsCli } from './run_tests/cli'; +export { startServersCli } from './start_servers/cli'; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap new file mode 100644 index 0000000000000..dd79687ea02c3 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap @@ -0,0 +1,137 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`display help for run tests CLI displays as expected 1`] = ` +"Run Functional Tests + +Usage: + node scripts/functional_tests --help + node scripts/functional_tests [--config [--config ...]] + node scripts/functional_tests [options] [-- --] + +Options: + --help Display this menu and exit. + --config Pass in a config. Can pass in multiple configs. + --esFrom Build Elasticsearch from source or run from snapshot. Default: snapshot + --kibana-install-dir Run Kibana from existing install directory instead of from source. + --bail Stop the test run at the first failure. + --grep Pattern to select which tests to run. + --updateBaselines Replace baseline screenshots with whatever is generated from the test. + --verbose Log everything. + --debug Run in debug mode. + --quiet Only log errors. + --silent Log nothing." +`; + +exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "updateBaselines": true, +} +`; + +exports[`process options for run tests CLI accepts debug option 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "debug": true, + "extraKbnOpts": undefined, +} +`; + +exports[`process options for run tests CLI accepts empty config value if default passed 1`] = ` +Object { + "config": "", + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, +} +`; + +exports[`process options for run tests CLI accepts extra server options 1`] = ` +Object { + "_": Object { + "server.foo": "bar", + }, + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": Object { + "server.foo": "bar", + }, +} +`; + +exports[`process options for run tests CLI accepts quiet option 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "quiet": true, +} +`; + +exports[`process options for run tests CLI accepts silent option 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "silent": true, +} +`; + +exports[`process options for run tests CLI accepts source value for esFrom 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "esFrom": "source", + "extraKbnOpts": undefined, +} +`; + +exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "kibana-install-dir": "foo", +} +`; + +exports[`process options for run tests CLI accepts value for grep 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "grep": "management", +} +`; + +exports[`process options for run tests CLI accepts verbose option 1`] = ` +Object { + "configs": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "verbose": true, +} +`; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap new file mode 100644 index 0000000000000..5ae52e54e23f5 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap @@ -0,0 +1,75 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`run tests CLI options accepts help option even if invalid options passed 1`] = ` +Array [ + Array [ + "Run Functional Tests + +Usage: + node scripts/functional_tests --help + node scripts/functional_tests [--config [--config ...]] + node scripts/functional_tests [options] [-- --] + +Options: + --help Display this menu and exit. + --config Pass in a config. Can pass in multiple configs. + --esFrom Build Elasticsearch from source or run from snapshot. Default: snapshot + --kibana-install-dir Run Kibana from existing install directory instead of from source. + --bail Stop the test run at the first failure. + --grep Pattern to select which tests to run. + --updateBaselines Replace baseline screenshots with whatever is generated from the test. + --verbose Log everything. + --debug Run in debug mode. + --quiet Only log errors. + --silent Log nothing.", + ], +] +`; + +exports[`run tests CLI options rejects boolean config value 1`] = ` +Array [ + Array [ + "Error: functional_tests: invalid argument [true] to option [config]", + ], +] +`; + +exports[`run tests CLI options rejects boolean value for kibana-install-dir 1`] = ` +Array [ + Array [ + "Error: functional_tests: invalid argument [true] to option [kibana-install-dir]", + ], +] +`; + +exports[`run tests CLI options rejects empty config value if no default passed 1`] = ` +Array [ + Array [ + "Error: functional_tests: config is required", + ], +] +`; + +exports[`run tests CLI options rejects invalid options even if valid options exist 1`] = ` +Array [ + Array [ + "Error: functional_tests: invalid option [aintnothang]", + ], +] +`; + +exports[`run tests CLI options rejects non-boolean value for bail 1`] = ` +Array [ + Array [ + "Error: functional_tests: invalid argument [peanut] to option [bail]", + ], +] +`; + +exports[`run tests CLI options rejects non-enum value for esFrom 1`] = ` +Array [ + Array [ + "Error: functional_tests: invalid argument [butter] to option [esFrom]", + ], +] +`; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js new file mode 100644 index 0000000000000..b6490171f5d16 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import dedent from 'dedent'; +import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; + +const options = { + help: { desc: 'Display this menu and exit.' }, + config: { + arg: '', + desc: 'Pass in a config. Can pass in multiple configs.', + }, + esFrom: { + arg: '', + choices: ['snapshot', 'source'], + desc: 'Build Elasticsearch from source or run from snapshot.', + default: 'snapshot', + }, + 'kibana-install-dir': { + arg: '', + desc: 'Run Kibana from existing install directory instead of from source.', + }, + bail: { desc: 'Stop the test run at the first failure.' }, + grep: { + arg: '', + desc: 'Pattern to select which tests to run.', + }, + updateBaselines: { + desc: + 'Replace baseline screenshots with whatever is generated from the test.', + }, + verbose: { desc: 'Log everything.' }, + debug: { desc: 'Run in debug mode.' }, + quiet: { desc: 'Only log errors.' }, + silent: { desc: 'Log nothing.' }, +}; + +export function displayHelp() { + const helpOptions = Object.keys(options) + .filter(name => name !== '_') + .map(name => { + const option = options[name]; + return { + ...option, + usage: `${name} ${option.arg || ''}`, + default: option.default ? `Default: ${option.default}` : '', + }; + }) + .map(option => { + return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`; + }) + .join(`\n `); + + return dedent(` + Run Functional Tests + + Usage: + node scripts/functional_tests --help + node scripts/functional_tests [--config [--config ...]] + node scripts/functional_tests [options] [-- --] + + Options: + ${helpOptions} + `); +} + +export function processOptions(userOptions, defaultConfigPaths) { + validateOptions(userOptions); + + let configs; + if (userOptions.config) { + configs = [].concat(userOptions.config); + } else { + if (!defaultConfigPaths || defaultConfigPaths.length === 0) { + throw new Error(`functional_tests: config is required`); + } else { + configs = defaultConfigPaths; + } + } + + function createLogger() { + const log = createToolingLog(pickLevelFromFlags(userOptions)); + log.pipe(process.stdout); + return log; + } + + return { + ...userOptions, + configs, + createLogger, + extraKbnOpts: userOptions._, + }; +} + +function validateOptions(userOptions) { + Object.entries(userOptions).forEach(([key, val]) => { + if (key === '_') return; + + // Validate flags passed + if (options[key] === undefined) { + throw new Error(`functional_tests: invalid option [${key}]`); + } + + if ( + // Validate boolean flags + (!options[key].arg && typeof val !== 'boolean') || + // Validate string/array flags + (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) || + // Validate enum flags + (options[key].choices && !options[key].choices.includes(val)) + ) { + throw new Error( + `functional_tests: invalid argument [${val}] to option [${key}]` + ); + } + }); +} diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js new file mode 100644 index 0000000000000..0cd119a2d8eb8 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { displayHelp, processOptions } from './args'; + +describe('display help for run tests CLI', () => { + it('displays as expected', () => { + expect(displayHelp()).toMatchSnapshot(); + }); +}); + +describe('process options for run tests CLI', () => { + it('rejects boolean config value', () => { + expect(() => { + processOptions({ config: true }); + }).toThrow('functional_tests: invalid argument [true] to option [config]'); + }); + + it('rejects empty config value if no default passed', () => { + expect(() => { + processOptions({}); + }).toThrow('functional_tests: config is required'); + }); + + it('accepts empty config value if default passed', () => { + const options = processOptions({ config: '' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects non-boolean value for bail', () => { + expect(() => { + processOptions({ bail: 'peanut' }, ['foo']); + }).toThrow('functional_tests: invalid argument [peanut] to option [bail]'); + }); + + it('accepts string value for kibana-install-dir', () => { + const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects boolean value for kibana-install-dir', () => { + expect(() => { + processOptions({ 'kibana-install-dir': true }, ['foo']); + }).toThrow( + 'functional_tests: invalid argument [true] to option [kibana-install-dir]' + ); + }); + + it('accepts boolean value for updateBaselines', () => { + const options = processOptions({ updateBaselines: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts source value for esFrom', () => { + const options = processOptions({ esFrom: 'source' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects non-enum value for esFrom', () => { + expect(() => { + processOptions({ esFrom: 'butter' }, ['foo']); + }).toThrow( + 'functional_tests: invalid argument [butter] to option [esFrom]' + ); + }); + + it('accepts value for grep', () => { + const options = processOptions({ grep: 'management' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts debug option', () => { + const options = processOptions({ debug: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts silent option', () => { + const options = processOptions({ silent: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts quiet option', () => { + const options = processOptions({ quiet: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts verbose option', () => { + const options = processOptions({ verbose: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts extra server options', () => { + const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects invalid options even if valid options exist', () => { + expect(() => { + processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']); + }).toThrow('functional_tests: invalid option [aintnothang]'); + }); +}); diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js new file mode 100644 index 0000000000000..2345dffbb2376 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import chalk from 'chalk'; +import getopts from 'getopts'; +import { runTests } from '../../tasks'; +import { processOptions, displayHelp } from './args'; + +/** + * Run servers and tests for each config + * Only cares about --config option. Other options + * are passed directly to functional_test_runner, such as + * --bail, --verbose, etc. + * @param {string[]} defaultConfigPaths Optional paths to configs + * if no config option is passed + */ +export async function runTestsCli(defaultConfigPaths) { + try { + const userOptions = getopts(process.argv.slice(2)) || {}; + if (userOptions.help) { + console.log(displayHelp()); + return undefined; + } + + const options = processOptions(userOptions, defaultConfigPaths); + await runTests(options); + } catch (err) { + console.log(chalk.red(err)); + process.exit(1); + } +} diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js new file mode 100644 index 0000000000000..ffaa223a64ff9 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js @@ -0,0 +1,202 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { runTestsCli } from './cli'; + +// Note: Stub the runTests function to keep testing only around the cli +// method and arguments. +jest.mock('../../tasks', () => ({ + runTests: jest.fn(), +})); + +describe('run tests CLI', () => { + describe('options', () => { + const originalObjects = {}; + const exitMock = jest.fn(); + const logMock = jest.fn(); // mock logging so we don't send output to the test results + const argvMock = ['foo', 'foo']; + + const processMock = { + exit: exitMock, + argv: argvMock, + stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() }, + cwd: jest.fn(), + }; + + beforeAll(() => { + originalObjects.process = process; + originalObjects.console = console; + global.process = processMock; + global.console = { log: logMock }; + }); + + afterAll(() => { + global.process = originalObjects.process; + global.console = originalObjects.console; + }); + + beforeEach(() => { + global.process.argv = [...argvMock]; + jest.resetAllMocks(); + }); + + it('rejects boolean config value', async () => { + global.process.argv.push('--config'); + + await runTestsCli(); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('rejects empty config value if no default passed', async () => { + global.process.argv.push('--config', ''); + + await runTestsCli(); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts empty config value if default passed', async () => { + global.process.argv.push('--config', ''); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('rejects non-boolean value for bail', async () => { + global.process.argv.push('--bail', 'peanut'); + + await runTestsCli(['foo']); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts string value for kibana-install-dir', async () => { + global.process.argv.push('--kibana-install-dir', 'foo'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('rejects boolean value for kibana-install-dir', async () => { + global.process.argv.push('--kibana-install-dir'); + + await runTestsCli(['foo']); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts boolean value for updateBaselines', async () => { + global.process.argv.push('--updateBaselines'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalledWith(); + }); + + it('accepts source value for esFrom', async () => { + global.process.argv.push('--esFrom', 'source'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('rejects non-enum value for esFrom', async () => { + global.process.argv.push('--esFrom', 'butter'); + + await runTestsCli(['foo']); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts value for grep', async () => { + global.process.argv.push('--grep', 'management'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts debug option', async () => { + global.process.argv.push('--debug'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts silent option', async () => { + global.process.argv.push('--silent'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts quiet option', async () => { + global.process.argv.push('--quiet'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts verbose option', async () => { + global.process.argv.push('--verbose'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts extra server options', async () => { + global.process.argv.push('--', '--server.foo=bar'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts help option even if invalid options passed', async () => { + global.process.argv.push('--debug', '--aintnothang', '--help'); + + await runTestsCli(['foo']); + + expect(exitMock).not.toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('rejects invalid options even if valid options exist', async () => { + global.process.argv.push('--debug', '--aintnothang', '--bail'); + + await runTestsCli(['foo']); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + }); +}); diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js deleted file mode 100644 index 0f366b6278dfd..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/run_tests_cli.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import dedent from 'dedent'; -import getopts from 'getopts'; -import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; -import { runTests } from '../../'; - -/** - * Run servers and tests for each config - * Only cares about --config option. Other options - * are passed directly to functional_test_runner, such as - * --bail, --verbose, etc. - * @param {string[]} defaultConfigPaths Array of paths to configs to use - * if no config option is passed - */ -export async function runTestsCli(defaultConfigPaths) { - const { configs, help, bail, log, installDir } = processArgs( - defaultConfigPaths - ); - - if (help) return displayHelp(); - - if (!configs || configs.length === 0) { - log.error( - `Run Tests requires at least one path to a config. Leave blank to use defaults.` - ); - process.exit(9); - } - - try { - await runTests(configs, { bail, log, installDir }); - } catch (err) { - log.error('FATAL ERROR'); - log.error(err); - process.exit(1); - } -} - -function processArgs(defaultConfigPaths) { - // If no args are passed, use {} - const options = getopts(process.argv.slice(2)) || {}; - - // If --config is passed without paths, it's "true", so use default - const configs = - typeof options.config === 'string' || Array.isArray(options.config) - ? [].concat(options.config) - : defaultConfigPaths; - - const log = createToolingLog(pickLevelFromFlags(options)); - log.pipe(process.stdout); - - return { - configs, - log, - help: options.help, - bail: options.bail, - installDir: options['kibana-install-dir'], - rest: options._, - }; -} - -function displayHelp() { - console.log( - dedent(` - Run Functional Tests - - Usage: node scripts/functional_tests [options] - - --config Option to pass in a config - Can pass in multiple configs with - --config file1 --config file2 --config file3 - --kibana-install-dir - Run Kibana from an existing install directory - Default: run from source - --bail Stop the test run at the first failure - --help Display this menu and exit - - Log level options: - - --verbose - --debug - --quiet Log errors - --silent - `) - ); -} diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap new file mode 100644 index 0000000000000..5c9f5a9a8d7fc --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap @@ -0,0 +1,111 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`display help for start servers CLI displays as expected 1`] = ` +"Start Functional Test Servers + +Usage: + node scripts/functional_tests_server --help + node scripts/functional_tests_server [--config ] + node scripts/functional_tests_server [options] [-- --] + +Options: + --help Display this menu and exit. + --config Pass in a config + --esFrom Build Elasticsearch from source or run from snapshot. Default: snapshot + --kibana-install-dir Run Kibana from existing install directory instead of from source. + --verbose Log everything. + --debug Run in debug mode. + --quiet Only log errors. + --silent Log nothing." +`; + +exports[`process options for start servers CLI accepts debug option 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "debug": true, + "extraKbnOpts": undefined, +} +`; + +exports[`process options for start servers CLI accepts empty config value if default passed 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, +} +`; + +exports[`process options for start servers CLI accepts extra server options 1`] = ` +Object { + "_": Object { + "server.foo": "bar", + }, + "config": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": Object { + "server.foo": "bar", + }, +} +`; + +exports[`process options for start servers CLI accepts quiet option 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "quiet": true, +} +`; + +exports[`process options for start servers CLI accepts silent option 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "silent": true, +} +`; + +exports[`process options for start servers CLI accepts source value for esFrom 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "esFrom": "source", + "extraKbnOpts": undefined, +} +`; + +exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "kibana-install-dir": "foo", +} +`; + +exports[`process options for start servers CLI accepts verbose option 1`] = ` +Object { + "config": Array [ + "foo", + ], + "createLogger": [Function], + "extraKbnOpts": undefined, + "verbose": true, +} +`; diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap new file mode 100644 index 0000000000000..f3fee1d0942bb --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap @@ -0,0 +1,57 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`start servers CLI options accepts boolean value for updateBaselines 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: invalid option [updateBaselines]", + ], +] +`; + +exports[`start servers CLI options rejects bail 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: invalid option [bail]", + ], +] +`; + +exports[`start servers CLI options rejects boolean config value 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: invalid argument [true] to option [config]", + ], +] +`; + +exports[`start servers CLI options rejects boolean value for kibana-install-dir 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: invalid argument [true] to option [kibana-install-dir]", + ], +] +`; + +exports[`start servers CLI options rejects empty config value if no default passed 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: config is required", + ], +] +`; + +exports[`start servers CLI options rejects invalid options even if valid options exist 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: invalid option [grep]", + ], +] +`; + +exports[`start servers CLI options rejects non-enum value for esFrom 1`] = ` +Array [ + Array [ + "Error: functional_tests_server: invalid argument [butter] to option [esFrom]", + ], +] +`; diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js new file mode 100644 index 0000000000000..467ab098918ed --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import dedent from 'dedent'; +import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; + +const options = { + help: { desc: 'Display this menu and exit.' }, + config: { + arg: '', + desc: 'Pass in a config', + }, + esFrom: { + arg: '', + choices: ['snapshot', 'source'], + desc: 'Build Elasticsearch from source or run from snapshot.', + default: 'snapshot', + }, + 'kibana-install-dir': { + arg: '', + desc: 'Run Kibana from existing install directory instead of from source.', + }, + verbose: { desc: 'Log everything.' }, + debug: { desc: 'Run in debug mode.' }, + quiet: { desc: 'Only log errors.' }, + silent: { desc: 'Log nothing.' }, +}; + +export function displayHelp() { + const helpOptions = Object.keys(options) + .filter(name => name !== '_') + .map(name => { + const option = options[name]; + return { + ...option, + usage: `${name} ${option.arg || ''}`, + default: option.default ? `Default: ${option.default}` : '', + }; + }) + .map(option => { + return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`; + }) + .join(`\n `); + + return dedent(` + Start Functional Test Servers + + Usage: + node scripts/functional_tests_server --help + node scripts/functional_tests_server [--config ] + node scripts/functional_tests_server [options] [-- --] + + Options: + ${helpOptions} + `); +} + +export function processOptions(userOptions, defaultConfigPath) { + validateOptions(userOptions); + + const config = userOptions.config || defaultConfigPath; + + if (!config) { + throw new Error(`functional_tests_server: config is required`); + } + + function createLogger() { + const log = createToolingLog(pickLevelFromFlags(userOptions)); + log.pipe(process.stdout); + return log; + } + + return { + ...userOptions, + config, + createLogger, + extraKbnOpts: userOptions._, + }; +} + +function validateOptions(userOptions) { + Object.entries(userOptions).forEach(([key, val]) => { + if (key === '_') return; + + // Validate flags passed + if (options[key] === undefined) { + throw new Error(`functional_tests_server: invalid option [${key}]`); + } + + if ( + // Validate boolean flags + (!options[key].arg && typeof val !== 'boolean') || + // Validate string/array flags + (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) || + // Validate enum flags + (options[key].choices && !options[key].choices.includes(val)) + ) { + throw new Error( + `functional_tests_server: invalid argument [${val}] to option [${key}]` + ); + } + }); +} diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js new file mode 100644 index 0000000000000..e4785a26b21b9 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { displayHelp, processOptions } from './args'; + +describe('display help for start servers CLI', () => { + it('displays as expected', () => { + expect(displayHelp()).toMatchSnapshot(); + }); +}); + +describe('process options for start servers CLI', () => { + it('rejects boolean config value', () => { + expect(() => { + processOptions({ config: true }); + }).toThrow( + 'functional_tests_server: invalid argument [true] to option [config]' + ); + }); + + it('rejects empty config value if no default passed', () => { + expect(() => { + processOptions({}); + }).toThrow('functional_tests_server: config is required'); + }); + + it('accepts empty config value if default passed', () => { + const options = processOptions({ config: '' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects invalid option', () => { + expect(() => { + processOptions({ bail: true }, ['foo']); + }).toThrow('functional_tests_server: invalid option [bail]'); + }); + + it('accepts string value for kibana-install-dir', () => { + const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects boolean value for kibana-install-dir', () => { + expect(() => { + processOptions({ 'kibana-install-dir': true }, ['foo']); + }).toThrow( + 'functional_tests_server: invalid argument [true] to option [kibana-install-dir]' + ); + }); + + it('accepts source value for esFrom', () => { + const options = processOptions({ esFrom: 'source' }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects non-enum value for esFrom', () => { + expect(() => { + processOptions({ esFrom: 'butter' }, ['foo']); + }).toThrow( + 'functional_tests_server: invalid argument [butter] to option [esFrom]' + ); + }); + + it('accepts debug option', () => { + const options = processOptions({ debug: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts silent option', () => { + const options = processOptions({ silent: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts quiet option', () => { + const options = processOptions({ quiet: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts verbose option', () => { + const options = processOptions({ verbose: true }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('accepts extra server options', () => { + const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']); + expect(options).toMatchSnapshot(); + }); + + it('rejects invalid options even if valid options exist', () => { + expect(() => { + processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']); + }).toThrow('functional_tests_server: invalid option [aintnothang]'); + }); +}); diff --git a/tasks/config/functional_test_runner.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js similarity index 53% rename from tasks/config/functional_test_runner.js rename to packages/kbn-test/src/functional_tests/cli/start_servers/cli.js index e57d823f1d4d5..33f266194eee9 100644 --- a/tasks/config/functional_test_runner.js +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js @@ -17,16 +17,28 @@ * under the License. */ -export const functional = { - options: { - logLevel: 'debug', - configFile: require.resolve('../../test/functional/config.js') - } -}; +import chalk from 'chalk'; +import getopts from 'getopts'; +import { startServers } from '../../tasks'; +import { processOptions, displayHelp } from './args'; + +/** + * Start servers + * @param {string} defaultConfigPath Optional path to config + * if no config option is passed + */ +export async function startServersCli(defaultConfigPath) { + try { + const userOptions = getopts(process.argv.slice(2)) || {}; + if (userOptions.help) { + console.log(displayHelp()); + return undefined; + } -export const apiIntegration = { - options: { - logLevel: 'debug', - configFile: require.resolve('../../test/api_integration/config.js') + const options = processOptions(userOptions, defaultConfigPath); + await startServers(options); + } catch (err) { + console.log(chalk.red(err)); + process.exit(1); } -}; +} diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js new file mode 100644 index 0000000000000..f5f29d6d4c8cb --- /dev/null +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js @@ -0,0 +1,194 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { startServersCli } from './cli'; + +// Note: Stub the startServers function to keep testing only around the cli +// method and arguments. +jest.mock('../../tasks', () => ({ + startServers: jest.fn(), +})); + +describe('start servers CLI', () => { + describe('options', () => { + const originalObjects = {}; + const exitMock = jest.fn(); + const logMock = jest.fn(); // mock logging so we don't send output to the test results + const argvMock = ['foo', 'foo']; + + const processMock = { + exit: exitMock, + argv: argvMock, + stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() }, + cwd: jest.fn(), + }; + + beforeAll(() => { + originalObjects.process = process; + originalObjects.console = console; + global.process = processMock; + global.console = { log: logMock }; + }); + + afterAll(() => { + global.process = originalObjects.process; + global.console = originalObjects.console; + }); + + beforeEach(() => { + global.process.argv = [...argvMock]; + jest.resetAllMocks(); + }); + + it('rejects boolean config value', async () => { + global.process.argv.push('--config'); + + await startServersCli(); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('rejects empty config value if no default passed', async () => { + global.process.argv.push('--config', ''); + + await startServersCli(); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts empty config value if default passed', async () => { + global.process.argv.push('--config', ''); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('rejects bail', async () => { + global.process.argv.push('--bail', true); + + await startServersCli('foo'); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts string value for kibana-install-dir', async () => { + global.process.argv.push('--kibana-install-dir', 'foo'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('rejects boolean value for kibana-install-dir', async () => { + global.process.argv.push('--kibana-install-dir'); + + await startServersCli('foo'); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts boolean value for updateBaselines', async () => { + global.process.argv.push('--updateBaselines'); + + await startServersCli('foo'); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts source value for esFrom', async () => { + global.process.argv.push('--esFrom', 'source'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('rejects non-enum value for esFrom', async () => { + global.process.argv.push('--esFrom', 'butter'); + + await startServersCli('foo'); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + + it('accepts debug option', async () => { + global.process.argv.push('--debug'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts silent option', async () => { + global.process.argv.push('--silent'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts quiet option', async () => { + global.process.argv.push('--quiet'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts verbose option', async () => { + global.process.argv.push('--verbose'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts extra server options', async () => { + global.process.argv.push('--', '--server.foo=bar'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalled(); + }); + + it('accepts help option even if invalid options passed', async () => { + global.process.argv.push('--debug', '--grep', '--help'); + + await startServersCli('foo'); + + expect(exitMock).not.toHaveBeenCalledWith(1); + }); + + it('rejects invalid options even if valid options exist', async () => { + global.process.argv.push('--debug', '--grep', '--bail'); + + await startServersCli('foo'); + + expect(exitMock).toHaveBeenCalledWith(1); + expect(logMock.mock.calls).toMatchSnapshot(); + }); + }); +}); diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js deleted file mode 100644 index 31266b8e7d0dc..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/start_servers_cli.js +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import chalk from 'chalk'; -import dedent from 'dedent'; -import getopts from 'getopts'; -import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; -import { startServers } from '../../'; - -/** - * Start servers - * @param {string} configPath path to config - */ -export async function startServersCli(defaultConfigPath) { - const { config, log, help, installDir } = processArgv(defaultConfigPath); - - if (help) return displayHelp(); - - if (!config) { - log.error( - `Start Servers requires one path to a config. Leave blank to use default.` - ); - process.exit(1); - } - - try { - await startServers(config, { log, installDir }); - } catch (err) { - log.error('FATAL ERROR'); - log.error(err); - process.exit(1); - } -} - -function processArgv(defaultConfigPath) { - const options = getopts(process.argv.slice(2)) || {}; - - if (Array.isArray(options.config)) { - console.log( - chalk.red( - `Starting servers requires a single config path. Multiple were passed.` - ) - ); - process.exit(9); - } - - const config = - typeof options.config === 'string' ? options.config : defaultConfigPath; - - const log = createToolingLog(pickLevelFromFlags(options)); - log.pipe(process.stdout); - - return { - config, - log, - installDir: options.kibanaInstallDir, - help: options.help, - rest: options._, - }; -} - -function displayHelp() { - console.log( - dedent(` - Start Functional Test Servers - - Usage: node scripts/functional_tests_server [options] - - --config Option to pass in a config - --kibana-install-dir - Run Kibana from an existing install directory - Default: run from source - --help Display this menu and exit - - Log level options: - - --verbose - --debug - --quiet Log errors - --silent - `) - ); -} diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js index 35b6d862a299c..efcc464faf059 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js @@ -23,7 +23,8 @@ import { createEsTestCluster } from '../../es'; import { setupUsers, DEFAULT_SUPERUSER_PASS } from './auth'; -export async function runElasticsearch({ config, log }) { +export async function runElasticsearch({ config, options }) { + const { log, esFrom } = options; const isOss = config.get('esTestCluster.license') === 'oss'; const cluster = createEsTestCluster({ @@ -34,7 +35,7 @@ export async function runElasticsearch({ config, log }) { license: config.get('esTestCluster.license'), log, basePath: resolve(KIBANA_ROOT, '.es'), - from: config.get('esTestCluster.from'), + esFrom: esFrom || config.get('esTestCluster.from'), }); const esArgs = config.get('esTestCluster.serverArgs'); diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js index 968ec3e1f678d..90a3289f72028 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -22,15 +22,16 @@ import { KIBANA_FTR_SCRIPT, PROJECT_ROOT } from './paths'; export async function runFtr({ procs, configPath, - bail, - log, cwd = PROJECT_ROOT, + options: { log, bail, grep, updateBaselines }, }) { const args = [KIBANA_FTR_SCRIPT]; if (getLogFlag(log)) args.push(`--${getLogFlag(log)}`); if (bail) args.push('--bail'); if (configPath) args.push('--config', configPath); + if (grep) args.push('--grep', grep); + if (updateBaselines) args.push('--updateBaselines'); await procs.run('ftr', { cmd: 'node', diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js index ae92fe9e34c97..730f816f9c727 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js +++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js @@ -25,7 +25,7 @@ export async function runKibanaServer({ procs, config, options }) { await procs.run('kibana', { cmd: getKibanaCmd(installDir), - args: getCliArgs(config, options), + args: collectCliArgs(config, options), env: { FORCE_COLOR: 1, ...process.env, @@ -45,14 +45,35 @@ function getKibanaCmd(installDir) { return KIBANA_EXEC; } -function getCliArgs(config, { devMode, installDir }) { +/* When installDir is passed, we run from a built version of Kibana, + * which uses different command line arguments. If installDir is not + * passed, we run from source code. We also allow passing in extra + * Kibana server options, so we tack those on here. + */ +function collectCliArgs(config, { installDir, extraKbnOpts }) { const buildArgs = config.get('kbnTestServer.buildArgs') || []; const sourceArgs = config.get('kbnTestServer.sourceArgs') || []; const serverArgs = config.get('kbnTestServer.serverArgs') || []; - if (devMode) serverArgs.push('--dev'); + return pipe( + serverArgs, + args => (installDir ? args.filter(a => a !== '--oss') : args), + args => { + return installDir + ? [...args, ...buildArgs] + : [KIBANA_EXEC_PATH, ...args, ...sourceArgs]; + }, + args => args.concat(extraKbnOpts || []) + ); +} - return installDir - ? [...serverArgs, ...buildArgs] - : [KIBANA_EXEC_PATH, ...serverArgs, ...sourceArgs]; +/* + * Apply each function in fns to the result of the + * previous function. The first function's input + * is the arr array. + */ +function pipe(arr, ...fns) { + return fns.reduce((acc, fn) => { + return fn(acc); + }, arr); } diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js index 3983df54d90c4..6f10dc7760c14 100644 --- a/packages/kbn-test/src/functional_tests/tasks.js +++ b/packages/kbn-test/src/functional_tests/tasks.js @@ -42,39 +42,47 @@ in another terminal session by running this command from this directory: /** * Run servers and tests for each config - * @param {string[]} configPaths Array of paths to configs - * @param {object} options Optional - * @param {Log} options.log Optional logger - * @param {string} options.installDir Optional installation dir - * from which to run Kibana - * @param {boolean} options.bail Whether to exit test run at the first failure + * @param {object} options Optional + * @property {string[]} configPaths Array of paths to configs + * @property {function} options.createLogger Optional logger creation function + * @property {string} options.installDir Optional installation dir from which to run Kibana + * @property {boolean} options.bail Whether to exit test run at the first failure + * @property {string} options.esFrom Optionally run from source instead of snapshot */ -export async function runTests(configPaths, options) { - for (const configPath of configPaths) { +export async function runTests(options) { + for (const configPath of options.configs) { await runSingleConfig(resolve(process.cwd(), configPath), options); } } /** * Start only servers using single config - * @param {string} configPath Path to a config file - * @param {object} options Optional - * @param {Log} options.log Optional logger - * @param {string} options.installDir Optional installation dir - * from which to run Kibana + * @param {object} options Optional + * @property {string} options.configPath Path to a config file + * @property {function} options.createLogger Optional logger creation function + * @property {string} options.installDir Optional installation dir from which to run Kibana + * @property {string} options.esFrom Optionally run from source instead of snapshot */ -export async function startServers(configPath, options) { - const { log } = options; - configPath = resolve(process.cwd(), configPath); +export async function startServers(options) { + const { config: configOption, createLogger } = options; + const configPath = resolve(process.cwd(), configOption); + const log = createLogger(); + const opts = { + ...options, + log, + }; await withProcRunner(log, async procs => { const config = await readConfigFile(log, configPath); - const es = await runElasticsearch({ config, log }); + const es = await runElasticsearch({ config, options: opts }); await runKibanaServer({ procs, config, - options: { ...options, devMode: true }, + options: { + ...opts, + extraKbnOpts: [...options.extraKbnOpts, '--dev'], + }, }); // wait for 5 seconds of silence before logging the @@ -97,22 +105,25 @@ async function silence(milliseconds, { log }) { * Start servers and run tests for single config */ async function runSingleConfig(configPath, options) { - const { bail, log } = options; + const log = options.createLogger(); + const opts = { + ...options, + log, + }; await withProcRunner(log, async procs => { const config = await readConfigFile(log, configPath); - const es = await runElasticsearch({ config, log }); - await runKibanaServer({ procs, config, options }); + const es = await runElasticsearch({ config, options: opts }); + await runKibanaServer({ procs, config, options: opts }); // Note: When solving how to incorporate functional_test_runner // clean this up await runFtr({ procs, configPath, - bail, - log, cwd: process.cwd(), + options: opts, }); await procs.stop('kibana'); diff --git a/tasks/config/run.js b/tasks/config/run.js index d4b97b89b9fc3..8fe7a49627236 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -17,12 +17,8 @@ * under the License. */ -import { esTestConfig, kbnTestConfig } from '@kbn/test'; import { resolve } from 'path'; -const SECOND = 1000; -const MINUTE = 60 * SECOND; -const HOUR = 60 * MINUTE; const PKG_VERSION = require('../../package.json').version; module.exports = function (grunt) { @@ -58,20 +54,6 @@ module.exports = function (grunt) { }; } - const apiTestServerFlags = [ - '--optimize.enabled=false', - '--elasticsearch.url=' + esTestConfig.getUrl(), - '--elasticsearch.healthCheck.delay=' + HOUR, - '--server.port=' + kbnTestConfig.getPort(), - '--server.xsrf.disableProtection=true', - ]; - - const funcTestServerFlags = [ - '--server.maxPayloadBytes=1648576', //default is 1048576 - '--elasticsearch.url=' + esTestConfig.getUrl(), - '--server.port=' + kbnTestConfig.getPort(), - ]; - const browserTestServerFlags = [ '--plugins.initialize=false', '--optimize.bundleFilter=tests', @@ -117,55 +99,6 @@ module.exports = function (grunt) { ] }, - // used by the test:api task - // runs the kibana server prepared for the api_integration tests - apiTestServer: createKbnServerTask({ - flags: [ - ...apiTestServerFlags - ] - }), - - // used by the test:api:server task - // runs the kibana server in --dev mode, prepared for developing api_integration tests - // and watching for changes so the server will restart when necessary - devApiTestServer: createKbnServerTask({ - flags: [ - '--dev', - '--no-base-path', - ...apiTestServerFlags, - ] - }), - - // used by test:ui task - // runs the kibana server prepared for the functional tests - funcTestServer: createKbnServerTask({ - flags: [ - ...funcTestServerFlags, - ] - }), - - // used by the test:ui:server task - // runs the kibana server in dev mode, prepared for the functional tests - devFuncTestServer: createKbnServerTask({ - flags: [ - ...funcTestServerFlags, - '--dev', - '--no-base-path', - '--optimize.watchPort=5611', - '--optimize.watchPrebuild=true', - '--optimize.bundleDir=' + resolve(__dirname, '../../optimize/testUiServer'), - ] - }), - - // used by test:uiRelease task - // runs the kibana server from the oss distributable prepared for the functional tests - ossDistFuncTestServer: createKbnServerTask({ - runBuild: `oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`, - flags: [ - ...funcTestServerFlags, - ] - }), - // used by the test:browser task // runs the kibana server to serve the browser test bundle browserTestServer: createKbnServerTask({ @@ -198,22 +131,6 @@ module.exports = function (grunt) { ] }), - testEsServer: { - options: { - wait: false, - ready: /started/, - quiet: false, - }, - cmd: process.execPath, - args: [ - 'scripts/es', - grunt.option('from') || 'snapshot', - '--license', 'oss', - '-E', `http.port=${esTestConfig.getPort()}`, - '-E', `discovery.zen.ping.unicast.hosts=localhost:${esTestConfig.getPort()}`, - ], - }, - verifyNotice: { options: { wait: true, @@ -223,6 +140,56 @@ module.exports = function (grunt) { 'scripts/notice', '--validate' ] - } + }, + + apiIntegrationTests: { + cmd: process.execPath, + args: [ + 'scripts/functional_tests', + '--config', 'test/api_integration/config.js', + '--esFrom', 'source', + '--bail', + '--debug', + ], + }, + + functionalTests: { + cmd: process.execPath, + args: [ + 'scripts/functional_tests', + '--config', 'test/functional/config.js', + '--esFrom', 'source', + '--bail', + '--debug', + '--', + '--server.maxPayloadBytes=1648576', + ], + }, + + functionalTestsRelease: { + cmd: process.execPath, + args: [ + 'scripts/functional_tests', + '--config', 'test/functional/config.js', + '--esFrom', 'source', + '--bail', + '--debug', + '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`, + '--', + '--server.maxPayloadBytes=1648576', + ], + }, + + functionalTestsDevServer: { + cmd: process.execPath, + args: [ + 'scripts/functional_tests_server', + '--config', 'test/functional/config.js', + '--esFrom', 'source', + '--debug', + '--', + '--server.maxPayloadBytes=1648576', + ], + }, }; }; diff --git a/tasks/jenkins.js b/tasks/jenkins.js index daece61ea4c78..2f7f1d5b92ae2 100644 --- a/tasks/jenkins.js +++ b/tasks/jenkins.js @@ -34,11 +34,10 @@ module.exports = function (grunt) { 'test:jest_integration', 'test:projects', 'test:browser-ci', - 'test:api', + 'run:apiIntegrationTests', 'verifyTranslations', ]); - grunt.config.set('functional_test_runner.functional.options.configOverrides.mochaOpts.bail', true); grunt.registerTask('jenkins:selenium', [ 'test:uiRelease' ]); diff --git a/tasks/test.js b/tasks/test.js index af6ac2e7779e8..f83207aeef15b 100644 --- a/tasks/test.js +++ b/tasks/test.js @@ -64,7 +64,7 @@ module.exports = function (grunt) { 'test:jest_integration', 'test:projects', 'test:browser', - 'test:api' + 'run:apiIntegrationTests' ]); grunt.registerTask('test:dev', [ @@ -75,45 +75,19 @@ module.exports = function (grunt) { grunt.registerTask('test:ui', [ 'checkPlugins', - 'run:testEsServer', - 'run:funcTestServer', - 'functional_test_runner:functional', - 'stop:testEsServer', - 'stop:funcTestServer' + 'run:functionalTests', ]); grunt.registerTask('test:uiRelease', [ 'checkPlugins', - 'run:testEsServer', - 'run:ossDistFuncTestServer', - 'functional_test_runner:functional', - 'stop:testEsServer', - 'stop:ossDistFuncTestServer' + 'run:functionalTestsRelease', ]); grunt.registerTask('test:ui:server', [ 'checkPlugins', - 'run:testEsServer', - 'run:devFuncTestServer:keepalive' + 'run:functionalTestsDevServer', ]); - grunt.registerTask('test:api', [ - 'run:testEsServer', - 'run:apiTestServer', - 'functional_test_runner:apiIntegration', - 'stop:testEsServer', - 'stop:apiTestServer' - ]); - - grunt.registerTask('test:api:server', [ - 'run:testEsServer', - 'run:devApiTestServer:keepalive' - ]); - - grunt.registerTask('test:api:runner', () => { - grunt.fail.fatal('test:api:runner has moved, use: `node scripts/functional_test_runner --config test/api_integration/config.js`'); - }); - grunt.registerTask('test', subTask => { if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`); diff --git a/test/common/config.js b/test/common/config.js index 2f2f7d8920a50..2d677e4b57f78 100644 --- a/test/common/config.js +++ b/test/common/config.js @@ -49,7 +49,7 @@ export default function () { `--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`, ], serverArgs: [ - '--env=development', + '--env.name=development', '--logging.json=false', `--server.port=${kbnTestConfig.getPort()}`, `--optimize.watchPort=${kbnTestConfig.getPort() + 10}`, diff --git a/test/scripts/jenkins_xpack.sh b/test/scripts/jenkins_xpack.sh index 6207a9fc78fb8..8f5d75e2cdc2a 100755 --- a/test/scripts/jenkins_xpack.sh +++ b/test/scripts/jenkins_xpack.sh @@ -37,6 +37,6 @@ tar -xzf "$linuxBuild" -C "$installDir" --strip=1 echo " -> Running functional and api tests" cd "$XPACK_DIR" -xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --es-from=source +xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --esFrom=source echo "" echo ""