diff --git a/.ci/jobs.yml b/.ci/jobs.yml index 1740e1db33f2d..f9302db4b2910 100644 --- a/.ci/jobs.yml +++ b/.ci/jobs.yml @@ -1,7 +1,26 @@ JOB: - - selenium - - intake - - x-pack + - kibana-intake + - x-pack-intake + # make sure all kibana-ciGRoups are listed in tasks/function_test_groups.js + - kibana-ciGroup1 + - kibana-ciGroup2 + - kibana-ciGroup3 + - kibana-ciGroup4 + - kibana-ciGroup5 + - kibana-ciGroup6 + - kibana-ciGroup7 + - kibana-ciGroup8 + - kibana-ciGroup9 + - kibana-ciGroup10 + - kibana-ciGroup11 + - kibana-ciGroup12 + # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh + - x-pack-ciGroup1 + - x-pack-ciGroup2 + - x-pack-ciGroup3 + - x-pack-ciGroup4 + - x-pack-ciGroup5 + - x-pack-ciGroup6 # `~` is yaml for `null` exclude: ~ diff --git a/.ci/run.sh b/.ci/run.sh index 32c138bd2f450..57cc4afd6a38a 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -6,15 +6,20 @@ set -e cd "$(dirname "$0")/.." case "$JOB" in -"selenium") - ./test/scripts/jenkins_selenium.sh - ;; -"intake") +kibana-intake) ./test/scripts/jenkins_unit.sh ;; -"x-pack") +kibana-ciGroup*) + export CI_GROUP="${JOB##kibana-ciGroup}" + ./test/scripts/jenkins_ci_group.sh + ;; +x-pack-intake) ./test/scripts/jenkins_xpack.sh ;; +x-pack-ciGroup*) + export CI_GROUP="${JOB##x-pack-ciGroup}" + ./test/scripts/jenkins_xpack_ci_group.sh + ;; *) echo "JOB '$JOB' is not implemented." exit 1 diff --git a/packages/kbn-test/src/functional_tests/lib/index.js b/packages/kbn-test/src/functional_tests/lib/index.js index 886a77ba6e2e4..ec381b56b0699 100644 --- a/packages/kbn-test/src/functional_tests/lib/index.js +++ b/packages/kbn-test/src/functional_tests/lib/index.js @@ -19,6 +19,6 @@ export { runKibanaServer } from './run_kibana_server'; export { runElasticsearch } from './run_elasticsearch'; -export { runFtr, hasTests, assertNoneExcluded } from './fun_ftr'; +export { runFtr, hasTests, assertNoneExcluded } from './run_ftr'; export { KIBANA_ROOT, KIBANA_FTR_SCRIPT, FUNCTIONAL_CONFIG_PATH, API_CONFIG_PATH } from './paths'; export { runCli } from './run_cli'; diff --git a/packages/kbn-test/src/functional_tests/lib/fun_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js similarity index 89% rename from packages/kbn-test/src/functional_tests/lib/fun_ftr.js rename to packages/kbn-test/src/functional_tests/lib/run_ftr.js index e3dbe6cef6d8a..05e5cbb3d4b55 100644 --- a/packages/kbn-test/src/functional_tests/lib/fun_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -39,14 +39,15 @@ export async function assertNoneExcluded({ configPath, options }) { const ftr = createFtr({ configPath, options }); const stats = await ftr.getTestStats(); - if (stats.excludedTests > 0) { + if (stats.excludedTests.length > 0) { throw new CliError(` - ${stats.excludedTests} tests in the ${configPath} config + ${stats.excludedTests.length} tests in the ${configPath} config are excluded when filtering by the tags run on CI. Make sure that all suites are tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh - ${JSON.stringify(options.suiteTags)} + tags: ${JSON.stringify(options.suiteTags)} + - ${stats.excludedTests.join('\n - ')} `); } } @@ -65,5 +66,5 @@ export async function runFtr({ configPath, options }) { export async function hasTests({ configPath, options }) { const ftr = createFtr({ configPath, options }); const stats = await ftr.getTestStats(); - return stats.tests > 0; + return stats.testCount > 0; } diff --git a/src/dev/jest/junit_reporter.js b/src/dev/jest/junit_reporter.js index 9f8f042b2ddb7..9d05e1dabf59c 100644 --- a/src/dev/jest/junit_reporter.js +++ b/src/dev/jest/junit_reporter.js @@ -49,7 +49,7 @@ export default class JestJUnitReporter { * @return {undefined} */ onRunComplete(contexts, results) { - if (!process.env.CI) { + if (!process.env.CI || !results.testResults.length) { return; } diff --git a/src/dev/mocha/junit_report_generation.js b/src/dev/mocha/junit_report_generation.js index e5c7aec947b8f..74b271e82f52e 100644 --- a/src/dev/mocha/junit_report_generation.js +++ b/src/dev/mocha/junit_report_generation.js @@ -27,6 +27,8 @@ import xmlBuilder from 'xmlbuilder'; import { getSnapshotOfRunnableLogs } from './log_cache'; import { escapeCdata } from '../xml'; +const dateNow = Date.now.bind(Date); + export function setupJUnitReportGeneration(runner, options = {}) { const { reportName = 'Unnamed Mocha Tests', @@ -47,11 +49,11 @@ export function setupJUnitReportGeneration(runner, options = {}) { ); const setStartTime = (node) => { - node.startTime = Date.now(); + node.startTime = dateNow(); }; const setEndTime = node => { - node.endTime = Date.now(); + node.endTime = dateNow(); }; const getFullTitle = node => { @@ -85,6 +87,9 @@ export function setupJUnitReportGeneration(runner, options = {}) { runner.on('end', () => { // crawl the test graph to collect all defined tests const allTests = findAllTests(runner.suite); + if (!allTests.length) { + return; + } // filter out just the failures const failures = results.filter(result => result.failed); diff --git a/src/functional_test_runner/functional_test_runner.js b/src/functional_test_runner/functional_test_runner.js index e7edb63777c1b..9356412ba0738 100644 --- a/src/functional_test_runner/functional_test_runner.js +++ b/src/functional_test_runner/functional_test_runner.js @@ -89,8 +89,8 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides }) ); return { - tests: countTests(mocha.suite), - excludedTests: mocha.excludedTests.length + testCount: countTests(mocha.suite), + excludedTests: mocha.excludedTests.map(t => t.fullTitle()) }; }); } diff --git a/src/functional_test_runner/lib/config/read_config_file.js b/src/functional_test_runner/lib/config/read_config_file.js index bd3cddcf8e25a..e733d2b989ff3 100644 --- a/src/functional_test_runner/lib/config/read_config_file.js +++ b/src/functional_test_runner/lib/config/read_config_file.js @@ -22,18 +22,17 @@ import { defaultsDeep } from 'lodash'; import { Config } from './config'; import { transformDeprecations } from './transform_deprecations'; -async function getSettingsFromFile(log, path, settingOverrides) { - log.debug('Loading config file from %j', path); +const cache = new WeakMap(); +async function getSettingsFromFile(log, path, settingOverrides) { const configModule = require(path); const configProvider = configModule.__esModule ? configModule.default : configModule; - const settingsWithDefaults = defaultsDeep( - {}, - settingOverrides, - await configProvider({ + if (!cache.has(configProvider)) { + log.debug('Loading config file from %j', path); + cache.set(configProvider, configProvider({ log, async readConfigFile(...args) { return new Config({ @@ -42,7 +41,13 @@ async function getSettingsFromFile(log, path, settingOverrides) { path, }); } - }) + })); + } + + const settingsWithDefaults = defaultsDeep( + {}, + settingOverrides, + await cache.get(configProvider) ); const logDeprecation = (...args) => log.error(...args); diff --git a/tasks/config/run.js b/tasks/config/run.js index 2934093a4d424..cd328b0f58ea8 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -18,8 +18,10 @@ */ import { resolve } from 'path'; +import { getFunctionalTestGroupRunConfigs } from '../function_test_groups'; -const PKG_VERSION = require('../../package.json').version; +const { version } = require('../../package.json'); +const KIBANA_INSTALL_DIR = `./build/oss/kibana-${version}-SNAPSHOT-${process.platform}-x86_64`; const KIBANA_BIN_PATH = process.platform.startsWith('win') ? '.\\bin\\kibana.bat' : './bin/kibana'; @@ -191,7 +193,7 @@ module.exports = function (grunt) { '--esFrom', esFrom, '--bail', '--debug', - '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`, + '--kibana-install-dir', KIBANA_INSTALL_DIR, ], }, @@ -203,7 +205,7 @@ module.exports = function (grunt) { '--esFrom', esFrom, '--bail', '--debug', - '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`, + '--kibana-install-dir', KIBANA_INSTALL_DIR, '--', '--server.maxPayloadBytes=1648576', ], @@ -222,18 +224,9 @@ module.exports = function (grunt) { ], }, - functionalTestsRelease: { - cmd: process.execPath, - args: [ - 'scripts/functional_tests', - '--config', 'test/functional/config.js', - '--esFrom', esFrom, - '--bail', - '--debug', - '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`, - '--', - '--server.maxPayloadBytes=1648576', - ], - }, + ...getFunctionalTestGroupRunConfigs({ + esFrom, + kibanaInstallDir: KIBANA_INSTALL_DIR + }) }; }; diff --git a/tasks/function_test_groups.js b/tasks/function_test_groups.js new file mode 100644 index 0000000000000..9ac7bcbf49877 --- /dev/null +++ b/tasks/function_test_groups.js @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import execa from 'execa'; +import grunt from 'grunt'; + +/** + * The list of tags that we use in the functional tests, if we add a new group we need to add it to this list + * and to the list of jobs in .ci/jobs.yml + */ +const TEST_TAGS = [ + 'ciGroup1', + 'ciGroup2', + 'ciGroup3', + 'ciGroup4', + 'ciGroup5', + 'ciGroup6', + 'ciGroup7', + 'ciGroup8', + 'ciGroup9', + 'ciGroup10', + 'ciGroup11', + 'ciGroup12' +]; + +export function getFunctionalTestGroupRunConfigs({ esFrom, kibanaInstallDir } = {}) { + return { + // include a run task for each test group + ...TEST_TAGS.reduce((acc, tag) => ({ + ...acc, + [`functionalTests_${tag}`]: { + cmd: process.execPath, + args: [ + 'scripts/functional_tests', + '--include-tag', tag, + '--config', 'test/functional/config.js', + '--esFrom', esFrom, + '--bail', + '--debug', + '--kibana-install-dir', kibanaInstallDir, + '--', + '--server.maxPayloadBytes=1648576', + ], + } + }), {}), + }; +} + +grunt.registerTask( + 'functionalTests:ensureAllTestsInCiGroup', + 'Check that all of the functional tests are in a CI group', + async function () { + const done = this.async(); + + try { + const stats = JSON.parse(await execa.stderr(process.execPath, [ + 'scripts/functional_test_runner', + ...TEST_TAGS.map(tag => `--include-tag=${tag}`), + '--config', 'test/functional/config.js', + '--test-stats' + ])); + + if (stats.excludedTests.length > 0) { + grunt.fail.fatal(` + ${stats.excludedTests.length} tests are excluded by the ciGroup tags, make sure that + all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js" + knows about the tag that you are using. + + tags: ${JSON.stringify({ include: TEST_TAGS })} + + - ${stats.excludedTests.join('\n - ')} + `); + return; + } + + done(); + } catch (error) { + grunt.fail.fatal(error.stack); + } + } +); diff --git a/tasks/jenkins.js b/tasks/jenkins.js index 69f3bc8205934..283e93e782b24 100644 --- a/tasks/jenkins.js +++ b/tasks/jenkins.js @@ -41,12 +41,6 @@ module.exports = function (grunt) { 'run:apiIntegrationTests', ]); - grunt.registerTask('jenkins:selenium', [ - 'checkPlugins', - 'run:functionalTestsRelease', - 'run:pluginFunctionalTestsRelease', - ]); - grunt.registerTask( 'jenkins:report', 'Reports failed tests found in junit xml files to Github issues', diff --git a/test/functional/apps/console/index.js b/test/functional/apps/console/index.js index ff7c200d3c779..abf758b40d4ff 100644 --- a/test/functional/apps/console/index.js +++ b/test/functional/apps/console/index.js @@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) { const remote = getService('remote'); describe('console app', function () { + this.tags('ciGroup1'); + before(async function () { await remote.setWindowSize(1300, 1100); }); diff --git a/test/functional/apps/context/index.js b/test/functional/apps/context/index.js index e912bafa68f22..0b699f2e404ca 100644 --- a/test/functional/apps/context/index.js +++ b/test/functional/apps/context/index.js @@ -24,6 +24,8 @@ export default function ({ getService, getPageObjects, loadTestFile }) { const kibanaServer = getService('kibanaServer'); describe('context app', function () { + this.tags('ciGroup1'); + before(async function () { await remote.setWindowSize(1200, 800); await esArchiver.loadIfNeeded('logstash_functional'); diff --git a/test/functional/apps/dashboard/index.js b/test/functional/apps/dashboard/index.js index d53364ecdd186..53ba2a77fd0a7 100644 --- a/test/functional/apps/dashboard/index.js +++ b/test/functional/apps/dashboard/index.js @@ -22,26 +22,30 @@ export default function ({ getService, loadTestFile, getPageObjects }) { const esArchiver = getService('esArchiver'); const PageObjects = getPageObjects(['dashboard']); + async function loadCurrentData() { + await remote.setWindowSize(1300, 900); + await PageObjects.dashboard.initTests({ + kibanaIndex: 'dashboard/current/kibana', + dataIndex: 'dashboard/current/data', + defaultIndex: 'logstash-*', + }); + await PageObjects.dashboard.preserveCrossAppState(); + } + + async function unloadCurrentData() { + await PageObjects.dashboard.clearSavedObjectsFromAppLinks(); + await esArchiver.unload('dashboard/current/kibana'); + await esArchiver.unload('dashboard/current/data'); + } + describe('dashboard app', function () { + // This has to be first since the other tests create some embeddables as side affects and our counting assumes + // a fresh index. describe('using current data', function () { - before(async () => { - await remote.setWindowSize(1300, 900); - await PageObjects.dashboard.initTests({ - kibanaIndex: 'dashboard/current/kibana', - dataIndex: 'dashboard/current/data', - defaultIndex: 'logstash-*', - }); - await PageObjects.dashboard.preserveCrossAppState(); - }); - - after(async function () { - await PageObjects.dashboard.clearSavedObjectsFromAppLinks(); - await esArchiver.unload('dashboard/current/kibana'); - await esArchiver.unload('dashboard/current/data'); - }); + this.tags('ciGroup2'); + before(loadCurrentData); + after(unloadCurrentData); - // This has to be first since the other tests create some embeddables as side affects and our counting assumes - // a fresh index. loadTestFile(require.resolve('./_empty_dashboard')); loadTestFile(require.resolve('./_dark_theme')); loadTestFile(require.resolve('./_embeddable_rendering')); @@ -50,6 +54,13 @@ export default function ({ getService, loadTestFile, getPageObjects }) { loadTestFile(require.resolve('./_dashboard_options')); loadTestFile(require.resolve('./_data_shared_attributes')); loadTestFile(require.resolve('./_embed_mode')); + }); + + describe('using current data', function () { + this.tags('ciGroup3'); + before(loadCurrentData); + after(unloadCurrentData); + loadTestFile(require.resolve('./_full_screen_mode')); loadTestFile(require.resolve('./_dashboard_filter_bar')); loadTestFile(require.resolve('./_dashboard_filtering')); @@ -63,12 +74,19 @@ export default function ({ getService, loadTestFile, getPageObjects }) { // the data once to save on time. Eventually, all of these tests should just use current data and we can reserve // legacy data only for specifically testing BWC situations. describe('using legacy data', function () { + this.tags('ciGroup4'); before(() => remote.setWindowSize(1200, 900)); loadTestFile(require.resolve('./_dashboard_time_picker')); loadTestFile(require.resolve('./_bwc_shared_urls')); loadTestFile(require.resolve('./_panel_controls')); loadTestFile(require.resolve('./_dashboard_state')); + }); + + describe('using legacy data', function () { + this.tags('ciGroup5'); + before(() => remote.setWindowSize(1200, 900)); + loadTestFile(require.resolve('./_dashboard_save')); loadTestFile(require.resolve('./_dashboard_time')); loadTestFile(require.resolve('./_dashboard_listing')); diff --git a/test/functional/apps/discover/index.js b/test/functional/apps/discover/index.js index 4c8441d5e40de..37e1853eea793 100644 --- a/test/functional/apps/discover/index.js +++ b/test/functional/apps/discover/index.js @@ -22,6 +22,8 @@ export default function ({ getService, loadTestFile }) { const remote = getService('remote'); describe('discover app', function () { + this.tags('ciGroup6'); + before(function () { return remote.setWindowSize(1200, 800); }); diff --git a/test/functional/apps/getting_started/index.js b/test/functional/apps/getting_started/index.js index 2683372b9df16..3a18bff81b81a 100644 --- a/test/functional/apps/getting_started/index.js +++ b/test/functional/apps/getting_started/index.js @@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) { const remote = getService('remote'); describe('Getting Started ', function () { + this.tags('ciGroup6'); + before(async function () { await remote.setWindowSize(1200, 800); }); diff --git a/test/functional/apps/home/index.js b/test/functional/apps/home/index.js index b4ac690c37e0a..496eaf9fd6af8 100644 --- a/test/functional/apps/home/index.js +++ b/test/functional/apps/home/index.js @@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) { const remote = getService('remote'); describe('homepage app', function () { + this.tags('ciGroup6'); + before(function () { return remote.setWindowSize(1200, 800); }); diff --git a/test/functional/apps/management/index.js b/test/functional/apps/management/index.js index 29ff8ddb9ad78..4d4031b4e489b 100644 --- a/test/functional/apps/management/index.js +++ b/test/functional/apps/management/index.js @@ -21,33 +21,38 @@ export default function ({ getService, loadTestFile }) { const esArchiver = getService('esArchiver'); describe('management', function () { - // on setup, we create an settingsPage instance - // that we will use for all the tests - before(async function () { + before(async () => { await esArchiver.unload('logstash_functional'); await esArchiver.load('empty_kibana'); await esArchiver.loadIfNeeded('makelogs'); }); - after(async function () { + after(async () => { await esArchiver.unload('makelogs'); await esArchiver.unload('empty_kibana'); }); - loadTestFile(require.resolve('./_create_index_pattern_wizard')); - loadTestFile(require.resolve('./_index_pattern_create_delete')); - loadTestFile(require.resolve('./_index_pattern_results_sort')); - loadTestFile(require.resolve('./_index_pattern_popularity')); - loadTestFile(require.resolve('./_kibana_settings')); - loadTestFile(require.resolve('./_scripted_fields')); - loadTestFile(require.resolve('./_scripted_fields_preview')); - loadTestFile(require.resolve('./_index_pattern_filter')); - loadTestFile(require.resolve('./_scripted_fields_filter')); - loadTestFile(require.resolve('./_import_objects')); - loadTestFile(require.resolve('./_mgmt_import_saved_objects')); - loadTestFile(require.resolve('./_test_huge_fields')); - loadTestFile(require.resolve('./_handle_alias')); - loadTestFile(require.resolve('./_handle_version_conflict')); - }); + describe('', function () { + this.tags('ciGroup7'); + + loadTestFile(require.resolve('./_create_index_pattern_wizard')); + loadTestFile(require.resolve('./_index_pattern_create_delete')); + loadTestFile(require.resolve('./_index_pattern_results_sort')); + loadTestFile(require.resolve('./_index_pattern_popularity')); + loadTestFile(require.resolve('./_kibana_settings')); + loadTestFile(require.resolve('./_scripted_fields')); + loadTestFile(require.resolve('./_scripted_fields_preview')); + }); + describe('', function () { + this.tags('ciGroup8'); + + loadTestFile(require.resolve('./_index_pattern_filter')); + loadTestFile(require.resolve('./_scripted_fields_filter')); + loadTestFile(require.resolve('./_import_objects')); + loadTestFile(require.resolve('./_test_huge_fields')); + loadTestFile(require.resolve('./_handle_alias')); + loadTestFile(require.resolve('./_handle_version_conflict')); + }); + }); } diff --git a/test/functional/apps/status_page/index.js b/test/functional/apps/status_page/index.js index 2d25e20042111..52d26cd9f3a45 100644 --- a/test/functional/apps/status_page/index.js +++ b/test/functional/apps/status_page/index.js @@ -25,6 +25,8 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common']); describe('status page', function () { + this.tags('ciGroup1'); + beforeEach(async () => { await PageObjects.common.navigateToApp('status_page'); }); diff --git a/test/functional/apps/timelion/index.js b/test/functional/apps/timelion/index.js index 73c9375460b34..286f14f0fbcdd 100644 --- a/test/functional/apps/timelion/index.js +++ b/test/functional/apps/timelion/index.js @@ -24,6 +24,8 @@ export default function ({ getService, loadTestFile }) { const kibanaServer = getService('kibanaServer'); describe('timelion app', function () { + this.tags('ciGroup1'); + before(async function () { log.debug('Starting timelion before method'); remote.setWindowSize(1280, 800); diff --git a/test/functional/apps/visualize/index.js b/test/functional/apps/visualize/index.js index 808d7fdfe5bf9..c8a7fafed0ad6 100644 --- a/test/functional/apps/visualize/index.js +++ b/test/functional/apps/visualize/index.js @@ -23,8 +23,9 @@ export default function ({ getService, loadTestFile }) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); + describe('visualize app', function () { - before(async function () { + before(async ()=> { log.debug('Starting visualize before method'); remote.setWindowSize(1280, 800); await esArchiver.loadIfNeeded('logstash_functional'); @@ -32,30 +33,49 @@ export default function ({ getService, loadTestFile }) { await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'UTC', 'defaultIndex': 'logstash-*' }); }); - loadTestFile(require.resolve('./_embedding_chart')); - loadTestFile(require.resolve('./_inspector')); - loadTestFile(require.resolve('./_chart_types')); - loadTestFile(require.resolve('./_experimental_vis')); - loadTestFile(require.resolve('./_gauge_chart')); - loadTestFile(require.resolve('./_area_chart')); - loadTestFile(require.resolve('./_line_chart')); - loadTestFile(require.resolve('./_data_table')); - loadTestFile(require.resolve('./_data_table_nontimeindex')); - loadTestFile(require.resolve('./_pie_chart')); - loadTestFile(require.resolve('./_tag_cloud')); - loadTestFile(require.resolve('./_tile_map')); - loadTestFile(require.resolve('./_region_map')); - loadTestFile(require.resolve('./_vertical_bar_chart')); - loadTestFile(require.resolve('./_vertical_bar_chart_nontimeindex')); - loadTestFile(require.resolve('./_heatmap_chart')); - loadTestFile(require.resolve('./_point_series_options')); - loadTestFile(require.resolve('./_markdown_vis')); - loadTestFile(require.resolve('./_tsvb_chart')); - loadTestFile(require.resolve('./_shared_item')); - loadTestFile(require.resolve('./_input_control_vis')); - loadTestFile(require.resolve('./_histogram_request_start')); - loadTestFile(require.resolve('./_vega_chart')); - loadTestFile(require.resolve('./_lab_mode')); - loadTestFile(require.resolve('./_linked_saved_searches.js')); + describe('', function () { + this.tags('ciGroup9'); + + loadTestFile(require.resolve('./_embedding_chart')); + loadTestFile(require.resolve('./_chart_types')); + loadTestFile(require.resolve('./_area_chart')); + loadTestFile(require.resolve('./_data_table')); + loadTestFile(require.resolve('./_data_table_nontimeindex')); + }); + + describe('', function () { + this.tags('ciGroup10'); + + loadTestFile(require.resolve('./_inspector')); + loadTestFile(require.resolve('./_experimental_vis')); + loadTestFile(require.resolve('./_gauge_chart')); + loadTestFile(require.resolve('./_heatmap_chart')); + loadTestFile(require.resolve('./_input_control_vis')); + loadTestFile(require.resolve('./_histogram_request_start')); + }); + + describe('', function () { + this.tags('ciGroup11'); + + loadTestFile(require.resolve('./_line_chart')); + loadTestFile(require.resolve('./_pie_chart')); + loadTestFile(require.resolve('./_region_map')); + loadTestFile(require.resolve('./_point_series_options')); + loadTestFile(require.resolve('./_markdown_vis')); + loadTestFile(require.resolve('./_shared_item')); + loadTestFile(require.resolve('./_lab_mode')); + loadTestFile(require.resolve('./_linked_saved_searches')); + }); + + describe('', function () { + this.tags('ciGroup12'); + + loadTestFile(require.resolve('./_tag_cloud')); + loadTestFile(require.resolve('./_tile_map')); + loadTestFile(require.resolve('./_vertical_bar_chart')); + loadTestFile(require.resolve('./_vertical_bar_chart_nontimeindex')); + loadTestFile(require.resolve('./_tsvb_chart')); + loadTestFile(require.resolve('./_vega_chart')); + }); }); } diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh new file mode 100755 index 0000000000000..ee678898f056e --- /dev/null +++ b/test/scripts/jenkins_ci_group.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e +source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh" +source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh" +source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh" + +"$(FORCE_COLOR=0 yarn bin)/grunt" functionalTests:ensureAllTestsInCiGroup; + +node scripts/build --debug --oss; + +export TEST_BROWSER_HEADLESS=1 +export TEST_ES_FROM=${TEST_ES_FROM:-source} + +"$(FORCE_COLOR=0 yarn bin)/grunt" "run:functionalTests_ciGroup${CI_GROUP}" --from=source; + +if [ "$CI_GROUP" == "1" ]; then + "$(FORCE_COLOR=0 yarn bin)/grunt" run:pluginFunctionalTestsRelease --from=source; +fi diff --git a/test/scripts/jenkins_selenium.sh b/test/scripts/jenkins_selenium.sh deleted file mode 100755 index 178d9657ba706..0000000000000 --- a/test/scripts/jenkins_selenium.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e -source "$(dirname $0)/../../src/dev/ci_setup/setup.sh" -source "$(dirname $0)/../../src/dev/ci_setup/git_setup.sh" -source "$(dirname $0)/../../src/dev/ci_setup/java_setup.sh" - -node scripts/build --release --debug --oss; - -export TEST_BROWSER_HEADLESS=1 -export TEST_ES_FROM=${TEST_ES_FROM:-source} -"$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:selenium --from=source; diff --git a/test/scripts/jenkins_xpack.sh b/test/scripts/jenkins_xpack.sh index 25fe08088cbbd..2af9241942c51 100755 --- a/test/scripts/jenkins_xpack.sh +++ b/test/scripts/jenkins_xpack.sh @@ -24,19 +24,3 @@ cd "$XPACK_DIR" node scripts/jest --ci --no-cache --verbose echo "" echo "" - - -echo " -> building and extracting default Kibana distributable for use in functional tests" -cd "$KIBANA_DIR" -node scripts/build --debug --no-oss -linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" -installDir="$PARENT_DIR/install/kibana" -mkdir -p "$installDir" -tar -xzf "$linuxBuild" -C "$installDir" --strip=1 - -export TEST_ES_FROM=${TEST_ES_FROM:-source} -echo " -> Running functional and api tests" -cd "$XPACK_DIR" -node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir" -echo "" -echo "" diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh new file mode 100755 index 0000000000000..62d2449451747 --- /dev/null +++ b/test/scripts/jenkins_xpack_ci_group.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh" +source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh" +source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh" + +export TEST_BROWSER_HEADLESS=1 +export XPACK_DIR="$(cd "$(dirname "$0")/../../x-pack"; pwd)" +echo "-> XPACK_DIR ${XPACK_DIR}" + +echo " -> Ensuring all functional tests are in a ciGroup" +cd "$XPACK_DIR" +node scripts/functional_tests --assert-none-excluded \ + --include-tag ciGroup1 \ + --include-tag ciGroup2 \ + --include-tag ciGroup3 \ + --include-tag ciGroup4 \ + --include-tag ciGroup5 \ + --include-tag ciGroup6 + +echo " -> building and extracting default Kibana distributable for use in functional tests" +cd "$KIBANA_DIR" +node scripts/build --debug --no-oss +linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" +installDir="$PARENT_DIR/install/kibana" +mkdir -p "$installDir" +tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + +export TEST_ES_FROM=${TEST_ES_FROM:-source} +echo " -> Running functional and api tests" +cd "$XPACK_DIR" +node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir" --include-tag "ciGroup$CI_GROUP" +echo "" +echo "" diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index aa2273ce64212..41d4cabc3b63f 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -5,7 +5,9 @@ */ export default function ({ loadTestFile }) { - describe('apis', () => { + describe('apis', function () { + this.tags('ciGroup5'); + loadTestFile(require.resolve('./es')); loadTestFile(require.resolve('./security')); loadTestFile(require.resolve('./monitoring')); diff --git a/x-pack/test/functional/apps/dashboard_mode/index.js b/x-pack/test/functional/apps/dashboard_mode/index.js index e7907f4fa6508..5953dd4924c5d 100644 --- a/x-pack/test/functional/apps/dashboard_mode/index.js +++ b/x-pack/test/functional/apps/dashboard_mode/index.js @@ -6,6 +6,8 @@ export default function ({ loadTestFile }) { describe('dashboard mode', function () { + this.tags('ciGroup3'); + loadTestFile(require.resolve('./dashboard_view_mode')); }); } diff --git a/x-pack/test/functional/apps/graph/index.js b/x-pack/test/functional/apps/graph/index.js index d77ec07969e63..98b360320c2c7 100644 --- a/x-pack/test/functional/apps/graph/index.js +++ b/x-pack/test/functional/apps/graph/index.js @@ -6,6 +6,8 @@ export default function ({ loadTestFile }) { describe('graph app', function () { + this.tags('ciGroup1'); + loadTestFile(require.resolve('./graph')); }); } diff --git a/x-pack/test/functional/apps/grok_debugger/index.js b/x-pack/test/functional/apps/grok_debugger/index.js index 3dc15acbe5e30..75c05f35abd28 100644 --- a/x-pack/test/functional/apps/grok_debugger/index.js +++ b/x-pack/test/functional/apps/grok_debugger/index.js @@ -5,7 +5,9 @@ */ export default function ({ loadTestFile }) { - describe('logstash', () => { + describe('logstash', function () { + this.tags('ciGroup2'); + loadTestFile(require.resolve('./grok_debugger')); }); } diff --git a/x-pack/test/functional/apps/infra/home_page.ts b/x-pack/test/functional/apps/infra/home_page.ts index d7efdb85838df..04382e4964cf0 100644 --- a/x-pack/test/functional/apps/infra/home_page.ts +++ b/x-pack/test/functional/apps/infra/home_page.ts @@ -15,6 +15,10 @@ export default ({ getPageObjects, getService }: KibanaFunctionalTestDefaultProvi const pageObjects = getPageObjects(['common', 'infraHome']); describe('Home page', () => { + before(async () => { + await esArchiver.load('empty_kibana'); + }); + describe('without metrics present', () => { before(async () => await esArchiver.unload('infra')); diff --git a/x-pack/test/functional/apps/infra/index.ts b/x-pack/test/functional/apps/infra/index.ts index efb3cc9f276bc..87abb2584b6c2 100644 --- a/x-pack/test/functional/apps/infra/index.ts +++ b/x-pack/test/functional/apps/infra/index.ts @@ -8,7 +8,9 @@ import { KibanaFunctionalTestDefaultProviders } from '../../../types/providers'; // tslint:disable-next-line:no-default-export export default ({ loadTestFile }: KibanaFunctionalTestDefaultProviders) => { - describe('InfraOps app', () => { + describe('InfraOps app', function() { + (this as any).tags('ciGroup4'); + loadTestFile(require.resolve('./home_page')); }); }; diff --git a/x-pack/test/functional/apps/logstash/index.js b/x-pack/test/functional/apps/logstash/index.js index bd7577e2b88f7..bc7d2941b7198 100644 --- a/x-pack/test/functional/apps/logstash/index.js +++ b/x-pack/test/functional/apps/logstash/index.js @@ -5,7 +5,9 @@ */ export default function ({ loadTestFile }) { - describe('logstash', () => { + describe('logstash', function () { + this.tags('ciGroup2'); + loadTestFile(require.resolve('./pipeline_list')); loadTestFile(require.resolve('./pipeline_create')); }); diff --git a/x-pack/test/functional/apps/monitoring/index.js b/x-pack/test/functional/apps/monitoring/index.js index 817467f847ddc..a1551f2829a98 100644 --- a/x-pack/test/functional/apps/monitoring/index.js +++ b/x-pack/test/functional/apps/monitoring/index.js @@ -5,7 +5,9 @@ */ export default function ({ loadTestFile }) { - describe('Monitoring app', () => { + describe('Monitoring app', function () { + this.tags('ciGroup1'); + loadTestFile(require.resolve('./cluster/list')); loadTestFile(require.resolve('./cluster/overview')); loadTestFile(require.resolve('./cluster/alerts')); diff --git a/x-pack/test/functional/apps/security/index.js b/x-pack/test/functional/apps/security/index.js index 7e3a1b2b813d5..90b58e57107e6 100644 --- a/x-pack/test/functional/apps/security/index.js +++ b/x-pack/test/functional/apps/security/index.js @@ -6,6 +6,8 @@ export default function ({ loadTestFile }) { describe('security app', function () { + this.tags('ciGroup4'); + loadTestFile(require.resolve('./security')); loadTestFile(require.resolve('./doc_level_security_roles')); loadTestFile(require.resolve('./management')); diff --git a/x-pack/test/functional/apps/spaces/index.ts b/x-pack/test/functional/apps/spaces/index.ts index 455692b86fc81..76eb7d911b17b 100644 --- a/x-pack/test/functional/apps/spaces/index.ts +++ b/x-pack/test/functional/apps/spaces/index.ts @@ -8,6 +8,8 @@ import { TestInvoker } from './lib/types'; // tslint:disable:no-default-export export default function spacesApp({ loadTestFile }: TestInvoker) { describe('Spaces app', function spacesAppTestSuite() { + (this as any).tags('ciGroup4'); + loadTestFile(require.resolve('./spaces_selection')); }); } diff --git a/x-pack/test/functional/apps/status_page/index.ts b/x-pack/test/functional/apps/status_page/index.ts index 15e6711090e09..6b6d5c1d97600 100644 --- a/x-pack/test/functional/apps/status_page/index.ts +++ b/x-pack/test/functional/apps/status_page/index.ts @@ -8,6 +8,8 @@ import { TestInvoker } from './lib/types'; // tslint:disable:no-default-export export default function statusPage({ loadTestFile }: TestInvoker) { describe('Status page', function statusPageTestSuite() { + (this as any).tags('ciGroup4'); + loadTestFile(require.resolve('./status_page')); }); } diff --git a/x-pack/test/functional/apps/watcher/index.js b/x-pack/test/functional/apps/watcher/index.js index 80e60c213d211..5542f5cb8f0b7 100644 --- a/x-pack/test/functional/apps/watcher/index.js +++ b/x-pack/test/functional/apps/watcher/index.js @@ -6,6 +6,8 @@ export default function ({ loadTestFile }) { describe('watcher app', function () { + this.tags('ciGroup1'); + //loadTestFile(require.resolve('./management')); loadTestFile(require.resolve('./watcher_test')); }); diff --git a/x-pack/test/reporting/api/chromium_tests.js b/x-pack/test/reporting/api/chromium_tests.js index d9da4ac802cbc..c9282eda4f0cc 100644 --- a/x-pack/test/reporting/api/chromium_tests.js +++ b/x-pack/test/reporting/api/chromium_tests.js @@ -10,7 +10,9 @@ export default function ({ loadTestFile, getService }) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); - describe('chromium', () => { + describe('chromium', function () { + this.tags('ciGroup6'); + before(async () => { await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH); await esArchiver.load(OSS_DATA_ARCHIVE_PATH); diff --git a/x-pack/test/reporting/api/phantom_tests.js b/x-pack/test/reporting/api/phantom_tests.js index 4a3212bff46ff..e0c8c11b326bd 100644 --- a/x-pack/test/reporting/api/phantom_tests.js +++ b/x-pack/test/reporting/api/phantom_tests.js @@ -10,7 +10,9 @@ export default function ({ loadTestFile, getService }) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); - describe('phantom', () => { + describe('phantom', function () { + this.tags('ciGroup6'); + before(async () => { await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH); await esArchiver.load(OSS_DATA_ARCHIVE_PATH); diff --git a/x-pack/test/reporting/configs/chromium_functional.js b/x-pack/test/reporting/configs/chromium_functional.js index f14d943746645..80a9024c06ab8 100644 --- a/x-pack/test/reporting/configs/chromium_functional.js +++ b/x-pack/test/reporting/configs/chromium_functional.js @@ -13,7 +13,7 @@ export default async function ({ readConfigFile }) { return { ...functionalConfig, junit: { - reportName: 'X-Pack Chromium API Reporting Tests', + reportName: 'X-Pack Chromium Functional Reporting Tests', }, testFiles: [require.resolve('../functional')], kbnTestServer: { diff --git a/x-pack/test/reporting/configs/phantom_functional.js b/x-pack/test/reporting/configs/phantom_functional.js index 48d2559479186..e9a61ee813cf2 100644 --- a/x-pack/test/reporting/configs/phantom_functional.js +++ b/x-pack/test/reporting/configs/phantom_functional.js @@ -13,7 +13,7 @@ export default async function ({ readConfigFile }) { return { ...functionalConfig, junit: { - reportName: 'X-Pack Phantom API Reporting Tests', + reportName: 'X-Pack Phantom Functional Reporting Tests', }, testFiles: [require.resolve('../functional')], kbnTestServer: { diff --git a/x-pack/test/reporting/functional/index.js b/x-pack/test/reporting/functional/index.js index d99cf9a900153..fa473f454a925 100644 --- a/x-pack/test/reporting/functional/index.js +++ b/x-pack/test/reporting/functional/index.js @@ -6,6 +6,7 @@ export default function ({ loadTestFile }) { describe('reporting app', function () { + this.tags('ciGroup6'); loadTestFile(require.resolve('./reporting')); }); } diff --git a/x-pack/test/saml_api_integration/apis/index.js b/x-pack/test/saml_api_integration/apis/index.js index ce7c48029e66a..ac08d2e078abf 100644 --- a/x-pack/test/saml_api_integration/apis/index.js +++ b/x-pack/test/saml_api_integration/apis/index.js @@ -5,7 +5,8 @@ */ export default function ({ loadTestFile }) { - describe('apis SAML', () => { + describe('apis SAML', function () { + this.tags('ciGroup6'); loadTestFile(require.resolve('./security')); }); } diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts index d25a9b852b789..b2b62b501e765 100644 --- a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts +++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts @@ -12,7 +12,9 @@ export default function({ getService, loadTestFile }: TestInvoker) { const es = getService('es'); const supertest = getService('supertest'); - describe('saved objects security and spaces enabled', () => { + describe('saved objects security and spaces enabled', function() { + (this as any).tags('ciGroup5'); + before(async () => { await createUsersAndRoles(es, supertest); }); diff --git a/x-pack/test/saved_object_api_integration/security_only/apis/index.ts b/x-pack/test/saved_object_api_integration/security_only/apis/index.ts index c9be7152f96ea..a0797300ebf2b 100644 --- a/x-pack/test/saved_object_api_integration/security_only/apis/index.ts +++ b/x-pack/test/saved_object_api_integration/security_only/apis/index.ts @@ -12,7 +12,9 @@ export default function({ getService, loadTestFile }: TestInvoker) { const es = getService('es'); const supertest = getService('supertest'); - describe('saved objects security only enabled', () => { + describe('saved objects security only enabled', function() { + (this as any).tags('ciGroup5'); + before(async () => { await createUsersAndRoles(es, supertest); }); diff --git a/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts b/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts index 113cf86454d5f..b1028fa6c74c9 100644 --- a/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts +++ b/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts @@ -8,7 +8,9 @@ import { TestInvoker } from '../../common/lib/types'; // tslint:disable:no-default-export export default function({ loadTestFile }: TestInvoker) { - describe('saved objects spaces only enabled', () => { + describe('saved objects spaces only enabled', function() { + (this as any).tags('ciGroup5'); + loadTestFile(require.resolve('./bulk_create')); loadTestFile(require.resolve('./bulk_get')); loadTestFile(require.resolve('./create')); diff --git a/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts b/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts index 65ab34e7ae8f8..6d8402fbb9244 100644 --- a/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts +++ b/x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts @@ -12,7 +12,9 @@ export default function({ loadTestFile, getService }: TestInvoker) { const es = getService('es'); const supertest = getService('supertest'); - describe('spaces api with security', () => { + describe('spaces api with security', function() { + (this as any).tags('ciGroup5'); + before(async () => { await createUsersAndRoles(es, supertest); }); diff --git a/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts b/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts index 6864ee7fbda94..75b546dd16022 100644 --- a/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts +++ b/x-pack/test/spaces_api_integration/spaces_only/apis/index.ts @@ -8,7 +8,9 @@ import { TestInvoker } from '../../common/lib/types'; // tslint:disable:no-default-export export default function spacesOnlyTestSuite({ loadTestFile }: TestInvoker) { - describe('spaces api without security', () => { + describe('spaces api without security', function() { + (this as any).tags('ciGroup5'); + loadTestFile(require.resolve('./create')); loadTestFile(require.resolve('./delete')); loadTestFile(require.resolve('./get_all'));