[ftr] allow filtering suites by tag

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -16,6 +16,8 @@ Options:
   --bail                         Stop the test run at the first failure. 
   --grep <pattern>               Pattern to select which tests to run. 
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
+  --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 
@@ -29,6 +31,10 @@ Object {
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
   "updateBaselines": true,
 }
 `;
@@ -41,6 +47,10 @@ Object {
   "createLogger": [Function],
   "debug": true,
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -52,6 +62,10 @@ Object {
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -67,6 +81,10 @@ Object {
   "extraKbnOpts": Object {
     "server.foo": "bar",
   },
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -78,6 +96,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "quiet": true,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -89,6 +111,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "silent": true,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -100,6 +126,10 @@ Object {
   "createLogger": [Function],
   "esFrom": "source",
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -111,6 +141,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "installDir": "foo",
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -122,6 +156,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "grep": "management",
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -132,6 +170,10 @@ Object {
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
   "verbose": true,
 }
 `;

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -16,6 +16,8 @@ Options:
   --bail                         Stop the test run at the first failure. 
   --grep <pattern>               Pattern to select which tests to run. 
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
+  --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -44,6 +44,14 @@ const options = {
   updateBaselines: {
     desc: 'Replace baseline screenshots with whatever is generated from the test.',
   },
+  'include-tag': {
+    arg: '<tag>',
+    desc: 'Tags that suites must include to be run, can be included multiple times.',
+  },
+  'exclude-tag': {
+    arg: '<tag>',
+    desc: 'Tags that suites must NOT include to be run, can be included multiple times.',
+  },
   verbose: { desc: 'Log everything.' },
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
@@ -98,6 +106,13 @@ export function processOptions(userOptions, defaultConfigPaths) {
     delete userOptions['kibana-install-dir'];
   }
 
+  userOptions.suiteTags = {
+    include: [].concat(userOptions['include-tag'] || []),
+    exclude: [].concat(userOptions['exclude-tag'] || []),
+  };
+  delete userOptions['include-tag'];
+  delete userOptions['exclude-tag'];
+
   function createLogger() {
     return new ToolingLog({
       level: pickLevelFromFlags(userOptions),
@@ -115,7 +130,9 @@ export function processOptions(userOptions, defaultConfigPaths) {
 
 function validateOptions(userOptions) {
   Object.entries(userOptions).forEach(([key, val]) => {
-    if (key === '_') return;
+    if (key === '_' || key === 'suiteTags') {
+      return;
+    }
 
     // Validate flags passed
     if (options[key] === undefined) {

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -20,7 +20,10 @@
 import { createFunctionalTestRunner } from '../../../../../src/functional_test_runner';
 import { CliError } from './run_cli';
 
-export async function runFtr({ configPath, options: { log, bail, grep, updateBaselines } }) {
+export async function runFtr({
+  configPath,
+  options: { log, bail, grep, updateBaselines, suiteTags },
+}) {
   const ftr = createFunctionalTestRunner({
     log,
     configFile: configPath,
@@ -30,6 +33,7 @@ export async function runFtr({ configPath, options: { log, bail, grep, updateBas
         grep,
       },
       updateBaselines,
+      suiteTags,
     },
   });
 

src/dev/jest/config.js

@@ -27,6 +27,7 @@ export default {
     '<rootDir>/src/cli',
     '<rootDir>/src/cli_keystore',
     '<rootDir>/src/cli_plugin',
+    '<rootDir>/src/functional_test_runner',
     '<rootDir>/src/dev',
     '<rootDir>/src/utils',
     '<rootDir>/src/setup_node_env',

src/functional_test_runner/cli.js

@@ -28,20 +28,26 @@ const cmd = new Command('node scripts/functional_test_runner');
 const resolveConfigPath = v => resolve(process.cwd(), v);
 const defaultConfigPath = resolveConfigPath('test/functional/config.js');
 
-const collectExcludePaths = () => {
+const createMultiArgCollector = (map) => () => {
   const paths = [];
   return (arg) => {
-    paths.push(resolve(arg));
+    paths.push(map ? map(arg) : arg);
     return paths;
   };
 };
 
+const collectExcludePaths = createMultiArgCollector(a => resolve(a));
+const collectIncludeTags = createMultiArgCollector();
+const collectExcludeTags = createMultiArgCollector();
+
 cmd
   .option('--config [path]', 'Path to a config file', resolveConfigPath, defaultConfigPath)
   .option('--bail', 'stop tests after the first failure', false)
   .option('--grep <pattern>', 'pattern used to select which tests to run')
   .option('--invert', 'invert grep to exclude tests', false)
   .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), [])
+  .option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), [])
+  .option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), [])
   .option('--verbose', 'Log everything', false)
   .option('--quiet', 'Only log errors', false)
   .option('--silent', 'Log nothing', false)
@@ -69,6 +75,10 @@ const functionalTestRunner = createFunctionalTestRunner({
       grep: cmd.grep,
       invert: cmd.invert,
     },
+    suiteTags: {
+      include: cmd.includeTag,
+      exclude: cmd.excludeTag,
+    },
     updateBaselines: cmd.updateBaselines,
     excludeTestFiles: cmd.exclude
   }

src/functional_test_runner/lib/config/schema.js

@@ -62,6 +62,11 @@ export const schema = Joi.object().keys({
 
   excludeTestFiles: Joi.array().items(Joi.string()).default([]),
 
+  suiteTags: Joi.object().keys({
+    include: Joi.array().items(Joi.string()).default([]),
+    exclude: Joi.array().items(Joi.string()).default([]),
+  }).default(),
+
   services: Joi.object().pattern(
     ID_PATTERN,
     Joi.func().required()

src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -61,6 +61,10 @@ export function decorateMochaUi(lifecycle, context) {
             await lifecycle.trigger('beforeTestSuite', this);
           });
 
+          this.tags = (tags) => {
+            this._tags = [].concat(this._tags || [], tags);
+          };
+
           provider.call(this);
 
           after(async () => {

src/functional_test_runner/lib/mocha/filter_suites_by_tags.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Given a mocha instance that has already loaded all of its suites, filter out
+ * the suites based on the include/exclude tags. If there are include tags then
+ * only suites which include the tag will be run, and if there are exclude tags
+ * then any suite with that tag will not be run.
+ *
+ * @param options.mocha instance of mocha that we are going to be running
+ * @param options.include an array of tags that suites must be tagged with to be run
+ * @param options.exclude an array of tags that will be used to exclude suites from the run
+ */
+export function filterSuitesByTags({ log, mocha, include, exclude }) {
+  // if include tags were provided, filter the tree once to
+  // only include branches that are included at some point
+  if (include.length) {
+    log.info('Only running suites (and their sub-suites) if they include the tag(s):', include);
+
+    const isIncluded = suite => !suite._tags ? false : suite._tags.some(t => include.includes(t));
+    const isChildIncluded = suite => suite.suites.some(s => isIncluded(s) || isChildIncluded(s));
+
+    (function recurse(parentSuite) {
+      const children = parentSuite.suites;
+      parentSuite.suites = [];
+
+      for (const child of children) {
+        // this suite is explicitly included
+        if (isIncluded(child)) {
+          parentSuite.suites.push(child);
+          continue;
+        }
+
+        // this suite has an included child but is not included
+        // itself, so strip out its tests and recurse to filter
+        // out child suites which are not included
+        if (isChildIncluded(child)) {
+          child.tests = [];
+          parentSuite.suites.push(child);
+          recurse(child);
+          continue;
+        }
+      }
+    }(mocha.suite));
+  }
+
+
+  // if exclude tags were provided, filter the possibly already
+  // filtered tree to remove branches that are excluded
+  if (exclude.length) {
+    log.info('Filtering out any suites that include the tag(s):', exclude);
+
+    const isNotExcluded = suite => !suite._tags || !suite._tags.some(t => exclude.includes(t));
+
+    (function recurse(parentSuite) {
+      const children = parentSuite.suites;
+      parentSuite.suites = [];
+
+      for (const child of children) {
+        // keep suites that are not explicitly excluded but
+        // recurse to remove excluded children
+        if (isNotExcluded(child)) {
+          parentSuite.suites.push(child);
+          recurse(child);
+        }
+      }
+    }(mocha.suite));
+  }
+}