Collecting code coverage from functional tests

package.json

@@ -70,7 +70,8 @@
     "core:acceptApiChanges": "node scripts/check_core_api_changes.js --accept",
     "kbn:bootstrap": "yarn build:types && node scripts/register_git_hook",
     "spec_to_console": "node scripts/spec_to_console",
-    "backport-skip-ci": "backport --prDescription \"[skip-ci]\""
+    "backport-skip-ci": "backport --prDescription \"[skip-ci]\"",
+    "cover:report": "nyc report --temp-dir target/kibana-coverage/functional --report-dir target/coverage/report --reporter=lcov && open ./target/coverage/report/lcov-report/index.html"
   },
   "repository": {
     "type": "git",
@@ -261,6 +262,7 @@
     "webpack": "4.41.0",
     "webpack-merge": "4.2.2",
     "whatwg-fetch": "^3.0.0",
+    "wrapper-webpack-plugin": "^2.1.0",
     "yauzl": "2.10.0"
   },
   "devDependencies": {
@@ -355,6 +357,7 @@
     "babel-eslint": "^10.0.3",
     "babel-jest": "^24.9.0",
     "babel-plugin-dynamic-import-node": "^2.3.0",
+    "babel-plugin-istanbul": "^5.2.0",
     "backport": "4.7.1",
     "chai": "3.5.0",
     "chance": "1.0.18",

packages/kbn-babel-preset/istanbul_preset.js

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+module.exports = () => {
+  return {
+    plugins: ['istanbul']
+  };
+};

packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts

@@ -36,7 +36,7 @@ export function createLifecycle() {
     phaseEnd: [] as Listener[],
   };
 
-  const cleanup$ = new Rx.ReplaySubject(1);
+  const cleanup$ = new Rx.ReplaySubject<undefined>(1);
 
   return {
     cleanup$: cleanup$.asObservable(),

src/legacy/server/config/schema.js

@@ -167,7 +167,7 @@ export default () => Joi.object({
     watchPort: Joi.number().default(5602),
     watchHost: Joi.string().hostname().default('localhost'),
     watchPrebuild: Joi.boolean().default(false),
-    watchProxyTimeout: Joi.number().default(5 * 60000),
+    watchProxyTimeout: Joi.number().default(10 * 60000),
     useBundleCache: Joi.boolean().default(Joi.ref('$prod')),
     sourceMaps: Joi.when('$prod', {
       is: true,
@@ -178,7 +178,7 @@ export default () => Joi.object({
           Joi.string().required(),
           Joi.boolean()
         )
-        .default('#cheap-source-map'),
+        .default(!!process.env.CODE_COVERAGE ? 'true' : '#cheap-source-map'),
     }),
     workers: Joi.number().min(1),
     profile: Joi.boolean().default(false)

src/optimize/base_optimizer.js

@@ -27,6 +27,7 @@ import Stats from 'webpack/lib/Stats';
 import * as threadLoader from 'thread-loader';
 import webpackMerge from 'webpack-merge';
 import { DynamicDllPlugin } from './dynamic_dll_plugin';
+import WrapperPlugin from 'wrapper-webpack-plugin';
 
 import { defaults } from 'lodash';
 
@@ -36,13 +37,15 @@ import { PUBLIC_PATH_PLACEHOLDER } from './public_path_placeholder';
 
 const POSTCSS_CONFIG_PATH = require.resolve('./postcss.config');
 const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');
+const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset');
 const BABEL_EXCLUDE_RE = [
   /[\/\\](webpackShims|node_modules|bower_components)[\/\\]/,
 ];
 const STATS_WARNINGS_FILTER = new RegExp([
   '(export .* was not found in)',
   '|(chunk .* \\[mini-css-extract-plugin\\]\\\nConflicting order between:)'
 ].join(''));
+const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE;
 
 function recursiveIssuer(m) {
   if (m.issuer) {
@@ -383,9 +386,7 @@ export default class BaseOptimizer {
                 loader: 'babel-loader',
                 options: {
                   babelrc: false,
-                  presets: [
-                    BABEL_PRESET_PATH,
-                  ],
+                  presets: this.getPresets()
                 },
               }
             ]),
@@ -455,6 +456,22 @@ export default class BaseOptimizer {
       ]
     };
 
+    const coverageConfig = {
+      plugins: [
+        new WrapperPlugin({
+          test: /commons\.bundle\.js$/, // only wrap output of bundle files with '.js' extension
+          header: `
+            window.flushCoverageToLog = function () {
+              if (window.__coverage__) {
+                console.log('coveragejson:' + btoa(JSON.stringify(window.__coverage__)));
+              }
+            };
+            window.addEventListener('beforeunload', window.flushCoverageToLog);
+          `
+        }),
+      ]
+    };
+
     // in production we set the process.env.NODE_ENV and run
     // the terser minimizer over our bundles
     const productionConfig = {
@@ -475,6 +492,9 @@ export default class BaseOptimizer {
 
     return this.uiBundles.getExtendedConfig(
       webpackMerge(
+        IS_CODE_COVERAGE
+          ? coverageConfig
+          : {},
         commonConfig,
         IS_KIBANA_DISTRIBUTABLE
           ? isDistributableConfig
@@ -534,4 +554,10 @@ export default class BaseOptimizer {
         return entryPoints;
       }, {});
   }
+
+  getPresets() {
+    return IS_CODE_COVERAGE
+      ? [ ISTANBUL_PRESET_PATH, BABEL_PRESET_PATH, ]
+      : [ BABEL_PRESET_PATH, ];
+  }
 }

test/functional/services/remote/poll_for_log_entry.ts

@@ -19,48 +19,90 @@
 
 import { WebDriver, logging } from 'selenium-webdriver';
 import * as Rx from 'rxjs';
-import { mergeMap, catchError, repeatWhen, mergeMapTo, delay } from 'rxjs/operators';
+import { mergeMap, catchError, mergeMapTo, delay, first } from 'rxjs/operators';
 
 /**
  * Create an observable that emits log entries representing the calls to log messages
  * available for a specific logger.
  */
-export function pollForLogEntry$(driver: WebDriver, type: string, ms: number) {
+export function pollForLogEntry$(
+  driver: WebDriver,
+  type: string,
+  ms: number,
+  stop$: Rx.Observable<undefined>
+) {
   const logCtrl = driver.manage().logs();
+  const poll$ = new Rx.BehaviorSubject(undefined);
 
-  // setup log polling
-  return Rx.defer(async () => await logCtrl.get(type)).pipe(
-    // filter and flatten list of entries
-    mergeMap(entries =>
-      entries.filter(entry => {
-        // ignore react devtools
-        if (entry.message.includes('Download the React DevTools')) {
-          return false;
-        }
+  const FINAL_MSG = '@@final@@';
 
-        // down-level inline script errors
-        if (entry.message.includes('Refused to execute inline script')) {
-          entry.level = logging.getLevel('INFO');
-        }
+  return new Rx.Observable<logging.Entry>(subscriber => {
+    subscriber.add(
+      stop$.pipe(first()).subscribe(() => {
+        driver
+          .executeScript(
+            `
+              if (window.flushCoverageToLog) {
+                window.flushCoverageToLog();
+              }
 
-        return true;
+              console.log(${JSON.stringify(FINAL_MSG)})
+            `
+          )
+          .catch(error => subscriber.error(error));
       })
-    ),
+    );
 
-    // repeat when parent completes, delayed by `ms` milliseconds
-    repeatWhen($ => $.pipe(delay(ms))),
+    subscriber.add(
+      poll$
+        .pipe(
+          delay(ms),
 
-    catchError((error, resubscribe) => {
-      return Rx.concat(
-        // log error as a log entry
-        [new logging.Entry('SEVERE', `ERROR FETCHING BROWSR LOGS: ${error.message}`)],
+          mergeMap(async () => await logCtrl.get(type)),
 
-        // pause 10 seconds then resubscribe
-        Rx.of(1).pipe(
-          delay(10 * 1000),
-          mergeMapTo(resubscribe)
+          // filter and flatten list of entries
+          mergeMap(entries => {
+            const filtered = entries.filter(entry => {
+              if (entry.message.includes(FINAL_MSG)) {
+                poll$.complete();
+                return false;
+              }
+
+              // ignore react devtools
+              if (entry.message.includes('Download the React DevTools')) {
+                return false;
+              }
+
+              // down-level inline script errors
+              if (entry.message.includes('Refused to execute inline script')) {
+                entry.level = logging.getLevel('INFO');
+              }
+
+              return true;
+            });
+
+            if (!poll$.isStopped) {
+              // schedule next poll
+              poll$.next(undefined);
+            }
+
+            return filtered;
+          }),
+
+          catchError((error, resubscribe) => {
+            return Rx.concat(
+              // log error as a log entry
+              [new logging.Entry('SEVERE', `ERROR FETCHING BROWSR LOGS: ${error.message}`)],
+
+              // pause 10 seconds then resubscribe
+              Rx.of(1).pipe(
+                delay(10 * 1000),
+                mergeMapTo(resubscribe)
+              )
+            );
+          })
         )
-      );
-    })
-  );
+        .subscribe(subscriber)
+    );
+  });
 }

test/functional/services/remote/remote.ts

@@ -17,15 +17,27 @@
  * under the License.
  */
 
+import Fs from 'fs';
+import { resolve } from 'path';
+
+import * as Rx from 'rxjs';
+import { mergeMap } from 'rxjs/operators';
+import { logging } from 'selenium-webdriver';
+
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { initWebDriver } from './webdriver';
 import { Browsers } from './browsers';
+import { pollForLogEntry$ } from './poll_for_log_entry';
 
 export async function RemoteProvider({ getService }: FtrProviderContext) {
   const lifecycle = getService('lifecycle');
   const log = getService('log');
   const config = getService('config');
   const browserType: Browsers = config.get('browser.type');
+  const collectCoverage: boolean = !!process.env.CODE_COVERAGE;
+  const coveragePrefix = 'coveragejson:';
+  const coverageDir = resolve(__dirname, '../../../../target/kibana-coverage/functional');
+  let logSubscription: undefined | Rx.Subscription;
 
   const { driver, By, until, consoleLog$ } = await initWebDriver(
     log,
@@ -42,8 +54,49 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
 
   if (browserType === Browsers.Chrome) {
     log.info(
-      `Chromedriver version: ${caps.get('chrome').chromedriverVersion}, w3c=${isW3CEnabled}`
+      `Chromedriver version: ${
+        caps.get('chrome').chromedriverVersion
+      }, w3c=${isW3CEnabled}, codeCoverage=${collectCoverage}`
     );
+
+    if (collectCoverage) {
+      let coverageCounter = 1;
+      // We are running xpack tests with different configs and cleanup will delete collected coverage
+      // del.sync(coverageDir);
+      Fs.mkdirSync(coverageDir, { recursive: true });
+
+      logSubscription = pollForLogEntry$(
+        driver,
+        logging.Type.BROWSER,
+        config.get('browser.logPollingMs'),
+        lifecycle.cleanup$ as any
+      )
+        .pipe(
+          mergeMap(logEntry => {
+            if (logEntry.message.includes(coveragePrefix)) {
+              const id = coverageCounter++;
+              const timestamp = Date.now();
+              const path = resolve(coverageDir, `${id}.${timestamp}.coverage.json`);
+              const [, coverageJsonBase64] = logEntry.message.split(coveragePrefix);
+              const coverageJson = Buffer.from(coverageJsonBase64, 'base64').toString('utf8');
+
+              log.info('writing coverage to', path);
+              Fs.writeFileSync(path, JSON.stringify(JSON.parse(coverageJson), null, 2));
+
+              // filter out this message
+              return [];
+            }
+
+            return [logEntry];
+          })
+        )
+        .subscribe({
+          next({ message, level: { name: level } }) {
+            const msg = message.replace(/\\n/g, '\n');
+            log[level === 'SEVERE' ? 'error' : 'debug'](`browser[${level}] ${msg}`);
+          },
+        });
+    }
   }
 
   lifecycle.on('beforeTests', async () => {
@@ -77,7 +130,13 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
       .setRect({ width, height });
   });
 
-  lifecycle.on('cleanup', async () => await driver.quit());
+  lifecycle.on('cleanup', async () => {
+    if (logSubscription) {
+      await new Promise(r => logSubscription!.add(r));
+    }
+
+    await driver.quit();
+  });
 
   return { driver, By, until, browserType, consoleLog$ };
 }

test/functional/services/remote/webdriver.ts

@@ -101,7 +101,12 @@ async function attemptToCreateCommand(
 
         return {
           session,
-          consoleLog$: pollForLogEntry$(session, logging.Type.BROWSER, logPollingMs).pipe(
+          consoleLog$: pollForLogEntry$(
+            session,
+            logging.Type.BROWSER,
+            logPollingMs,
+            lifecycle.cleanup$
+          ).pipe(
             takeUntil(lifecycle.cleanup$),
             map(({ message, level: { name: level } }) => ({
               message: message.replace(/\\n/g, '\n'),