[plugin-helpers] improve 3rd party KP plugin support (#75019)

Co-authored-by: Tyler Smalley <[email protected]>
Co-authored-by: spalger <[email protected]>

Author: 3 people

docs/developer/plugin/external-plugin-functional-tests.asciidoc

@@ -13,7 +13,7 @@ To get started copy and paste this example to `test/functional/config.js`:
 ["source","js"]
 -----------
 import { resolve } from 'path';
-import { resolveKibanaPath } from '@kbn/plugin-helpers';
+import { REPO_ROOT } from '@kbn/dev-utils';
 
 import { MyServiceProvider } from './services/my_service';
 import { MyAppPageProvider } from './services/my_app_page';
@@ -24,7 +24,7 @@ export default async function ({ readConfigFile }) {
 
   // read the {kib} config file so that we can utilize some of
   // its services and PageObjects
-  const kibanaConfig = await readConfigFile(resolveKibanaPath('test/functional/config.js'));
+  const kibanaConfig = await readConfigFile(resolve(REPO_ROOT, 'test/functional/config.js'));
 
   return {
     // list paths to the files that contain your plugins tests

packages/kbn-dev-utils/package.json

@@ -1,32 +1,38 @@
 {
   "name": "@kbn/dev-utils",
-  "main": "./target/index.js",
   "version": "1.0.0",
-  "license": "Apache-2.0",
   "private": true,
+  "license": "Apache-2.0",
+  "main": "./target/index.js",
   "scripts": {
     "build": "tsc",
     "kbn:bootstrap": "yarn build",
     "kbn:watch": "yarn build --watch"
   },
   "dependencies": {
+    "@babel/core": "^7.11.1",
     "axios": "^0.19.0",
     "chalk": "^4.1.0",
+    "cheerio": "0.22.0",
     "dedent": "^0.7.0",
     "execa": "^4.0.2",
     "exit-hook": "^2.2.0",
     "getopts": "^2.2.5",
+    "globby": "^8.0.1",
     "load-json-file": "^6.2.0",
-    "normalize-path": "^3.0.0",
+    "markdown-it": "^10.0.0",
     "moment": "^2.24.0",
+    "normalize-path": "^3.0.0",
     "rxjs": "^6.5.5",
     "strip-ansi": "^6.0.0",
     "tree-kill": "^1.2.2",
-    "tslib": "^2.0.0"
+    "vinyl": "^2.2.0"
   },
   "devDependencies": {
-    "typescript": "4.0.2",
+    "@kbn/babel-preset": "1.0.0",
     "@kbn/expect": "1.0.0",
-    "chance": "1.0.18"
+    "@types/vinyl": "^2.0.4",
+    "chance": "1.0.18",
+    "typescript": "4.0.2"
   }
 }

packages/kbn-dev-utils/src/babel.ts

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import File from 'vinyl';
+import * as Babel from '@babel/core';
+
+const transformedFiles = new WeakSet<File>();
+
+/**
+ * Returns a promise that resolves when the file has been
+ * mutated so the contents of the file are tranformed with
+ * babel, include inline sourcemaps, and the filename has
+ * been updated to use .js.
+ *
+ * If the file was previously transformed with this function
+ * the promise will just resolve immediately.
+ */
+export async function transformFileWithBabel(file: File) {
+  if (!(file.contents instanceof Buffer)) {
+    throw new Error('file must be buffered');
+  }
+
+  if (transformedFiles.has(file)) {
+    return;
+  }
+
+  const source = file.contents.toString('utf8');
+  const result = await Babel.transformAsync(source, {
+    babelrc: false,
+    configFile: false,
+    sourceMaps: 'inline',
+    filename: file.path,
+    presets: [require.resolve('@kbn/babel-preset/node_preset')],
+  });
+
+  if (!result || typeof result.code !== 'string') {
+    throw new Error('babel transformation failed without an error...');
+  }
+
+  file.contents = Buffer.from(result.code);
+  file.extname = '.js';
+  transformedFiles.add(file);
+}

packages/kbn-dev-utils/src/index.ts

@@ -41,3 +41,6 @@ export * from './stdio';
 export * from './ci_stats_reporter';
 export * from './plugin_list';
 export * from './simple_kibana_platform_plugin_discovery';
+export * from './streams';
+export * from './babel';
+export * from './parse_kibana_platform_plugin';

packages/kbn-dev-utils/src/parse_kibana_platform_plugin.ts

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+import loadJsonFile from 'load-json-file';
+
+export interface KibanaPlatformPlugin {
+  readonly directory: string;
+  readonly manifestPath: string;
+  readonly manifest: {
+    id: string;
+    ui: boolean;
+    server: boolean;
+    [key: string]: unknown;
+  };
+}
+
+export function parseKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin {
+  if (!Path.isAbsolute(manifestPath)) {
+    throw new TypeError('expected new platform manifest path to be absolute');
+  }
+
+  const manifest = loadJsonFile.sync(manifestPath);
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
+    throw new TypeError('expected new platform plugin manifest to be a JSON encoded object');
+  }
+
+  if (typeof manifest.id !== 'string') {
+    throw new TypeError('expected new platform plugin manifest to have a string id');
+  }
+
+  return {
+    directory: Path.dirname(manifestPath),
+    manifestPath,
+    manifest: {
+      ...manifest,
+
+      ui: !!manifest.ui,
+      server: !!manifest.server,
+      id: manifest.id,
+    },
+  };
+}

packages/kbn-dev-utils/src/run/flags.ts

@@ -52,8 +52,8 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions =
     boolean: [...(global.boolean || []), ...(local.boolean || [])],
     string: [...(global.string || []), ...(local.string || [])],
     default: {
-      ...global.alias,
-      ...local.alias,
+      ...global.default,
+      ...local.default,
     },
 
     help: local.help,

packages/kbn-dev-utils/src/serializers/index.ts

@@ -21,3 +21,4 @@ export * from './absolute_path_serializer';
 export * from './strip_ansi_serializer';
 export * from './recursive_serializer';
 export * from './any_instance_serizlizer';
+export * from './replace_serializer';

packages/kbn-dev-utils/src/serializers/replace_serializer.ts

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createRecursiveSerializer } from './recursive_serializer';
+
+type Replacer = (substring: string, ...args: any[]) => string;
+
+export function createReplaceSerializer(
+  toReplace: string | RegExp,
+  replaceWith: string | Replacer
+) {
+  return createRecursiveSerializer(
+    typeof toReplace === 'string'
+      ? (v: any) => typeof v === 'string' && v.includes(toReplace)
+      : (v: any) => typeof v === 'string' && toReplace.test(v),
+    typeof replaceWith === 'string'
+      ? (v: string) => v.replace(toReplace, replaceWith)
+      : (v: string) => v.replace(toReplace, replaceWith)
+  );
+}

packages/kbn-dev-utils/src/simple_kibana_platform_plugin_discovery.ts

@@ -20,67 +20,37 @@
 import Path from 'path';
 
 import globby from 'globby';
-import loadJsonFile from 'load-json-file';
 
-export interface KibanaPlatformPlugin {
-  readonly directory: string;
-  readonly manifestPath: string;
-  readonly manifest: {
-    id: string;
-    [key: string]: unknown;
-  };
-}
+import { parseKibanaPlatformPlugin } from './parse_kibana_platform_plugin';
 
 /**
  * Helper to find the new platform plugins.
  */
-export function simpleKibanaPlatformPluginDiscovery(scanDirs: string[], paths: string[]) {
+export function simpleKibanaPlatformPluginDiscovery(scanDirs: string[], pluginPaths: string[]) {
   const patterns = Array.from(
     new Set([
       // find kibana.json files up to 5 levels within the scan dir
       ...scanDirs.reduce(
         (acc: string[], dir) => [
           ...acc,
-          `${dir}/*/kibana.json`,
-          `${dir}/*/*/kibana.json`,
-          `${dir}/*/*/*/kibana.json`,
-          `${dir}/*/*/*/*/kibana.json`,
-          `${dir}/*/*/*/*/*/kibana.json`,
+          Path.resolve(dir, '*/kibana.json'),
+          Path.resolve(dir, '*/*/kibana.json'),
+          Path.resolve(dir, '*/*/*/kibana.json'),
+          Path.resolve(dir, '*/*/*/*/kibana.json'),
+          Path.resolve(dir, '*/*/*/*/*/kibana.json'),
         ],
         []
       ),
-      ...paths.map((path) => `${path}/kibana.json`),
+      ...pluginPaths.map((path) => Path.resolve(path, `kibana.json`)),
     ])
   );
 
   const manifestPaths = globby.sync(patterns, { absolute: true }).map((path) =>
-    // absolute paths returned from globby are using normalize or something so the path separators are `/` even on windows, Path.resolve solves this
+    // absolute paths returned from globby are using normalize or
+    // something so the path separators are `/` even on windows,
+    // Path.resolve solves this
     Path.resolve(path)
   );
 
-  return manifestPaths.map(
-    (manifestPath): KibanaPlatformPlugin => {
-      if (!Path.isAbsolute(manifestPath)) {
-        throw new TypeError('expected new platform manifest path to be absolute');
-      }
-
-      const manifest = loadJsonFile.sync(manifestPath);
-      if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
-        throw new TypeError('expected new platform plugin manifest to be a JSON encoded object');
-      }
-
-      if (typeof manifest.id !== 'string') {
-        throw new TypeError('expected new platform plugin manifest to have a string id');
-      }
-
-      return {
-        directory: Path.dirname(manifestPath),
-        manifestPath,
-        manifest: {
-          ...manifest,
-          id: manifest.id,
-        },
-      };
-    }
-  );
+  return manifestPaths.map(parseKibanaPlatformPlugin);
 }

packages/kbn-plugin-generator/src/streams.ts renamed to packages/kbn-dev-utils/src/streams.ts

@@ -20,7 +20,6 @@
 import { Transform } from 'stream';
 
 import File from 'vinyl';
-import { Minimatch } from 'minimatch';
 
 interface BufferedFile extends File {
   contents: Buffer;
@@ -33,41 +32,31 @@ interface BufferedFile extends File {
  * mutate the file, replace it with another file (return a new File
  * object), or drop it from the stream (return null)
  */
-export const tapFileStream = (
+export const transformFileStream = (
   fn: (file: BufferedFile) => File | void | null | Promise<File | void | null>
 ) =>
   new Transform({
     objectMode: true,
-    transform(file: BufferedFile, _, cb) {
-      Promise.resolve(file)
-        .then(fn)
-        .then(
-          (result) => {
-            // drop the file when null is returned
-            if (result === null) {
-              cb();
-            } else {
-              cb(undefined, result || file);
-            }
-          },
-          (error) => cb(error)
-        );
-    },
-  });
+    transform(file: File, _, cb) {
+      Promise.resolve()
+        .then(async () => {
+          if (file.isDirectory()) {
+            return cb(undefined, file);
+          }
 
-export const excludeFiles = (globs: string[]) => {
-  const patterns = globs.map(
-    (g) =>
-      new Minimatch(g, {
-        matchBase: true,
-      })
-  );
+          if (!(file.contents instanceof Buffer)) {
+            throw new Error('files must be buffered to use transformFileStream()');
+          }
 
-  return tapFileStream((file) => {
-    const path = file.relative.replace(/\.ejs$/, '');
-    const exclude = patterns.some((p) => p.match(path));
-    if (exclude) {
-      return null;
-    }
+          const result = await fn(file as BufferedFile);
+
+          if (result === null) {
+            // explicitly drop file if null is returned
+            cb();
+          } else {
+            cb(undefined, result || file);
+          }
+        })
+        .catch(cb);
+    },
   });
-};