[pkgs/peggy] automatically transform peggy files with babel-register and webpack

.github/CODEOWNERS

@@ -885,6 +885,7 @@ packages/home/sample_data_tab @elastic/kibana-global-experience
 packages/home/sample_data_types @elastic/kibana-global-experience
 packages/kbn-ace @elastic/platform-deployment-management
 packages/kbn-alerts @elastic/security-solution
+packages/kbn-ambient-common-types @elastic/kibana-operations
 packages/kbn-ambient-storybook-types @elastic/kibana-operations
 packages/kbn-ambient-ui-types @elastic/kibana-operations
 packages/kbn-analytics @elastic/kibana-core
@@ -956,6 +957,8 @@ packages/kbn-monaco @elastic/kibana-app-services
 packages/kbn-optimizer @elastic/kibana-operations
 packages/kbn-optimizer-webpack-helpers @elastic/kibana-operations
 packages/kbn-osquery-io-ts-types @elastic/security-asset-management
+packages/kbn-peggy @elastic/kibana-operations
+packages/kbn-peggy-loader @elastic/kibana-operations
 packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing
 packages/kbn-plugin-discovery @elastic/kibana-operations
 packages/kbn-plugin-generator @elastic/kibana-operations

dev_docs/operations/operations_landing.mdx

@@ -64,6 +64,7 @@ layout: landing
     { pageId: "kibDevDocsOpsExpect" },
     { pageId: "kibDevDocsOpsAmbientStorybookTypes" },
     { pageId: "kibDevDocsOpsAmbientUiTypes" },
+    { pageId: "kibDevDocsOpsAmbientCommonTypes" },
     { pageId: "kibDevDocsOpsTestSubjSelector" },
     { pageId: "kibDevDocsOpsBazelRunner" },
     { pageId: "kibDevDocsOpsCliDevMode" },
@@ -77,5 +78,7 @@ layout: landing
     { pageId: "kibDevDocsOpsManagedVscodeConfigCli" },
     { pageId: "kibDevDocsOpsTest" },
     { pageId: "kibDevDocsOpsEsArchiver" },
+    { pageId: "kibDevDocsOpsPeggy" },
+    { pageId: "kibDevDocsOpsPeggyLoader" },
   ]}
 />

package.json

@@ -718,7 +718,9 @@
     "@istanbuljs/schema": "^0.1.2",
     "@jest/console": "^29.3.1",
     "@jest/reporters": "^29.3.1",
+    "@jest/transform": "^29.3.1",
     "@jest/types": "^29.3.1",
+    "@kbn/ambient-common-types": "link:bazel-bin/packages/kbn-ambient-common-types",
     "@kbn/ambient-storybook-types": "link:bazel-bin/packages/kbn-ambient-storybook-types",
     "@kbn/ambient-ui-types": "link:bazel-bin/packages/kbn-ambient-ui-types",
     "@kbn/apm-synthtrace": "link:bazel-bin/packages/kbn-apm-synthtrace",
@@ -757,6 +759,8 @@
     "@kbn/managed-vscode-config-cli": "link:bazel-bin/packages/kbn-managed-vscode-config-cli",
     "@kbn/optimizer": "link:bazel-bin/packages/kbn-optimizer",
     "@kbn/optimizer-webpack-helpers": "link:bazel-bin/packages/kbn-optimizer-webpack-helpers",
+    "@kbn/peggy": "link:bazel-bin/packages/kbn-peggy",
+    "@kbn/peggy-loader": "link:bazel-bin/packages/kbn-peggy-loader",
     "@kbn/performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor",
     "@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator",
     "@kbn/plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers",
@@ -1060,7 +1064,7 @@
     "jsondiffpatch": "0.4.1",
     "license-checker": "^25.0.1",
     "listr": "^0.14.1",
-    "lmdb-store": "^1.6.11",
+    "lmdb-store": "^1",
     "loader-utils": "^2.0.4",
     "marge": "^1.0.1",
     "micromatch": "^4.0.5",

packages/BUILD.bazel

@@ -194,6 +194,7 @@ filegroup(
     "//packages/home/sample_data_types:build",
     "//packages/kbn-ace:build",
     "//packages/kbn-alerts:build",
+    "//packages/kbn-ambient-common-types:build",
     "//packages/kbn-ambient-storybook-types:build",
     "//packages/kbn-ambient-ui-types:build",
     "//packages/kbn-analytics:build",
@@ -265,6 +266,8 @@ filegroup(
     "//packages/kbn-optimizer:build",
     "//packages/kbn-optimizer-webpack-helpers:build",
     "//packages/kbn-osquery-io-ts-types:build",
+    "//packages/kbn-peggy:build",
+    "//packages/kbn-peggy-loader:build",
     "//packages/kbn-performance-testing-dataset-extractor:build",
     "//packages/kbn-plugin-discovery:build",
     "//packages/kbn-plugin-generator:build",
@@ -617,6 +620,8 @@ filegroup(
     "//packages/kbn-optimizer:build_types",
     "//packages/kbn-optimizer-webpack-helpers:build_types",
     "//packages/kbn-osquery-io-ts-types:build_types",
+    "//packages/kbn-peggy:build_types",
+    "//packages/kbn-peggy-loader:build_types",
     "//packages/kbn-performance-testing-dataset-extractor:build_types",
     "//packages/kbn-plugin-discovery:build_types",
     "//packages/kbn-plugin-generator:build_types",

packages/kbn-ambient-common-types/BUILD.bazel

@@ -0,0 +1,58 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "pkg_npm")
+
+PKG_DIRNAME = "kbn-ambient-common-types"
+PKG_REQUIRE_NAME = "@kbn/ambient-common-types"
+
+SRCS = glob(
+  [
+    "*.d.ts",
+  ]
+)
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+]
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = SRCS + NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS,
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+alias(
+  name = "npm_module_types",
+  actual = ":" + PKG_DIRNAME,
+  visibility = ["//visibility:public"],
+)

packages/kbn-ambient-common-types/README.mdx

@@ -0,0 +1,20 @@
+---
+id: kibDevDocsOpsAmbientCommonTypes
+slug: /kibana-dev-docs/ops/ambient-common-types
+title: "@kbn/ambient-common-types"
+description: A package holding ambient type definitions for files that are expected to run on the server and the browser
+date: 2022-05-18
+tags: ['kibana', 'dev', 'contributor', 'operations', 'ambient', 'ui', 'common', 'server', 'types']
+---
+
+This package holds ambient typescript definitions which should be included in projects which are expected to run on the server and the browser.
+
+## Plugins
+These types will automatically be included for plugins.
+
+## Packages
+
+To include these types in a package:
+
+- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file.
+- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file.

packages/kbn-ambient-common-types/index.d.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * These ambient types are used to define default types for anything which is
+ * supported in both server/browser environments.
+ */
+
+/**
+ * peggy grammars are built automatically on import in both browser/server
+ */
+declare module '*.peggy' {
+  export interface ParserOptions {
+    [key: string]: any;
+    /**
+     * Object that will be attached to the each `LocationRange` object created by
+     * the parser. For example, this can be path to the parsed file or even the
+     * File object.
+     */
+    grammarSource?: any;
+    startRule?: string;
+    tracer?: ParserTracer;
+  }
+
+  /**
+   * parse `input` using the peggy grammer
+   * @param input code to parse
+   * @param options parse options
+   */
+  export function parse(input: string, options?: ParserOptions): any;
+}

packages/kbn-ambient-common-types/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-ambient-common-types'],
+};

packages/kbn-ambient-common-types/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/ambient-common-types",
+  "owner": "@elastic/kibana-operations",
+  "devOnly": true,
+  "runtimeDeps": [],
+  "typeDeps": [],
+}

packages/kbn-ambient-common-types/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@kbn/ambient-common-types",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "types": "./target_types/index.d.ts",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-ambient-common-types/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ]
+}

packages/kbn-es-query/BUILD.bazel

@@ -1,5 +1,4 @@
 load("@npm//@bazel/typescript:index.bzl", "ts_config")
-load("@npm//peggy:index.bzl", "peggy")
 load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
 load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
 
@@ -9,6 +8,8 @@ PKG_REQUIRE_NAME = "@kbn/es-query"
 SOURCE_FILES = glob(
   [
     "**/*.ts",
+    "**/grammar.peggy.config.json",
+    "**/grammar.peggy",
   ],
   exclude = [
     "**/*.config.js",
@@ -51,6 +52,7 @@ RUNTIME_DEPS = [
 TYPES_DEPS = [
   "//packages/kbn-utility-types:npm_module_types",
   "//packages/kbn-i18n:npm_module_types",
+  "//packages/kbn-ambient-common-types:npm_module_types",
   "@npm//@elastic/elasticsearch",
   "@npm//tslib",
   "@npm//@types/jest",
@@ -59,33 +61,25 @@ TYPES_DEPS = [
   "@npm//@types/node",
 ]
 
-peggy(
-  name = "grammar",
-  data = [
-    ":grammar/grammar.peggy",
-    ":grammar/grammar.config.json"
-  ],
-  output_dir = True,
-  args = [
-    "--extra-options-file",
-    "./%s/grammar/grammar.config.json" % package_name(),
-    "-o",
-    "$(@D)/built_grammar.js",
-    "./%s/grammar/grammar.peggy" % package_name()
-  ],
-)
-
 jsts_transpiler(
   name = "target_node",
   srcs = SRCS,
   build_pkg_name = package_name(),
+  additional_args = [
+    "--copy-files",
+    "--quiet"
+  ],
 )
 
 jsts_transpiler(
   name = "target_web",
   srcs = SRCS,
   build_pkg_name = package_name(),
   web = True,
+  additional_args = [
+    "--copy-files",
+    "--quiet"
+  ],
 )
 
 ts_config(
@@ -110,15 +104,15 @@ ts_project(
 
 js_library(
   name = PKG_DIRNAME,
-  srcs = NPM_MODULE_EXTRA_FILES + [":grammar"],
+  srcs = NPM_MODULE_EXTRA_FILES,
   deps = RUNTIME_DEPS + [":target_node", ":target_web"],
   package_name = PKG_REQUIRE_NAME,
   visibility = ["//visibility:public"],
 )
 
 js_library(
   name = "npm_module_types",
-  srcs = NPM_MODULE_EXTRA_FILES + [":grammar"],
+  srcs = NPM_MODULE_EXTRA_FILES,
   deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"],
   package_name = PKG_REQUIRE_NAME,
   visibility = ["//visibility:public"],

packages/kbn-es-query/src/kuery/grammar/index.ts

@@ -6,4 +6,4 @@
  * Side Public License, v 1.
  */
 
-export { parse } from '../../../../grammar/built_grammar.js';
+export { parse } from './grammar.peggy';

packages/kbn-es-query/tsconfig.json

@@ -6,7 +6,8 @@
     "outDir": "target_types",
     "types": [
       "jest",
-      "node"
+      "node",
+      "@kbn/ambient-common-types"
     ]
   },
   "include": [

packages/kbn-import-resolver/src/import_resolver.ts

@@ -125,14 +125,10 @@ export class ImportResolver {
       return true;
     }
 
-    // ignore requests to grammar/built_grammar.js files or bazel target dirs, these files are only
-    // available in the build output and will never resolve in dev. We will validate that people don't
-    // import these files from outside the package in another rule
-    if (
-      req.endsWith('grammar/built_grammar.js') ||
-      req.includes('/target_workers/') ||
-      req.includes('/target_node/')
-    ) {
+    // ignore requests to bazel target dirs, these files are only available in the build output
+    // and will never resolve in dev. We will validate that people don't import these files from
+    // outside the package in another rule
+    if (req.includes('/target_workers/') || req.includes('/target_node/')) {
       return true;
     }
 

packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts

@@ -77,12 +77,6 @@ describe('#resolve()', () => {
   });
 
   it('returns ignore results for known unresolvable but okay import statements', () => {
-    expect(resolver.resolve('../../grammar/built_grammar.js', FIXTURES_DIR)).toMatchInlineSnapshot(`
-      Object {
-        "type": "ignore",
-      }
-    `);
-
     expect(resolver.resolve('kibana-buildkite-library', FIXTURES_DIR)).toMatchInlineSnapshot(`
       Object {
         "type": "ignore",