Merge remote-tracking branch 'origin/master' into HEAD

Author: wylieconlon

.github/workflows/project-assigner.yml

@@ -11,7 +11,7 @@ jobs:
         uses: elastic/github-actions/[email protected]
         id: project_assigner
         with:
-          issue-mappings: '[{"label": "Team:AppArch", "projectNumber": 37, "columnName": "To triage"}, {"label": "Feature:Lens", "projectNumber": 32, "columnName": "Long-term goals"}, {"label": "Feature:Canvas", "projectNumber": 38, "columnName": "Inbox"}, {"label": "Feature:Dashboard", "projectNumber": 68, "columnName": "Inbox"}, {"label": "Feature:Drilldowns", "projectNumber": 68, "columnName": "Inbox"}]'
+          issue-mappings: '[{"label": "Feature:Lens", "projectNumber": 32, "columnName": "Long-term goals"}, {"label": "Feature:Canvas", "projectNumber": 38, "columnName": "Inbox"}, {"label": "Feature:Dashboard", "projectNumber": 68, "columnName": "Inbox"}, {"label": "Feature:Drilldowns", "projectNumber": 68, "columnName": "Inbox"}]'
           ghToken: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}
 
 

docs/discover/kuery.asciidoc

@@ -147,7 +147,7 @@ To match multiple fields:
 machine.os*:windows 10
 -------------------
 
-This sytax is handy when you have text and keyword
+This syntax is handy when you have text and keyword
 versions of a field. The query checks machine.os and machine.os.keyword
 for the term
 `windows 10`.

docs/migration/migrate_8_0.asciidoc

@@ -36,6 +36,12 @@ for example, `logstash-*`.
 === Settings changes
 
 // tag::notable-breaking-changes[]
+[float]
+==== Multitenancy by changing `kibana.index` is no longer supported
+*Details:* `kibana.index`, `xpack.reporting.index` and `xpack.task_manager.index` can no longer be specified. 
+
+*Impact:* Users who relied on changing these settings to achieve multitenancy should use *Spaces*, cross-cluster replication, or cross-cluster search instead. To migrate to *Spaces*, users are encouraged to use saved object management to export their saved objects from a tenant into the default tenant in a space. Improvements are planned to improve on this workflow. See https://github.com/elastic/kibana/issues/82020 for more details.   
+
 [float]
 ==== Legacy browsers are now rejected by default
 *Details:* `csp.strict` is now enabled by default, so Kibana will fail to load for older, legacy browsers that do not enforce basic Content Security Policy protections - notably Internet Explorer 11.

docs/user/alerting/action-types.asciidoc

@@ -23,6 +23,9 @@ a| <<jira-action-type, Jira>>
 
 | Create an incident in Jira.
 
+a| <<teams-action-type, Microsoft Teams>>
+
+| Send a message to a Microsoft Teams channel.
 
 a| <<pagerduty-action-type, PagerDuty>>
 
@@ -65,6 +68,7 @@ include::action-types/email.asciidoc[]
 include::action-types/resilient.asciidoc[]
 include::action-types/index.asciidoc[]
 include::action-types/jira.asciidoc[]
+include::action-types/teams.asciidoc[]
 include::action-types/pagerduty.asciidoc[]
 include::action-types/server-log.asciidoc[]
 include::action-types/servicenow.asciidoc[]

docs/user/alerting/action-types/teams.asciidoc

@@ -0,0 +1,58 @@
+[role="xpack"]
+[[teams-action-type]]
+=== Microsoft Teams action
+
+The Microsoft Teams action type uses https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook[Incoming Webhooks].
+
+[float]
+[[teams-connector-configuration]]
+==== Connector configuration
+
+Microsoft Teams connectors have the following configuration properties:
+
+Name::      The name of the connector. The name is used to identify a  connector in the management UI connector listing, or in the connector list when configuring an action.
+Webhook URL::   The URL of the incoming webhook. See https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook#add-an-incoming-webhook-to-a-teams-channel[Add Incoming Webhooks] for instructions on generating this URL. If you are using the <<action-settings, `xpack.actions.allowedHosts`>> setting, make sure the hostname is added to the allowed hosts.
+
+[float]
+[[Preconfigured-teams-configuration]]
+==== Preconfigured action type
+
+[source,text]
+--
+ my-teams:
+   name: preconfigured-teams-action-type
+   actionTypeId: .teams
+   config:
+     webhookUrl: 'https://outlook.office.com/webhook/abcd@0123456/IncomingWebhook/abcdefgh/ijklmnopqrstuvwxyz'
+--
+
+`config` defines the action type specific to the configuration.
+`config` contains
+`webhookUrl`, a string that corresponds to *Webhook URL*.
+
+
+[float]
+[[teams-action-configuration]]
+==== Action configuration
+
+Microsoft Teams actions have the following properties:
+
+Message::   The message	text, converted to the `text` field in the Webhook JSON payload. Currently only the text field is supported. Markdown, images, and other advanced formatting are not yet supported.
+
+[[configuring-teams]]
+==== Configuring Microsoft Teams Accounts
+
+You need a https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook[Microsoft Teams webhook URL] to 
+configure a Microsoft Teams action. To create a webhook
+URL, add the **Incoming Webhook App** through the Microsoft Teams console:
+
+. Log in to http://teams.microsoft.com[teams.microsoft.com] as a team administrator.
+. Navigate to the Apps directory, search for and select the *Incoming Webhook* app.
+. Choose _Add to team_ and select a team and channel for the app.
+. Enter a name for your webhook and (optionally) upload a custom icon.
++
+image::images/teams-add-webhook-integration.png[]
+. Click *Create*.
+. Copy the generated webhook URL so you can paste it into your Teams connector form.
++
+image::images/teams-copy-webhook-url.png[]

docs/user/alerting/images/teams-add-webhook-integration.png

@@ -723,7 +723,7 @@
     "less": "npm:@elastic/[email protected]",
     "license-checker": "^16.0.0",
     "listr": "^0.14.1",
-    "lmdb-store": "^0.6.10",
+    "lmdb-store": "^0.8.15",
     "load-grunt-config": "^3.0.1",
     "loader-utils": "^1.2.3",
     "log-symbols": "^2.2.0",

docs/user/alerting/images/teams-copy-webhook-url.png

@@ -18,9 +18,7 @@
  */
 
 import { InjectedIntl as _InjectedIntl, InjectedIntlProps as _InjectedIntlProps } from 'react-intl';
-
-export type InjectedIntl = _InjectedIntl;
-export type InjectedIntlProps = _InjectedIntlProps;
+export type { InjectedIntl, InjectedIntlProps } from 'react-intl';
 
 export {
   intlShape,

package.json

@@ -17,91 +17,67 @@
  * under the License.
  */
 
-import Path from 'path';
-import Fs from 'fs';
+import { Writable } from 'stream';
 
-// @ts-expect-error no types available
+import chalk from 'chalk';
 import * as LmdbStore from 'lmdb-store';
-import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';
-
-const LMDB_PKG = JSON.parse(
-  Fs.readFileSync(Path.resolve(REPO_ROOT, 'node_modules/lmdb-store/package.json'), 'utf8')
-);
-const CACHE_DIR = Path.resolve(
-  REPO_ROOT,
-  `data/node_auto_transpilation_cache/lmdb-${LMDB_PKG.version}/${UPSTREAM_BRANCH}`
-);
-
-const reportError = () => {
-  // right now I'm not sure we need to worry about errors, the cache isn't actually
-  // necessary, and if the cache is broken it should just rebuild on the next restart
-  // of the process. We don't know how often errors occur though and what types of
-  // things might fail on different machines so we probably want some way to signal
-  // to users that something is wrong
-};
 
 const GLOBAL_ATIME = `${Date.now()}`;
 const MINUTE = 1000 * 60;
 const HOUR = MINUTE * 60;
 const DAY = HOUR * 24;
 
-interface Lmdb<T> {
-  name: string;
-  get(key: string): T | undefined;
-  put(key: string, value: T, version?: number, ifVersion?: number): Promise<boolean>;
-  remove(key: string, ifVersion?: number): Promise<boolean>;
-  removeSync(key: string): void;
-  openDB<T2>(options: {
-    name: string;
-    encoding: 'msgpack' | 'string' | 'json' | 'binary';
-  }): Lmdb<T2>;
-  getRange(options?: {
-    start?: T;
-    end?: T;
-    reverse?: boolean;
-    limit?: number;
-    versions?: boolean;
-  }): Iterable<{ key: string; value: T }>;
-}
+const dbName = (db: LmdbStore.Database) =>
+  // @ts-expect-error db.name is not a documented/typed property
+  db.name;
 
 export class Cache {
-  private readonly codes: Lmdb<string>;
-  private readonly atimes: Lmdb<string>;
-  private readonly mtimes: Lmdb<string>;
-  private readonly sourceMaps: Lmdb<any>;
+  private readonly codes: LmdbStore.RootDatabase<string, string>;
+  private readonly atimes: LmdbStore.Database<string, string>;
+  private readonly mtimes: LmdbStore.Database<string, string>;
+  private readonly sourceMaps: LmdbStore.Database<string, string>;
   private readonly prefix: string;
+  private readonly log?: Writable;
+  private readonly timer: NodeJS.Timer;
 
-  constructor(config: { prefix: string }) {
+  constructor(config: { dir: string; prefix: string; log?: Writable }) {
     this.prefix = config.prefix;
+    this.log = config.log;
 
-    this.codes = LmdbStore.open({
+    this.codes = LmdbStore.open(config.dir, {
       name: 'codes',
-      path: CACHE_DIR,
+      encoding: 'string',
       maxReaders: 500,
     });
 
-    this.atimes = this.codes.openDB({
+    // TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
+    this.atimes = this.codes.openDB('atimes', {
       name: 'atimes',
       encoding: 'string',
     });
 
-    this.mtimes = this.codes.openDB({
+    this.mtimes = this.codes.openDB('mtimes', {
       name: 'mtimes',
       encoding: 'string',
     });
 
-    this.sourceMaps = this.codes.openDB({
+    this.sourceMaps = this.codes.openDB('sourceMaps', {
       name: 'sourceMaps',
-      encoding: 'msgpack',
+      encoding: 'string',
     });
 
     // after the process has been running for 30 minutes prune the
     // keys which haven't been used in 30 days. We use `unref()` to
     // make sure this timer doesn't hold other processes open
     // unexpectedly
-    setTimeout(() => {
+    this.timer = setTimeout(() => {
       this.pruneOldKeys();
-    }, 30 * MINUTE).unref();
+    }, 30 * MINUTE);
+
+    // timer.unref is not defined in jest which emulates the dom by default
+    if (typeof this.timer.unref === 'function') {
+      this.timer.unref();
+    }
   }
 
   getMtime(path: string) {
@@ -110,45 +86,78 @@ export class Cache {
 
   getCode(path: string) {
     const key = this.getKey(path);
+    const code = this.safeGet(this.codes, key);
 
-    // when we use a file from the cache set the "atime" of that cache entry
-    // so that we know which cache items we use and which haven't been
-    // touched in a long time (currently 30 days)
-    this.atimes.put(key, GLOBAL_ATIME).catch(reportError);
+    if (code !== undefined) {
+      // when we use a file from the cache set the "atime" of that cache entry
+      // so that we know which cache items we use and which haven't been
+      // touched in a long time (currently 30 days)
+      this.safePut(this.atimes, key, GLOBAL_ATIME);
+    }
 
-    return this.safeGet(this.codes, key);
+    return code;
   }
 
   getSourceMap(path: string) {
-    return this.safeGet(this.sourceMaps, this.getKey(path));
+    const map = this.safeGet(this.sourceMaps, this.getKey(path));
+    if (typeof map === 'string') {
+      return JSON.parse(map);
+    }
   }
 
-  update(path: string, file: { mtime: string; code: string; map: any }) {
+  async update(path: string, file: { mtime: string; code: string; map: any }) {
     const key = this.getKey(path);
 
-    Promise.all([
-      this.atimes.put(key, GLOBAL_ATIME),
-      this.mtimes.put(key, file.mtime),
-      this.codes.put(key, file.code),
-      this.sourceMaps.put(key, file.map),
-    ]).catch(reportError);
+    await Promise.all([
+      this.safePut(this.atimes, key, GLOBAL_ATIME),
+      this.safePut(this.mtimes, key, file.mtime),
+      this.safePut(this.codes, key, file.code),
+      this.safePut(this.sourceMaps, key, JSON.stringify(file.map)),
+    ]);
+  }
+
+  close() {
+    clearTimeout(this.timer);
   }
 
   private getKey(path: string) {
     return `${this.prefix}${path}`;
   }
 
-  private safeGet<V>(db: Lmdb<V>, key: string) {
+  private safeGet<V>(db: LmdbStore.Database<V, string>, key: string) {
     try {
-      return db.get(key);
+      const value = db.get(key);
+      this.debug(value === undefined ? 'MISS' : 'HIT', db, key);
+      return value;
     } catch (error) {
-      process.stderr.write(
-        `failed to read node transpilation [${db.name}] cache for [${key}]: ${error.stack}\n`
-      );
-      db.removeSync(key);
+      this.logError('GET', db, key, error);
     }
   }
 
+  private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
+    try {
+      await db.put(key, value);
+      this.debug('PUT', db, key);
+    } catch (error) {
+      this.logError('PUT', db, key, error);
+    }
+  }
+
+  private debug(type: string, db: LmdbStore.Database, key: LmdbStore.Key) {
+    if (this.log) {
+      this.log.write(`${type}   [${dbName(db)}]   ${String(key)}\n`);
+    }
+  }
+
+  private logError(type: 'GET' | 'PUT', db: LmdbStore.Database, key: LmdbStore.Key, error: Error) {
+    this.debug(`ERROR/${type}`, db, `${String(key)}: ${error.stack}`);
+    process.stderr.write(
+      chalk.red(
+        `[@kbn/optimizer/node] ${type} error [${dbName(db)}/${String(key)}]: ${error.stack}\n`
+      )
+    );
+  }
+
   private async pruneOldKeys() {
     try {
       const ATIME_LIMIT = Date.now() - 30 * DAY;
@@ -157,9 +166,10 @@ export class Cache {
       const validKeys: string[] = [];
       const invalidKeys: string[] = [];
 
+      // @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
       for (const { key, value } of this.atimes.getRange()) {
-        const atime = parseInt(value, 10);
-        if (atime < ATIME_LIMIT) {
+        const atime = parseInt(`${value}`, 10);
+        if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
           invalidKeys.push(key);
         } else {
           validKeys.push(key);