[7.x] [Alerting] Preconfigured alert history index connector (#94909)

docs/settings/alert-action-settings.asciidoc

@@ -53,8 +53,12 @@ You can configure the following settings in the `kibana.yml` file.
   +
   Disabled action types will not appear as an option when creating new connectors, but existing connectors and actions of that type will remain in {kib} and will not function.
 
+| `xpack.actions`
+`.preconfiguredAlertHistoryEsIndex` {ess-icon}
+  | Enables a preconfigured alert history {es} <<index-action-type, Index>> connector. Defaults to `false`.
+
 | `xpack.actions.preconfigured`
-  | Specifies preconfigured action IDs and configs. Defaults to {}.
+  | Specifies preconfigured connector IDs and configs. Defaults to {}.
 
 | `xpack.actions.proxyUrl` {ess-icon}
   | Specifies the proxy URL to use, if using a proxy for actions. By default, no proxy is used.

docs/user/alerting/action-types/index.asciidoc

@@ -82,3 +82,38 @@ PUT test
     }
 }
 --------------------------------------------------
+
+[float]
+[[preconfigured-connector-alert-history]]
+=== Alert history {es} index connector
+
+experimental[] {kib} offers a preconfigured index connector to facilitate indexing active alert data into {es}.
+
+[WARNING]
+==================================================
+This functionality is experimental and may be changed or removed completely in a future release.
+==================================================
+
+To use this connector, set the <<action-settings, `xpack.actions.preconfiguredAlertHistoryEsIndex`>> configuration to `true`. 
+
+```js
+  xpack.actions.preconfiguredAlertHistoryEsIndex: true
+```
+
+When creating a new rule, add an <<index-action-type, Index action>> and select the `Alert history Elasticsearch index (preconfigured)` connector.
+
+[role="screenshot"]
+image::images/pre-configured-alert-history-connector.png[Select pre-configured alert history connectors]
+
+Documents are indexed using a preconfigured schema that captures the <<defining-alerts-actions-variables, action variables>> available for the rule. By default, these documents are indexed into the `kibana-alert-history-default` index, but you can specify a different index. Index names must start with `kibana-alert-history-` to take advantage of the preconfigured alert history index template.
+
+[IMPORTANT]
+==============================================
+To write documents to the preconfigured index, you must have `all` or `write` privileges to the `kibana-alert-history-*` indices. Refer to <<kibana-role-management>> for more information.
+==============================================
+
+[NOTE]
+==================================================
+The `kibana-alert-history-*` indices are not configured to use ILM so they must be maintained manually. If the index size grows large,
+consider using the {ref}/docs-delete-by-query.html[delete by query] API to clean up older documents in the index.
+==================================================

docs/user/alerting/action-types/pre-configured-connectors.asciidoc

@@ -51,6 +51,14 @@ two out-of-the box connectors: <<slack-action-type, Slack>> and <<webhook-action
 Sensitive properties, such as passwords, can also be stored in the <<creating-keystore, {kib} keystore>>.
 ==============================================
 
+[float]
+[[build-in-preconfigured-connectors]]
+==== Built-in preconfigured connectors
+
+{kib} provides one built-in preconfigured connector:
+
+* <<preconfigured-connector-alert-history, Alert history preconfigured {es} index connector>>
+
 [float]
 [[managing-pre-configured-connectors]]
 ==== View preconfigured connectors
@@ -63,4 +71,4 @@ image::images/pre-configured-connectors-managing.png[Connectors managing tab wit
 Clicking a preconfigured connector shows the description, but not the configuration. A message indicates that this is a preconfigured connector.
 
 [role="screenshot"]
-image::images/pre-configured-connectors-view-screen.png[Pre-configured connector view details]
+image::images/pre-configured-connectors-view-screen.png[Pre-configured connector view details]

src/core/public/doc_links/doc_links_service.ts

@@ -210,6 +210,7 @@ export class DocLinksService {
           indexThreshold: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/rule-type-index-threshold.html`,
           pagerDutyAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/pagerduty-action-type.html`,
           preconfiguredConnectors: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/pre-configured-connectors.html`,
+          preconfiguredAlertHistoryConnector: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/index-action-type.html#preconfigured-connector-alert-history`,
           serviceNowAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/servicenow-action-type.html#configuring-servicenow`,
           setupPrerequisites: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/alerting-getting-started.html#alerting-setup-prerequisites`,
           slackAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/slack-action-type.html#configuring-slack`,

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -159,6 +159,7 @@ kibana_vars=(
     xpack.actions.allowedHosts
     xpack.actions.enabled
     xpack.actions.enabledActionTypes
+    xpack.actions.preconfiguredAlertHistoryEsIndex
     xpack.actions.preconfigured
     xpack.actions.proxyHeaders
     xpack.actions.proxyRejectUnauthorizedCertificates

x-pack/plugins/actions/common/alert_history_schema.test.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { buildAlertHistoryDocument } from './alert_history_schema';
+
+function getVariables(overrides = {}) {
+  return {
+    date: '2021-01-01T00:00:00.000Z',
+    rule: {
+      id: 'rule-id',
+      name: 'rule-name',
+      type: 'rule-type',
+      spaceId: 'space-id',
+    },
+    context: {
+      contextVar1: 'contextValue1',
+      contextVar2: 'contextValue2',
+    },
+    params: {
+      ruleParam: 1,
+      ruleParamString: 'another param',
+    },
+    tags: ['abc', 'def'],
+    alert: {
+      id: 'alert-id',
+      actionGroup: 'action-group-id',
+      actionGroupName: 'Action Group',
+    },
+    ...overrides,
+  };
+}
+
+describe('buildAlertHistoryDocument', () => {
+  it('handles empty variables', () => {
+    expect(buildAlertHistoryDocument({})).toBeNull();
+  });
+
+  it('returns null if rule type is not defined', () => {
+    expect(buildAlertHistoryDocument(getVariables({ rule: { type: undefined } }))).toBeNull();
+  });
+
+  it('returns null if alert variables are not defined', () => {
+    expect(buildAlertHistoryDocument(getVariables({ alert: undefined }))).toBeNull();
+  });
+
+  it('returns null if rule variables are not defined', () => {
+    expect(buildAlertHistoryDocument(getVariables({ rule: undefined }))).toBeNull();
+  });
+
+  it('includes @timestamp field if date is null', () => {
+    const alertHistoryDoc = buildAlertHistoryDocument(getVariables({ date: undefined }));
+    expect(alertHistoryDoc).not.toBeNull();
+    expect(alertHistoryDoc!['@timestamp']).toBeTruthy();
+  });
+
+  it(`doesn't include context if context is empty`, () => {
+    const alertHistoryDoc = buildAlertHistoryDocument(getVariables({ context: {} }));
+    expect(alertHistoryDoc).not.toBeNull();
+    expect(alertHistoryDoc!.kibana?.alert?.context).toBeFalsy();
+  });
+
+  it(`doesn't include params if params is empty`, () => {
+    const alertHistoryDoc = buildAlertHistoryDocument(getVariables({ params: {} }));
+    expect(alertHistoryDoc).not.toBeNull();
+    expect(alertHistoryDoc!.rule?.params).toBeFalsy();
+  });
+
+  it(`doesn't include tags if tags is empty array`, () => {
+    const alertHistoryDoc = buildAlertHistoryDocument(getVariables({ tags: [] }));
+    expect(alertHistoryDoc).not.toBeNull();
+    expect(alertHistoryDoc!.tags).toBeFalsy();
+  });
+
+  it(`included message if context contains message`, () => {
+    const alertHistoryDoc = buildAlertHistoryDocument(
+      getVariables({
+        context: { contextVar1: 'contextValue1', contextVar2: 'contextValue2', message: 'hello!' },
+      })
+    );
+    expect(alertHistoryDoc).not.toBeNull();
+    expect(alertHistoryDoc!.message).toEqual('hello!');
+  });
+
+  it('builds alert history document from variables', () => {
+    expect(buildAlertHistoryDocument(getVariables())).toEqual({
+      '@timestamp': '2021-01-01T00:00:00.000Z',
+      kibana: {
+        alert: {
+          actionGroup: 'action-group-id',
+          actionGroupName: 'Action Group',
+          context: {
+            'rule-type': {
+              contextVar1: 'contextValue1',
+              contextVar2: 'contextValue2',
+            },
+          },
+          id: 'alert-id',
+        },
+      },
+      event: {
+        kind: 'alert',
+      },
+      rule: {
+        id: 'rule-id',
+        name: 'rule-name',
+        params: {
+          'rule-type': {
+            ruleParam: 1,
+            ruleParamString: 'another param',
+          },
+        },
+        space: 'space-id',
+        type: 'rule-type',
+      },
+      tags: ['abc', 'def'],
+    });
+  });
+});

x-pack/plugins/actions/common/alert_history_schema.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isEmpty } from 'lodash';
+
+export const ALERT_HISTORY_PREFIX = 'kibana-alert-history-';
+export const AlertHistoryDefaultIndexName = `${ALERT_HISTORY_PREFIX}default`;
+export const AlertHistoryEsIndexConnectorId = 'preconfigured-alert-history-es-index';
+
+export const buildAlertHistoryDocument = (variables: Record<string, unknown>) => {
+  const { date, alert: alertVariables, context, params, tags, rule: ruleVariables } = variables as {
+    date: string;
+    alert: Record<string, unknown>;
+    context: Record<string, unknown>;
+    params: Record<string, unknown>;
+    rule: Record<string, unknown>;
+    tags: string[];
+  };
+
+  if (!alertVariables || !ruleVariables) {
+    return null;
+  }
+
+  const { actionGroup, actionGroupName, id: alertId } = alertVariables as {
+    actionGroup: string;
+    actionGroupName: string;
+    id: string;
+  };
+
+  const { id: ruleId, name, spaceId, type } = ruleVariables as {
+    id: string;
+    name: string;
+    spaceId: string;
+    type: string;
+  };
+
+  if (!type) {
+    // can't build the document without a type
+    return null;
+  }
+
+  const ruleType = type.replace(/\./g, '__');
+
+  const rule = {
+    ...(ruleId ? { id: ruleId } : {}),
+    ...(name ? { name } : {}),
+    ...(!isEmpty(params) ? { params: { [ruleType]: params } } : {}),
+    ...(spaceId ? { space: spaceId } : {}),
+    ...(type ? { type } : {}),
+  };
+  const alert = {
+    ...(alertId ? { id: alertId } : {}),
+    ...(!isEmpty(context) ? { context: { [ruleType]: context } } : {}),
+    ...(actionGroup ? { actionGroup } : {}),
+    ...(actionGroupName ? { actionGroupName } : {}),
+  };
+
+  const alertHistoryDoc = {
+    '@timestamp': date ? date : new Date().toISOString(),
+    ...(tags && tags.length > 0 ? { tags } : {}),
+    ...(context?.message ? { message: context.message } : {}),
+    ...(!isEmpty(rule) ? { rule } : {}),
+    ...(!isEmpty(alert) ? { kibana: { alert } } : {}),
+  };
+
+  return !isEmpty(alertHistoryDoc) ? { ...alertHistoryDoc, event: { kind: 'alert' } } : null;
+};
+
+export const AlertHistoryDocumentTemplate = Object.freeze(
+  buildAlertHistoryDocument({
+    rule: {
+      id: '{{rule.id}}',
+      name: '{{rule.name}}',
+      type: '{{rule.type}}',
+      spaceId: '{{rule.spaceId}}',
+    },
+    context: '{{context}}',
+    params: '{{params}}',
+    tags: '{{rule.tags}}',
+    alert: {
+      id: '{{alert.id}}',
+      actionGroup: '{{alert.actionGroup}}',
+      actionGroupName: '{{alert.actionGroupName}}',
+    },
+  })
+);

x-pack/plugins/actions/common/index.ts

@@ -6,7 +6,7 @@
  */
 
 export * from './types';
+export * from './alert_history_schema';
+export * from './rewrite_request_case';
 
 export const BASE_ACTION_API_PATH = '/api/actions';
-
-export * from './rewrite_request_case';

x-pack/plugins/actions/server/actions_client.test.ts

@@ -405,6 +405,7 @@ describe('create()', () => {
       enabled: true,
       enabledActionTypes: ['some-not-ignored-action-type'],
       allowedHosts: ['*'],
+      preconfiguredAlertHistoryEsIndex: false,
       preconfigured: {},
       proxyRejectUnauthorizedCertificates: true,
       rejectUnauthorized: true,

x-pack/plugins/actions/server/actions_config.test.ts

@@ -18,6 +18,7 @@ const defaultActionsConfig: ActionsConfig = {
   enabled: false,
   allowedHosts: [],
   enabledActionTypes: [],
+  preconfiguredAlertHistoryEsIndex: false,
   preconfigured: {},
   proxyRejectUnauthorizedCertificates: true,
   rejectUnauthorized: true,