From 456bc370aae42b5f6705e9e324b9dcd78ac9d421 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 15 Mar 2021 08:58:27 +0100 Subject: [PATCH 01/24] Rule registry --- .../server/task_runner/task_runner.ts | 11 +- x-pack/plugins/apm/kibana.json | 4 +- .../apm/server/lib/alerts/action_variables.ts | 12 +- .../server/lib/alerts/alerting_es_client.ts | 15 +- .../server/lib/alerts/register_apm_alerts.ts | 34 +- ...egister_transaction_duration_alert_type.ts | 43 ++- x-pack/plugins/apm/server/plugin.ts | 48 ++- .../server/es/cluster_client_adapter.test.ts | 3 +- .../server/es/cluster_client_adapter.ts | 28 +- x-pack/plugins/event_log/server/es/context.ts | 2 +- x-pack/plugins/event_log/server/es/names.ts | 6 +- x-pack/plugins/observability/kibana.json | 2 +- x-pack/plugins/observability/server/plugin.ts | 32 +- x-pack/plugins/observability/tsconfig.json | 1 + x-pack/plugins/rule_registry/common/index.ts | 8 + x-pack/plugins/rule_registry/common/types.ts | 11 + x-pack/plugins/rule_registry/kibana.json | 13 + x-pack/plugins/rule_registry/server/index.ts | 23 ++ x-pack/plugins/rule_registry/server/plugin.ts | 46 +++ .../rule_registry/check_service/index.ts | 51 +++ .../rule_registry/defaults/field_map.ts | 31 ++ .../rule_registry/defaults/ilm_policy.ts | 28 ++ .../field_map/mapping_from_field_map.ts | 32 ++ .../field_map/merge_field_maps.ts | 50 +++ .../field_map/schema_from_field_map.ts | 75 ++++ .../server/rule_registry/index.ts | 345 ++++++++++++++++++ .../server/rule_registry/types.ts | 44 +++ x-pack/plugins/rule_registry/server/types.ts | 119 ++++++ x-pack/plugins/rule_registry/tsconfig.json | 15 + 29 files changed, 1029 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/index.ts create mode 100644 x-pack/plugins/rule_registry/common/types.ts create mode 100644 x-pack/plugins/rule_registry/kibana.json create mode 100644 x-pack/plugins/rule_registry/server/index.ts create mode 100644 x-pack/plugins/rule_registry/server/plugin.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/types.ts create mode 100644 x-pack/plugins/rule_registry/server/types.ts create mode 100644 x-pack/plugins/rule_registry/tsconfig.json diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 744be16451999..817942f32db40 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -539,11 +539,16 @@ export class TaskRunner< }; }, (err: Error) => { - const message = `Executing Alert "${alertId}" has resulted in Error: ${err.message}`; + const error = new Error( + `Executing Alert "${alertId}" has resulted in Error: ${err.message}` + ); + + Object.assign(error, { wrapped: err }); + if (isAlertSavedObjectNotFoundError(err, alertId)) { - this.logger.debug(message); + this.logger.debug(error.message); } else { - this.logger.error(message); + this.logger.error(error); } return originalState; } diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index a9a0149e72ce7..da17000a4d667 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -9,7 +9,8 @@ "licensing", "triggersActionsUi", "embeddable", - "infra" + "infra", + "observability" ], "optionalPlugins": [ "cloud", @@ -17,7 +18,6 @@ "taskManager", "actions", "alerting", - "observability", "security", "ml", "home", diff --git a/x-pack/plugins/apm/server/lib/alerts/action_variables.ts b/x-pack/plugins/apm/server/lib/alerts/action_variables.ts index 473912c4177a9..b065da7123dec 100644 --- a/x-pack/plugins/apm/server/lib/alerts/action_variables.ts +++ b/x-pack/plugins/apm/server/lib/alerts/action_variables.ts @@ -13,28 +13,28 @@ export const apmActionVariables = { 'xpack.apm.alerts.action_variables.serviceName', { defaultMessage: 'The service the alert is created for' } ), - name: 'serviceName', + name: 'serviceName' as const, }, transactionType: { description: i18n.translate( 'xpack.apm.alerts.action_variables.transactionType', { defaultMessage: 'The transaction type the alert is created for' } ), - name: 'transactionType', + name: 'transactionType' as const, }, environment: { description: i18n.translate( 'xpack.apm.alerts.action_variables.environment', { defaultMessage: 'The transaction type the alert is created for' } ), - name: 'environment', + name: 'environment' as const, }, threshold: { description: i18n.translate('xpack.apm.alerts.action_variables.threshold', { defaultMessage: 'Any trigger value above this value will cause the alert to fire', }), - name: 'threshold', + name: 'threshold' as const, }, triggerValue: { description: i18n.translate( @@ -44,7 +44,7 @@ export const apmActionVariables = { 'The value that breached the threshold and triggered the alert', } ), - name: 'triggerValue', + name: 'triggerValue' as const, }, interval: { description: i18n.translate( @@ -54,6 +54,6 @@ export const apmActionVariables = { 'The length and unit of the time period where the alert conditions were met', } ), - name: 'interval', + name: 'interval' as const, }, }; diff --git a/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts b/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts index 727b0c1f04cf4..04850114eca3b 100644 --- a/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts +++ b/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts @@ -5,26 +5,17 @@ * 2.0. */ -import { ThresholdMetActionGroupId } from '../../../common/alert_types'; import { ESSearchRequest, ESSearchResponse, } from '../../../../../typings/elasticsearch'; -import { - AlertInstanceContext, - AlertInstanceState, - AlertServices, -} from '../../../../alerting/server'; +import { AlertServices } from '../../../../alerting/server'; export function alertingEsClient( - services: AlertServices< - AlertInstanceState, - AlertInstanceContext, - ThresholdMetActionGroupId - >, + callCluster: AlertServices['callCluster'], params: TParams ): Promise> { - return services.callCluster('search', { + return callCluster('search', { ...params, ignore_unavailable: true, }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts index a9824c130faa5..9650264f24971 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts @@ -6,38 +6,36 @@ */ import { Observable } from 'rxjs'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { ActionsPlugin } from '../../../../actions/server'; import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; import { registerErrorCountAlertType } from './register_error_count_alert_type'; import { APMConfig } from '../..'; import { MlPluginSetup } from '../../../../ml/server'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; +import { APMRuleRegistry } from '../../plugin'; interface Params { - alerting: AlertingPlugin['setup']; - actions: ActionsPlugin['setup']; + registry: APMRuleRegistry; ml?: MlPluginSetup; config$: Observable; } export function registerApmAlerts(params: Params) { registerTransactionDurationAlertType({ - alerting: params.alerting, - config$: params.config$, - }); - registerTransactionDurationAnomalyAlertType({ - alerting: params.alerting, - ml: params.ml, - config$: params.config$, - }); - registerErrorCountAlertType({ - alerting: params.alerting, - config$: params.config$, - }); - registerTransactionErrorRateAlertType({ - alerting: params.alerting, + registry: params.registry, config$: params.config$, }); + // registerTransactionDurationAnomalyAlertType({ + // alerting: params.alerting, + // ml: params.ml, + // config$: params.config$, + // }); + // registerErrorCountAlertType({ + // alerting: params.alerting, + // config$: params.config$, + // }); + // registerTransactionErrorRateAlertType({ + // alerting: params.alerting, + // config$: params.config$, + // }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index bb8e67574e9ad..721c77c4f0f2a 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -9,7 +9,6 @@ import { schema } from '@kbn/config-schema'; import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; import { APMConfig } from '../..'; -import { AlertingPlugin } from '../../../../alerting/server'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { PROCESSOR_EVENT, @@ -21,12 +20,13 @@ import { import { ProcessorEvent } from '../../../common/processor_event'; import { getDurationFormatter } from '../../../common/utils/formatters'; import { environmentQuery } from '../../../server/utils/queries'; +import { APMRuleRegistry } from '../../plugin'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; + registry: APMRuleRegistry; config$: Observable; } @@ -47,10 +47,10 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDuration]; export function registerTransactionDurationAlertType({ - alerting, + registry, config$, }: RegisterAlertParams) { - alerting.registerType({ + registry.registerType({ id: AlertType.TransactionDuration, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, @@ -122,10 +122,13 @@ export function registerTransactionDurationAlertType({ }, }; - const response = await alertingEsClient(services, searchParams); + const response = await alertingEsClient( + services.callCluster, + searchParams + ); if (!response.aggregations) { - return; + return {}; } const { agg, environments } = response.aggregations; @@ -143,20 +146,28 @@ export function registerTransactionDurationAlertType({ environments.buckets.map((bucket) => { const environment = bucket.key; - const alertInstance = services.alertInstanceFactory( - `${AlertType.TransactionDuration}_${environment}` - ); - - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - transactionType: alertParams.transactionType, - serviceName: alertParams.serviceName, - environment, + services.check.warning({ + name: `${AlertType.TransactionDuration}_${environment}`, threshold, - triggerValue: transactionDurationFormatted, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + value: transactionDuration, + context: { + transactionType: alertParams.transactionType, + serviceName: alertParams.serviceName, + environment, + threshold, + triggerValue: transactionDurationFormatted, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }, + fields: { + 'service.name': alertParams.serviceName, + 'service.environment': environment, + 'transaction.type': alertParams.transactionType, + }, }); }); } + + return {}; }, }); } diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index f556374179c51..df3ded6a08a9a 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { combineLatest, Observable } from 'rxjs'; +import { combineLatest } from 'rxjs'; import { map, take } from 'rxjs/operators'; import { CoreSetup, @@ -45,15 +45,8 @@ import { createElasticCloudInstructions } from './tutorial/elastic_cloud'; import { uiSettings } from './ui_settings'; import type { ApmPluginRequestHandlerContext } from './routes/typings'; -export interface APMPluginSetup { - config$: Observable; - getApmIndices: () => ReturnType; - createApmEventClient: (params: { - debug?: boolean; - request: KibanaRequest; - context: ApmPluginRequestHandlerContext; - }) => Promise>; -} +export type APMPluginSetup = ReturnType; +export type APMRuleRegistry = APMPluginSetup['registry']; export class APMPlugin implements Plugin { private currentConfig?: APMConfig; @@ -73,7 +66,7 @@ export class APMPlugin implements Plugin { taskManager?: TaskManagerSetupContract; alerting?: AlertingPlugin['setup']; actions?: ActionsPlugin['setup']; - observability?: ObservabilityPluginSetup; + observability: ObservabilityPluginSetup; features: FeaturesPluginSetup; security?: SecurityPluginSetup; ml?: MlPluginSetup; @@ -90,15 +83,6 @@ export class APMPlugin implements Plugin { core.uiSettings.register(uiSettings); - if (plugins.actions && plugins.alerting) { - registerApmAlerts({ - alerting: plugins.alerting, - actions: plugins.actions, - ml: plugins.ml, - config$: mergedConfig$, - }); - } - this.currentConfig = mergeConfigs( plugins.apmOss.config, this.initContext.config.get() @@ -161,6 +145,29 @@ export class APMPlugin implements Plugin { config: await mergedConfig$.pipe(take(1)).toPromise(), }); + const apmRuleRegistry = plugins.observability.registry.create({ + namespace: 'apm', + fieldMap: { + 'service.environment': { + type: 'keyword', + }, + 'processor.event': { + type: 'keyword', + }, + 'transaction.type': { + type: 'keyword', + }, + }, + }); + + if (plugins.actions && plugins.alerting) { + registerApmAlerts({ + registry: apmRuleRegistry, + ml: plugins.ml, + config$: mergedConfig$, + }); + } + return { config$: mergedConfig$, getApmIndices: boundGetApmIndices, @@ -190,6 +197,7 @@ export class APMPlugin implements Plugin { }, }); }, + registry: apmRuleRegistry, }; } diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 73bf17195a5db..80b51ef802f2a 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -12,7 +12,6 @@ import { IClusterClientAdapter, EVENT_BUFFER_LENGTH, } from './cluster_client_adapter'; -import { contextMock } from './context.mock'; import { findOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; import { times } from 'lodash'; @@ -31,7 +30,7 @@ beforeEach(() => { clusterClientAdapter = new ClusterClientAdapter({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), - context: contextMock.create(), + wait: () => Promise.resolve(true), }); }); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index f025801a45955..3b2b9a53c8b9d 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -10,7 +10,6 @@ import { bufferTime, filter as rxFilter, switchMap } from 'rxjs/operators'; import { reject, isUndefined } from 'lodash'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger, ElasticsearchClient } from 'src/core/server'; -import { EsContext } from '.'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; import { FindOptionsType } from '../event_log_client'; import { esKuery } from '../../../../../src/plugins/data/server'; @@ -25,10 +24,12 @@ export interface Doc { body: IEvent; } +type Wait = () => Promise; + export interface ConstructorOpts { logger: Logger; elasticsearchClientPromise: Promise; - context: EsContext; + wait: Wait; } export interface QueryEventsBySavedObjectResult { @@ -38,18 +39,21 @@ export interface QueryEventsBySavedObjectResult { data: IValidatedEvent[]; } -export class ClusterClientAdapter { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AliasAny = any; + +export class ClusterClientAdapter { private readonly logger: Logger; private readonly elasticsearchClientPromise: Promise; - private readonly docBuffer$: Subject; - private readonly context: EsContext; + private readonly docBuffer$: Subject; + private readonly wait: Wait; private readonly docsBufferedFlushed: Promise; constructor(opts: ConstructorOpts) { this.logger = opts.logger; this.elasticsearchClientPromise = opts.elasticsearchClientPromise; - this.context = opts.context; - this.docBuffer$ = new Subject(); + this.wait = opts.wait; + this.docBuffer$ = new Subject(); // buffer event log docs for time / buffer length, ignore empty // buffers, then index the buffered docs; kick things off with a @@ -74,15 +78,15 @@ export class ClusterClientAdapter { await this.docsBufferedFlushed; } - public indexDocument(doc: Doc): void { + public indexDocument(doc: TDoc): void { this.docBuffer$.next(doc); } - async indexDocuments(docs: Doc[]): Promise { + async indexDocuments(docs: TDoc[]): Promise { // If es initialization failed, don't try to index. // Also, don't log here, we log the failure case in plugin startup // instead, otherwise we'd be spamming the log (if done here) - if (!(await this.context.waitTillReady())) { + if (!(await this.wait())) { return; } @@ -156,7 +160,9 @@ export class ClusterClientAdapter { // instances at the same time. const existsNow = await this.doesIndexTemplateExist(name); if (!existsNow) { - throw new Error(`error creating index template: ${err.message}`); + const error = new Error(`Could not create index template: ${err.message}`); + Object.assign(error, { wrapped: err }); + throw error; } } } diff --git a/x-pack/plugins/event_log/server/es/context.ts b/x-pack/plugins/event_log/server/es/context.ts index 6d3b2208b3408..f6ae0a2002dd4 100644 --- a/x-pack/plugins/event_log/server/es/context.ts +++ b/x-pack/plugins/event_log/server/es/context.ts @@ -53,7 +53,7 @@ class EsContextImpl implements EsContext { this.esAdapter = new ClusterClientAdapter({ logger: params.logger, elasticsearchClientPromise: params.elasticsearchClientPromise, - context: this, + wait: () => this.readySignal.wait(), }); } diff --git a/x-pack/plugins/event_log/server/es/names.ts b/x-pack/plugins/event_log/server/es/names.ts index 2b98c7ceb95f7..6540bf3676e03 100644 --- a/x-pack/plugins/event_log/server/es/names.ts +++ b/x-pack/plugins/event_log/server/es/names.ts @@ -17,7 +17,11 @@ export interface EsNames { indexTemplate: string; } -export function getEsNames(baseName: string, kibanaVersion: string): EsNames { +export function getEsNames( + baseName: string, + kibanaVersion: string, + suffix: string = EVENT_LOG_NAME_SUFFIX +): EsNames { const EVENT_LOG_VERSION_SUFFIX = `-${kibanaVersion.toLocaleLowerCase()}`; const eventLogName = `${baseName}${EVENT_LOG_NAME_SUFFIX}`; const eventLogNameWithVersion = `${eventLogName}${EVENT_LOG_VERSION_SUFFIX}`; diff --git a/x-pack/plugins/observability/kibana.json b/x-pack/plugins/observability/kibana.json index 84aa1be9a8d87..aaa179e09c28a 100644 --- a/x-pack/plugins/observability/kibana.json +++ b/x-pack/plugins/observability/kibana.json @@ -3,7 +3,7 @@ "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "observability"], - "optionalPlugins": ["licensing", "home", "usageCollection"], + "optionalPlugins": ["licensing", "home", "usageCollection", "ruleRegistry"], "requiredPlugins": ["data"], "ui": true, "server": true, diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index a5843d1c4ade1..08e3fe483d2dc 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -9,25 +9,24 @@ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; import { ObservabilityConfig } from '.'; import { bootstrapAnnotations, - ScopedAnnotationsClient, ScopedAnnotationsClientFactory, AnnotationsAPI, } from './lib/annotations/bootstrap_annotations'; +import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server'; -type LazyScopedAnnotationsClientFactory = ( - ...args: Parameters -) => Promise; - -export interface ObservabilityPluginSetup { - getScopedAnnotationsClient: LazyScopedAnnotationsClientFactory; -} +export type ObservabilityPluginSetup = ReturnType; export class ObservabilityPlugin implements Plugin { constructor(private readonly initContext: PluginInitializerContext) { this.initContext = initContext; } - public setup(core: CoreSetup, plugins: {}): ObservabilityPluginSetup { + public setup( + core: CoreSetup, + plugins: { + ruleRegistry: RuleRegistryPluginSetupContract; + } + ) { const config = this.initContext.config.get(); let annotationsApiPromise: Promise | undefined; @@ -45,10 +44,23 @@ export class ObservabilityPlugin implements Plugin { } return { - getScopedAnnotationsClient: async (...args) => { + getScopedAnnotationsClient: async (...args: Parameters) => { const api = await annotationsApiPromise; return api?.getScopedAnnotationsClient(...args); }, + registry: plugins.ruleRegistry.create({ + namespace: 'observability', + fieldMap: { + 'host.hostname': { + type: 'keyword', + required: false, + }, + 'service.name': { + type: 'keyword', + required: false, + }, + }, + }), }; } diff --git a/x-pack/plugins/observability/tsconfig.json b/x-pack/plugins/observability/tsconfig.json index 5c7528610a0b1..22efbe62b5fff 100644 --- a/x-pack/plugins/observability/tsconfig.json +++ b/x-pack/plugins/observability/tsconfig.json @@ -16,6 +16,7 @@ { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, { "path": "../alerting/tsconfig.json" }, + { "path": "../rule_registry/tsconfig.json" }, { "path": "../licensing/tsconfig.json" }, { "path": "../translations/tsconfig.json" } ] diff --git a/x-pack/plugins/rule_registry/common/index.ts b/x-pack/plugins/rule_registry/common/index.ts new file mode 100644 index 0000000000000..6cc0ccaa93a6d --- /dev/null +++ b/x-pack/plugins/rule_registry/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts new file mode 100644 index 0000000000000..638a63e70c1da --- /dev/null +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum AlertSeverityLevel { + warning = 'warning', + critical = 'critical', +} diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json new file mode 100644 index 0000000000000..9d766bf61816c --- /dev/null +++ b/x-pack/plugins/rule_registry/kibana.json @@ -0,0 +1,13 @@ +{ + "id": "ruleRegistry", + "version": "8.0.0", + "kibanaVersion": "kibana", + "configPath": [ + "xpack", + "rule_registry" + ], + "requiredPlugins": [ + "alerting" + ], + "server": true +} diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts new file mode 100644 index 0000000000000..503cced4268c3 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from 'src/core/server'; +import { RuleRegistryPlugin, RuleRegistryPluginSetupContract } from './plugin'; + +export const config = { + schema: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + }), +}; + +export type RuleRegistryConfig = TypeOf; + +export const plugin = (initContext: PluginInitializerContext) => + new RuleRegistryPlugin(initContext); + +export { RuleRegistryPluginSetupContract }; diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts new file mode 100644 index 0000000000000..51f34b916875a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../alerting/server'; +import { RuleRegistry } from './rule_registry'; +import { defaultIlmPolicy } from './rule_registry/defaults/ilm_policy'; +import { defaultFieldMap } from './rule_registry/defaults/field_map'; + +export type RuleRegistryPluginSetupContract = RuleRegistry; + +export class RuleRegistryPlugin implements Plugin { + constructor(private readonly initContext: PluginInitializerContext) { + this.initContext = initContext; + } + + public setup( + core: CoreSetup, + plugins: { alerting: AlertingPluginSetupContract } + ): RuleRegistryPluginSetupContract { + const globalConfig = this.initContext.config.legacy.get(); + + const logger = this.initContext.logger.get(); + + const rootRegistry = new RuleRegistry({ + core, + ilmPolicy: defaultIlmPolicy, + fieldMap: defaultFieldMap, + kibanaIndex: globalConfig.kibana.index, + namespace: 'alert-history', + kibanaVersion: this.initContext.env.packageInfo.version, + logger: logger.get('root'), + alertingPluginSetupContract: plugins.alerting, + }); + + return rootRegistry; + } + + public start() {} + + public stop() {} +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts new file mode 100644 index 0000000000000..ed5ec66a70914 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ActionVariable } from '../../../../alerting/common'; +import { AlertExecutorOptions } from '../../../../alerting/server'; +import { AlertSeverityLevel } from '../../../common'; +import { AlertCheck, AlertContext } from '../../types'; +import { DefaultFieldMap } from '../defaults/field_map'; + +type AlertInstanceFactory = AlertExecutorOptions< + never, + never, + never, + never, + string +>['services']['alertInstanceFactory']; + +export function createCheckService({ + levels, + alertInstanceFactory, +}: { + levels: Array<{ level: AlertSeverityLevel; actionGroupId: string }>; + alertInstanceFactory: AlertInstanceFactory; +}) { + const alerts: Record< + string, + { level: AlertSeverityLevel } & AlertCheck + > = {}; + + const getCheckFunction = (level: { level: AlertSeverityLevel; actionGroupId: string }) => { + return (alert: AlertCheck) => { + const instance = alertInstanceFactory(alert.name); + instance.scheduleActions(level.actionGroupId); + alerts[alert.name] = { + level: level.level, + ...alert, + }; + }; + }; + + return { + check: Object.fromEntries( + levels.map((level) => [level.level, getCheckFunction(level)]) + ) as Record>, + getAlerts: () => alerts, + }; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts new file mode 100644 index 0000000000000..db8cd46500d55 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const defaultFieldMap = { + 'event.kind': { type: 'keyword', required: true }, + '@timestamp': { type: 'date', required: true }, + 'alert.id': { type: 'keyword', required: true }, + 'alert.created': { type: 'date', required: true }, + 'alert.active': { type: 'boolean', required: true }, + // 'alert.muted': { type: 'boolean', required: true }, + 'alert.type': { type: 'keyword', required: true }, + 'alert.name': { type: 'keyword', required: true }, + 'alert.series_id': { type: 'keyword', required: true }, // rule.id + alert.name + 'alert.check.severity': { type: 'keyword', required: true }, + 'alert.check.value': { type: 'scaled_float', scaling_factor: 100 }, + 'alert.check.threshold': { type: 'scaled_float', scaling_factor: 100 }, + 'alert.check.influencers': { type: 'flattened' }, + 'rule.id': { type: 'keyword', required: true }, + 'rule.namespace': { type: 'keyword' }, + 'rule.name': { type: 'keyword', required: true }, + 'rule.interval.ms': { type: 'long', required: true }, + 'rule_type.id': { type: 'keyword', required: true }, + 'rule_type.name': { type: 'keyword', required: true }, + 'rule_type.producer': { type: 'keyword', required: true }, +} as const; + +export type DefaultFieldMap = typeof defaultFieldMap; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts new file mode 100644 index 0000000000000..c80f7e772f308 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ILMPolicy } from '../types'; + +export const defaultIlmPolicy: ILMPolicy = { + policy: { + phases: { + hot: { + actions: { + rollover: { + max_age: '90d', + max_size: '50gb', + }, + }, + }, + delete: { + actions: { + delete: {}, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts new file mode 100644 index 0000000000000..6e4e13b01d2c5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@elastic/safer-lodash-set'; +import { FieldMap, Mappings } from '../types'; + +export function mappingFromFieldMap(fieldMap: FieldMap): Mappings { + const mappings = { + dynamic: 'strict' as const, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + const { name, required, array, ...rest } = field; + + set(mappings.properties, field.name.split('.').join('.properties.'), rest); + }); + + return mappings; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts new file mode 100644 index 0000000000000..6c9dd965f0d35 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { isEqual } from 'lodash'; +import { FieldMap } from '../types'; + +export function mergeFieldMaps( + first: T1, + second: T2 +): T1 & T2 { + const conflicts: Array> = []; + + Object.keys(second).forEach((name) => { + const field = second[name]; + + const parts = name.split('.'); + + const parents = parts.slice(0, parts.length - 2).map((part, index, array) => { + return [...array.slice(0, index - 1), part].join('.'); + }); + + parents + .filter((parent) => first[parent] !== undefined) + .forEach((parent) => { + conflicts.push({ + [parent]: [{ type: 'object' }, first[parent]!], + }); + }); + + if (first[name] && !isEqual(first[name], field)) { + conflicts.push({ + fields: [field, first[name]], + }); + } + }); + + if (conflicts.length) { + const err = new Error(`Could not merge mapping due to conflicts`); + Object.assign(err, { conflicts }); + throw err; + } + + return { + ...first, + ...second, + }; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts new file mode 100644 index 0000000000000..042955a7b7388 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues } from 'lodash'; +import { ObjectType, schema, Type, TypeOf } from '@kbn/config-schema'; +import { FieldMap } from '../types'; + +type MaybeArrayType = TOptions extends { array: true } + ? T[] + : T; + +type MaybeRequiredType< + T, + TOptions extends { required?: boolean; array?: boolean } +> = TOptions extends { + required: true; +} + ? MaybeArrayType + : MaybeArrayType | undefined | null; + +const map = { + keyword: schema.string(), + text: schema.string(), + date: schema.string(), + boolean: schema.boolean(), + byte: schema.number(), + long: schema.number(), + integer: schema.number(), + short: schema.number(), + double: schema.number(), + float: schema.number(), + scaled_float: schema.number(), + unsigned_long: schema.number(), + flattened: schema.mapOf(schema.string(), schema.arrayOf(schema.string())), +}; + +type SchemaMap = typeof map; + +type TypeOfField = Type< + MaybeRequiredType< + (Record & + { + [key in keyof SchemaMap]: TypeOf; + })[T['type']], + T + > +>; + +export type SchemaOf = ObjectType< + { + [key in keyof TFieldMap]: TypeOfField; + } +>; + +export function schemaFromFieldMap(fieldMap: TFieldMap) { + return (schema.object( + mapValues(fieldMap, ({ type, array, required }) => { + let schemaType: Type = type in map ? map[type as keyof SchemaMap] : schema.never(); + + if (array) { + schemaType = schema.arrayOf(schemaType); + } + + if (!required) { + schemaType = schema.maybe(schemaType); + } + + return schemaType; + }) + ) as unknown) as SchemaOf; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts new file mode 100644 index 0000000000000..2daf61090b83b --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -0,0 +1,345 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, Logger } from 'kibana/server'; +import { mapValues, omitBy, compact } from 'lodash'; +import { schema } from '@kbn/config-schema'; +import { inspect } from 'util'; +import uuid from 'uuid'; +import { ESSearchRequest, ESSearchResponse } from '../../../../typings/elasticsearch'; +import { createReadySignal } from '../../../event_log/server/lib/ready_signal'; +import { ClusterClientAdapter } from '../../../event_log/server/es/cluster_client_adapter'; +import { FieldMap, ILMPolicy } from './types'; +import { RegisterRuleType, RuleState, RuleAlertState } from '../types'; +import { mergeFieldMaps } from './field_map/merge_field_maps'; +import { schemaFromFieldMap, SchemaOf } from './field_map/schema_from_field_map'; +import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; +import { createCheckService } from './check_service'; +import { AlertSeverityLevel } from '../../common'; + +interface RuleRegistryOptions { + kibanaIndex: string; + kibanaVersion: string; + namespace: string; + logger: Logger; + core: CoreSetup; + fieldMap: TFieldMap; + ilmPolicy: ILMPolicy; + alertingPluginSetupContract: AlertingPluginSetupContract; + parent?: RuleRegistry; +} + +export class RuleRegistry { + private readonly esAdapter: ClusterClientAdapter; + private readonly docSchema: SchemaOf; + private readonly children: Array> = []; + + constructor(private readonly options: RuleRegistryOptions) { + const { logger, core } = options; + + const { wait, signal } = createReadySignal(); + + this.esAdapter = new ClusterClientAdapter({ + wait, + elasticsearchClientPromise: core + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + logger, + }); + + this.docSchema = schemaFromFieldMap(options.fieldMap); + + this.initialize() + .then(() => { + signal(true); + }) + .catch((err) => { + logger.error(inspect(err, { depth: null })); + signal(false); + }); + } + + private getEsNames() { + const base = [this.options.kibanaIndex, this.options.namespace]; + const indexAliasName = [...base, this.options.kibanaVersion].join('-'); + const policyName = [...base, 'policy'].join('-'); + + return { + indexAliasName, + policyName, + }; + } + + private async initialize() { + const { indexAliasName, policyName } = this.getEsNames(); + + const ilmPolicyExists = await this.esAdapter.doesIlmPolicyExist(policyName); + + if (!ilmPolicyExists) { + await this.esAdapter.createIlmPolicy( + policyName, + (this.options.ilmPolicy as unknown) as Record + ); + } + + const templateExists = await this.esAdapter.doesIndexTemplateExist(indexAliasName); + + if (!templateExists) { + await this.esAdapter.createIndexTemplate(indexAliasName, { + index_patterns: [`${indexAliasName}-*`], + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + 'index.lifecycle.name': policyName, + 'index.lifecycle.rollover_alias': indexAliasName, + }, + mappings: mappingFromFieldMap(this.options.fieldMap), + }); + } + + const aliasExists = await this.esAdapter.doesAliasExist(indexAliasName); + + if (!aliasExists) { + await this.esAdapter.createIndex(`${indexAliasName}-000001`, { + aliases: { + [indexAliasName]: { + is_write_index: true, + }, + }, + }); + } + } + + async search( + ruleIds: string[], + request: TSearchRequest + ): Promise> { + const [{ elasticsearch }] = await this.options.core.getStartServices(); + + const query = { + bool: { + filter: [ + { + terms: { + 'rule.id': ruleIds, + }, + }, + ...compact([request.body?.query]), + ], + }, + }; + + const response = await elasticsearch.client.asInternalUser.search({ + ...request, + body: { + ...request.body, + query, + }, + }); + + return response.body as ESSearchResponse; + } + + registerType: RegisterRuleType = (type) => { + this.options.alertingPluginSetupContract.registerType< + Record, + RuleState, + Record, + Record, + string, + string + >({ + ...type, + executor: async (options) => { + const { + services, + previousStartedAt, + startedAt, + state: maybePrevAlertState, + alertId: ruleId, + name: ruleName, + params, + namespace, + } = options; + + const prevAlertState = + maybePrevAlertState && 'alerts' in maybePrevAlertState + ? maybePrevAlertState + : { alerts: {}, wrappedRuleState: maybePrevAlertState }; + + const { alertInstanceFactory, ...passthroughServices } = services; + + const checkService = createCheckService({ + alertInstanceFactory: services.alertInstanceFactory as any, + levels: [{ level: AlertSeverityLevel.warning, actionGroupId: type.defaultActionGroupId }], + }); + + const executorOptions = { + previousStartedAt, + startedAt, + params: params as any, + services: { + ...passthroughServices, + check: checkService.check as any, + }, + }; + + const ruleState = await type.executor(executorOptions); + + const activeAlerts = checkService.getAlerts(); + const previousAlertStates = prevAlertState.alerts; + + const previousAlertNames = Object.keys(previousAlertStates); + const activeAlertNames = Object.keys(activeAlerts); + + const newAlertNames = activeAlertNames.filter( + (alertName) => !previousAlertNames.includes(alertName) + ); + + const mergedAlertStates = { + ...previousAlertStates, + ...newAlertNames.reduce((prev, alertName) => { + prev[alertName] = { + alertId: uuid.v4(), // for log-type alerts, use alertName + created: startedAt.getTime(), + }; + return prev; + }, {} as Record), + }; + + const common = { + 'event.kind': 'alert', + '@timestamp': startedAt.toISOString(), + 'rule.id': ruleId, + 'rule.name': ruleName, + 'rule.namespace': namespace, + 'rule_type.id': type.id, + 'rule_type.name': type.name, + 'rule_type.producer': type.producer, + // 'rule.interval.ms': prev + }; + + const idsOfLastAlertEventsToFetch = Object.values(mergedAlertStates).map( + (state) => state.alertId + ); + + const start = new Date().getTime() - 60 * 60 * 1000; + + const response = await this.search([ruleId], { + body: { + size: idsOfLastAlertEventsToFetch.length, + query: { + bool: { + filter: [ + { + terms: { + 'alert.id': idsOfLastAlertEventsToFetch, + }, + }, + { + range: { + '@timestamp': { + gte: start, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + collapse: { + field: 'alert.id', + }, + sort: { + '@timestamp': 'desc', + }, + _source: false, + fields: Object.keys(this.options.fieldMap), + }, + }); + + const lastEventByAlertId = response.hits.hits.reduce((prev, hit) => { + const alertId = hit.fields['alert.id']![0] as string; + prev[alertId] = hit.fields as Record; + return prev; + }, {} as Record>); + + const index = this.getEsNames().indexAliasName; + + const updates = Object.entries(mergedAlertStates).map(([alertName, state]) => { + const active = activeAlertNames.includes(alertName); + + const lastEvent = lastEventByAlertId[state.alertId] ?? {}; + const nextState = active ? activeAlerts[alertName] : undefined; + + return { + index, + body: { + ...lastEvent, + ...(nextState + ? { + ...nextState.fields, + 'alert.check.severity': nextState.level, + 'alert.check.value': nextState.value, + 'alert.check.threshold': nextState.threshold, + } + : {}), + ...common, + 'alert.active': active, + 'alert.id': state.alertId, + 'alert.created': state.created, + 'alert.type': 'threshold', // or, log + 'alert.name': alertName, + 'alert.series_id': [ruleId, alertName].join('|'), + }, + }; + }); + + if (updates.length) { + await this.esAdapter.indexDocuments(updates); + } + + const nextState = omitBy(mergedAlertStates, (_, alertName) => { + return activeAlerts[alertName] === undefined; + }); + + return { + wrappedRuleState: ruleState, + alerts: nextState, + }; + }, + }); + }; + + create({ + namespace, + fieldMap, + ilmPolicy, + }: { + namespace: string; + fieldMap: TNextFieldMap; + ilmPolicy?: ILMPolicy; + }): RuleRegistry { + const mergedFieldMap = fieldMap + ? mergeFieldMaps(this.options.fieldMap, fieldMap) + : this.options.fieldMap; + + const child = new RuleRegistry({ + ...this.options, + logger: this.options.logger.get(namespace), + namespace: [this.options.namespace, namespace].filter(Boolean).join('-'), + fieldMap: mergedFieldMap, + ...(ilmPolicy ? { ilmPolicy } : {}), + parent: this, + }); + + this.children.push(child); + + return child; + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/types.ts new file mode 100644 index 0000000000000..f6baf8bcecbd0 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface Mappings { + dynamic: 'strict' | boolean; + properties: Record; +} + +enum ILMPolicyPhase { + hot = 'hot', + delete = 'delete', +} + +enum ILMPolicyAction { + rollover = 'rollover', + delete = 'delete', +} + +interface ILMActionOptions { + [ILMPolicyAction.rollover]: { + max_size: string; + max_age: string; + }; + [ILMPolicyAction.delete]: {}; +} + +export interface ILMPolicy { + policy: { + phases: Record< + ILMPolicyPhase, + { + actions: { + [key in keyof ILMActionOptions]?: ILMActionOptions[key]; + }; + } + >; + }; +} + +export type FieldMap = Record; diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts new file mode 100644 index 0000000000000..773d366dfa0bb --- /dev/null +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Type, TypeOf } from '@kbn/config-schema'; +import { ActionVariable, AlertTypeState } from '../../alerting/common'; +import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; +import { AlertSeverityLevel } from '../common'; +import { DefaultFieldMap } from './rule_registry/defaults/field_map'; +import { SchemaOf } from './rule_registry/field_map/schema_from_field_map'; +import { FieldMap } from './rule_registry/types'; + +enum ESFieldType { + keyword = 'keyword', + text = 'text', + date = 'date', + boolean = 'boolean', + long = 'long', + integer = 'integer', + short = 'short', + byte = 'byte', + double = 'double', + float = 'half_float', + scaled_float = 'scaled_float', + unsigned_long = 'unsigned_long', +} + +type RuleTypeFieldMap = Record; + +type RuleParams = Type; + +export type AlertContext = Record< + string, + { + description: string; + field?: TFieldName; + type: Type; + } +>; + +export interface AlertCheck { + name: string; + value?: number; + threshold?: number; + context: { + [key in TActionVariable['name']]: any; + }; + fields: Omit>>, keyof DefaultFieldMap>; +} + +type TypeOfRuleParams = TypeOf; + +type RuleExecutorServices< + TFieldMap extends FieldMap, + TActionVariable extends ActionVariable +> = Omit & { + check: { warning: (check: AlertCheck) => void }; +}; + +type PassthroughAlertExecutorOptions = Pick< + AlertExecutorOptions, + 'previousStartedAt' | 'startedAt' +>; + +type RuleExecutorFunction< + TFieldMap extends FieldMap, + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable +> = ( + options: (PassthroughAlertExecutorOptions & { + services: RuleExecutorServices; + }) & + (TRuleParams extends RuleParams ? { params: TypeOfRuleParams } : {}) +) => Promise>; + +export interface RuleType { + id: string; + name: string; + fields?: RuleTypeFieldMap; + params?: RuleParams; + levels?: AlertSeverityLevel[]; + context?: AlertContext; + actionGroups: Array>; + defaultActionGroupId: string; + producer: string; + minimumLicenseRequired: 'basic' | 'gold' | 'trial'; + executor: RuleExecutorFunction; +} + +export type RegisterRuleType = < + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable +>(ruleType: { + id: string; + name: string; + validate: { + params: TRuleParams; + }; + actionVariables: { + context: TActionVariable[]; + }; + actionGroups: Array>; + defaultActionGroupId: string; + producer: string; + minimumLicenseRequired: 'basic' | 'gold' | 'trial'; + executor: RuleExecutorFunction; +}) => void; + +export interface RuleAlertState { + created: number; + alertId: string; +} + +export type RuleState = AlertTypeState & { + wrappedRuleState: Record; + alerts: Record; +}; diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json new file mode 100644 index 0000000000000..7f38665320050 --- /dev/null +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["common/**/*", "server/**/*" ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../alerting/tsconfig.json" }, + ] +} From b01fb639f98e321ac991f417e9fe5cad013ff1a9 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 26 Mar 2021 11:26:31 +0100 Subject: [PATCH 02/24] Index alert events --- .../server/es/cluster_client_adapter.ts | 12 +- x-pack/plugins/event_log/server/index.ts | 4 + x-pack/plugins/observability/server/plugin.ts | 11 +- x-pack/plugins/rule_registry/common/types.ts | 9 + .../scripts/generate_ecs_fieldmap/index.js | 81 + .../server/generated/ecs_field_map.ts | 3374 ++++++++++++++++ .../server/generated/ecs_mappings.json | 3416 +++++++++++++++++ .../rule_registry/check_service/index.ts | 2 +- .../rule_registry/defaults/field_map.ts | 33 +- .../field_map/merge_field_maps.ts | 7 +- .../field_map/pick_with_patterns.ts | 63 + .../field_map/runtime_type_from_fieldmap.ts | 90 + .../field_map/schema_from_field_map.ts | 75 - .../server/rule_registry/index.ts | 230 +- x-pack/plugins/rule_registry/server/types.ts | 10 +- 15 files changed, 7206 insertions(+), 211 deletions(-) create mode 100644 x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js create mode 100644 x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/generated/ecs_mappings.json create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts delete mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 3b2b9a53c8b9d..4ca9ba9416cc2 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -10,6 +10,7 @@ import { bufferTime, filter as rxFilter, switchMap } from 'rxjs/operators'; import { reject, isUndefined } from 'lodash'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger, ElasticsearchClient } from 'src/core/server'; +import util from 'util'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; import { FindOptionsType } from '../event_log_client'; import { esKuery } from '../../../../../src/plugins/data/server'; @@ -87,9 +88,12 @@ export class ClusterClientAdapter> = []; for (const doc of docs) { @@ -101,7 +105,13 @@ export class ClusterClientAdapter new Plugin(context); diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 1e943e8a4be77..5d9e4bc7fbc5c 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -6,6 +6,7 @@ */ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { pickWithPatterns } from '../../rule_registry/server/rule_registry/field_map/pick_with_patterns'; import { ObservabilityConfig } from '.'; import { bootstrapAnnotations, @@ -14,6 +15,7 @@ import { } from './lib/annotations/bootstrap_annotations'; import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server'; import { uiSettings } from './ui_settings'; +import { ecsFieldMap } from '../../rule_registry/server/generated/ecs_field_map'; export type ObservabilityPluginSetup = ReturnType; @@ -54,14 +56,7 @@ export class ObservabilityPlugin implements Plugin { registry: plugins.ruleRegistry.create({ namespace: 'observability', fieldMap: { - 'host.hostname': { - type: 'keyword', - required: false, - }, - 'service.name': { - type: 'keyword', - required: false, - }, + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), }, }), }; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts index 638a63e70c1da..d0d15d86a2248 100644 --- a/x-pack/plugins/rule_registry/common/types.ts +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -9,3 +9,12 @@ export enum AlertSeverityLevel { warning = 'warning', critical = 'critical', } + +const alertSeverityLevelValues = { + [AlertSeverityLevel.warning]: 70, + [AlertSeverityLevel.critical]: 90, +}; + +export function getAlertSeverityLevelValue(level: AlertSeverityLevel) { + return alertSeverityLevelValues[level]; +} diff --git a/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js new file mode 100644 index 0000000000000..6e3a8f7cbe663 --- /dev/null +++ b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +const path = require('path'); +const fs = require('fs'); +const util = require('util'); +const yaml = require('js-yaml'); +const { exec: execCb } = require('child_process'); +const { mapValues } = require('lodash'); + +const exists = util.promisify(fs.exists); +const readFile = util.promisify(fs.readFile); +const writeFile = util.promisify(fs.writeFile); +const mkdir = util.promisify(fs.mkdir); +const rmdir = util.promisify(fs.rmdir); +const exec = util.promisify(execCb); + +const ecsDir = path.resolve(__dirname, '../../../../../../ecs'); +const ecsTemplateFilename = path.join(ecsDir, 'generated/elasticsearch/7/template.json'); +const flatYamlFilename = path.join(ecsDir, 'generated/ecs/ecs_flat.yml'); + +const outputDir = path.join(__dirname, '../../server/generated'); + +const outputFieldMapFilename = path.join(outputDir, 'ecs_field_map.ts'); +const outputMappingFilename = path.join(outputDir, 'ecs_mappings.json'); + +async function generate() { + const allExists = await Promise.all([exists(ecsDir), exists(ecsTemplateFilename)]); + + if (!allExists.every(Boolean)) { + throw new Error( + `Directory not found: ${ecsDir} - did you checkout elastic/ecs as a peer of this repo?` + ); + } + + const [template, flatYaml] = await Promise.all([ + readFile(ecsTemplateFilename, { encoding: 'utf-8' }).then((str) => JSON.parse(str)), + (async () => yaml.safeLoad(await readFile(flatYamlFilename)))(), + ]); + + const mappings = { + properties: template.mappings.properties, + }; + + const fields = mapValues(flatYaml, (description) => { + return { + type: description.type, + array: description.normalize.includes('array'), + required: !!description.required, + }; + }); + + const hasOutputDir = await exists(outputDir); + + if (hasOutputDir) { + await rmdir(outputDir, { recursive: true }); + } + + await mkdir(outputDir); + + await Promise.all([ + writeFile( + outputFieldMapFilename, + ` + export const ecsFieldMap = ${JSON.stringify(fields, null, 2)} as const + `, + { encoding: 'utf-8' } + ).then(() => { + return exec(`node scripts/eslint --fix ${outputFieldMapFilename}`); + }), + writeFile(outputMappingFilename, JSON.stringify(mappings, null, 2)), + ]); +} + +generate().catch((err) => { + console.log(err); + process.exit(1); +}); diff --git a/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts b/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts new file mode 100644 index 0000000000000..cd8865a3f57c2 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts @@ -0,0 +1,3374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ecsFieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: true, + }, + 'agent.build.original': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.ephemeral_id': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.id': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.type': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.version': { + type: 'keyword', + array: false, + required: false, + }, + 'client.address': { + type: 'keyword', + array: false, + required: false, + }, + 'client.as.number': { + type: 'long', + array: false, + required: false, + }, + 'client.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.bytes': { + type: 'long', + array: false, + required: false, + }, + 'client.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'client.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.ip': { + type: 'ip', + array: false, + required: false, + }, + 'client.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'client.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'client.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'client.packets': { + type: 'long', + array: false, + required: false, + }, + 'client.port': { + type: 'long', + array: false, + required: false, + }, + 'client.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'cloud.account.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.account.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.availability_zone': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.instance.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.instance.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.machine.type': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.project.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.project.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.region': { + type: 'keyword', + array: false, + required: false, + }, + 'container.id': { + type: 'keyword', + array: false, + required: false, + }, + 'container.image.name': { + type: 'keyword', + array: false, + required: false, + }, + 'container.image.tag': { + type: 'keyword', + array: true, + required: false, + }, + 'container.labels': { + type: 'object', + array: false, + required: false, + }, + 'container.name': { + type: 'keyword', + array: false, + required: false, + }, + 'container.runtime': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.address': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.as.number': { + type: 'long', + array: false, + required: false, + }, + 'destination.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.bytes': { + type: 'long', + array: false, + required: false, + }, + 'destination.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'destination.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.ip': { + type: 'ip', + array: false, + required: false, + }, + 'destination.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'destination.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'destination.packets': { + type: 'long', + array: false, + required: false, + }, + 'destination.port': { + type: 'long', + array: false, + required: false, + }, + 'destination.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'dll.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.path': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers': { + type: 'object', + array: true, + required: false, + }, + 'dns.answers.class': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.data': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.ttl': { + type: 'long', + array: false, + required: false, + }, + 'dns.answers.type': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.header_flags': { + type: 'keyword', + array: true, + required: false, + }, + 'dns.id': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.op_code': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.class': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.type': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.resolved_ip': { + type: 'ip', + array: true, + required: false, + }, + 'dns.response_code': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.type': { + type: 'keyword', + array: false, + required: false, + }, + 'ecs.version': { + type: 'keyword', + array: false, + required: true, + }, + 'error.code': { + type: 'keyword', + array: false, + required: false, + }, + 'error.id': { + type: 'keyword', + array: false, + required: false, + }, + 'error.message': { + type: 'text', + array: false, + required: false, + }, + 'error.stack_trace': { + type: 'keyword', + array: false, + required: false, + }, + 'error.type': { + type: 'keyword', + array: false, + required: false, + }, + 'event.action': { + type: 'keyword', + array: false, + required: false, + }, + 'event.category': { + type: 'keyword', + array: true, + required: false, + }, + 'event.code': { + type: 'keyword', + array: false, + required: false, + }, + 'event.created': { + type: 'date', + array: false, + required: false, + }, + 'event.dataset': { + type: 'keyword', + array: false, + required: false, + }, + 'event.duration': { + type: 'long', + array: false, + required: false, + }, + 'event.end': { + type: 'date', + array: false, + required: false, + }, + 'event.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'event.id': { + type: 'keyword', + array: false, + required: false, + }, + 'event.ingested': { + type: 'date', + array: false, + required: false, + }, + 'event.kind': { + type: 'keyword', + array: false, + required: false, + }, + 'event.module': { + type: 'keyword', + array: false, + required: false, + }, + 'event.original': { + type: 'keyword', + array: false, + required: false, + }, + 'event.outcome': { + type: 'keyword', + array: false, + required: false, + }, + 'event.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'event.reason': { + type: 'keyword', + array: false, + required: false, + }, + 'event.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'event.risk_score': { + type: 'float', + array: false, + required: false, + }, + 'event.risk_score_norm': { + type: 'float', + array: false, + required: false, + }, + 'event.sequence': { + type: 'long', + array: false, + required: false, + }, + 'event.severity': { + type: 'long', + array: false, + required: false, + }, + 'event.start': { + type: 'date', + array: false, + required: false, + }, + 'event.timezone': { + type: 'keyword', + array: false, + required: false, + }, + 'event.type': { + type: 'keyword', + array: true, + required: false, + }, + 'event.url': { + type: 'keyword', + array: false, + required: false, + }, + 'file.accessed': { + type: 'date', + array: false, + required: false, + }, + 'file.attributes': { + type: 'keyword', + array: true, + required: false, + }, + 'file.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'file.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'file.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'file.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'file.created': { + type: 'date', + array: false, + required: false, + }, + 'file.ctime': { + type: 'date', + array: false, + required: false, + }, + 'file.device': { + type: 'keyword', + array: false, + required: false, + }, + 'file.directory': { + type: 'keyword', + array: false, + required: false, + }, + 'file.drive_letter': { + type: 'keyword', + array: false, + required: false, + }, + 'file.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'file.gid': { + type: 'keyword', + array: false, + required: false, + }, + 'file.group': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'file.inode': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mode': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mtime': { + type: 'date', + array: false, + required: false, + }, + 'file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.owner': { + type: 'keyword', + array: false, + required: false, + }, + 'file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'file.size': { + type: 'long', + array: false, + required: false, + }, + 'file.target_path': { + type: 'keyword', + array: false, + required: false, + }, + 'file.type': { + type: 'keyword', + array: false, + required: false, + }, + 'file.uid': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'file.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'file.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'file.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'file.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'host.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'host.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.hostname': { + type: 'keyword', + array: false, + required: false, + }, + 'host.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.ip': { + type: 'ip', + array: true, + required: false, + }, + 'host.mac': { + type: 'keyword', + array: true, + required: false, + }, + 'host.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'host.type': { + type: 'keyword', + array: false, + required: false, + }, + 'host.uptime': { + type: 'long', + array: false, + required: false, + }, + 'host.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'http.request.body.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.request.body.content': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.request.method': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.referrer': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.body.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.response.body.content': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.response.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.status_code': { + type: 'long', + array: false, + required: false, + }, + 'http.version': { + type: 'keyword', + array: false, + required: false, + }, + labels: { + type: 'object', + array: false, + required: false, + }, + 'log.file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'log.level': { + type: 'keyword', + array: false, + required: false, + }, + 'log.logger': { + type: 'keyword', + array: false, + required: false, + }, + 'log.origin.file.line': { + type: 'integer', + array: false, + required: false, + }, + 'log.origin.file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'log.origin.function': { + type: 'keyword', + array: false, + required: false, + }, + 'log.original': { + type: 'keyword', + array: false, + required: false, + }, + 'log.syslog': { + type: 'object', + array: false, + required: false, + }, + 'log.syslog.facility.code': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.facility.name': { + type: 'keyword', + array: false, + required: false, + }, + 'log.syslog.priority': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.severity.code': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.severity.name': { + type: 'keyword', + array: false, + required: false, + }, + message: { + type: 'text', + array: false, + required: false, + }, + 'network.application': { + type: 'keyword', + array: false, + required: false, + }, + 'network.bytes': { + type: 'long', + array: false, + required: false, + }, + 'network.community_id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.direction': { + type: 'keyword', + array: false, + required: false, + }, + 'network.forwarded_ip': { + type: 'ip', + array: false, + required: false, + }, + 'network.iana_number': { + type: 'keyword', + array: false, + required: false, + }, + 'network.inner': { + type: 'object', + array: false, + required: false, + }, + 'network.inner.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.inner.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'network.name': { + type: 'keyword', + array: false, + required: false, + }, + 'network.packets': { + type: 'long', + array: false, + required: false, + }, + 'network.protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'network.transport': { + type: 'keyword', + array: false, + required: false, + }, + 'network.type': { + type: 'keyword', + array: false, + required: false, + }, + 'network.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress': { + type: 'object', + array: false, + required: false, + }, + 'observer.egress.interface.alias': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.interface.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.interface.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.zone': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'observer.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.hostname': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress': { + type: 'object', + array: false, + required: false, + }, + 'observer.ingress.interface.alias': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.interface.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.interface.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.zone': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ip': { + type: 'ip', + array: true, + required: false, + }, + 'observer.mac': { + type: 'keyword', + array: true, + required: false, + }, + 'observer.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.product': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.type': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.vendor': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.version': { + type: 'keyword', + array: false, + required: false, + }, + 'organization.id': { + type: 'keyword', + array: false, + required: false, + }, + 'organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'package.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'package.build_version': { + type: 'keyword', + array: false, + required: false, + }, + 'package.checksum': { + type: 'keyword', + array: false, + required: false, + }, + 'package.description': { + type: 'keyword', + array: false, + required: false, + }, + 'package.install_scope': { + type: 'keyword', + array: false, + required: false, + }, + 'package.installed': { + type: 'date', + array: false, + required: false, + }, + 'package.license': { + type: 'keyword', + array: false, + required: false, + }, + 'package.name': { + type: 'keyword', + array: false, + required: false, + }, + 'package.path': { + type: 'keyword', + array: false, + required: false, + }, + 'package.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'package.size': { + type: 'long', + array: false, + required: false, + }, + 'package.type': { + type: 'keyword', + array: false, + required: false, + }, + 'package.version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.args': { + type: 'keyword', + array: true, + required: false, + }, + 'process.args_count': { + type: 'long', + array: false, + required: false, + }, + 'process.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'process.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'process.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'process.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'process.command_line': { + type: 'keyword', + array: false, + required: false, + }, + 'process.entity_id': { + type: 'keyword', + array: false, + required: false, + }, + 'process.executable': { + type: 'keyword', + array: false, + required: false, + }, + 'process.exit_code': { + type: 'long', + array: false, + required: false, + }, + 'process.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'process.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.args': { + type: 'keyword', + array: true, + required: false, + }, + 'process.parent.args_count': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.command_line': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.entity_id': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.executable': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.exit_code': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pgid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.pid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.ppid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.start': { + type: 'date', + array: false, + required: false, + }, + 'process.parent.thread.id': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.thread.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.title': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.uptime': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.working_directory': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pgid': { + type: 'long', + array: false, + required: false, + }, + 'process.pid': { + type: 'long', + array: false, + required: false, + }, + 'process.ppid': { + type: 'long', + array: false, + required: false, + }, + 'process.start': { + type: 'date', + array: false, + required: false, + }, + 'process.thread.id': { + type: 'long', + array: false, + required: false, + }, + 'process.thread.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.title': { + type: 'keyword', + array: false, + required: false, + }, + 'process.uptime': { + type: 'long', + array: false, + required: false, + }, + 'process.working_directory': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.data.bytes': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.data.strings': { + type: 'keyword', + array: true, + required: false, + }, + 'registry.data.type': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.hive': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.key': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.path': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.value': { + type: 'keyword', + array: false, + required: false, + }, + 'related.hash': { + type: 'keyword', + array: true, + required: false, + }, + 'related.hosts': { + type: 'keyword', + array: true, + required: false, + }, + 'related.ip': { + type: 'ip', + array: true, + required: false, + }, + 'related.user': { + type: 'keyword', + array: true, + required: false, + }, + 'rule.author': { + type: 'keyword', + array: true, + required: false, + }, + 'rule.category': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.description': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.id': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.license': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.name': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.ruleset': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.uuid': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.version': { + type: 'keyword', + array: false, + required: false, + }, + 'server.address': { + type: 'keyword', + array: false, + required: false, + }, + 'server.as.number': { + type: 'long', + array: false, + required: false, + }, + 'server.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.bytes': { + type: 'long', + array: false, + required: false, + }, + 'server.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'server.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.ip': { + type: 'ip', + array: false, + required: false, + }, + 'server.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'server.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'server.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'server.packets': { + type: 'long', + array: false, + required: false, + }, + 'server.port': { + type: 'long', + array: false, + required: false, + }, + 'server.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'service.ephemeral_id': { + type: 'keyword', + array: false, + required: false, + }, + 'service.id': { + type: 'keyword', + array: false, + required: false, + }, + 'service.name': { + type: 'keyword', + array: false, + required: false, + }, + 'service.node.name': { + type: 'keyword', + array: false, + required: false, + }, + 'service.state': { + type: 'keyword', + array: false, + required: false, + }, + 'service.type': { + type: 'keyword', + array: false, + required: false, + }, + 'service.version': { + type: 'keyword', + array: false, + required: false, + }, + 'source.address': { + type: 'keyword', + array: false, + required: false, + }, + 'source.as.number': { + type: 'long', + array: false, + required: false, + }, + 'source.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.bytes': { + type: 'long', + array: false, + required: false, + }, + 'source.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'source.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.ip': { + type: 'ip', + array: false, + required: false, + }, + 'source.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'source.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'source.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'source.packets': { + type: 'long', + array: false, + required: false, + }, + 'source.port': { + type: 'long', + array: false, + required: false, + }, + 'source.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'span.id': { + type: 'keyword', + array: false, + required: false, + }, + tags: { + type: 'keyword', + array: true, + required: false, + }, + 'threat.framework': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.tactic.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.tactic.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.tactic.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.cipher': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.certificate': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.certificate_chain': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.issuer': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.ja3': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.server_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.subject': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.supported_ciphers': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'tls.client.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'tls.client.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.established': { + type: 'boolean', + array: false, + required: false, + }, + 'tls.next_protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.resumed': { + type: 'boolean', + array: false, + required: false, + }, + 'tls.server.certificate': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.certificate_chain': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.issuer': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.ja3s': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.subject': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'tls.server.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'tls.server.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.version': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.version_protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'trace.id': { + type: 'keyword', + array: false, + required: false, + }, + 'transaction.id': { + type: 'keyword', + array: false, + required: false, + }, + 'url.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'url.fragment': { + type: 'keyword', + array: false, + required: false, + }, + 'url.full': { + type: 'keyword', + array: false, + required: false, + }, + 'url.original': { + type: 'keyword', + array: false, + required: false, + }, + 'url.password': { + type: 'keyword', + array: false, + required: false, + }, + 'url.path': { + type: 'keyword', + array: false, + required: false, + }, + 'url.port': { + type: 'long', + array: false, + required: false, + }, + 'url.query': { + type: 'keyword', + array: false, + required: false, + }, + 'url.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.scheme': { + type: 'keyword', + array: false, + required: false, + }, + 'url.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.username': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.target.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user_agent.device.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.original': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.version': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.category': { + type: 'keyword', + array: true, + required: false, + }, + 'vulnerability.classification': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.description': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.enumeration': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.id': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.report_id': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.scanner.vendor': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.score.base': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.environmental': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.temporal': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.version': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.severity': { + type: 'keyword', + array: false, + required: false, + }, +} as const; diff --git a/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json b/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json new file mode 100644 index 0000000000000..f7cbfc3dfaae3 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json @@ -0,0 +1,3416 @@ +{ + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "build": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resolved_ip": { + "type": "ip" + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "integer" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "span": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "changes": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "effective": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts index ed5ec66a70914..5f594a2a79d20 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts @@ -8,7 +8,7 @@ import { ActionVariable } from '../../../../alerting/common'; import { AlertExecutorOptions } from '../../../../alerting/server'; import { AlertSeverityLevel } from '../../../common'; -import { AlertCheck, AlertContext } from '../../types'; +import { AlertCheck } from '../../types'; import { DefaultFieldMap } from '../defaults/field_map'; type AlertInstanceFactory = AlertExecutorOptions< diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts index db8cd46500d55..8f72fce0cb195 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts @@ -5,27 +5,26 @@ * 2.0. */ +import { Mutable } from 'utility-types'; +import { ecsFieldMap } from '../../generated/ecs_field_map'; +import { pickWithPatterns } from '../field_map/pick_with_patterns'; + export const defaultFieldMap = { - 'event.kind': { type: 'keyword', required: true }, - '@timestamp': { type: 'date', required: true }, - 'alert.id': { type: 'keyword', required: true }, - 'alert.created': { type: 'date', required: true }, - 'alert.active': { type: 'boolean', required: true }, - // 'alert.muted': { type: 'boolean', required: true }, - 'alert.type': { type: 'keyword', required: true }, - 'alert.name': { type: 'keyword', required: true }, - 'alert.series_id': { type: 'keyword', required: true }, // rule.id + alert.name - 'alert.check.severity': { type: 'keyword', required: true }, + ...pickWithPatterns(ecsFieldMap, '@timestamp', 'event.*', 'rule.*'), + 'alert.id': { type: 'keyword' }, + 'alert.type': { type: 'keyword' }, + 'alert.name': { type: 'keyword' }, + 'alert.series_id': { type: 'keyword' }, // rule.id + alert.name 'alert.check.value': { type: 'scaled_float', scaling_factor: 100 }, 'alert.check.threshold': { type: 'scaled_float', scaling_factor: 100 }, 'alert.check.influencers': { type: 'flattened' }, - 'rule.id': { type: 'keyword', required: true }, - 'rule.namespace': { type: 'keyword' }, - 'rule.name': { type: 'keyword', required: true }, - 'rule.interval.ms': { type: 'long', required: true }, - 'rule_type.id': { type: 'keyword', required: true }, - 'rule_type.name': { type: 'keyword', required: true }, 'rule_type.producer': { type: 'keyword', required: true }, } as const; -export type DefaultFieldMap = typeof defaultFieldMap; +type DefaultFieldMapReadOnly = typeof defaultFieldMap; + +export type DefaultFieldMap = Mutable< + { + [key in keyof DefaultFieldMapReadOnly]: Mutable; + } +>; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts index 6c9dd965f0d35..b64250740e2fa 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts @@ -5,6 +5,7 @@ * 2.0. */ import { isEqual } from 'lodash'; +import util from 'util'; import { FieldMap } from '../types'; export function mergeFieldMaps( @@ -30,16 +31,16 @@ export function mergeFieldMaps( }); }); - if (first[name] && !isEqual(first[name], field)) { + if (first[name]) { conflicts.push({ - fields: [field, first[name]], + [name]: [field, first[name]], }); } }); if (conflicts.length) { const err = new Error(`Could not merge mapping due to conflicts`); - Object.assign(err, { conflicts }); + Object.assign(err, { conflicts: util.inspect(conflicts, { depth: null }) }); throw err; } diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts new file mode 100644 index 0000000000000..96c027c5ed144 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ValuesType, SetIntersection, OmitByValueExact } from 'utility-types'; +import { pick, memoize } from 'lodash'; +import { ecsFieldMap } from '../../generated/ecs_field_map'; + +type SplitByDot< + TPath extends string, + TPrefix extends string = '' +> = TPath extends `${infer TKey}.${infer TRest}` + ? [`${TPrefix}${TKey}.*`, ...SplitByDot] + : [`${TPrefix}${TPath}`]; + +type PatternMapOf> = { + [TKey in keyof T]: ValuesType : never>; +}; + +type PickByPatterns, TPatterns extends string[]> = OmitByValueExact< + { + [TFieldName in keyof T]: SetIntersection< + ValuesType, + PatternMapOf[TFieldName] + > extends never + ? undefined + : T[TFieldName]; + }, + undefined +>; + +const allEcsFields = Object.keys(ecsFieldMap) as Array; + +export function pickWithPatterns< + T extends Record, + TPatterns extends Array>> +>(map: T, ...patterns: TPatterns): PickByPatterns { + const matchedFields = allEcsFields.filter((field) => + patterns.some((pattern) => { + if (pattern === field) { + return true; + } + + const fieldParts = field.split('.'); + const patternParts = pattern.split('.'); + + if (patternParts.indexOf('*') !== patternParts.length - 1) { + return false; + } + + return fieldParts.every((fieldPart, index) => { + const patternPart = patternParts.length < index ? '*' : patternParts[index]; + + return fieldPart === patternPart || patternPart === '*'; + }); + }) + ); + + return (pick(ecsFieldMap, matchedFields) as unknown) as PickByPatterns; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts new file mode 100644 index 0000000000000..e8e3cb2076f5e --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues, pickBy } from 'lodash'; +import * as t from 'io-ts'; +import { RequiredKeys, OptionalKeys, PickByValueExact, OmitByValueExact } from 'utility-types'; +import { FieldMap } from '../types'; + +const esFieldTypeMap = { + keyword: t.string, + text: t.string, + date: t.string, + boolean: t.boolean, + byte: t.number, + long: t.number, + integer: t.number, + short: t.number, + double: t.number, + float: t.number, + scaled_float: t.number, + unsigned_long: t.number, + flattened: t.record(t.string, t.array(t.string)), +}; + +type EsFieldTypeMap = typeof esFieldTypeMap; + +type EsFieldTypeOf = T extends keyof EsFieldTypeMap + ? EsFieldTypeMap[T] + : t.UnknownC; + +type RequiredKeysOf> = keyof PickByValueExact< + { + [key in keyof T]: T[key]['required']; + }, + true +>; + +type OptionalKeysOf> = keyof OmitByValueExact< + { + [key in keyof T]: T[key]['required']; + }, + true +>; + +type IntersectionTypeOf< + T extends Record +> = t.IntersectionC< + [ + t.TypeC>>, + t.PartialC>> + ] +>; + +type MapTypeValues = { + [key in keyof T]: { + required: T[key]['required']; + type: T[key]['array'] extends true + ? t.ArrayC> + : EsFieldTypeOf; + }; +}; +export type FieldMapType = IntersectionTypeOf>; + +export type TypeOfFieldMap = t.TypeOf>; + +export function runtimeTypeFromFieldMap( + fieldMap: TFieldMap +): FieldMapType { + function mapToType(fields: FieldMap) { + return mapValues(fields, (field) => { + const type = + field.type in esFieldTypeMap + ? esFieldTypeMap[field.type as keyof EsFieldTypeMap] + : t.unknown; + + return field.array ? t.array(type) : type; + }); + } + + const required = pickBy(fieldMap, (field) => field.required); + + return (t.intersection([ + t.exact(t.partial(mapToType(fieldMap))), + t.type(mapToType(required)), + ]) as unknown) as FieldMapType; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts deleted file mode 100644 index 042955a7b7388..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/field_map/schema_from_field_map.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { mapValues } from 'lodash'; -import { ObjectType, schema, Type, TypeOf } from '@kbn/config-schema'; -import { FieldMap } from '../types'; - -type MaybeArrayType = TOptions extends { array: true } - ? T[] - : T; - -type MaybeRequiredType< - T, - TOptions extends { required?: boolean; array?: boolean } -> = TOptions extends { - required: true; -} - ? MaybeArrayType - : MaybeArrayType | undefined | null; - -const map = { - keyword: schema.string(), - text: schema.string(), - date: schema.string(), - boolean: schema.boolean(), - byte: schema.number(), - long: schema.number(), - integer: schema.number(), - short: schema.number(), - double: schema.number(), - float: schema.number(), - scaled_float: schema.number(), - unsigned_long: schema.number(), - flattened: schema.mapOf(schema.string(), schema.arrayOf(schema.string())), -}; - -type SchemaMap = typeof map; - -type TypeOfField = Type< - MaybeRequiredType< - (Record & - { - [key in keyof SchemaMap]: TypeOf; - })[T['type']], - T - > ->; - -export type SchemaOf = ObjectType< - { - [key in keyof TFieldMap]: TypeOfField; - } ->; - -export function schemaFromFieldMap(fieldMap: TFieldMap) { - return (schema.object( - mapValues(fieldMap, ({ type, array, required }) => { - let schemaType: Type = type in map ? map[type as keyof SchemaMap] : schema.never(); - - if (array) { - schemaType = schema.arrayOf(schemaType); - } - - if (!required) { - schemaType = schema.maybe(schemaType); - } - - return schemaType; - }) - ) as unknown) as SchemaOf; -} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index 2daf61090b83b..15e3bdf0ddf4d 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -6,21 +6,26 @@ */ import { CoreSetup, Logger } from 'kibana/server'; -import { mapValues, omitBy, compact } from 'lodash'; -import { schema } from '@kbn/config-schema'; +import { omitBy, compact } from 'lodash'; import { inspect } from 'util'; import uuid from 'uuid'; -import { ESSearchRequest, ESSearchResponse } from '../../../../typings/elasticsearch'; -import { createReadySignal } from '../../../event_log/server/lib/ready_signal'; -import { ClusterClientAdapter } from '../../../event_log/server/es/cluster_client_adapter'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { isLeft } from 'fp-ts/lib/Either'; +import { ESSearchRequest, ESSearchResponse } from '../../../../../typings/elasticsearch'; +import { createReadySignal, ClusterClientAdapter } from '../../../event_log/server'; import { FieldMap, ILMPolicy } from './types'; import { RegisterRuleType, RuleState, RuleAlertState } from '../types'; import { mergeFieldMaps } from './field_map/merge_field_maps'; -import { schemaFromFieldMap, SchemaOf } from './field_map/schema_from_field_map'; +import { + FieldMapType, + runtimeTypeFromFieldMap, + TypeOfFieldMap, +} from './field_map/runtime_type_from_fieldmap'; import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; import { createCheckService } from './check_service'; -import { AlertSeverityLevel } from '../../common'; +import { AlertSeverityLevel, getAlertSeverityLevelValue } from '../../common'; +import { DefaultFieldMap } from './defaults/field_map'; interface RuleRegistryOptions { kibanaIndex: string; @@ -31,31 +36,38 @@ interface RuleRegistryOptions { fieldMap: TFieldMap; ilmPolicy: ILMPolicy; alertingPluginSetupContract: AlertingPluginSetupContract; - parent?: RuleRegistry; + parent?: RuleRegistry; } export class RuleRegistry { - private readonly esAdapter: ClusterClientAdapter; - private readonly docSchema: SchemaOf; - private readonly children: Array> = []; - - constructor(private readonly options: RuleRegistryOptions) { + private readonly esAdapter: ClusterClientAdapter<{ + body: TypeOfFieldMap; + index: string; + }>; + private readonly docRt: FieldMapType; + private readonly children: Array> = []; + + constructor(private readonly options: RuleRegistryOptions) { const { logger, core } = options; const { wait, signal } = createReadySignal(); - this.esAdapter = new ClusterClientAdapter({ + this.esAdapter = new ClusterClientAdapter<{ + body: TypeOfFieldMap; + index: string; + }>({ wait, elasticsearchClientPromise: core .getStartServices() .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), - logger, + logger: logger.get('esAdapter'), }); - this.docSchema = schemaFromFieldMap(options.fieldMap); + this.docRt = runtimeTypeFromFieldMap(options.fieldMap); this.initialize() .then(() => { + this.options.logger.debug('Bootstrapped alerts index'); signal(true); }) .catch((err) => { @@ -97,6 +109,8 @@ export class RuleRegistry { auto_expand_replicas: '0-1', 'index.lifecycle.name': policyName, 'index.lifecycle.rollover_alias': indexAliasName, + 'sort.field': '@timestamp', + 'sort.order': 'desc', }, mappings: mappingFromFieldMap(this.options.fieldMap), }); @@ -126,7 +140,7 @@ export class RuleRegistry { filter: [ { terms: { - 'rule.id': ruleIds, + 'rule.uuid': ruleIds, }, }, ...compact([request.body?.query]), @@ -145,12 +159,12 @@ export class RuleRegistry { return response.body as ESSearchResponse; } - registerType: RegisterRuleType = (type) => { + registerType: RegisterRuleType = (type) => { this.options.alertingPluginSetupContract.registerType< - Record, + Record, RuleState, - Record, - Record, + Record, + Record, string, string >({ @@ -164,7 +178,7 @@ export class RuleRegistry { alertId: ruleId, name: ruleName, params, - namespace, + // namespace, } = options; const prevAlertState = @@ -182,7 +196,7 @@ export class RuleRegistry { const executorOptions = { previousStartedAt, startedAt, - params: params as any, + params, services: { ...passthroughServices, check: checkService.check as any, @@ -212,96 +226,110 @@ export class RuleRegistry { }, {} as Record), }; - const common = { - 'event.kind': 'alert', - '@timestamp': startedAt.toISOString(), - 'rule.id': ruleId, - 'rule.name': ruleName, - 'rule.namespace': namespace, - 'rule_type.id': type.id, - 'rule_type.name': type.name, - 'rule_type.producer': type.producer, - // 'rule.interval.ms': prev - }; - - const idsOfLastAlertEventsToFetch = Object.values(mergedAlertStates).map( - (state) => state.alertId - ); - - const start = new Date().getTime() - 60 * 60 * 1000; - - const response = await this.search([ruleId], { - body: { - size: idsOfLastAlertEventsToFetch.length, - query: { - bool: { - filter: [ - { - terms: { - 'alert.id': idsOfLastAlertEventsToFetch, - }, - }, - { - range: { - '@timestamp': { - gte: start, - format: 'epoch_millis', - }, - }, - }, - ], - }, - }, - collapse: { - field: 'alert.id', - }, - sort: { - '@timestamp': 'desc', - }, - _source: false, - fields: Object.keys(this.options.fieldMap), - }, - }); - - const lastEventByAlertId = response.hits.hits.reduce((prev, hit) => { - const alertId = hit.fields['alert.id']![0] as string; - prev[alertId] = hit.fields as Record; - return prev; - }, {} as Record>); + // const idsOfLastAlertEventsToFetch = Object.values(mergedAlertStates).map( + // (state) => state.alertId + // ); + + // const start = new Date().getTime() - 60 * 60 * 1000; + + // const response = await this.search([ruleId], { + // body: { + // size: idsOfLastAlertEventsToFetch.length, + // query: { + // bool: { + // filter: [ + // { + // terms: { + // 'alert.id': idsOfLastAlertEventsToFetch, + // }, + // }, + // { + // range: { + // '@timestamp': { + // gte: start, + // format: 'epoch_millis', + // }, + // }, + // }, + // ], + // }, + // }, + // collapse: { + // field: 'alert.id', + // }, + // sort: { + // '@timestamp': 'desc', + // }, + // _source: false, + // fields: Object.keys(this.options.fieldMap), + // }, + // }); + + // const lastEventByAlertId = response.hits.hits.reduce((prev, hit) => { + // const alertId = hit.fields['alert.id']![0] as string; + // prev[alertId] = hit.fields as Record; + // return prev; + // }, {} as Record>); const index = this.getEsNames().indexAliasName; const updates = Object.entries(mergedAlertStates).map(([alertName, state]) => { const active = activeAlertNames.includes(alertName); - const lastEvent = lastEventByAlertId[state.alertId] ?? {}; const nextState = active ? activeAlerts[alertName] : undefined; + const createdAt = new Date(state.created); + + const body: TypeOfFieldMap = { + ...(nextState + ? { + ...nextState.fields, + 'event.severity': getAlertSeverityLevelValue(nextState.level), + 'alert.check.value': nextState.value, + 'alert.check.threshold': nextState.threshold, + 'event.action': 'active-alert', + } + : { + 'event.end': startedAt.toISOString(), + 'event.action': 'recovered-alert', + }), + 'event.kind': 'alert', + '@timestamp': startedAt.toISOString(), + 'rule.uuid': ruleId, + 'rule.id': type.id, + 'rule.category': type.name, + 'rule.name': ruleName, + // 'rule.namespace': namespace, + 'rule_type.producer': type.producer, + 'alert.id': state.alertId, + 'event.start': createdAt.toISOString(), + 'event.duration': (startedAt.getTime() - createdAt.getTime()) * 1000, + 'alert.name': alertName, + 'alert.series_id': [ruleId, alertName].join('|'), + }; + return { index, - body: { - ...lastEvent, - ...(nextState - ? { - ...nextState.fields, - 'alert.check.severity': nextState.level, - 'alert.check.value': nextState.value, - 'alert.check.threshold': nextState.threshold, - } - : {}), - ...common, - 'alert.active': active, - 'alert.id': state.alertId, - 'alert.created': state.created, - 'alert.type': 'threshold', // or, log - 'alert.name': alertName, - 'alert.series_id': [ruleId, alertName].join('|'), - }, + body, }; }); - if (updates.length) { - await this.esAdapter.indexDocuments(updates); + let indexedCount = 0; + + updates.forEach((update) => { + const decode = this.docRt.decode(update.body); + if (isLeft(decode)) { + const error = new Error(`Failed to validate alert event`); + error.stack += '\n' + PathReporter.report(decode).join('\n'); + this.options.logger.error(error); + } else { + this.esAdapter.indexDocument(update); + indexedCount++; + } + }); + + if (indexedCount > 0) { + this.options.logger.debug(`Indexed ${indexedCount} events`); } const nextState = omitBy(mergedAlertStates, (_, alertName) => { @@ -329,7 +357,7 @@ export class RuleRegistry { ? mergeFieldMaps(this.options.fieldMap, fieldMap) : this.options.fieldMap; - const child = new RuleRegistry({ + const child = new RuleRegistry({ ...this.options, logger: this.options.logger.get(namespace), namespace: [this.options.namespace, namespace].filter(Boolean).join('-'), diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index 773d366dfa0bb..c8b47313df77c 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -9,7 +9,7 @@ import { ActionVariable, AlertTypeState } from '../../alerting/common'; import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; import { AlertSeverityLevel } from '../common'; import { DefaultFieldMap } from './rule_registry/defaults/field_map'; -import { SchemaOf } from './rule_registry/field_map/schema_from_field_map'; +import { TypeOfFieldMap } from './rule_registry/field_map/runtime_type_from_fieldmap'; import { FieldMap } from './rule_registry/types'; enum ESFieldType { @@ -47,7 +47,7 @@ export interface AlertCheck>>, keyof DefaultFieldMap>; + fields: Omit>, keyof DefaultFieldMap>; } type TypeOfRuleParams = TypeOf; @@ -69,10 +69,10 @@ type RuleExecutorFunction< TRuleParams extends RuleParams, TActionVariable extends ActionVariable > = ( - options: (PassthroughAlertExecutorOptions & { + options: PassthroughAlertExecutorOptions & { services: RuleExecutorServices; - }) & - (TRuleParams extends RuleParams ? { params: TypeOfRuleParams } : {}) + params: TypeOfRuleParams; + } ) => Promise>; export interface RuleType { From d7e194ce0985dfb9e9d46049b717608996213989 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 30 Mar 2021 17:58:38 +0200 Subject: [PATCH 03/24] Scoped rule registry client --- .../apm/common/environment_filter_values.ts | 18 + .../scripts/optimize-tsconfig/tsconfig.json | 1 + .../alerts/create_apm_lifecycle_rule_type.ts | 11 + .../register_error_count_alert_type.test.ts | 189 +++++----- .../alerts/register_error_count_alert_type.ts | 211 +++++------ ...egister_transaction_duration_alert_type.ts | 203 +++++----- ...action_duration_anomaly_alert_type.test.ts | 299 +++++++-------- ...transaction_duration_anomaly_alert_type.ts | 349 +++++++++--------- ..._transaction_error_rate_alert_type.test.ts | 278 +++----------- ...ister_transaction_error_rate_alert_type.ts | 264 ++++++------- x-pack/plugins/rule_registry/jest.config.js | 12 + x-pack/plugins/rule_registry/server/index.ts | 3 +- .../rule_registry/check_service/index.ts | 51 --- .../index.ts | 150 ++++++++ .../types.ts | 51 +++ .../rule_registry/defaults/field_map.ts | 39 +- .../field_map/pick_with_patterns.test.ts | 71 ++++ .../field_map/pick_with_patterns.ts | 25 +- .../runtime_type_from_fieldmap.test.ts | 95 +++++ .../field_map/runtime_type_from_fieldmap.ts | 37 +- .../server/rule_registry/index.ts | 276 +++----------- .../create_lifecycle_rule_type_factory.ts | 206 +++++++++++ x-pack/plugins/rule_registry/server/types.ts | 124 +++---- 23 files changed, 1588 insertions(+), 1375 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts create mode 100644 x-pack/plugins/rule_registry/jest.config.js delete mode 100644 x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts diff --git a/x-pack/plugins/apm/common/environment_filter_values.ts b/x-pack/plugins/apm/common/environment_filter_values.ts index e091b53b2e5b8..c80541ee1ba6b 100644 --- a/x-pack/plugins/apm/common/environment_filter_values.ts +++ b/x-pack/plugins/apm/common/environment_filter_values.ts @@ -22,11 +22,13 @@ const environmentLabels: Record = { }; export const ENVIRONMENT_ALL = { + esFieldValue: undefined, value: ENVIRONMENT_ALL_VALUE, text: environmentLabels[ENVIRONMENT_ALL_VALUE], }; export const ENVIRONMENT_NOT_DEFINED = { + esFieldValue: undefined, value: ENVIRONMENT_NOT_DEFINED_VALUE, text: environmentLabels[ENVIRONMENT_NOT_DEFINED_VALUE], }; @@ -35,6 +37,22 @@ export function getEnvironmentLabel(environment: string) { return environmentLabels[environment] || environment; } +export function parseEnvironmentUrlParam(environment: string) { + if (environment === ENVIRONMENT_ALL_VALUE) { + return ENVIRONMENT_ALL; + } + + if (environment === ENVIRONMENT_NOT_DEFINED_VALUE) { + return ENVIRONMENT_NOT_DEFINED; + } + + return { + esFieldValue: environment, + value: environment, + text: environment, + }; +} + // returns the environment url param that should be used // based on the requested environment. If the requested // environment is different from the URL parameter, we'll diff --git a/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json b/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json index 319eb53313231..40d42298b967b 100644 --- a/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json +++ b/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json @@ -2,6 +2,7 @@ "include": [ "./x-pack/plugins/apm/**/*", "./x-pack/plugins/observability/**/*", + "./x-pack/plugins/rule_registry/**/*", "./typings/**/*" ], "exclude": [ diff --git a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts new file mode 100644 index 0000000000000..78f2fd6fcfed8 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; +import { APMRuleRegistry } from '../../plugin'; + +export const createAPMLifecyleRuleType = createLifecycleRuleTypeFactory(); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts index d7dd7aee3ca25..470330016cc6f 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts @@ -8,14 +8,13 @@ import { Observable } from 'rxjs'; import * as Rx from 'rxjs'; import { toArray, map } from 'rxjs/operators'; - -import { AlertingPlugin } from '../../../../alerting/server'; import { APMConfig } from '../..'; import { registerErrorCountAlertType } from './register_error_count_alert_type'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; +import { APMRuleRegistry } from '../../plugin'; type Operator = (source: Rx.Observable) => Rx.Observable; const pipeClosure = (fn: Operator): Operator => { @@ -33,22 +32,28 @@ const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( describe('Error count alert', () => { it("doesn't send an alert when error count is less than threshold", async () => { let alertExecutor: any; - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; registerErrorCountAlertType({ - alerting, + registry, config$: mockedConfig$, + logger: {} as any, }); expect(alertExecutor).toBeDefined(); const services = { scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, alertInstanceFactory: jest.fn(), + alertWithLifecycle: jest.fn(), }; + const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -71,30 +76,37 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params }); + await alertExecutor!({ services, params, startedAt: new Date() }); expect(services.alertInstanceFactory).not.toBeCalled(); }); - it('sends alerts with service name and environment', async () => { + it('sends alerts with service name and environment for those that exceeded the threshold', async () => { let alertExecutor: any; - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; registerErrorCountAlertType({ - alerting, + registry, config$: mockedConfig$, + logger: {} as any, }); expect(alertExecutor).toBeDefined(); const scheduleActions = jest.fn(); + const services = { scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), }; - const params = { threshold: 1, windowSize: 5, windowUnit: 'm' }; + + const params = { threshold: 2, windowSize: 5, windowUnit: 'm' }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -106,18 +118,62 @@ describe('Error count alert', () => { }, }, aggregations: { - services: { + error_counts: { buckets: [ { - key: 'foo', - environments: { - buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], + key: ['foo', 'env-foo'], + doc_count: 5, + latest: { + top: [ + { + metrics: { + 'service.name': 'foo', + 'service.environment': 'env-foo', + }, + }, + ], }, }, { - key: 'bar', - environments: { - buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], + key: ['foo', 'env-foo-2'], + doc_count: 4, + latest: { + top: [ + { + metrics: { + 'service.name': 'foo', + 'service.environment': 'env-foo-2', + }, + }, + ], + }, + }, + { + key: ['bar', 'env-bar'], + doc_count: 3, + latest: { + top: [ + { + metrics: { + 'service.name': 'bar', + 'service.environment': 'env-bar', + }, + }, + ], + }, + }, + { + key: ['bar', 'env-bar-2'], + doc_count: 1, + latest: { + top: [ + { + metrics: { + 'service.name': 'bar', + 'service.environment': 'env-bar-2', + }, + }, + ], }, }, ], @@ -134,115 +190,36 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params }); + await alertExecutor!({ services, params, startedAt: new Date() }); [ 'apm.error_rate_foo_env-foo', 'apm.error_rate_foo_env-foo-2', 'apm.error_rate_bar_env-bar', - 'apm.error_rate_bar_env-bar-2', ].forEach((instanceName) => expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) ); + expect(scheduleActions).toHaveBeenCalledTimes(3); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', environment: 'env-foo', - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 5, interval: '5m', }); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', environment: 'env-foo-2', - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 4, interval: '5m', }); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'bar', environment: 'env-bar', - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - environment: 'env-bar-2', - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - }); - it('sends alerts with service name', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 1, windowSize: 5, windowUnit: 'm' }; - - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - relation: 'eq', - value: 2, - }, - }, - aggregations: { - services: { - buckets: [ - { - key: 'foo', - }, - { - key: 'bar', - }, - ], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); - - await alertExecutor!({ services, params }); - ['apm.error_rate_foo', 'apm.error_rate_bar'].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - environment: undefined, - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - environment: undefined, - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 3, interval: '5m', }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index 485aec996ee61..d4e3ac70439d9 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -21,6 +21,7 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; import { RegisterRuleDependencies } from './register_apm_alerts'; +import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -36,124 +37,128 @@ export function registerErrorCountAlertType({ registry, config$, }: RegisterRuleDependencies) { - registry.registerType({ - id: AlertType.ErrorCount, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); + registry.registerType( + createAPMLifecyleRuleType({ + id: AlertType.ErrorCount, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.errorIndices'], - size: 0, - body: { - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.errorIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, - ...(alertParams.serviceName - ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] - : []), - ...environmentQuery(alertParams.environment), - ], - }, - }, - aggs: { - error_counts: { - multi_terms: { - terms: [ - { field: SERVICE_NAME }, - { field: SERVICE_ENVIRONMENT, missing: '' }, + { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...environmentQuery(alertParams.environment), ], - size: 10000, }, - aggs: { - latest: { - top_metrics: { - metrics: asMutableArray([ - { field: SERVICE_NAME }, - { field: SERVICE_ENVIRONMENT }, - ] as const), - sort: { - '@timestamp': 'desc' as const, + }, + aggs: { + error_counts: { + multi_terms: { + terms: [ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT, missing: '' }, + ], + size: 10000, + }, + aggs: { + latest: { + top_metrics: { + metrics: asMutableArray([ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT }, + ] as const), + sort: { + '@timestamp': 'desc' as const, + }, }, }, }, }, }, }, - }, - }; + }; - const response = await alertingEsClient( - services.scopedClusterClient, - searchParams - ); + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - const errorCountResults = - response.aggregations?.error_counts.buckets.map((bucket) => { - const latest = bucket.latest.top[0].metrics; + const errorCountResults = + response.aggregations?.error_counts.buckets.map((bucket) => { + const latest = bucket.latest.top[0].metrics; - return { - serviceName: latest['service.name'], - environment: latest['service.environment'], - errorCount: bucket.doc_count, - }; - }) ?? []; + return { + serviceName: latest['service.name'] as string, + environment: latest['service.environment'] as string | undefined, + errorCount: bucket.doc_count, + }; + }) ?? []; - errorCountResults - .filter((result) => result.errorCount >= alertParams.threshold) - .forEach((result) => { - const { serviceName, environment, errorCount } = result; - services.check.warning({ - name: [AlertType.ErrorCount, serviceName, environment] - .filter((name) => name) - .join('_'), - threshold: alertParams.threshold, - value: errorCount, - context: { - serviceName, - environment: environment || ENVIRONMENT_NOT_DEFINED.text, - threshold: alertParams.threshold, - triggerValue: errorCount, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }, - fields: { - 'service.name': serviceName, - ...(environment ? { 'service.environment': environment } : {}), - 'processor.event': 'error', - }, + errorCountResults + .filter((result) => result.errorCount >= alertParams.threshold) + .forEach((result) => { + const { serviceName, environment, errorCount } = result; + + services + .alertWithLifecycle({ + id: [AlertType.ErrorCount, serviceName, environment] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(environment + ? { [SERVICE_ENVIRONMENT]: environment } + : {}), + [PROCESSOR_EVENT]: 'error', + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + environment: environment || ENVIRONMENT_NOT_DEFINED.text, + threshold: alertParams.threshold, + triggerValue: errorCount, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); }); - }); - return {}; - }, - }); + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index 63dbd006714ea..110e7e117907b 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -7,6 +7,8 @@ import { schema } from '@kbn/config-schema'; import { take } from 'rxjs/operators'; +import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { PROCESSOR_EVENT, @@ -22,6 +24,7 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; import { RegisterRuleDependencies } from './register_apm_alerts'; +import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.string(), @@ -43,115 +46,121 @@ export function registerTransactionDurationAlertType({ registry, config$, }: RegisterRuleDependencies) { - registry.registerType({ - id: AlertType.TransactionDuration, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; + registry.registerType( + createAPMLifecyleRuleType({ + id: AlertType.TransactionDuration, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.transactionIndices'], - size: 0, - body: { - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.transactionIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - { term: { [SERVICE_NAME]: alertParams.serviceName } }, - { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, - ...environmentQuery(alertParams.environment), - ], + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { term: { [SERVICE_NAME]: alertParams.serviceName } }, + { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, + ...environmentQuery(alertParams.environment), + ] as QueryContainer[], + }, }, - }, - aggs: { - metric: - alertParams.aggregationType === 'avg' - ? { avg: { field: TRANSACTION_DURATION } } - : { - percentiles: { - field: TRANSACTION_DURATION, - percents: [ - alertParams.aggregationType === '95th' ? 95 : 99, - ], + aggs: { + metric: + alertParams.aggregationType === 'avg' + ? { avg: { field: TRANSACTION_DURATION } } + : { + percentiles: { + field: TRANSACTION_DURATION, + percents: [ + alertParams.aggregationType === '95th' ? 95 : 99, + ], + }, }, - }, + }, }, - }, - }; + }; - const response = await alertingEsClient( - services.scopedClusterClient, - searchParams - ); + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - if (!response.aggregations) { - return {}; - } + if (!response.aggregations) { + return {}; + } - const { metric } = response.aggregations; + const { metric } = response.aggregations; - const transactionDuration = - 'values' in metric ? Object.values(metric.values)[0] : metric?.value; + const transactionDuration = + 'values' in metric ? Object.values(metric.values)[0] : metric?.value; - const threshold = alertParams.threshold * 1000; + const threshold = alertParams.threshold * 1000; - if (transactionDuration && transactionDuration > threshold) { - const durationFormatter = getDurationFormatter(transactionDuration); - const transactionDurationFormatted = durationFormatter( - transactionDuration - ).formatted; + if (transactionDuration && transactionDuration > threshold) { + const durationFormatter = getDurationFormatter(transactionDuration); + const transactionDurationFormatted = durationFormatter( + transactionDuration + ).formatted; - services.check.warning({ - name: `${AlertType.TransactionDuration}_${environment}`, - threshold, - value: transactionDuration, - context: { - transactionType: alertParams.transactionType, - serviceName: alertParams.serviceName, - environment, - threshold, - triggerValue: transactionDurationFormatted, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }, - fields: { - 'service.name': alertParams.serviceName, - 'service.environment': environment, - 'transaction.type': alertParams.transactionType, - }, - }); - } + const environmentParsed = parseEnvironmentUrlParam( + alertParams.environment + ); - return {}; - }, - }); + services + .alertWithLifecycle({ + id: `${AlertType.TransactionDuration}_${environmentParsed.text}`, + fields: { + [SERVICE_NAME]: alertParams.serviceName, + ...(environmentParsed.esFieldValue + ? { [SERVICE_ENVIRONMENT]: environmentParsed.esFieldValue } + : {}), + [TRANSACTION_TYPE]: alertParams.transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + transactionType: alertParams.transactionType, + serviceName: alertParams.serviceName, + environment: environmentParsed.text, + threshold, + triggerValue: transactionDurationFormatted, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); + } + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts index 5f6c07cae4b8f..3d2e534e75bee 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts @@ -8,12 +8,13 @@ import { Observable } from 'rxjs'; import * as Rx from 'rxjs'; import { toArray, map } from 'rxjs/operators'; -import { AlertingPlugin } from '../../../../alerting/server'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; import { APMConfig } from '../..'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { Job, MlPluginSetup } from '../../../../ml/server'; import * as GetServiceAnomalies from '../service_map/get_service_anomalies'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { APMRuleRegistry } from '../../plugin'; type Operator = (source: Rx.Observable) => Rx.Observable; const pipeClosure = (fn: Operator): Operator => { @@ -35,27 +36,36 @@ describe('Transaction duration anomaly alert', () => { describe("doesn't send alert", () => { it('ml is not defined', async () => { let alertExecutor: any; - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; registerTransactionDurationAnomalyAlertType({ - alerting, + registry, ml: undefined, config$: mockedConfig$, + logger: {} as any, }); expect(alertExecutor).toBeDefined(); + const scheduleActions = jest.fn(); + const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); + await alertExecutor!({ services, params, startedAt: new Date() }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); @@ -65,130 +75,78 @@ describe('Transaction duration anomaly alert', () => { .mockReturnValue(Promise.resolve([])); let alertExecutor: any; - - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; - const ml = ({ - mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }), - anomalyDetectorsProvider: jest.fn(), - } as unknown) as MlPluginSetup; - - registerTransactionDurationAnomalyAlertType({ - alerting, - ml, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + const scheduleActions = jest.fn(); const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); - expect(services.alertInstanceFactory).not.toHaveBeenCalled(); - }); - - it('anomaly is less than threshold', async () => { - jest - .spyOn(GetServiceAnomalies, 'getMLJobs') - .mockReturnValue( - Promise.resolve([{ job_id: '1' }, { job_id: '2' }] as Job[]) - ); - - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), }, - } as AlertingPlugin['setup']; + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), + }; const ml = ({ - mlSystemProvider: () => ({ - mlAnomalySearch: () => ({ - hits: { total: { value: 0 } }, - }), - }), + mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }), anomalyDetectorsProvider: jest.fn(), } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + registry, ml, config$: mockedConfig$, + logger: {} as any, }); + expect(alertExecutor).toBeDefined(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); + await alertExecutor!({ services, params, startedAt: new Date() }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); - }); - describe('sends alert', () => { - it('with service name, environment and transaction type', async () => { + it('anomaly is less than threshold', async () => { jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( - Promise.resolve([ + Promise.resolve(([ { job_id: '1', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, + custom_settings: { job_tags: { environment: 'development' } }, + }, { job_id: '2', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, - ] as Job[]) + custom_settings: { job_tags: { environment: 'production' } }, + }, + ] as unknown) as Job[]) ); let alertExecutor: any; - - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: () => ({ - hits: { total: { value: 2 } }, aggregations: { - services: { + anomaly_groups: { buckets: [ { - key: 'foo', - transaction_types: { - buckets: [{ key: 'type-foo' }], + doc_count: 1, + latest_score: { + top: [{ metrics: { record_score: 0, job_id: '1' } }], }, - record_avg: { value: 80 }, - }, - { - key: 'bar', - transaction_types: { - buckets: [{ key: 'type-bar' }], - }, - record_avg: { value: 20 }, }, ], }, @@ -199,84 +157,100 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + registry, ml, config$: mockedConfig$, + logger: {} as any, }); + expect(alertExecutor).toBeDefined(); const scheduleActions = jest.fn(); + const services = { - callCluster: jest.fn(), + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), }; - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); + registerTransactionDurationAnomalyAlertType({ + registry, + ml, + config$: mockedConfig$, + logger: {} as any, + }); + expect(alertExecutor).toBeDefined(); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_duration_anomaly_foo_production_type-foo', - 'apm.transaction_duration_anomaly_bar_production_type-bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'production', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'production', - threshold: 'minor', - thresholdValue: 'warning', - }); + await alertExecutor!({ services, params, startedAt: new Date() }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); + }); - it('with service name', async () => { + describe('sends alert', () => { + it('for all services that exceeded the threshold', async () => { jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( - Promise.resolve([ + Promise.resolve(([ { job_id: '1', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, + custom_settings: { job_tags: { environment: 'development' } }, + }, { job_id: '2', - custom_settings: { - job_tags: { - environment: 'testing', - }, - }, - } as unknown, - ] as Job[]) + custom_settings: { job_tags: { environment: 'production' } }, + }, + ] as unknown) as Job[]) ); let alertExecutor: any; - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: () => ({ - hits: { total: { value: 2 } }, aggregations: { - services: { + anomaly_groups: { buckets: [ - { key: 'foo', record_avg: { value: 80 } }, - { key: 'bar', record_avg: { value: 20 } }, + { + latest_score: { + top: [ + { + metrics: { + record_score: 80, + job_id: '1', + partition_field_value: 'foo', + by_field_value: 'type-foo', + }, + }, + ], + }, + }, + { + latest_score: { + top: [ + { + metrics: { + record_score: 20, + job_id: '2', + parttition_field_value: 'bar', + by_field_value: 'type-bar', + }, + }, + ], + }, + }, ], }, }, @@ -286,58 +260,41 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + registry, ml, config$: mockedConfig$, + logger: {} as any, }); + expect(alertExecutor).toBeDefined(); const scheduleActions = jest.fn(); + const services = { - callCluster: jest.fn(), + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), }; + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); + await alertExecutor!({ services, params, startedAt: new Date() }); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_duration_anomaly_foo_production', - 'apm.transaction_duration_anomaly_foo_testing', - 'apm.transaction_duration_anomaly_bar_production', - 'apm.transaction_duration_anomaly_bar_testing', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); + + expect(services.alertInstanceFactory).toHaveBeenCalledWith( + 'apm.transaction_duration_anomaly_foo_development_type-foo' ); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', - transactionType: undefined, - environment: 'production', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: 'production', - threshold: 'minor', - thresholdValue: 'warning', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: undefined, - environment: 'testing', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: 'testing', + transactionType: 'type-foo', + environment: 'development', threshold: 'minor', - thresholdValue: 'warning', + triggerValue: 'critical', }); }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index a2cdc47e8b77a..96fa27a87a51c 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -9,6 +9,12 @@ import { schema } from '@kbn/config-schema'; import { compact } from 'lodash'; import { ESSearchResponse } from 'typings/elasticsearch'; import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { getSeverity } from '../../../common/anomaly_detection'; +import { + SERVICE_ENVIRONMENT, + SERVICE_NAME, + TRANSACTION_TYPE, +} from '../../../common/elasticsearch_fieldnames'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { KibanaRequest } from '../../../../../../src/core/server'; @@ -20,6 +26,8 @@ import { import { getMLJobs } from '../service_map/get_service_anomalies'; import { apmActionVariables } from './action_variables'; import { RegisterRuleDependencies } from './register_apm_alerts'; +import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; +import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.maybe(schema.string()), @@ -43,186 +51,195 @@ export function registerTransactionDurationAnomalyAlertType({ ml, logger, }: RegisterRuleDependencies) { - registry.registerType({ - id: AlertType.TransactionDurationAnomaly, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - if (!ml) { - return {}; - } - const alertParams = params; - const request = {} as KibanaRequest; - const { mlAnomalySearch } = ml.mlSystemProvider( - request, - services.savedObjectsClient - ); - const anomalyDetectors = ml.anomalyDetectorsProvider( - request, - services.savedObjectsClient - ); - - const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment); - - const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( - (option) => option.type === alertParams.anomalySeverityType - ); - - if (!selectedOption) { - throw new Error( - `Anomaly alert severity type ${alertParams.anomalySeverityType} is not supported.` + registry.registerType( + createAPMLifecyleRuleType({ + id: AlertType.TransactionDurationAnomaly, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + if (!ml) { + return {}; + } + const alertParams = params; + const request = {} as KibanaRequest; + const { mlAnomalySearch } = ml.mlSystemProvider( + request, + services.savedObjectsClient + ); + const anomalyDetectors = ml.anomalyDetectorsProvider( + request, + services.savedObjectsClient ); - } - const threshold = selectedOption.threshold; + const mlJobs = await getMLJobs( + anomalyDetectors, + alertParams.environment + ); - if (mlJobs.length === 0) { - return {}; - } - - const jobIds = mlJobs.map((job) => job.job_id); - const anomalySearchParams = { - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { result_type: 'record' } }, - { terms: { job_id: jobIds } }, - { term: { is_interim: false } }, - { - range: { - timestamp: { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, - format: 'epoch_millis', + const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( + (option) => option.type === alertParams.anomalySeverityType + ); + + if (!selectedOption) { + throw new Error( + `Anomaly alert severity type ${alertParams.anomalySeverityType} is not supported.` + ); + } + + const threshold = selectedOption.threshold; + + if (mlJobs.length === 0) { + return {}; + } + + const jobIds = mlJobs.map((job) => job.job_id); + const anomalySearchParams = { + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { result_type: 'record' } }, + { terms: { job_id: jobIds } }, + { term: { is_interim: false } }, + { + range: { + timestamp: { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + format: 'epoch_millis', + }, }, }, - }, - ...(alertParams.serviceName - ? [ - { - term: { - partition_field_value: alertParams.serviceName, + ...(alertParams.serviceName + ? [ + { + term: { + partition_field_value: alertParams.serviceName, + }, }, - }, - ] - : []), - ...(alertParams.transactionType - ? [ - { - term: { - by_field_value: alertParams.transactionType, + ] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + by_field_value: alertParams.transactionType, + }, }, - }, - ] - : []), - ] as QueryContainer[], - }, - }, - aggs: { - anomaly_groups: { - multi_terms: { - terms: [ - { field: 'partition_field_value' }, - { field: 'by_field_value' }, - { field: 'job_id' }, - ], - size: 10000, + ] + : []), + ] as QueryContainer[], }, - aggs: { - latest_score: { - top_metrics: { - metrics: asMutableArray([ - { field: 'record_score' }, - { field: 'partition_field_value' }, - { field: 'by_field_value' }, - { field: 'job_id' }, - ] as const), - sort: { - '@timestamp': 'desc' as const, + }, + aggs: { + anomaly_groups: { + multi_terms: { + terms: [ + { field: 'partition_field_value' }, + { field: 'by_field_value' }, + { field: 'job_id' }, + ], + size: 10000, + }, + aggs: { + latest_score: { + top_metrics: { + metrics: asMutableArray([ + { field: 'record_score' }, + { field: 'partition_field_value' }, + { field: 'by_field_value' }, + { field: 'job_id' }, + ] as const), + sort: { + '@timestamp': 'desc' as const, + }, }, }, }, }, }, }, - }, - }; - - const response: ESSearchResponse< - unknown, - typeof anomalySearchParams - > = (await mlAnomalySearch(anomalySearchParams, [])) as any; - - const anomalies = - response.aggregations?.anomaly_groups.buckets - .map((bucket) => { - const latest = bucket.latest_score.top[0].metrics; - - const job = mlJobs.find((j) => j.job_id === latest.job_id); - - if (!job) { - logger.warn( - `Could not find matching job for job id ${latest.job_id}` - ); - return undefined; - } - - return { - serviceName: latest.partition_field_value, - transactionType: latest.by_field_value, - environment: job.custom_settings?.job_tags?.environment, - score: latest.record_score as number, - }; - }) - .filter((anomaly) => - anomaly ? anomaly.score >= threshold : false - ) ?? []; - - compact(anomalies).forEach((anomaly) => { - const { serviceName, environment, transactionType, score } = anomaly; - services.check.warning({ - name: [ - AlertType.TransactionDurationAnomaly, - anomaly.serviceName, - anomaly.environment, - anomaly.transactionType, - ] - .filter((name) => name) - .join('_'), - threshold, - value: score, - context: { - serviceName, - transactionType, - environment, - threshold, - triggerValue: score, - }, - fields: { - 'service.name': serviceName, - 'service.environment': environment, - 'transaction.type': transactionType, - }, + }; + + const response: ESSearchResponse< + unknown, + typeof anomalySearchParams + > = (await mlAnomalySearch(anomalySearchParams, [])) as any; + + const anomalies = + response.aggregations?.anomaly_groups.buckets + .map((bucket) => { + const latest = bucket.latest_score.top[0].metrics; + + const job = mlJobs.find((j) => j.job_id === latest.job_id); + + if (!job) { + logger.warn( + `Could not find matching job for job id ${latest.job_id}` + ); + return undefined; + } + + return { + serviceName: latest.partition_field_value as string, + transactionType: latest.by_field_value as string, + environment: job.custom_settings!.job_tags!.environment, + score: latest.record_score as number, + }; + }) + .filter((anomaly) => + anomaly ? anomaly.score >= threshold : false + ) ?? []; + + compact(anomalies).forEach((anomaly) => { + const { serviceName, environment, transactionType, score } = anomaly; + + const parsedEnvironment = parseEnvironmentUrlParam(environment); + + services + .alertWithLifecycle({ + id: [ + AlertType.TransactionDurationAnomaly, + serviceName, + environment, + transactionType, + ] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(parsedEnvironment.esFieldValue + ? { [SERVICE_ENVIRONMENT]: environment } + : {}), + [TRANSACTION_TYPE]: transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: selectedOption?.label, + triggerValue: getSeverity(score), + }); }); - }); - return {}; - }, - }); + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts index 148cd813a8a22..3425a9b24a3e0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts @@ -8,12 +8,12 @@ import { Observable } from 'rxjs'; import * as Rx from 'rxjs'; import { toArray, map } from 'rxjs/operators'; -import { AlertingPlugin } from '../../../../alerting/server'; import { APMConfig } from '../..'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; +import { APMRuleRegistry } from '../../plugin'; type Operator = (source: Rx.Observable) => Rx.Observable; const pipeClosure = (fn: Operator): Operator => { @@ -31,22 +31,31 @@ const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( describe('Transaction error rate alert', () => { it("doesn't send an alert when rate is less than threshold", async () => { let alertExecutor: any; - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; registerTransactionErrorRateAlertType({ - alerting, + registry, config$: mockedConfig$, + logger: {} as any, }); + expect(alertExecutor).toBeDefined(); + const scheduleActions = jest.fn(); + const services = { scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), }; + const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -60,6 +69,11 @@ describe('Transaction error rate alert', () => { }, took: 0, timed_out: false, + aggregations: { + series: { + buckets: [], + }, + }, _shards: { failed: 0, skipped: 0, @@ -69,30 +83,36 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params }); + await alertExecutor!({ services, params, startedAt: new Date() }); expect(services.alertInstanceFactory).not.toBeCalled(); }); - it('sends alerts with service name, transaction type and environment', async () => { + it('sends alerts for services that exceeded the threshold', async () => { let alertExecutor: any; - const alerting = { + const registry = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as AlertingPlugin['setup']; + } as APMRuleRegistry; registerTransactionErrorRateAlertType({ - alerting, + registry, config$: mockedConfig$, + logger: {} as any, }); + expect(alertExecutor).toBeDefined(); const scheduleActions = jest.fn(); + const services = { scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), }; - const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -100,37 +120,38 @@ describe('Transaction error rate alert', () => { hits: [], total: { relation: 'eq', - value: 4, + value: 0, }, }, aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { + series: { buckets: [ { - key: 'foo', - transaction_types: { + key: ['foo', 'env-foo', 'type-foo'], + outcomes: { buckets: [ { - key: 'type-foo', - environments: { - buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], - }, + key: 'success', + doc_count: 90, + }, + { + key: 'failure', + doc_count: 10, }, ], }, }, { - key: 'bar', - transaction_types: { + key: ['bar', 'env-bar', 'type-bar'], + outcomes: { buckets: [ { - key: 'type-bar', - environments: { - buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], - }, + key: 'success', + doc_count: 90, + }, + { + key: 'failure', + doc_count: 1, }, ], }, @@ -149,208 +170,25 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo_type-foo_env-foo', - 'apm.transaction_error_rate_foo_type-foo_env-foo-2', - 'apm.transaction_error_rate_bar_type-bar_env-bar', - 'apm.transaction_error_rate_bar_type-bar_env-bar-2', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'env-foo', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'env-foo-2', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'env-bar', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'env-bar-2', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - }); - it('sends alerts with service name and transaction type', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - relation: 'eq', - value: 4, - }, - }, - aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { - buckets: [ - { - key: 'foo', - transaction_types: { - buckets: [{ key: 'type-foo' }], - }, - }, - { - key: 'bar', - transaction_types: { - buckets: [{ key: 'type-bar' }], - }, - }, - ], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo_type-foo', - 'apm.transaction_error_rate_bar_type-bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); + await alertExecutor!({ services, params, startedAt: new Date() }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - }); + expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); - it('sends alerts with service name', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; - - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - value: 4, - relation: 'eq', - }, - }, - aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { - buckets: [{ key: 'foo' }, { key: 'bar' }], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) + expect(services.alertInstanceFactory).toHaveBeenCalledWith( + 'apm.transaction_error_rate_foo_type-foo_env-foo' ); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo', - 'apm.transaction_error_rate_bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + expect(services.alertInstanceFactory).not.toHaveBeenCalledWith( + 'apm.transaction_error_rate_bar_type-bar_env-bar' ); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', - transactionType: undefined, - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: undefined, + transactionType: 'type-foo', + environment: 'env-foo', threshold: 10, - triggerValue: '50', + triggerValue: '10', interval: '5m', }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index c0f12405f72af..813c6885efd56 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -22,6 +22,7 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; +import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; import { RegisterRuleDependencies } from './register_apm_alerts'; const paramsSchema = schema.object({ @@ -39,155 +40,162 @@ export function registerTransactionErrorRateAlertType({ registry, config$, }: RegisterRuleDependencies) { - registry.registerType({ - id: AlertType.TransactionErrorRate, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.transactionType, - apmActionVariables.serviceName, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params: alertParams }) => { - const config = await config$.pipe(take(1)).toPromise(); - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); + registry.registerType( + createAPMLifecyleRuleType({ + id: AlertType.TransactionErrorRate, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.transactionType, + apmActionVariables.serviceName, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params: alertParams }) => { + const config = await config$.pipe(take(1)).toPromise(); + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.transactionIndices'], - size: 0, - body: { - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.transactionIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - { - terms: { - [EVENT_OUTCOME]: [ - EventOutcome.failure, - EventOutcome.success, - ], + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { + terms: { + [EVENT_OUTCOME]: [ + EventOutcome.failure, + EventOutcome.success, + ], + }, }, - }, - ...(alertParams.serviceName - ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] - : []), - ...(alertParams.transactionType - ? [ - { - term: { - [TRANSACTION_TYPE]: alertParams.transactionType, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + [TRANSACTION_TYPE]: alertParams.transactionType, + }, }, - }, - ] - : []), - ...environmentQuery(alertParams.environment), - ], - }, - }, - aggs: { - series: { - multi_terms: { - terms: [ - { field: SERVICE_NAME }, - { field: SERVICE_ENVIRONMENT }, - { field: TRANSACTION_TYPE }, + ] + : []), + ...environmentQuery(alertParams.environment), ], - size: 10000, }, - aggs: { - outcomes: { - terms: { - field: EVENT_OUTCOME, + }, + aggs: { + series: { + multi_terms: { + terms: [ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT }, + { field: TRANSACTION_TYPE }, + ], + size: 10000, + }, + aggs: { + outcomes: { + terms: { + field: EVENT_OUTCOME, + }, }, }, }, }, }, - }, - }; + }; - const response = await alertingEsClient( - services.scopedClusterClient, - searchParams - ); + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - if (!response.aggregations) { - return {}; - } + if (!response.aggregations) { + return {}; + } - const results = response.aggregations.series.buckets - .map((bucket) => { - const [serviceName, environment, transactionType] = bucket.key; + const results = response.aggregations.series.buckets + .map((bucket) => { + const [serviceName, environment, transactionType] = bucket.key; - const failed = - bucket.outcomes.buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.doc_count ?? 0; - const succesful = - bucket.outcomes.buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.success - )?.doc_count ?? 0; + const failed = + bucket.outcomes.buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.doc_count ?? 0; + const succesful = + bucket.outcomes.buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.success + )?.doc_count ?? 0; - return { - serviceName, - environment, - transactionType, - errorRate: (failed / failed + succesful) * 100, - }; - }) - .filter((result) => result.errorRate >= alertParams.threshold); + return { + serviceName, + environment, + transactionType, + errorRate: (failed / (failed + succesful)) * 100, + }; + }) + .filter((result) => result.errorRate >= alertParams.threshold); - results.forEach((result) => { - const { serviceName, environment, transactionType, errorRate } = result; - services.check.warning({ - name: [ - AlertType.TransactionErrorRate, + results.forEach((result) => { + const { serviceName, - transactionType, environment, - ] - .filter((name) => name) - .join('_'), - threshold: alertParams.threshold, - value: errorRate, - context: { - serviceName, transactionType, - environment, - threshold: alertParams.threshold, - triggerValue: asDecimalOrInteger(errorRate), - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }, - fields: { - [SERVICE_NAME]: serviceName, - ...(environment ? { [SERVICE_ENVIRONMENT]: environment } : {}), - [TRANSACTION_TYPE]: transactionType, - }, + errorRate, + } = result; + + services + .alertWithLifecycle({ + id: [ + AlertType.TransactionErrorRate, + serviceName, + transactionType, + environment, + ] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(environment ? { [SERVICE_ENVIRONMENT]: environment } : {}), + [TRANSACTION_TYPE]: transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: alertParams.threshold, + triggerValue: asDecimalOrInteger(errorRate), + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); }); - }); - return {}; - }, - }); + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/rule_registry/jest.config.js b/x-pack/plugins/rule_registry/jest.config.js new file mode 100644 index 0000000000000..df8ac522e4b5d --- /dev/null +++ b/x-pack/plugins/rule_registry/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/rule_registry'], +}; diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 503cced4268c3..33cb22ff33687 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -8,6 +8,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginInitializerContext } from 'src/core/server'; import { RuleRegistryPlugin, RuleRegistryPluginSetupContract } from './plugin'; +import { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; export const config = { schema: schema.object({ @@ -20,4 +21,4 @@ export type RuleRegistryConfig = TypeOf; export const plugin = (initContext: PluginInitializerContext) => new RuleRegistryPlugin(initContext); -export { RuleRegistryPluginSetupContract }; +export { RuleRegistryPluginSetupContract, createLifecycleRuleTypeFactory }; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts deleted file mode 100644 index 5f594a2a79d20..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/check_service/index.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ActionVariable } from '../../../../alerting/common'; -import { AlertExecutorOptions } from '../../../../alerting/server'; -import { AlertSeverityLevel } from '../../../common'; -import { AlertCheck } from '../../types'; -import { DefaultFieldMap } from '../defaults/field_map'; - -type AlertInstanceFactory = AlertExecutorOptions< - never, - never, - never, - never, - string ->['services']['alertInstanceFactory']; - -export function createCheckService({ - levels, - alertInstanceFactory, -}: { - levels: Array<{ level: AlertSeverityLevel; actionGroupId: string }>; - alertInstanceFactory: AlertInstanceFactory; -}) { - const alerts: Record< - string, - { level: AlertSeverityLevel } & AlertCheck - > = {}; - - const getCheckFunction = (level: { level: AlertSeverityLevel; actionGroupId: string }) => { - return (alert: AlertCheck) => { - const instance = alertInstanceFactory(alert.name); - instance.scheduleActions(level.actionGroupId); - alerts[alert.name] = { - level: level.level, - ...alert, - }; - }; - }; - - return { - check: Object.fromEntries( - levels.map((level) => [level.level, getCheckFunction(level)]) - ) as Record>, - getAlerts: () => alerts, - }; -} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts new file mode 100644 index 0000000000000..099f487bfd0e4 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Either, isLeft, isRight } from 'fp-ts/lib/Either'; +import { Errors } from 'io-ts'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { Logger, SavedObjectsClientContract } from 'kibana/server'; +import { IScopedClusterClient as ScopedClusterClient } from 'src/core/server'; +import { compact } from 'lodash'; +import { ClusterClientAdapter } from '../../../../event_log/server'; +import { runtimeTypeFromFieldMap, OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; +import { ScopedRuleRegistryClient } from './types'; +import { DefaultFieldMap } from '../defaults/field_map'; + +const getRuleIds = async ({ + savedObjectsClient, + namespace, +}: { + savedObjectsClient: SavedObjectsClientContract; + namespace?: string; +}) => { + const options = { + type: 'alert', + ...(namespace ? { namespace } : {}), + }; + + const pitFinder = savedObjectsClient.createPointInTimeFinder(options); + + const ruleIds: string[] = []; + + for await (const response of pitFinder.find()) { + ruleIds.push(...response.saved_objects.map((object) => object.id)); + } + + await pitFinder.close(); + + return ruleIds; +}; + +const createPathReporterError = (either: Either) => { + const error = new Error(`Failed to validate alert event`); + error.stack += '\n' + PathReporter.report(either).join('\n'); + return error; +}; + +export function createScopedRuleRegistryClient({ + fieldMap, + scopedClusterClient, + savedObjectsClient, + namespace, + clusterClientAdapter, + index, + logger, + rule, + producer, +}: { + fieldMap: TFieldMap; + scopedClusterClient: ScopedClusterClient; + savedObjectsClient: SavedObjectsClientContract; + namespace?: string; + clusterClientAdapter: ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>; + index: string; + logger: Logger; + rule: { + id: string; + uuid: string; + category: string; + name: string; + }; + producer: string; +}): Promise> { + const docRt = runtimeTypeFromFieldMap(fieldMap); + + const defaults = { + 'rule.uuid': rule.uuid, + 'rule.id': rule.id, + 'rule.name': rule.name, + 'rule.category': rule.category, + producer, + }; + + const createClient = async () => { + const ruleIds = await getRuleIds({ + savedObjectsClient, + namespace, + }); + + const client: ScopedRuleRegistryClient = { + search: async (searchRequest) => { + const response = await scopedClusterClient.asInternalUser.search({ + index, + body: { + query: { + bool: { + filter: [ + { terms: { 'rule.uuid': ruleIds } }, + ...(searchRequest.body?.query ? [searchRequest.body.query] : []), + ], + }, + }, + }, + }); + + return response.body as any; + }, + index: (doc) => { + const validation = docRt.decode({ + ...doc, + ...defaults, + }); + + if (isLeft(validation)) { + throw createPathReporterError(validation); + } + + clusterClientAdapter.indexDocument({ body: validation.right, index }); + }, + bulkIndex: (docs) => { + const validations = docs.map((doc) => { + return docRt.decode(doc); + }); + + const errors = compact( + validations.map((validation) => + isLeft(validation) ? createPathReporterError(validation) : null + ) + ); + + errors.forEach((error) => { + logger.error(error); + }); + + const operations = compact( + validations.map((validation) => (isRight(validation) ? validation.right : null)) + ).map((doc) => ({ body: doc, index })); + + return clusterClientAdapter.indexDocuments(operations); + }, + }; + return client; + }; + + return createClient(); +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts new file mode 100644 index 0000000000000..08aba0a056f46 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESSearchRequest, ESSearchResponse } from 'typings/elasticsearch'; +import { DefaultFieldMap } from '../defaults/field_map'; +import { PatternsUnionOf, PickWithPatterns } from '../field_map/pick_with_patterns'; +import { OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; + +export type PrepopulatedRuleEventFields = + | 'rule.uuid' + | 'rule.id' + | 'rule.name' + | 'rule.type' + | 'rule.category' + | 'producer'; + +type FieldsOf = + | Array<{ field: PatternsUnionOf } | PatternsUnionOf> + | PatternsUnionOf; + +type Fields = Array<{ field: TPattern } | TPattern> | TPattern; + +type FieldsESSearchRequest = ESSearchRequest & { + body?: { fields: FieldsOf }; +}; + +type EventsOf< + TFieldsESSearchRequest extends ESSearchRequest, + TFieldMap extends DefaultFieldMap +> = TFieldsESSearchRequest extends { body: { fields: infer TFields } } + ? TFields extends Fields + ? Array>> + : never + : never; + +export interface ScopedRuleRegistryClient { + search>( + request: TSearchRequest + ): Promise<{ + body: ESSearchResponse; + events: EventsOf; + }>; + index(doc: Omit, PrepopulatedRuleEventFields>): void; + bulkIndex( + doc: Array, PrepopulatedRuleEventFields>> + ): Promise; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts index 8f72fce0cb195..3fb85ec58d05f 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts @@ -5,26 +5,33 @@ * 2.0. */ -import { Mutable } from 'utility-types'; import { ecsFieldMap } from '../../generated/ecs_field_map'; import { pickWithPatterns } from '../field_map/pick_with_patterns'; export const defaultFieldMap = { - ...pickWithPatterns(ecsFieldMap, '@timestamp', 'event.*', 'rule.*'), + ...pickWithPatterns( + ecsFieldMap, + '@timestamp', + 'event.kind', + 'event.action', + 'rule.uuid', + 'rule.id', + 'rule.name', + 'rule.category', + 'tags' + ), + producer: { type: 'keyword' }, + 'alert.uuid': { type: 'keyword' }, 'alert.id': { type: 'keyword' }, - 'alert.type': { type: 'keyword' }, - 'alert.name': { type: 'keyword' }, - 'alert.series_id': { type: 'keyword' }, // rule.id + alert.name - 'alert.check.value': { type: 'scaled_float', scaling_factor: 100 }, - 'alert.check.threshold': { type: 'scaled_float', scaling_factor: 100 }, - 'alert.check.influencers': { type: 'flattened' }, - 'rule_type.producer': { type: 'keyword', required: true }, + 'alert.start': { type: 'date' }, + 'alert.end': { type: 'date' }, + 'alert.duration.us': { type: 'long' }, + 'alert.severity.level': { type: 'keyword' }, + 'alert.severity.value': { type: 'long' }, + 'alert.status': { type: 'keyword' }, + 'evaluation.value': { type: 'scaled_float', scaling_factor: 100 }, + 'evaluation.threshold': { type: 'scaled_float', scaling_factor: 100 }, + 'evaluation.status': { type: 'keyword' }, } as const; -type DefaultFieldMapReadOnly = typeof defaultFieldMap; - -export type DefaultFieldMap = Mutable< - { - [key in keyof DefaultFieldMapReadOnly]: Mutable; - } ->; +export type DefaultFieldMap = typeof defaultFieldMap; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts new file mode 100644 index 0000000000000..48ba7c873db25 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickWithPatterns } from './pick_with_patterns'; + +describe('pickWithPatterns', () => { + const fieldMap = { + 'event.category': { type: 'keyword' }, + 'event.kind': { type: 'keyword' }, + 'destination.bytes': { + type: 'long', + array: false, + required: false, + }, + 'destination.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + } as const; + + it('picks a single field', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'event.category'))).toEqual(['event.category']); + }); + + it('picks event fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'event.*')).sort()).toEqual([ + 'event.category', + 'event.kind', + ]); + }); + + it('picks destination.geo fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'destination.geo.*')).sort()).toEqual([ + 'destination.geo.city_name', + ]); + }); + + it('picks all destination fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'destination.*')).sort()).toEqual([ + 'destination.bytes', + 'destination.domain', + 'destination.geo.city_name', + ]); + }); + + it('picks fields from multiple patterns', () => { + expect( + Object.keys(pickWithPatterns(fieldMap, 'destination.geo.*', 'event.category')).sort() + ).toEqual(['destination.geo.city_name', 'event.category']); + }); + + it('picks all fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, '*')).sort()).toEqual([ + 'destination.bytes', + 'destination.domain', + 'destination.geo.city_name', + 'event.category', + 'event.kind', + ]); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts index cf27eb0728e76..f8a88957fceb5 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts @@ -7,7 +7,6 @@ import { ValuesType, SetIntersection, OmitByValueExact } from 'utility-types'; import { pick } from 'lodash'; -import { ecsFieldMap } from '../../generated/ecs_field_map'; type SplitByDot< TPath extends string, @@ -17,28 +16,32 @@ type SplitByDot< : [`${TPrefix}${TPath}`]; type PatternMapOf> = { - [TKey in keyof T]: ValuesType : never>; + [TKey in keyof T]: ValuesType] : never>; }; -type PickByPatterns, TPatterns extends string[]> = OmitByValueExact< +export type PickWithPatterns< + T extends Record, + TPatterns extends string[] +> = OmitByValueExact< { [TFieldName in keyof T]: SetIntersection< ValuesType, PatternMapOf[TFieldName] > extends never - ? undefined + ? never : T[TFieldName]; }, - undefined + never >; -const allEcsFields = Object.keys(ecsFieldMap) as Array; +export type PatternsUnionOf> = '*' | ValuesType>; export function pickWithPatterns< T extends Record, - TPatterns extends Array>> ->(map: T, ...patterns: TPatterns): PickByPatterns { - const matchedFields = allEcsFields.filter((field) => + TPatterns extends Array> +>(map: T, ...patterns: TPatterns): PickWithPatterns { + const allFields = Object.keys(map); + const matchedFields = allFields.filter((field) => patterns.some((pattern) => { if (pattern === field) { return true; @@ -52,12 +55,12 @@ export function pickWithPatterns< } return fieldParts.every((fieldPart, index) => { - const patternPart = patternParts.length < index ? '*' : patternParts[index]; + const patternPart = patternParts.length - 1 < index ? '*' : patternParts[index]; return fieldPart === patternPart || patternPart === '*'; }); }) ); - return (pick(ecsFieldMap, matchedFields) as unknown) as PickByPatterns; + return (pick(map, matchedFields) as unknown) as PickWithPatterns; } diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts new file mode 100644 index 0000000000000..0acf80bfb42e5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { runtimeTypeFromFieldMap } from './runtime_type_from_fieldmap'; + +describe('runtimeTypeFromFieldMap', () => { + const fieldmapRt = runtimeTypeFromFieldMap({ + keywordField: { type: 'keyword' }, + longField: { type: 'long' }, + requiredKeywordField: { type: 'keyword', required: true }, + multiKeywordField: { type: 'keyword', array: true }, + } as const); + + it('accepts both singular and array fields', () => { + expect( + fieldmapRt.is({ + requiredKeywordField: 'keyword', + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + multiKeywordField: 'keyword', + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + multiKeywordField: ['keyword'], + }) + ).toBe(true); + }); + + it('fails on invalid data types', () => { + expect( + fieldmapRt.is({ + requiredKeywordField: 2, + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: [2], + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: ['keyword'], + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: [3], + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: 3, + }) + ).toBe(true); + }); + + it('outputs to single or array values', () => { + expect( + fieldmapRt.encode({ + requiredKeywordField: ['required'], + keywordField: 'keyword', + longField: [3, 2], + multiKeywordField: ['keyword', 'foo'], + }) + ).toEqual({ + requiredKeywordField: 'required', + keywordField: 'keyword', + longField: 3, + multiKeywordField: ['keyword', 'foo'], + }); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts index 1ecfeae0d21a0..422747282beaa 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts @@ -7,7 +7,7 @@ import { mapValues, pickBy } from 'lodash'; import * as t from 'io-ts'; -import { PickByValueExact } from 'utility-types'; +import { Mutable, PickByValueExact } from 'utility-types'; import { FieldMap } from '../types'; const esFieldTypeMap = { @@ -48,17 +48,42 @@ type IntersectionTypeOf< ] >; +type CastArray> = t.Type< + t.TypeOf | Array>, + Array>, + unknown +>; +type CastSingle> = t.Type< + t.TypeOf | Array>, + t.TypeOf, + unknown +>; + +const createCastArrayRt = >(type: T): CastArray => { + const union = t.union([type, t.array(type)]); + + return new t.Type('castArray', union.is, union.validate, (a) => (Array.isArray(a) ? a : [a])); +}; + +const createCastSingleRt = >(type: T): CastSingle => { + const union = t.union([type, t.array(type)]); + + return new t.Type('castSingle', union.is, union.validate, (a) => (Array.isArray(a) ? a[0] : a)); +}; + type MapTypeValues = { [key in keyof T]: { required: T[key]['required']; type: T[key]['array'] extends true - ? t.ArrayC> - : EsFieldTypeOf; + ? CastArray> + : CastSingle>; }; }; -export type FieldMapType = IntersectionTypeOf>; -export type TypeOfFieldMap = t.TypeOf>; +type FieldMapType = IntersectionTypeOf>; + +export type TypeOfFieldMap = Mutable>>; +export type OutputOfFieldMap = Mutable>>; export function runtimeTypeFromFieldMap( fieldMap: TFieldMap @@ -70,7 +95,7 @@ export function runtimeTypeFromFieldMap( ? esFieldTypeMap[field.type as keyof EsFieldTypeMap] : t.unknown; - return field.array ? t.array(type) : type; + return field.array ? createCastArrayRt(type) : createCastSingleRt(type); }); } diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index b6bf9aa95ae98..34a3607a0048e 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -6,25 +6,21 @@ */ import { CoreSetup, Logger } from 'kibana/server'; -import { omitBy, compact } from 'lodash'; import { inspect } from 'util'; -import uuid from 'uuid'; -import { PathReporter } from 'io-ts/lib/PathReporter'; -import { isLeft } from 'fp-ts/lib/Either'; -import { ESSearchRequest, ESSearchResponse } from '../../../../../typings/elasticsearch'; +import { + ActionVariable, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../../alerting/common'; import { createReadySignal, ClusterClientAdapter } from '../../../event_log/server'; import { FieldMap, ILMPolicy } from './types'; -import { RegisterRuleType, RuleState, RuleAlertState } from '../types'; +import { RuleParams, RuleType } from '../types'; import { mergeFieldMaps } from './field_map/merge_field_maps'; -import { - FieldMapType, - runtimeTypeFromFieldMap, - TypeOfFieldMap, -} from './field_map/runtime_type_from_fieldmap'; +import { OutputOfFieldMap } from './field_map/runtime_type_from_fieldmap'; import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; -import { createCheckService } from './check_service'; -import { AlertSeverityLevel, getAlertSeverityLevelValue } from '../../common'; +import { createScopedRuleRegistryClient } from './create_scoped_rule_registry_client'; import { DefaultFieldMap } from './defaults/field_map'; interface RuleRegistryOptions { @@ -39,21 +35,20 @@ interface RuleRegistryOptions { parent?: RuleRegistry; } -export class RuleRegistry { +export class RuleRegistry { private readonly esAdapter: ClusterClientAdapter<{ - body: TypeOfFieldMap; + body: OutputOfFieldMap; index: string; }>; - private readonly docRt: FieldMapType; - private readonly children: Array> = []; + private readonly children: Array> = []; - constructor(private readonly options: RuleRegistryOptions) { + constructor(private readonly options: RuleRegistryOptions) { const { logger, core } = options; const { wait, signal } = createReadySignal(); this.esAdapter = new ClusterClientAdapter<{ - body: TypeOfFieldMap; + body: OutputOfFieldMap; index: string; }>({ wait, @@ -63,8 +58,6 @@ export class RuleRegistry { logger: logger.get('esAdapter'), }); - this.docRt = runtimeTypeFromFieldMap(options.fieldMap); - this.initialize() .then(() => { this.options.logger.debug('Bootstrapped alerts index'); @@ -129,220 +122,53 @@ export class RuleRegistry { } } - async search( - ruleIds: string[], - request: TSearchRequest - ): Promise> { - const [{ elasticsearch }] = await this.options.core.getStartServices(); - - const query = { - bool: { - filter: [ - { - terms: { - 'rule.uuid': ruleIds, - }, - }, - ...compact([request.body?.query]), - ], - }, - }; - - const response = await elasticsearch.client.asInternalUser.search({ - ...request, - body: { - ...request.body, - query, - }, - }); - - return (response.body as unknown) as ESSearchResponse; - } - - registerType: RegisterRuleType = (type) => { + registerType( + type: RuleType + ) { this.options.alertingPluginSetupContract.registerType< - Record, - RuleState, - Record, - Record, - string, + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + { [key in TActionVariable['name']]: any }, string >({ ...type, - executor: async (options) => { - const { - services, - previousStartedAt, - startedAt, - state: maybePrevAlertState, - alertId: ruleId, - name: ruleName, - params, - // namespace, - } = options; - - const prevAlertState = - maybePrevAlertState && 'alerts' in maybePrevAlertState - ? maybePrevAlertState - : { alerts: {}, wrappedRuleState: maybePrevAlertState }; - - const { alertInstanceFactory, ...passthroughServices } = services; + executor: async (executorOptions) => { + const { services, namespace, alertId, name } = executorOptions; + + const rule = { + id: type.id, + uuid: alertId, + category: type.name, + name, + }; - const checkService = createCheckService({ - alertInstanceFactory: services.alertInstanceFactory as any, - levels: [{ level: AlertSeverityLevel.warning, actionGroupId: type.defaultActionGroupId }], + const producer = type.producer; + + const scopedRuleRegistryClient = await createScopedRuleRegistryClient({ + savedObjectsClient: services.savedObjectsClient, + scopedClusterClient: services.scopedClusterClient, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + index: this.getEsNames().indexAliasName, + namespace, + producer, + rule, + logger: this.options.logger, }); - const executorOptions = { - previousStartedAt, - startedAt, - params, + return type.executor({ + ...executorOptions, + rule, + producer, services: { - ...passthroughServices, - check: checkService.check as any, + ...services, + scopedRuleRegistryClient, }, - }; - - const ruleState = await type.executor(executorOptions); - - const activeAlerts = checkService.getAlerts(); - const previousAlertStates = prevAlertState.alerts; - - const previousAlertNames = Object.keys(previousAlertStates); - const activeAlertNames = Object.keys(activeAlerts); - - const newAlertNames = activeAlertNames.filter( - (alertName) => !previousAlertNames.includes(alertName) - ); - - const mergedAlertStates = { - ...previousAlertStates, - ...newAlertNames.reduce((prev, alertName) => { - prev[alertName] = { - alertId: uuid.v4(), // for log-type alerts, use alertName - created: startedAt.getTime(), - }; - return prev; - }, {} as Record), - }; - - // const idsOfLastAlertEventsToFetch = Object.values(mergedAlertStates).map( - // (state) => state.alertId - // ); - - // const start = new Date().getTime() - 60 * 60 * 1000; - - // const response = await this.search([ruleId], { - // body: { - // size: idsOfLastAlertEventsToFetch.length, - // query: { - // bool: { - // filter: [ - // { - // terms: { - // 'alert.id': idsOfLastAlertEventsToFetch, - // }, - // }, - // { - // range: { - // '@timestamp': { - // gte: start, - // format: 'epoch_millis', - // }, - // }, - // }, - // ], - // }, - // }, - // collapse: { - // field: 'alert.id', - // }, - // sort: { - // '@timestamp': 'desc', - // }, - // _source: false, - // fields: Object.keys(this.options.fieldMap), - // }, - // }); - - // const lastEventByAlertId = response.hits.hits.reduce((prev, hit) => { - // const alertId = hit.fields['alert.id']![0] as string; - // prev[alertId] = hit.fields as Record; - // return prev; - // }, {} as Record>); - - const index = this.getEsNames().indexAliasName; - - const updates = Object.entries(mergedAlertStates).map(([alertName, state]) => { - const active = activeAlertNames.includes(alertName); - - const nextState = active ? activeAlerts[alertName] : undefined; - - const createdAt = new Date(state.created); - - const body: TypeOfFieldMap = { - ...(nextState - ? { - ...nextState.fields, - 'event.severity': getAlertSeverityLevelValue(nextState.level), - 'alert.check.value': nextState.value, - 'alert.check.threshold': nextState.threshold, - 'event.action': 'active-alert', - } - : { - 'event.end': startedAt.toISOString(), - 'event.action': 'recovered-alert', - }), - 'event.kind': 'alert', - '@timestamp': startedAt.toISOString(), - 'rule.uuid': ruleId, - 'rule.id': type.id, - 'rule.category': type.name, - 'rule.name': ruleName, - // 'rule.namespace': namespace, - 'rule_type.producer': type.producer, - 'alert.id': state.alertId, - 'event.start': createdAt.toISOString(), - 'event.duration': (startedAt.getTime() - createdAt.getTime()) * 1000, - 'alert.name': alertName, - 'alert.series_id': [ruleId, alertName].join('|'), - }; - - return { - index, - body, - }; - }); - - let indexedCount = 0; - - updates.forEach((update) => { - const decode = this.docRt.decode(update.body); - if (isLeft(decode)) { - const error = new Error(`Failed to validate alert event`); - error.stack += '\n' + PathReporter.report(decode).join('\n'); - this.options.logger.error(error); - } else { - this.esAdapter.indexDocument(update); - indexedCount++; - } - }); - - if (indexedCount > 0) { - this.options.logger.debug(`Indexed ${indexedCount} events`); - } - - const nextState = omitBy(mergedAlertStates, (_, alertName) => { - return activeAlerts[alertName] === undefined; }); - - return { - wrappedRuleState: ruleState, - alerts: nextState, - }; }, }); - }; + } create({ namespace, @@ -357,17 +183,19 @@ export class RuleRegistry { ? mergeFieldMaps(this.options.fieldMap, fieldMap) : this.options.fieldMap; - const child = new RuleRegistry({ + const child = new RuleRegistry({ ...this.options, logger: this.options.logger.get(namespace), namespace: [this.options.namespace, namespace].filter(Boolean).join('-'), fieldMap: mergedFieldMap, ...(ilmPolicy ? { ilmPolicy } : {}), + // @ts-expect-error Types of property 'body' are incompatible. parent: this, }); this.children.push(child); + // @ts-expect-error could be instantiated with a different subtype of constraint return child; } } diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts new file mode 100644 index 0000000000000..fd63a886d42b0 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import * as t from 'io-ts'; +import { isLeft } from 'fp-ts/lib/Either'; +import v4 from 'uuid/v4'; +import { AlertInstance } from '../../../../alerting/server'; +import { ActionVariable, AlertInstanceState } from '../../../../alerting/common'; +import { RuleParams, RuleType } from '../../types'; +import { DefaultFieldMap } from '../defaults/field_map'; +import { OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; +import { PrepopulatedRuleEventFields } from '../create_scoped_rule_registry_client/types'; +import { RuleRegistry } from '..'; + +type UserDefinedAlertFields = Omit< + OutputOfFieldMap, + PrepopulatedRuleEventFields | 'alert.id' | 'alert.uuid' | '@timestamp' +>; + +type LifecycleAlertService< + TFieldMap extends DefaultFieldMap, + TActionVariable extends ActionVariable +> = (alert: { + id: string; + fields: UserDefinedAlertFields; +}) => AlertInstance; + +type CreateLifecycleRuleType = < + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable +>( + type: RuleType< + TFieldMap, + TRuleParams, + TActionVariable, + { alertWithLifecycle: LifecycleAlertService } + > +) => RuleType; + +const trackedAlertStateRt = t.type({ + alertId: t.string, + alertUuid: t.string, + started: t.number, +}); + +const wrappedStateRt = t.type({ + wrapped: t.record(t.string, t.unknown), + trackedAlerts: t.record(t.string, trackedAlertStateRt), +}); + +export function createLifecycleRuleTypeFactory< + TRuleRegistry extends RuleRegistry +>(): TRuleRegistry extends RuleRegistry + ? CreateLifecycleRuleType + : never; + +export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { + return (type) => { + return { + ...type, + executor: async (options) => { + const { + services: { scopedRuleRegistryClient, alertInstanceFactory }, + state: previousState, + rule, + } = options; + + const decodedState = wrappedStateRt.decode(previousState); + + const state = isLeft(decodedState) + ? { + wrapped: previousState, + trackedAlerts: {}, + } + : decodedState.right; + + const currentAlerts: Record< + string, + UserDefinedAlertFields & { 'alert.id': string } + > = {}; + + const timestamp = options.startedAt.toISOString(); + + const nextWrappedState = await type.executor({ + ...options, + state: state.wrapped, + services: { + ...options.services, + alertWithLifecycle: ({ id, fields }) => { + currentAlerts[id] = { + ...fields, + 'alert.id': id, + }; + return alertInstanceFactory(id); + }, + }, + }); + + const currentAlertIds = Object.keys(currentAlerts); + const trackedAlertIds = Object.keys(state.trackedAlerts); + + const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; + + const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter( + (trackedAlertState) => currentAlerts[trackedAlertState.alertId] + ); + + const alertsDataMap: Record> = { + ...currentAlerts, + }; + + if (trackedAlertStatesOfRecovered.length) { + const { events } = await scopedRuleRegistryClient.search({ + body: { + query: { + bool: { + filter: [ + { + term: { + 'rule.id': rule.id, + }, + }, + { + terms: { + 'alert.uuid': trackedAlertStatesOfRecovered.map( + (trackedAlertState) => trackedAlertState.alertUuid + ), + }, + }, + ], + }, + }, + size: trackedAlertStatesOfRecovered.length, + collapse: { + field: 'alert.uuid', + }, + _source: false, + fields: ['*'], + sort: { + '@timestamp': 'desc' as const, + }, + }, + }); + + events.forEach((event) => { + const alertId = event['alert.id']!; + alertsDataMap[alertId] = event; + }); + } + + const eventsToIndex: Array> = allAlertIds.map( + (alertId) => { + const alertData = alertsDataMap[alertId]; + + const event: OutputOfFieldMap = { + ...alertData, + '@timestamp': timestamp, + 'event.kind': 'state', + 'alert.id': alertId, + }; + + const isNew = !state.trackedAlerts[alertId]; + const isRecovered = !currentAlerts[alertId]; + const isActiveButNotNew = !isNew && !isRecovered; + const isActive = !isRecovered; + + if (isNew) { + event['alert.uuid'] = v4(); + event['alert.start'] = timestamp; + event['event.action'] = 'open'; + } + + if (isRecovered) { + event['alert.end'] = timestamp; + event['event.action'] = 'close'; + event['alert.status'] = 'closed'; + } + + if (isActiveButNotNew) { + event['event.action'] = 'active'; + } + + if (isActive) { + event['alert.status'] = 'open'; + } + + event['alert.duration.us'] = + (options.startedAt.getTime() - new Date(event['alert.start']!).getTime()) * 1000; + + return event; + } + ); + + await scopedRuleRegistryClient.bulkIndex(eventsToIndex); + + return { + wrapped: nextWrappedState, + trackedAlerts: eventsToIndex, + }; + }, + }; + }; +} diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index c8b47313df77c..d2f4c54b99a59 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -5,115 +5,89 @@ * 2.0. */ import { Type, TypeOf } from '@kbn/config-schema'; -import { ActionVariable, AlertTypeState } from '../../alerting/common'; +import { + ActionVariable, + AlertInstanceContext, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../alerting/common'; import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; -import { AlertSeverityLevel } from '../common'; +import { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; import { DefaultFieldMap } from './rule_registry/defaults/field_map'; -import { TypeOfFieldMap } from './rule_registry/field_map/runtime_type_from_fieldmap'; -import { FieldMap } from './rule_registry/types'; -enum ESFieldType { - keyword = 'keyword', - text = 'text', - date = 'date', - boolean = 'boolean', - long = 'long', - integer = 'integer', - short = 'short', - byte = 'byte', - double = 'double', - float = 'half_float', - scaled_float = 'scaled_float', - unsigned_long = 'unsigned_long', -} - -type RuleTypeFieldMap = Record; - -type RuleParams = Type; - -export type AlertContext = Record< - string, - { - description: string; - field?: TFieldName; - type: Type; - } ->; - -export interface AlertCheck { - name: string; - value?: number; - threshold?: number; - context: { - [key in TActionVariable['name']]: any; - }; - fields: Omit>, keyof DefaultFieldMap>; -} +export type RuleParams = Type; type TypeOfRuleParams = TypeOf; type RuleExecutorServices< - TFieldMap extends FieldMap, + TFieldMap extends DefaultFieldMap, TActionVariable extends ActionVariable -> = Omit & { - check: { warning: (check: AlertCheck) => void }; +> = AlertExecutorOptions< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + { [key in TActionVariable['name']]: any }, + string +>['services'] & { + scopedRuleRegistryClient: ScopedRuleRegistryClient; }; type PassthroughAlertExecutorOptions = Pick< - AlertExecutorOptions, - 'previousStartedAt' | 'startedAt' + AlertExecutorOptions< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + AlertInstanceContext, + string + >, + 'previousStartedAt' | 'startedAt' | 'state' >; type RuleExecutorFunction< - TFieldMap extends FieldMap, + TFieldMap extends DefaultFieldMap, TRuleParams extends RuleParams, - TActionVariable extends ActionVariable + TActionVariable extends ActionVariable, + TAdditionalRuleExecutorServices extends Record > = ( options: PassthroughAlertExecutorOptions & { - services: RuleExecutorServices; + services: RuleExecutorServices & TAdditionalRuleExecutorServices; params: TypeOfRuleParams; + rule: { + id: string; + uuid: string; + name: string; + category: string; + }; + producer: string; } ) => Promise>; -export interface RuleType { +interface RuleTypeBase { id: string; name: string; - fields?: RuleTypeFieldMap; - params?: RuleParams; - levels?: AlertSeverityLevel[]; - context?: AlertContext; actionGroups: Array>; defaultActionGroupId: string; producer: string; minimumLicenseRequired: 'basic' | 'gold' | 'trial'; - executor: RuleExecutorFunction; } -export type RegisterRuleType = < +export type RuleType< + TFieldMap extends DefaultFieldMap, TRuleParams extends RuleParams, - TActionVariable extends ActionVariable ->(ruleType: { - id: string; - name: string; + TActionVariable extends ActionVariable, + TAdditionalRuleExecutorServices extends Record = {} +> = RuleTypeBase & { validate: { params: TRuleParams; }; actionVariables: { context: TActionVariable[]; }; - actionGroups: Array>; - defaultActionGroupId: string; - producer: string; - minimumLicenseRequired: 'basic' | 'gold' | 'trial'; - executor: RuleExecutorFunction; -}) => void; - -export interface RuleAlertState { - created: number; - alertId: string; -} - -export type RuleState = AlertTypeState & { - wrappedRuleState: Record; - alerts: Record; + executor: RuleExecutorFunction< + TFieldMap, + TRuleParams, + TActionVariable, + TAdditionalRuleExecutorServices + >; }; From 1433e7c832e17521845054862b01c196d7faefa8 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 31 Mar 2021 10:10:34 +0200 Subject: [PATCH 04/24] Clean up APM rule type tests --- .../register_error_count_alert_type.test.ts | 74 ++------- ...action_duration_anomaly_alert_type.test.ts | 141 +++--------------- ..._transaction_error_rate_alert_type.test.ts | 77 ++-------- .../apm/server/lib/alerts/test_utils/index.ts | 64 ++++++++ .../index.ts | 72 ++++++--- .../types.ts | 2 +- .../server/rule_registry/index.ts | 12 +- .../create_lifecycle_rule_type_factory.ts | 44 +++++- x-pack/plugins/rule_registry/server/types.ts | 2 + 9 files changed, 206 insertions(+), 282 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts index 470330016cc6f..ad1b5fa43a638 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts @@ -5,54 +5,18 @@ * 2.0. */ -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; -import { APMConfig } from '../..'; - import { registerErrorCountAlertType } from './register_error_count_alert_type'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; -import { APMRuleRegistry } from '../../plugin'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Error count alert', () => { it("doesn't send an alert when error count is less than threshold", async () => { - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { services, dependencies, executor } = createRuleTypeMocks(); registerErrorCountAlertType({ - registry, - config$: mockedConfig$, - logger: {} as any, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(), - alertWithLifecycle: jest.fn(), - }; const params = { threshold: 1 }; @@ -76,35 +40,21 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); expect(services.alertInstanceFactory).not.toBeCalled(); }); it('sends alerts with service name and environment for those that exceeded the threshold', async () => { - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); registerErrorCountAlertType({ - registry, - config$: mockedConfig$, - logger: {} as any, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; const params = { threshold: 2, windowSize: 5, windowUnit: 'm' }; @@ -190,7 +140,7 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); [ 'apm.error_rate_foo_env-foo', 'apm.error_rate_foo_env-foo-2', diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts index 3d2e534e75bee..b9346b2bf4649 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts @@ -4,30 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; -import { APMConfig } from '../..'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { Job, MlPluginSetup } from '../../../../ml/server'; import * as GetServiceAnomalies from '../service_map/get_service_anomalies'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; -import { APMRuleRegistry } from '../../plugin'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Transaction duration anomaly alert', () => { afterEach(() => { @@ -35,37 +16,21 @@ describe('Transaction duration anomaly alert', () => { }); describe("doesn't send alert", () => { it('ml is not defined', async () => { - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { services, dependencies, executor } = createRuleTypeMocks(); registerTransactionDurationAnomalyAlertType({ - registry, + ...dependencies, ml: undefined, - config$: mockedConfig$, - logger: {} as any, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); + expect( services.scopedClusterClient.asCurrentUser.search ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); @@ -74,23 +39,7 @@ describe('Transaction duration anomaly alert', () => { .spyOn(GetServiceAnomalies, 'getMLJobs') .mockReturnValue(Promise.resolve([])); - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; - - const scheduleActions = jest.fn(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; + const { services, dependencies, executor } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }), @@ -98,20 +47,17 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - registry, + ...dependencies, ml, - config$: mockedConfig$, - logger: {} as any, }); - expect(alertExecutor).toBeDefined(); - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); expect( services.scopedClusterClient.asCurrentUser.search ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); @@ -129,12 +75,7 @@ describe('Transaction duration anomaly alert', () => { ] as unknown) as Job[]) ); - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { services, dependencies, executor } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ @@ -157,36 +98,14 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - registry, + ...dependencies, ml, - config$: mockedConfig$, - logger: {} as any, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; - - registerTransactionDurationAnomalyAlertType({ - registry, - ml, - config$: mockedConfig$, - logger: {} as any, - }); - expect(alertExecutor).toBeDefined(); - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); + expect( services.scopedClusterClient.asCurrentUser.search ).not.toHaveBeenCalled(); @@ -209,13 +128,12 @@ describe('Transaction duration anomaly alert', () => { ] as unknown) as Job[]) ); - let alertExecutor: any; - - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ @@ -260,28 +178,13 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - registry, + ...dependencies, ml, - config$: mockedConfig$, - logger: {} as any, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts index 3425a9b24a3e0..be5f4705482d0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts @@ -5,57 +5,19 @@ * 2.0. */ -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; -import { APMConfig } from '../..'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; -import { APMRuleRegistry } from '../../plugin'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Transaction error rate alert', () => { it("doesn't send an alert when rate is less than threshold", async () => { - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { services, dependencies, executor } = createRuleTypeMocks(); registerTransactionErrorRateAlertType({ - registry, - config$: mockedConfig$, - logger: {} as any, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; - const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -83,37 +45,22 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); expect(services.alertInstanceFactory).not.toBeCalled(); }); it('sends alerts for services that exceeded the threshold', async () => { - let alertExecutor: any; - const registry = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as APMRuleRegistry; + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); registerTransactionErrorRateAlertType({ - registry, - config$: mockedConfig$, - logger: {} as any, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - alertWithLifecycle: jest.fn(), - }; - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ hits: { @@ -172,7 +119,7 @@ describe('Transaction error rate alert', () => { const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; - await alertExecutor!({ services, params, startedAt: new Date() }); + await executor({ params }); expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts new file mode 100644 index 0000000000000..f7581eb1da371 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import { Observable, of } from 'rxjs'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { APMConfig } from '../../..'; +import { APMRuleRegistry } from '../../../plugin'; + +export const createRuleTypeMocks = () => { + let alertExecutor: (...args: any[]) => Promise; + + const mockedConfig$ = of({ + /* eslint-disable @typescript-eslint/naming-convention */ + 'apm_oss.errorIndices': 'apm-*', + 'apm_oss.transactionIndices': 'apm-*', + /* eslint-enable @typescript-eslint/naming-convention */ + }) as Observable; + + const loggerMock = ({ + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + } as unknown) as Logger; + + const registry = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as APMRuleRegistry; + + const scheduleActions = jest.fn(); + + const services = { + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), + logger: loggerMock, + }; + + return { + dependencies: { + registry, + config$: mockedConfig$, + logger: loggerMock, + }, + services, + scheduleActions, + executor: async ({ params }: { params: Record }) => { + return alertExecutor({ + services, + params, + startedAt: new Date(), + }); + }, + }; +}; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts index 099f487bfd0e4..0af0d6ecf1186 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -10,12 +10,13 @@ import { PathReporter } from 'io-ts/lib/PathReporter'; import { Logger, SavedObjectsClientContract } from 'kibana/server'; import { IScopedClusterClient as ScopedClusterClient } from 'src/core/server'; import { compact } from 'lodash'; +import { ESSearchRequest } from 'typings/elasticsearch'; import { ClusterClientAdapter } from '../../../../event_log/server'; import { runtimeTypeFromFieldMap, OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; -import { ScopedRuleRegistryClient } from './types'; +import { ScopedRuleRegistryClient, EventsOf } from './types'; import { DefaultFieldMap } from '../defaults/field_map'; -const getRuleIds = async ({ +const getRuleUuids = async ({ savedObjectsClient, namespace, }: { @@ -29,15 +30,15 @@ const getRuleIds = async ({ const pitFinder = savedObjectsClient.createPointInTimeFinder(options); - const ruleIds: string[] = []; + const ruleUuids: string[] = []; for await (const response of pitFinder.find()) { - ruleIds.push(...response.saved_objects.map((object) => object.id)); + ruleUuids.push(...response.saved_objects.map((object) => object.id)); } await pitFinder.close(); - return ruleIds; + return ruleUuids; }; const createPathReporterError = (either: Either) => { @@ -54,8 +55,7 @@ export function createScopedRuleRegistryClient; index: string; logger: Logger; - rule: { - id: string; - uuid: string; - category: string; - name: string; + ruleData?: { + rule: { + id: string; + uuid: string; + category: string; + name: string; + }; + producer: string; + tags: string[]; }; - producer: string; }): Promise> { const docRt = runtimeTypeFromFieldMap(fieldMap); - const defaults = { - 'rule.uuid': rule.uuid, - 'rule.id': rule.id, - 'rule.name': rule.name, - 'rule.category': rule.category, - producer, - }; + const defaults = ruleData + ? { + 'rule.uuid': ruleData.rule.uuid, + 'rule.id': ruleData.rule.id, + 'rule.name': ruleData.rule.name, + 'rule.category': ruleData.rule.category, + producer: ruleData.producer, + tags: ruleData?.tags, + } + : {}; const createClient = async () => { - const ruleIds = await getRuleIds({ + const ruleUuids = await getRuleUuids({ savedObjectsClient, namespace, }); @@ -94,12 +100,14 @@ export function createScopedRuleRegistryClient = { search: async (searchRequest) => { const response = await scopedClusterClient.asInternalUser.search({ + ...searchRequest, index, body: { + ...searchRequest.body, query: { bool: { filter: [ - { terms: { 'rule.uuid': ruleIds } }, + { terms: { 'rule.uuid': ruleUuids } }, ...(searchRequest.body?.query ? [searchRequest.body.query] : []), ], }, @@ -107,7 +115,20 @@ export function createScopedRuleRegistryClient { + const validation = docRt.decode(hit.fields); + if (isLeft(validation)) { + const error = createPathReporterError(validation); + logger.error(error); + return undefined; + } + return docRt.encode(validation.right); + }) + ) as EventsOf, + }; }, index: (doc) => { const validation = docRt.decode({ @@ -123,7 +144,10 @@ export function createScopedRuleRegistryClient { const validations = docs.map((doc) => { - return docRt.decode(doc); + return docRt.decode({ + ...doc, + ...defaults, + }); }); const errors = compact( diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts index 08aba0a056f46..95aa180709a51 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts @@ -28,7 +28,7 @@ type FieldsESSearchRequest = ESSearchRequest body?: { fields: FieldsOf }; }; -type EventsOf< +export type EventsOf< TFieldsESSearchRequest extends ESSearchRequest, TFieldMap extends DefaultFieldMap > = TFieldsESSearchRequest extends { body: { fields: infer TFields } } diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index 34a3607a0048e..f05c210dac287 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -125,6 +125,8 @@ export class RuleRegistry { registerType( type: RuleType ) { + const logger = this.options.logger.get(type.id); + this.options.alertingPluginSetupContract.registerType< AlertTypeParams, AlertTypeState, @@ -134,7 +136,7 @@ export class RuleRegistry { >({ ...type, executor: async (executorOptions) => { - const { services, namespace, alertId, name } = executorOptions; + const { services, namespace, alertId, name, tags } = executorOptions; const rule = { id: type.id, @@ -152,8 +154,11 @@ export class RuleRegistry { fieldMap: this.options.fieldMap, index: this.getEsNames().indexAliasName, namespace, - producer, - rule, + ruleData: { + producer, + rule, + tags, + }, logger: this.options.logger, }); @@ -163,6 +168,7 @@ export class RuleRegistry { producer, services: { ...services, + logger, scopedRuleRegistryClient, }, }); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts index fd63a886d42b0..425be8dc00ea0 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts @@ -43,7 +43,7 @@ type CreateLifecycleRuleType = < const trackedAlertStateRt = t.type({ alertId: t.string, alertUuid: t.string, - started: t.number, + started: t.string, }); const wrappedStateRt = t.type({ @@ -63,7 +63,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { const { - services: { scopedRuleRegistryClient, alertInstanceFactory }, + services: { scopedRuleRegistryClient, alertInstanceFactory, logger }, state: previousState, rule, } = options; @@ -101,11 +101,16 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType !trackedAlertIds.includes(alertId)); const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter( - (trackedAlertState) => currentAlerts[trackedAlertState.alertId] + (trackedAlertState) => !currentAlerts[trackedAlertState.alertId] + ); + + logger.debug( + `Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStatesOfRecovered.length} recovered)` ); const alertsDataMap: Record> = { @@ -120,7 +125,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { const alertData = alertsDataMap[alertId]; + if (!alertData) { + logger.warn(`Could not find alert data for ${alertId}`); + } + const event: OutputOfFieldMap = { ...alertData, '@timestamp': timestamp, @@ -167,9 +176,15 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType event['alert.status'] !== 'closed') + .map((event) => { + const alertId = event['alert.id']!; + const alertUuid = event['alert.uuid']!; + const started = new Date(event['alert.start']!).toISOString(); + return [alertId, { alertId, alertUuid, started }]; + }) + ); return { wrapped: nextWrappedState, - trackedAlerts: eventsToIndex, + trackedAlerts: nextTrackedAlerts, }; }, }; diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index d2f4c54b99a59..9ea017cddae86 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -5,6 +5,7 @@ * 2.0. */ import { Type, TypeOf } from '@kbn/config-schema'; +import { Logger } from 'kibana/server'; import { ActionVariable, AlertInstanceContext, @@ -30,6 +31,7 @@ type RuleExecutorServices< { [key in TActionVariable['name']]: any }, string >['services'] & { + logger: Logger; scopedRuleRegistryClient: ScopedRuleRegistryClient; }; From 1d656ad877d7fe972c3f7af762cd20c7fdd48406 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 31 Mar 2021 15:24:12 +0200 Subject: [PATCH 05/24] create scoped client for request --- x-pack/plugins/observability/server/plugin.ts | 4 +- x-pack/plugins/rule_registry/server/index.ts | 10 +- .../index.ts | 136 +++++++++--------- .../server/rule_registry/index.ts | 23 ++- 4 files changed, 95 insertions(+), 78 deletions(-) diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 5d9e4bc7fbc5c..77448913e720e 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -6,7 +6,7 @@ */ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; -import { pickWithPatterns } from '../../rule_registry/server/rule_registry/field_map/pick_with_patterns'; +import { pickWithPatterns } from '../../rule_registry/server'; import { ObservabilityConfig } from '.'; import { bootstrapAnnotations, @@ -15,7 +15,7 @@ import { } from './lib/annotations/bootstrap_annotations'; import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server'; import { uiSettings } from './ui_settings'; -import { ecsFieldMap } from '../../rule_registry/server/generated/ecs_field_map'; +import { ecsFieldMap } from '../../rule_registry/server'; export type ObservabilityPluginSetup = ReturnType; diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 33cb22ff33687..435119cbd9500 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -7,8 +7,12 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginInitializerContext } from 'src/core/server'; -import { RuleRegistryPlugin, RuleRegistryPluginSetupContract } from './plugin'; -import { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; +import { RuleRegistryPlugin } from './plugin'; + +export { RuleRegistryPluginSetupContract } from './plugin'; +export { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; +export { ecsFieldMap } from './generated/ecs_field_map'; +export { pickWithPatterns } from './rule_registry/field_map/pick_with_patterns'; export const config = { schema: schema.object({ @@ -20,5 +24,3 @@ export type RuleRegistryConfig = TypeOf; export const plugin = (initContext: PluginInitializerContext) => new RuleRegistryPlugin(initContext); - -export { RuleRegistryPluginSetupContract, createLifecycleRuleTypeFactory }; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts index 0af0d6ecf1186..14cad9b470694 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -77,7 +77,7 @@ export function createScopedRuleRegistryClient> { +}): ScopedRuleRegistryClient { const docRt = runtimeTypeFromFieldMap(fieldMap); const defaults = ruleData @@ -91,84 +91,80 @@ export function createScopedRuleRegistryClient { - const ruleUuids = await getRuleUuids({ - savedObjectsClient, - namespace, - }); - - const client: ScopedRuleRegistryClient = { - search: async (searchRequest) => { - const response = await scopedClusterClient.asInternalUser.search({ - ...searchRequest, - index, - body: { - ...searchRequest.body, - query: { - bool: { - filter: [ - { terms: { 'rule.uuid': ruleUuids } }, - ...(searchRequest.body?.query ? [searchRequest.body.query] : []), - ], - }, + const client: ScopedRuleRegistryClient = { + search: async (searchRequest) => { + const ruleUuids = await getRuleUuids({ + savedObjectsClient, + namespace, + }); + + const response = await scopedClusterClient.asInternalUser.search({ + ...searchRequest, + index, + body: { + ...searchRequest.body, + query: { + bool: { + filter: [ + { terms: { 'rule.uuid': ruleUuids } }, + ...(searchRequest.body?.query ? [searchRequest.body.query] : []), + ], }, }, - }); + }, + }); + + return { + body: response.body as any, + events: compact( + response.body.hits.hits.map((hit) => { + const validation = docRt.decode(hit.fields); + if (isLeft(validation)) { + const error = createPathReporterError(validation); + logger.error(error); + return undefined; + } + return docRt.encode(validation.right); + }) + ) as EventsOf, + }; + }, + index: (doc) => { + const validation = docRt.decode({ + ...doc, + ...defaults, + }); + + if (isLeft(validation)) { + throw createPathReporterError(validation); + } - return { - body: response.body as any, - events: compact( - response.body.hits.hits.map((hit) => { - const validation = docRt.decode(hit.fields); - if (isLeft(validation)) { - const error = createPathReporterError(validation); - logger.error(error); - return undefined; - } - return docRt.encode(validation.right); - }) - ) as EventsOf, - }; - }, - index: (doc) => { - const validation = docRt.decode({ + clusterClientAdapter.indexDocument({ body: validation.right, index }); + }, + bulkIndex: (docs) => { + const validations = docs.map((doc) => { + return docRt.decode({ ...doc, ...defaults, }); + }); - if (isLeft(validation)) { - throw createPathReporterError(validation); - } - - clusterClientAdapter.indexDocument({ body: validation.right, index }); - }, - bulkIndex: (docs) => { - const validations = docs.map((doc) => { - return docRt.decode({ - ...doc, - ...defaults, - }); - }); + const errors = compact( + validations.map((validation) => + isLeft(validation) ? createPathReporterError(validation) : null + ) + ); - const errors = compact( - validations.map((validation) => - isLeft(validation) ? createPathReporterError(validation) : null - ) - ); + errors.forEach((error) => { + logger.error(error); + }); - errors.forEach((error) => { - logger.error(error); - }); + const operations = compact( + validations.map((validation) => (isRight(validation) ? validation.right : null)) + ).map((doc) => ({ body: doc, index })); - const operations = compact( - validations.map((validation) => (isRight(validation) ? validation.right : null)) - ).map((doc) => ({ body: doc, index })); - - return clusterClientAdapter.indexDocuments(operations); - }, - }; - return client; + return clusterClientAdapter.indexDocuments(operations); + }, }; - - return createClient(); + return client; } diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index f05c210dac287..e73a2e9821539 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -5,8 +5,9 @@ * 2.0. */ -import { CoreSetup, Logger } from 'kibana/server'; +import { CoreSetup, KibanaRequest, Logger, RequestHandlerContext } from 'kibana/server'; import { inspect } from 'util'; +import { getSpaceIdFromPath } from '../../../spaces/common'; import { ActionVariable, AlertInstanceState, @@ -22,6 +23,7 @@ import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; import { createScopedRuleRegistryClient } from './create_scoped_rule_registry_client'; import { DefaultFieldMap } from './defaults/field_map'; +import { ScopedRuleRegistryClient } from './create_scoped_rule_registry_client/types'; interface RuleRegistryOptions { kibanaIndex: string; @@ -122,6 +124,23 @@ export class RuleRegistry { } } + createScopedRuleRegistryClient( + request: KibanaRequest, + context: RequestHandlerContext + ): ScopedRuleRegistryClient { + const { spaceId: namespace } = getSpaceIdFromPath(request.url.pathname); + + return createScopedRuleRegistryClient({ + savedObjectsClient: context.core.savedObjects.client, + scopedClusterClient: context.core.elasticsearch.client, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + index: this.getEsNames().indexAliasName, + namespace, + logger: this.options.logger, + }); + } + registerType( type: RuleType ) { @@ -147,7 +166,7 @@ export class RuleRegistry { const producer = type.producer; - const scopedRuleRegistryClient = await createScopedRuleRegistryClient({ + const scopedRuleRegistryClient = createScopedRuleRegistryClient({ savedObjectsClient: services.savedObjectsClient, scopedClusterClient: services.scopedClusterClient, clusterClientAdapter: this.esAdapter, From c94260373864e13748b785ae85605ec78b77bdf1 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 31 Mar 2021 15:45:18 +0200 Subject: [PATCH 06/24] Remove parent reference --- x-pack/plugins/rule_registry/server/rule_registry/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index e73a2e9821539..09422d123fb0c 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -34,7 +34,6 @@ interface RuleRegistryOptions { fieldMap: TFieldMap; ilmPolicy: ILMPolicy; alertingPluginSetupContract: AlertingPluginSetupContract; - parent?: RuleRegistry; } export class RuleRegistry { @@ -214,8 +213,6 @@ export class RuleRegistry { namespace: [this.options.namespace, namespace].filter(Boolean).join('-'), fieldMap: mergedFieldMap, ...(ilmPolicy ? { ilmPolicy } : {}), - // @ts-expect-error Types of property 'body' are incompatible. - parent: this, }); this.children.push(child); From 5166848818a062435bc43552e4cdd807edf1376f Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 31 Mar 2021 15:54:49 +0200 Subject: [PATCH 07/24] Add README --- x-pack/plugins/rule_registry/README.md | 68 ++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 x-pack/plugins/rule_registry/README.md diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md new file mode 100644 index 0000000000000..7322dade8bdaf --- /dev/null +++ b/x-pack/plugins/rule_registry/README.md @@ -0,0 +1,68 @@ +The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts. + +A rule registry creates a template, an ILM policy, and an alias. The template mappings can be configured. It also injects a client scoped to these indices. + +It also supports inheritance, which means that producers can create a registry specific to their solution or rule type, and specify additional mappings to be used. + +The rule registry plugin creates a root rule registry, with the mappings defined needed to create a unified experience. Rule type producers can use the plugin to access the root rule registry, and create their own registry that branches off of the root rule registry. The rule registry client sees data from its own registry, and all registries that branches off of it. It does not see data from its parents. + +Creating a rule registry + +To create a rule registry, producers should add the `ruleRegistry` plugin to their dependencies. They can then use the `ruleRegistry.create` method to create a child registry, with the additional mappings that should be used by specifying `fieldMap`: + +```ts +const observabilityRegistry = plugins.ruleRegistry.create({ + namespace: 'observability', + fieldMap: { + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), + }, +}) +``` + +`fieldMap` is a key-value map of field names and mapping options: + +```ts +{ + '@timestamp': { + type: 'date', + array: false, + required: true, + } +} +``` + +ECS mappings are generated via a script in the rule registry plugin directory. These mappings are available in x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts. + +To pick many fields, you can use `pickWithPatterns`, which supports wildcards with full type support. + +If a registry is created, it will initialise as soon as the core services needed become available. It will create a (versioned) template, alias, and ILM policy, but only if these do not exist yet. + +### Rule registry client + +The rule registry client can either be injected in the executor, or created in the scope of a request. It exposes a `search` method and a `bulkIndex` method. When `search` is called, it first gets all the rules the current user has access to, and adds these ids to the search request that it executes. This means that the user can only see data from rules they have access to. + +Both `search` and `bulkIndex` are fully typed, in the sense that they reflect the mappings defined for the registry. + +### Schema + +The following fields are available in the root rule registry: + +- `@timestamp`: the ISO timestamp of the alert event. For the lifecycle rule type helper, it is always the value of `startedAt` that is injected by the Kibana alerting framework. +- `event.kind`: signal (for the changeable alert document), state (for the state changes of the alert, e.g. when it opens, recovers, or changes in severity), or metric (individual evaluations that might be related to an alert). +- `event.action`: the reason for the event. This might be `open`, `close`, `active`, or `evaluate`. +- `tags`: tags attached to the alert. Right now they are copied over from the rule. +- `producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. +- `rule.id`: the identifier of the rule type, e.g. `apm.transaction_duration` +- `rule.uuid`: the saved objects id of the rule. +- `rule.name`: the name of the rule (as specified by the user). +- `rule.category`: the name of the rule type (as defined by the rule type producer) +- `alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. +- `alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. +- `alert.status`: the status of the alert. Can be `open` or `closed`. +- `alert.start`: the ISO timestamp of the time at which the alert started. +- `alert.end`: the ISO timestamp of the time at which the alert recovered. +- `alert.duration.us`: the duration of the alert, in microseconds. This is always the difference between either the current time, or the time when the alert recovered. +- `alert.severity.level`: the severity of the alert, as a keyword (e.g. critical). +- `alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting. + +This list is not final - just a start. Field names might change or moved to a scoped registry. If we implement log and sequence based rule types the list of fields will grow. If a rule type needs additional fields, the recommendation would be to have the field in its own registry first (or in its producer’s registry), and if usage is more broadly adopted, it can be moved to the root registry. From a85cacb14e48d5d1ae6aa6d82c6ee330d516fef4 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 14:22:46 +0200 Subject: [PATCH 08/24] Namespace alert fields --- x-pack/plugins/rule_registry/README.md | 16 ++--- x-pack/plugins/rule_registry/server/index.ts | 1 + x-pack/plugins/rule_registry/server/plugin.ts | 3 + .../rule_registry/defaults/field_map.ts | 21 +++---- .../server/rule_registry/index.ts | 63 +++++++++++-------- .../create_lifecycle_rule_type_factory.ts | 39 ++++++------ x-pack/plugins/rule_registry/server/types.ts | 2 +- 7 files changed, 79 insertions(+), 66 deletions(-) diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md index 7322dade8bdaf..eea3962461f8a 100644 --- a/x-pack/plugins/rule_registry/README.md +++ b/x-pack/plugins/rule_registry/README.md @@ -56,13 +56,13 @@ The following fields are available in the root rule registry: - `rule.uuid`: the saved objects id of the rule. - `rule.name`: the name of the rule (as specified by the user). - `rule.category`: the name of the rule type (as defined by the rule type producer) -- `alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. -- `alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. -- `alert.status`: the status of the alert. Can be `open` or `closed`. -- `alert.start`: the ISO timestamp of the time at which the alert started. -- `alert.end`: the ISO timestamp of the time at which the alert recovered. -- `alert.duration.us`: the duration of the alert, in microseconds. This is always the difference between either the current time, or the time when the alert recovered. -- `alert.severity.level`: the severity of the alert, as a keyword (e.g. critical). -- `alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting. +- `kibana.rac.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. +- `kibana.rac.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. +- `kibana.rac.alert.status`: the status of the alert. Can be `open` or `closed`. +- `kibana.rac.alert.start`: the ISO timestamp of the time at which the alert started. +- `kibana.rac.alert.end`: the ISO timestamp of the time at which the alert recovered. +- `kibana.rac.alert.duration.us`: the duration of the alert, in microseconds. This is always the difference between either the current time, or the time when the alert recovered. +- `kibana.rac.alert.severity.level`: the severity of the alert, as a keyword (e.g. critical). +- `kibana.rac.alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting. This list is not final - just a start. Field names might change or moved to a scoped registry. If we implement log and sequence based rule types the list of fields will grow. If a rule type needs additional fields, the recommendation would be to have the field in its own registry first (or in its producer’s registry), and if usage is more broadly adopted, it can be moved to the root registry. diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 435119cbd9500..7b43ae675faf7 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -17,6 +17,7 @@ export { pickWithPatterns } from './rule_registry/field_map/pick_with_patterns'; export const config = { schema: schema.object({ enabled: schema.boolean({ defaultValue: true }), + writeEnabled: schema.boolean({ defaultValue: false }), }), }; diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 51f34b916875a..85cbbe5a6d672 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -10,6 +10,7 @@ import { PluginSetupContract as AlertingPluginSetupContract } from '../../alerti import { RuleRegistry } from './rule_registry'; import { defaultIlmPolicy } from './rule_registry/defaults/ilm_policy'; import { defaultFieldMap } from './rule_registry/defaults/field_map'; +import { RuleRegistryConfig } from '.'; export type RuleRegistryPluginSetupContract = RuleRegistry; @@ -23,6 +24,7 @@ export class RuleRegistryPlugin implements Plugin(); const logger = this.initContext.logger.get(); @@ -35,6 +37,7 @@ export class RuleRegistryPlugin implements Plugin { fieldMap: TFieldMap; ilmPolicy: ILMPolicy; alertingPluginSetupContract: AlertingPluginSetupContract; + writeEnabled: boolean; } export class RuleRegistry { @@ -59,15 +60,19 @@ export class RuleRegistry { logger: logger.get('esAdapter'), }); - this.initialize() - .then(() => { - this.options.logger.debug('Bootstrapped alerts index'); - signal(true); - }) - .catch((err) => { - logger.error(inspect(err, { depth: null })); - signal(false); - }); + if (!this.options.writeEnabled) { + this.initialize() + .then(() => { + this.options.logger.debug('Bootstrapped alerts index'); + signal(true); + }) + .catch((err) => { + logger.error(inspect(err, { depth: null })); + signal(false); + }); + } else { + logger.debug('Write disabled, indices are not being bootstrapped'); + } } private getEsNames() { @@ -126,7 +131,11 @@ export class RuleRegistry { createScopedRuleRegistryClient( request: KibanaRequest, context: RequestHandlerContext - ): ScopedRuleRegistryClient { + ): ScopedRuleRegistryClient | undefined { + if (!this.options.writeEnabled) { + return undefined; + } + const { spaceId: namespace } = getSpaceIdFromPath(request.url.pathname); return createScopedRuleRegistryClient({ @@ -165,21 +174,6 @@ export class RuleRegistry { const producer = type.producer; - const scopedRuleRegistryClient = createScopedRuleRegistryClient({ - savedObjectsClient: services.savedObjectsClient, - scopedClusterClient: services.scopedClusterClient, - clusterClientAdapter: this.esAdapter, - fieldMap: this.options.fieldMap, - index: this.getEsNames().indexAliasName, - namespace, - ruleData: { - producer, - rule, - tags, - }, - logger: this.options.logger, - }); - return type.executor({ ...executorOptions, rule, @@ -187,7 +181,24 @@ export class RuleRegistry { services: { ...services, logger, - scopedRuleRegistryClient, + ...(this.options.writeEnabled + ? { + scopedRuleRegistryClient: createScopedRuleRegistryClient({ + savedObjectsClient: services.savedObjectsClient, + scopedClusterClient: services.scopedClusterClient, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + index: this.getEsNames().indexAliasName, + namespace, + ruleData: { + producer, + rule, + tags, + }, + logger: this.options.logger, + }), + } + : {}), }, }); }, diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts index 425be8dc00ea0..44ce7e8c7d613 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts @@ -79,7 +79,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType & { 'alert.id': string } + UserDefinedAlertFields & { 'kibana.rac.alert.id': string } > = {}; const timestamp = options.startedAt.toISOString(); @@ -92,7 +92,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { currentAlerts[id] = { ...fields, - 'alert.id': id, + 'kibana.rac.alert.id': id, }; return alertInstanceFactory(id); }, @@ -117,7 +117,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType trackedAlertState.alertUuid ), }, @@ -140,7 +140,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { - const alertId = event['alert.id']!; + const alertId = event['kibana.rac.alert.id']!; alertsDataMap[alertId] = event; }); } @@ -168,7 +168,7 @@ export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType event['alert.status'] !== 'closed') + .filter((event) => event['kibana.rac.alert.status'] !== 'closed') .map((event) => { - const alertId = event['alert.id']!; - const alertUuid = event['alert.uuid']!; - const started = new Date(event['alert.start']!).toISOString(); + const alertId = event['kibana.rac.alert.id']!; + const alertUuid = event['kibana.rac.alert.uuid']!; + const started = new Date(event['kibana.rac.alert.start']!).toISOString(); return [alertId, { alertId, alertUuid, started }]; }) ); diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index 9ea017cddae86..da355a412207c 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -32,7 +32,7 @@ type RuleExecutorServices< string >['services'] & { logger: Logger; - scopedRuleRegistryClient: ScopedRuleRegistryClient; + scopedRuleRegistryClient?: ScopedRuleRegistryClient; }; type PassthroughAlertExecutorOptions = Pick< From f64a93016b1adfd6f785364abfb11008609b6319 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 14:29:13 +0200 Subject: [PATCH 09/24] Update allowed user defined alert fields --- .../rule_type_helpers/create_lifecycle_rule_type_factory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts index 44ce7e8c7d613..9c64e85f839bb 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts @@ -17,7 +17,7 @@ import { RuleRegistry } from '..'; type UserDefinedAlertFields = Omit< OutputOfFieldMap, - PrepopulatedRuleEventFields | 'alert.id' | 'alert.uuid' | '@timestamp' + PrepopulatedRuleEventFields | 'kibana.rac.alert.id' | 'kibana.rac.alert.uuid' | '@timestamp' >; type LifecycleAlertService< From 3fe959edabbee23e23e13fee2de3344ecb12f61f Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 14:41:38 +0200 Subject: [PATCH 10/24] Review feedback --- .../alerting/server/task_runner/task_runner.ts | 11 +++-------- .../lib/alerts/create_apm_lifecycle_rule_type.ts | 2 +- .../alerts/register_error_count_alert_type.test.ts | 8 ++------ .../lib/alerts/register_error_count_alert_type.ts | 4 ++-- .../register_transaction_duration_alert_type.ts | 12 +++++++----- ...gister_transaction_duration_anomaly_alert_type.ts | 4 ++-- .../register_transaction_error_rate_alert_type.ts | 4 ++-- .../apm/server/lib/alerts/test_utils/index.ts | 4 ++-- x-pack/plugins/apm/server/plugin.ts | 6 +++--- x-pack/plugins/observability/server/plugin.ts | 2 +- 10 files changed, 25 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 817942f32db40..744be16451999 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -539,16 +539,11 @@ export class TaskRunner< }; }, (err: Error) => { - const error = new Error( - `Executing Alert "${alertId}" has resulted in Error: ${err.message}` - ); - - Object.assign(error, { wrapped: err }); - + const message = `Executing Alert "${alertId}" has resulted in Error: ${err.message}`; if (isAlertSavedObjectNotFoundError(err, alertId)) { - this.logger.debug(error.message); + this.logger.debug(message); } else { - this.logger.error(error); + this.logger.error(message); } return originalState; } diff --git a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts index 78f2fd6fcfed8..8d250a5765cce 100644 --- a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts @@ -8,4 +8,4 @@ import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; import { APMRuleRegistry } from '../../plugin'; -export const createAPMLifecyleRuleType = createLifecycleRuleTypeFactory(); +export const createAPMLifecycleRuleType = createLifecycleRuleTypeFactory(); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts index ad1b5fa43a638..5758dea1860b2 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts @@ -14,9 +14,7 @@ describe('Error count alert', () => { it("doesn't send an alert when error count is less than threshold", async () => { const { services, dependencies, executor } = createRuleTypeMocks(); - registerErrorCountAlertType({ - ...dependencies, - }); + registerErrorCountAlertType(dependencies); const params = { threshold: 1 }; @@ -52,9 +50,7 @@ describe('Error count alert', () => { scheduleActions, } = createRuleTypeMocks(); - registerErrorCountAlertType({ - ...dependencies, - }); + registerErrorCountAlertType(dependencies); const params = { threshold: 2, windowSize: 5, windowUnit: 'm' }; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index d4e3ac70439d9..8240e0c369d1f 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -21,7 +21,7 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; import { RegisterRuleDependencies } from './register_apm_alerts'; -import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -38,7 +38,7 @@ export function registerErrorCountAlertType({ config$, }: RegisterRuleDependencies) { registry.registerType( - createAPMLifecyleRuleType({ + createAPMLifecycleRuleType({ id: AlertType.ErrorCount, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index 110e7e117907b..6ca1c4370d6ae 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -24,7 +24,7 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; import { RegisterRuleDependencies } from './register_apm_alerts'; -import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.string(), @@ -47,7 +47,7 @@ export function registerTransactionDurationAlertType({ config$, }: RegisterRuleDependencies) { registry.registerType( - createAPMLifecyleRuleType({ + createAPMLifecycleRuleType({ id: AlertType.TransactionDuration, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, @@ -97,7 +97,7 @@ export function registerTransactionDurationAlertType({ }, }, aggs: { - metric: + latency: alertParams.aggregationType === 'avg' ? { avg: { field: TRANSACTION_DURATION } } : { @@ -121,10 +121,12 @@ export function registerTransactionDurationAlertType({ return {}; } - const { metric } = response.aggregations; + const { latency } = response.aggregations; const transactionDuration = - 'values' in metric ? Object.values(metric.values)[0] : metric?.value; + 'values' in latency + ? Object.values(latency.values)[0] + : latency?.value; const threshold = alertParams.threshold * 1000; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index 96fa27a87a51c..15f4a8ea07801 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -27,7 +27,7 @@ import { getMLJobs } from '../service_map/get_service_anomalies'; import { apmActionVariables } from './action_variables'; import { RegisterRuleDependencies } from './register_apm_alerts'; import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; -import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.maybe(schema.string()), @@ -52,7 +52,7 @@ export function registerTransactionDurationAnomalyAlertType({ logger, }: RegisterRuleDependencies) { registry.registerType( - createAPMLifecyleRuleType({ + createAPMLifecycleRuleType({ id: AlertType.TransactionDurationAnomaly, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index 813c6885efd56..095c08de026e2 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -22,7 +22,7 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; -import { createAPMLifecyleRuleType } from './create_apm_lifecycle_rule_type'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; import { RegisterRuleDependencies } from './register_apm_alerts'; const paramsSchema = schema.object({ @@ -41,7 +41,7 @@ export function registerTransactionErrorRateAlertType({ config$, }: RegisterRuleDependencies) { registry.registerType( - createAPMLifecyleRuleType({ + createAPMLifecycleRuleType({ id: AlertType.TransactionErrorRate, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts index f7581eb1da371..37b3e282d0a59 100644 --- a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -6,7 +6,7 @@ */ import { Logger } from 'kibana/server'; -import { Observable, of } from 'rxjs'; +import { of } from 'rxjs'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { APMConfig } from '../../..'; import { APMRuleRegistry } from '../../../plugin'; @@ -19,7 +19,7 @@ export const createRuleTypeMocks = () => { 'apm_oss.errorIndices': 'apm-*', 'apm_oss.transactionIndices': 'apm-*', /* eslint-enable @typescript-eslint/naming-convention */ - }) as Observable; + } as APMConfig); const loggerMock = ({ debug: jest.fn(), diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index eef3cda440636..d919d07f35949 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -47,7 +47,7 @@ import { uiSettings } from './ui_settings'; import type { ApmPluginRequestHandlerContext } from './routes/typings'; export type APMPluginSetup = ReturnType; -export type APMRuleRegistry = APMPluginSetup['registry']; +export type APMRuleRegistry = APMPluginSetup['ruleRegistry']; export class APMPlugin implements Plugin { private currentConfig?: APMConfig; @@ -143,7 +143,7 @@ export class APMPlugin implements Plugin { config: await mergedConfig$.pipe(take(1)).toPromise(), }); - const apmRuleRegistry = plugins.observability.registry.create({ + const apmRuleRegistry = plugins.observability.ruleRegistry.create({ namespace: 'apm', fieldMap: { 'service.environment': { @@ -194,7 +194,7 @@ export class APMPlugin implements Plugin { }, }); }, - registry: apmRuleRegistry, + ruleRegistry: apmRuleRegistry, }; } diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 77448913e720e..dcf98829a3289 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -53,7 +53,7 @@ export class ObservabilityPlugin implements Plugin { const api = await annotationsApiPromise; return api?.getScopedAnnotationsClient(...args); }, - registry: plugins.ruleRegistry.create({ + ruleRegistry: plugins.ruleRegistry.create({ namespace: 'observability', fieldMap: { ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), From e21813d8d47904c344ff2b738100b78ea324848b Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 14:58:33 +0200 Subject: [PATCH 11/24] Namespace > name --- x-pack/plugins/apm/server/plugin.ts | 2 +- x-pack/plugins/observability/server/plugin.ts | 2 +- x-pack/plugins/rule_registry/README.md | 2 +- x-pack/plugins/rule_registry/server/plugin.ts | 2 +- .../rule_registry/server/rule_registry/index.ts | 12 ++++++------ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index d919d07f35949..63c3212047b2e 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -144,7 +144,7 @@ export class APMPlugin implements Plugin { }); const apmRuleRegistry = plugins.observability.ruleRegistry.create({ - namespace: 'apm', + name: 'apm', fieldMap: { 'service.environment': { type: 'keyword', diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index dcf98829a3289..c59b4dbe373dd 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -54,7 +54,7 @@ export class ObservabilityPlugin implements Plugin { return api?.getScopedAnnotationsClient(...args); }, ruleRegistry: plugins.ruleRegistry.create({ - namespace: 'observability', + name: 'observability', fieldMap: { ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), }, diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md index eea3962461f8a..fc6d28497c939 100644 --- a/x-pack/plugins/rule_registry/README.md +++ b/x-pack/plugins/rule_registry/README.md @@ -12,7 +12,7 @@ To create a rule registry, producers should add the `ruleRegistry` plugin to the ```ts const observabilityRegistry = plugins.ruleRegistry.create({ - namespace: 'observability', + name: 'observability', fieldMap: { ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), }, diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 85cbbe5a6d672..06c16302b4b0a 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -33,7 +33,7 @@ export class RuleRegistryPlugin implements Plugin { kibanaIndex: string; kibanaVersion: string; - namespace: string; + name: string; logger: Logger; core: CoreSetup; fieldMap: TFieldMap; @@ -76,7 +76,7 @@ export class RuleRegistry { } private getEsNames() { - const base = [this.options.kibanaIndex, this.options.namespace]; + const base = [this.options.kibanaIndex, this.options.name]; const indexAliasName = [...base, this.options.kibanaVersion].join('-'); const policyName = [...base, 'policy'].join('-'); @@ -206,11 +206,11 @@ export class RuleRegistry { } create({ - namespace, + name, fieldMap, ilmPolicy, }: { - namespace: string; + name: string; fieldMap: TNextFieldMap; ilmPolicy?: ILMPolicy; }): RuleRegistry { @@ -220,8 +220,8 @@ export class RuleRegistry { const child = new RuleRegistry({ ...this.options, - logger: this.options.logger.get(namespace), - namespace: [this.options.namespace, namespace].filter(Boolean).join('-'), + logger: this.options.logger.get(name), + name: [this.options.name, name].filter(Boolean).join('-'), fieldMap: mergedFieldMap, ...(ilmPolicy ? { ilmPolicy } : {}), }); From 6762c6ac3c3a926024e5afbc56ba98be93b2e6d7 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 15:14:33 +0200 Subject: [PATCH 12/24] Add rule registry plugin to docs --- docs/developer/plugin-list.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index bcf74936077ec..ef3e3ed0bc148 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -489,6 +489,10 @@ Elastic. |Welcome to the Kibana rollup plugin! This plugin provides Kibana support for Elasticsearch's rollup feature. Please refer to the Elasticsearch documentation to understand rollup indices and how to create rollup jobs. +|{kib-repo}blob/{branch}/x-pack/plugins/rule_registry/README.md[ruleRegistry] +|The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts. + + |{kib-repo}blob/{branch}/x-pack/plugins/runtime_fields/README.md[runtimeFields] |Welcome to the home of the runtime field editor and everything related to runtime fields! From 83448269cbc3ca189f454a957e0d2715d8878268 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 16:30:15 +0200 Subject: [PATCH 13/24] configurable writes --- x-pack/plugins/rule_registry/kibana.json | 2 +- .../create_scoped_rule_registry_client/index.ts | 6 +++--- x-pack/plugins/rule_registry/server/rule_registry/index.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json index 9d766bf61816c..dea6ef560cc2d 100644 --- a/x-pack/plugins/rule_registry/kibana.json +++ b/x-pack/plugins/rule_registry/kibana.json @@ -4,7 +4,7 @@ "kibanaVersion": "kibana", "configPath": [ "xpack", - "rule_registry" + "ruleRegistry" ], "requiredPlugins": [ "alerting" diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts index 14cad9b470694..2a9eecc77ba58 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -80,14 +80,14 @@ export function createScopedRuleRegistryClient { const docRt = runtimeTypeFromFieldMap(fieldMap); - const defaults = ruleData + const defaults: Partial> = ruleData ? { 'rule.uuid': ruleData.rule.uuid, 'rule.id': ruleData.rule.id, 'rule.name': ruleData.rule.name, 'rule.category': ruleData.rule.category, - producer: ruleData.producer, - tags: ruleData?.tags, + 'kibana.rac.producer': ruleData.producer, + tags: ruleData.tags, } : {}; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index 3953fa1c1cc82..361baf7d4a73e 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -60,7 +60,7 @@ export class RuleRegistry { logger: logger.get('esAdapter'), }); - if (!this.options.writeEnabled) { + if (this.options.writeEnabled) { this.initialize() .then(() => { this.options.logger.debug('Bootstrapped alerts index'); From bd770295ea11ffb06bfb5b545f686075e017fced Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 16:57:10 +0200 Subject: [PATCH 14/24] Update producer field --- x-pack/plugins/rule_registry/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md index fc6d28497c939..17fe2b20f74fa 100644 --- a/x-pack/plugins/rule_registry/README.md +++ b/x-pack/plugins/rule_registry/README.md @@ -51,11 +51,11 @@ The following fields are available in the root rule registry: - `event.kind`: signal (for the changeable alert document), state (for the state changes of the alert, e.g. when it opens, recovers, or changes in severity), or metric (individual evaluations that might be related to an alert). - `event.action`: the reason for the event. This might be `open`, `close`, `active`, or `evaluate`. - `tags`: tags attached to the alert. Right now they are copied over from the rule. -- `producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. - `rule.id`: the identifier of the rule type, e.g. `apm.transaction_duration` - `rule.uuid`: the saved objects id of the rule. - `rule.name`: the name of the rule (as specified by the user). - `rule.category`: the name of the rule type (as defined by the rule type producer) +- `kibana.rac.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. - `kibana.rac.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. - `kibana.rac.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. - `kibana.rac.alert.status`: the status of the alert. Can be `open` or `closed`. From 2b243f9573fd916a84465bdc948a6776b9dd3126 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 1 Apr 2021 17:39:11 +0200 Subject: [PATCH 15/24] Revert changes to index template error --- x-pack/plugins/event_log/server/es/cluster_client_adapter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 40da5c0a5e1b4..dd6ac6350d6e3 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -170,7 +170,7 @@ export class ClusterClientAdapter Date: Wed, 7 Apr 2021 16:55:25 +0200 Subject: [PATCH 16/24] Add rule registry project to apm tsconfig --- x-pack/plugins/apm/tsconfig.json | 1 + x-pack/plugins/rule_registry/tsconfig.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/tsconfig.json b/x-pack/plugins/apm/tsconfig.json index ffbf11c23f63a..bb341059e2d43 100644 --- a/x-pack/plugins/apm/tsconfig.json +++ b/x-pack/plugins/apm/tsconfig.json @@ -38,6 +38,7 @@ { "path": "../ml/tsconfig.json" }, { "path": "../observability/tsconfig.json" }, { "path": "../reporting/tsconfig.json" }, + { "path": "../rule_registry/tsconfig.json" }, { "path": "../security/tsconfig.json" }, { "path": "../task_manager/tsconfig.json" }, { "path": "../triggers_actions_ui/tsconfig.json" } diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json index 7f38665320050..2961abe6cfecd 100644 --- a/x-pack/plugins/rule_registry/tsconfig.json +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -7,7 +7,7 @@ "declaration": true, "declarationMap": true }, - "include": ["common/**/*", "server/**/*" ], + "include": ["common/**/*", "server/**/*", "../../../typings/**/*"], "references": [ { "path": "../../../src/core/tsconfig.json" }, { "path": "../alerting/tsconfig.json" }, From 7ebaff65b4da27a65fae9bbe19d18d62abfded47 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 12:52:45 +0200 Subject: [PATCH 17/24] Add simple integration tests for rule registry --- ...ister_transaction_error_rate_alert_type.ts | 4 +- x-pack/plugins/rule_registry/server/plugin.ts | 4 +- .../server/rule_registry/index.ts | 23 +- .../test/apm_api_integration/common/config.ts | 13 +- .../test/apm_api_integration/configs/index.ts | 6 + .../test/apm_api_integration/rules/config.ts | 10 + .../tests/alerts/rule_registry.ts | 390 ++++++++++++++++++ .../test/apm_api_integration/tests/index.ts | 4 + 8 files changed, 437 insertions(+), 17 deletions(-) create mode 100644 x-pack/test/apm_api_integration/rules/config.ts create mode 100644 x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index 095c08de026e2..0865bed41142e 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -70,7 +70,7 @@ export function registerTransactionErrorRateAlertType({ const searchParams = { index: indices['apm_oss.transactionIndices'], - size: 0, + size: 1, body: { query: { bool: { @@ -112,7 +112,7 @@ export function registerTransactionErrorRateAlertType({ multi_terms: { terms: [ { field: SERVICE_NAME }, - { field: SERVICE_ENVIRONMENT }, + { field: SERVICE_ENVIRONMENT, missing: '' }, { field: TRANSACTION_TYPE }, ], size: 10000, diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 06c16302b4b0a..9e83d938d508b 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -29,11 +29,11 @@ export class RuleRegistryPlugin implements Plugin { kibanaVersion: string; name: string; logger: Logger; - core: CoreSetup; + coreSetup: CoreSetup; + spacesStart?: SpacesServiceStart; fieldMap: TFieldMap; ilmPolicy: ILMPolicy; alertingPluginSetupContract: AlertingPluginSetupContract; @@ -45,7 +46,7 @@ export class RuleRegistry { private readonly children: Array> = []; constructor(private readonly options: RuleRegistryOptions) { - const { logger, core } = options; + const { logger, coreSetup } = options; const { wait, signal } = createReadySignal(); @@ -54,7 +55,7 @@ export class RuleRegistry { index: string; }>({ wait, - elasticsearchClientPromise: core + elasticsearchClientPromise: coreSetup .getStartServices() .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), logger: logger.get('esAdapter'), @@ -128,23 +129,23 @@ export class RuleRegistry { } } - createScopedRuleRegistryClient( - request: KibanaRequest, - context: RequestHandlerContext - ): ScopedRuleRegistryClient | undefined { + createScopedRuleRegistryClient({ + context, + request, + }: { + request: KibanaRequest; + context: RequestHandlerContext; + }): ScopedRuleRegistryClient | undefined { if (!this.options.writeEnabled) { return undefined; } - const { spaceId: namespace } = getSpaceIdFromPath(request.url.pathname); - return createScopedRuleRegistryClient({ savedObjectsClient: context.core.savedObjects.client, scopedClusterClient: context.core.elasticsearch.client, clusterClientAdapter: this.esAdapter, fieldMap: this.options.fieldMap, index: this.getEsNames().indexAliasName, - namespace, logger: this.options.logger, }); } diff --git a/x-pack/test/apm_api_integration/common/config.ts b/x-pack/test/apm_api_integration/common/config.ts index 04ce83323ee66..732f14d2a7bc8 100644 --- a/x-pack/test/apm_api_integration/common/config.ts +++ b/x-pack/test/apm_api_integration/common/config.ts @@ -18,6 +18,7 @@ import { registry } from './registry'; interface Config { name: APMFtrConfigName; license: 'basic' | 'trial'; + kibanaConfig?: Record; } const supertestAsApmUser = (kibanaServer: UrlObject, apmUser: ApmUser) => async ( @@ -37,7 +38,7 @@ const supertestAsApmUser = (kibanaServer: UrlObject, apmUser: ApmUser) => async }; export function createTestConfig(config: Config) { - const { license, name } = config; + const { license, name, kibanaConfig } = config; return async ({ readConfigFile }: FtrConfigProviderContext) => { const xPackAPITestsConfig = await readConfigFile( @@ -79,7 +80,15 @@ export function createTestConfig(config: Config) { ...xPackAPITestsConfig.get('esTestCluster'), license, }, - kbnTestServer: xPackAPITestsConfig.get('kbnTestServer'), + kbnTestServer: { + ...xPackAPITestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'), + ...(kibanaConfig + ? Object.entries(kibanaConfig).map(([key, value]) => `--${key}=${value}`) + : []), + ], + }, }; }; } diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index 4eeb57e3c86c4..91437a2d22e27 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -15,6 +15,12 @@ const apmFtrConfigs = { trial: { license: 'trial' as const, }, + rules: { + license: 'trial' as const, + kibanaConfig: { + 'xpack.ruleRegistry.writeEnabled': 'true', + }, + }, }; export type APMFtrConfigName = keyof typeof apmFtrConfigs; diff --git a/x-pack/test/apm_api_integration/rules/config.ts b/x-pack/test/apm_api_integration/rules/config.ts new file mode 100644 index 0000000000000..9830d516eb80a --- /dev/null +++ b/x-pack/test/apm_api_integration/rules/config.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { configs } from '../configs'; + +export default configs.rules; diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts new file mode 100644 index 0000000000000..d87058ec28d73 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -0,0 +1,390 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { get, merge, omit } from 'lodash'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +interface Alert { + schedule: { + interval: string; + }; + updatedAt: string; + executionStatus: { + lastExecutionDate: string; + status: string; + }; + updatedBy: string; + id: string; + params: Record; + scheduledTaskId: string; +} + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertestAsApmWriteUser'); + const es = getService('es'); + + const MAX_POLLS = 5; + const BULK_INDEX_DELAY = 1000; + const INDEXING_DELAY = 5000; + + const ALERTS_INDEX_TARGET = '.kibana-alerts-*-apm*'; + const ALERTS_INDEX_NAME = '.kibana-alerts-observability-apm-8.0.0-000001'; + const APM_TRANSACTION_INDEX_NAME = 'apm-8.0.0-transaction'; + + const createTransactionEvent = (override: Record) => { + const now = Date.now(); + + const time = now - INDEXING_DELAY; + + return merge( + { + '@timestamp': new Date(time).toISOString(), + service: { + name: 'opbeans-go', + }, + event: { + outcome: 'success', + }, + transaction: { + duration: { + us: 1000000, + }, + type: 'request', + }, + processor: { + event: 'transaction', + }, + observer: { + version_major: 7, + }, + }, + override + ); + }; + + async function waitUntilNextExecution( + alert: Alert, + intervalInSeconds: number = 1, + count: number = 0 + ): Promise { + await new Promise((resolve) => { + setTimeout(resolve, intervalInSeconds * 1000); + }); + + const { body, status } = await supertest + .get(`/api/alerts/alert/${alert.id}`) + .set('kbn-xsrf', 'foo'); + + if (status >= 300) { + const error = new Error('Error getting alert'); + Object.assign(error, { response: { body, status } }); + throw error; + } + + const nextAlert = body as Alert; + + if (nextAlert.executionStatus.lastExecutionDate !== alert.executionStatus.lastExecutionDate) { + await new Promise((resolve) => { + setTimeout(resolve, BULK_INDEX_DELAY); + }); + await es.indices.refresh({ + index: ALERTS_INDEX_NAME, + }); + + return nextAlert; + } + + if (count >= MAX_POLLS) { + throw new Error('Maximum number of polls exceeded'); + } + + return waitUntilNextExecution(alert, intervalInSeconds, count + 1); + } + + registry.when('Rule registry with write enabled', { config: 'rules', archives: [] }, () => { + it('bootstraps the apm alert indices', async () => { + const { body } = await es.indices.get({ + index: ALERTS_INDEX_TARGET, + expand_wildcards: 'open', + allow_no_indices: false, + }); + + const indices = Object.entries(body).map(([indexName, index]) => { + return { + indexName, + index, + }; + }); + + const indexNames = indices.map((index) => index.indexName); + + const apmIndex = indices[0]; + + // make sure it only creates one index + expect(indices.length).to.be(1); + + const apmIndexName = apmIndex.indexName; + + expect(apmIndexName.split('-').includes('observability')).to.be(true); + expect(apmIndexName.split('-').includes('apm')).to.be(true); + + expectSnapshot(indexNames[0]).toMatchInline( + `".kibana-alerts-observability-apm-8.0.0-000001"` + ); + + expect(get(apmIndex, 'index.mappings.properties.service.properties.environment.type')).to.be( + 'keyword' + ); + }); + + describe('when creating a rule', () => { + let createResponse: { + alert: Alert; + status: number; + }; + + before(async () => { + await es.indices.create({ + index: APM_TRANSACTION_INDEX_NAME, + body: { + mappings: { + dynamic: 'strict', + properties: { + event: { + properties: { + outcome: { + type: 'keyword', + }, + }, + }, + processor: { + properties: { + event: { + type: 'keyword', + }, + }, + }, + observer: { + properties: { + version_major: { + type: 'byte', + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + }, + environment: { + type: 'keyword', + }, + }, + }, + transaction: { + properties: { + type: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + }, + }, + '@timestamp': { + type: 'date', + }, + }, + }, + }, + }); + + const body = { + params: { + threshold: 30, + windowSize: 5, + windowUnit: 'm', + transactionType: 'request', + environment: 'ENVIRONMENT_ALL', + serviceName: 'opbeans-go', + }, + consumer: 'apm', + alertTypeId: 'apm.transaction_error_rate', + schedule: { interval: '5s' }, + actions: [], + tags: ['apm', 'service.name:opbeans-go'], + notifyWhen: 'onActionGroupChange', + name: 'Transaction error rate threshold | opbeans-go', + }; + + const { body: response, status } = await supertest + .post('/api/alerts/alert') + .send(body) + .set('kbn-xsrf', 'foo'); + + createResponse = { + alert: response, + status, + }; + }); + + after(async () => { + if (createResponse.alert) { + const { body, status } = await supertest + .delete(`/api/alerts/alert/${createResponse.alert.id}`) + .set('kbn-xsrf', 'foo'); + + if (status >= 300) { + const error = new Error('Error deleting alert'); + Object.assign(error, { response: { body, status } }); + throw error; + } + } + + await es.deleteByQuery({ + index: ALERTS_INDEX_NAME, + body: { + query: { + match_all: {}, + }, + }, + refresh: true, + }); + + await es.indices.delete({ + index: APM_TRANSACTION_INDEX_NAME, + }); + }); + + it('writes alerts data to the alert indices', async () => { + expect(createResponse.status).to.be.below(299); + + expect(createResponse.alert).not.to.be(undefined); + + let alert = await waitUntilNextExecution(createResponse.alert); + + const beforeDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(beforeDataResponse.body.hits.hits.length).to.be(0); + + await es.index({ + index: APM_TRANSACTION_INDEX_NAME, + body: createTransactionEvent({ + event: { + outcome: 'success', + }, + }), + refresh: true, + }); + + alert = await waitUntilNextExecution(alert); + + const afterInitialDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(afterInitialDataResponse.body.hits.hits.length).to.be(0); + + await es.index({ + index: APM_TRANSACTION_INDEX_NAME, + body: createTransactionEvent({ + event: { + outcome: 'failure', + }, + }), + refresh: true, + }); + + alert = await waitUntilNextExecution(alert); + + const afterViolatingDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(afterViolatingDataResponse.body.hits.hits.length).to.be(1); + + const alertEvent = afterViolatingDataResponse.body.hits.hits[0]._source as Record< + string, + any + >; + + const toCompare = omit( + alertEvent, + '@timestamp', + 'kibana.rac.alert.start', + 'kibana.rac.alert.uuid', + 'rule.uuid' + ); + + expectSnapshot(toCompare).toMatchInline(` + Object { + "event.action": "open", + "event.kind": "state", + "kibana.rac.alert.duration.us": 0, + "kibana.rac.alert.id": "apm.transaction_error_rate_opbeans-go_request", + "kibana.rac.alert.status": "open", + "kibana.rac.producer": "apm", + "rule.category": "Transaction error rate threshold", + "rule.id": "apm.transaction_error_rate", + "rule.name": "Transaction error rate threshold | opbeans-go", + "service.name": "opbeans-go", + "tags": Array [ + "apm", + "service.name:opbeans-go", + ], + "transaction.type": "request", + } + `); + }); + }); + }); + + registry.when('Rule registry with write not enabled', { config: 'basic', archives: [] }, () => { + it('does not bootstrap the apm rule indices', async () => { + const errorOrUndefined = await es.indices + .get({ + index: ALERTS_INDEX_TARGET, + expand_wildcards: 'open', + allow_no_indices: false, + }) + .then(() => {}) + .catch((error) => { + return error.toString(); + }); + + expect(errorOrUndefined).not.to.be(undefined); + + expect(errorOrUndefined).to.be(`ResponseError: index_not_found_exception`); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index 7c69d5b996cea..53ec61b8d9b61 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -24,6 +24,10 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte loadTestFile(require.resolve('./alerts/chart_preview')); }); + describe('alerts/rule_registry', function () { + loadTestFile(require.resolve('./alerts/rule_registry')); + }); + describe('correlations/latency_slow_transactions', function () { loadTestFile(require.resolve('./correlations/latency_slow_transactions')); }); From 74d3eddcb04746bc3bcb92096c03f6f50cfb38a8 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 12:59:36 +0200 Subject: [PATCH 18/24] Revert change to getEsNames --- x-pack/plugins/event_log/server/es/names.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/names.ts b/x-pack/plugins/event_log/server/es/names.ts index 6540bf3676e03..2b98c7ceb95f7 100644 --- a/x-pack/plugins/event_log/server/es/names.ts +++ b/x-pack/plugins/event_log/server/es/names.ts @@ -17,11 +17,7 @@ export interface EsNames { indexTemplate: string; } -export function getEsNames( - baseName: string, - kibanaVersion: string, - suffix: string = EVENT_LOG_NAME_SUFFIX -): EsNames { +export function getEsNames(baseName: string, kibanaVersion: string): EsNames { const EVENT_LOG_VERSION_SUFFIX = `-${kibanaVersion.toLocaleLowerCase()}`; const eventLogName = `${baseName}${EVENT_LOG_NAME_SUFFIX}`; const eventLogNameWithVersion = `${eventLogName}${EVENT_LOG_VERSION_SUFFIX}`; From 45b5fffd8dc7aff4730c3fe2a6e92a07a5ddc3bb Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 14:33:33 +0200 Subject: [PATCH 19/24] Add rules integration tests to config --- x-pack/scripts/functional_tests.js | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 90306466a9753..1f6fe310bfa7c 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -34,6 +34,7 @@ const onlyNotInCoverageTests = [ require.resolve('../test/case_api_integration/basic/config.ts'), require.resolve('../test/apm_api_integration/basic/config.ts'), require.resolve('../test/apm_api_integration/trial/config.ts'), + require.resolve('../test/apm_api_integration/rules/config.ts'), require.resolve('../test/detection_engine_api_integration/security_and_spaces/config.ts'), require.resolve('../test/detection_engine_api_integration/basic/config.ts'), require.resolve('../test/lists_api_integration/security_and_spaces/config.ts'), From 4f43921e59eb4c6b894dce3abddd9cacfe948f4a Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 15:00:04 +0200 Subject: [PATCH 20/24] Export APMRuleRegistry type again --- x-pack/plugins/apm/server/plugin.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 96b7727c81df2..cb94b18a1ecf9 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -43,6 +43,8 @@ import { import { registerRoutes } from './routes/register_routes'; import { getGlobalApmServerRouteRepository } from './routes/get_global_apm_server_route_repository'; +export type APMRuleRegistry = ReturnType['ruleRegistry']; + export class APMPlugin implements Plugin< From 585e83afb97e3207a7f44de29393fb04697fa701 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 17:22:42 +0200 Subject: [PATCH 21/24] Lowercase index alias name --- x-pack/plugins/rule_registry/server/rule_registry/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index 1769373f239c4..dec9efdbd33b2 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -78,7 +78,7 @@ export class RuleRegistry { private getEsNames() { const base = [this.options.kibanaIndex, this.options.name]; - const indexAliasName = [...base, this.options.kibanaVersion].join('-'); + const indexAliasName = [...base, this.options.kibanaVersion.toLowerCase()].join('-'); const policyName = [...base, 'policy'].join('-'); return { @@ -131,9 +131,7 @@ export class RuleRegistry { createScopedRuleRegistryClient({ context, - request, }: { - request: KibanaRequest; context: RequestHandlerContext; }): ScopedRuleRegistryClient | undefined { if (!this.options.writeEnabled) { From 36d47863d52df389fd63126b808fe7b32f8ab7f4 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 17:28:49 +0200 Subject: [PATCH 22/24] Removed unused import --- x-pack/plugins/rule_registry/server/rule_registry/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index dec9efdbd33b2..fc3a2ab022815 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CoreSetup, KibanaRequest, Logger, RequestHandlerContext } from 'kibana/server'; +import { CoreSetup, Logger, RequestHandlerContext } from 'kibana/server'; import { inspect } from 'util'; import { SpacesServiceStart } from '../../../spaces/server'; import { From 31f97de365d268f1bc4dbb66fa6cdf019ee15adb Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 8 Apr 2021 20:17:00 +0200 Subject: [PATCH 23/24] Include alert types in SO client --- x-pack/plugins/rule_registry/server/index.ts | 2 ++ .../create_scoped_rule_registry_client/index.ts | 16 ++++++++++------ .../field_map/runtime_type_from_fieldmap.ts | 2 +- .../rule_registry/server/rule_registry/index.ts | 13 ++++++++++--- x-pack/plugins/rule_registry/server/types.ts | 5 +++++ .../tests/alerts/rule_registry.ts | 4 +--- 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 7b43ae675faf7..7c46717300819 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -13,6 +13,8 @@ export { RuleRegistryPluginSetupContract } from './plugin'; export { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; export { ecsFieldMap } from './generated/ecs_field_map'; export { pickWithPatterns } from './rule_registry/field_map/pick_with_patterns'; +export { FieldMapOf } from './types'; +export { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; export const config = { schema: schema.object({ diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts index 2a9eecc77ba58..9a3d4a38d2ad6 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -28,7 +28,9 @@ const getRuleUuids = async ({ ...(namespace ? { namespace } : {}), }; - const pitFinder = savedObjectsClient.createPointInTimeFinder(options); + const pitFinder = savedObjectsClient.createPointInTimeFinder({ + ...options, + }); const ruleUuids: string[] = []; @@ -53,7 +55,8 @@ export function createScopedRuleRegistryClient; index: string; }>; - index: string; + indexAliasName: string; + indexTarget: string; logger: Logger; ruleData?: { rule: { @@ -100,7 +104,7 @@ export function createScopedRuleRegistryClient { const validations = docs.map((doc) => { @@ -161,7 +165,7 @@ export function createScopedRuleRegistryClient (isRight(validation) ? validation.right : null)) - ).map((doc) => ({ body: doc, index })); + ).map((doc) => ({ body: doc, index: indexAliasName })); return clusterClientAdapter.indexDocuments(operations); }, diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts index 422747282beaa..6dc557c016d1a 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts @@ -89,7 +89,7 @@ export function runtimeTypeFromFieldMap( fieldMap: TFieldMap ): FieldMapType { function mapToType(fields: FieldMap) { - return mapValues(fields, (field) => { + return mapValues(fields, (field, key) => { const type = field.type in esFieldTypeMap ? esFieldTypeMap[field.type as keyof EsFieldTypeMap] diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts index fc3a2ab022815..f1d24550ade0a 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -78,11 +78,13 @@ export class RuleRegistry { private getEsNames() { const base = [this.options.kibanaIndex, this.options.name]; + const indexTarget = `${base.join('-')}*`; const indexAliasName = [...base, this.options.kibanaVersion.toLowerCase()].join('-'); const policyName = [...base, 'policy'].join('-'); return { indexAliasName, + indexTarget, policyName, }; } @@ -137,13 +139,15 @@ export class RuleRegistry { if (!this.options.writeEnabled) { return undefined; } + const { indexAliasName, indexTarget } = this.getEsNames(); return createScopedRuleRegistryClient({ - savedObjectsClient: context.core.savedObjects.client, + savedObjectsClient: context.core.savedObjects.getClient({ includedHiddenTypes: ['alert'] }), scopedClusterClient: context.core.elasticsearch.client, clusterClientAdapter: this.esAdapter, fieldMap: this.options.fieldMap, - index: this.getEsNames().indexAliasName, + indexAliasName, + indexTarget, logger: this.options.logger, }); } @@ -153,6 +157,8 @@ export class RuleRegistry { ) { const logger = this.options.logger.get(type.id); + const { indexAliasName, indexTarget } = this.getEsNames(); + this.options.alertingPluginSetupContract.registerType< AlertTypeParams, AlertTypeState, @@ -187,7 +193,8 @@ export class RuleRegistry { scopedClusterClient: services.scopedClusterClient, clusterClientAdapter: this.esAdapter, fieldMap: this.options.fieldMap, - index: this.getEsNames().indexAliasName, + indexAliasName, + indexTarget, namespace, ruleData: { producer, diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index da355a412207c..e6b53a8558964 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -14,6 +14,7 @@ import { AlertTypeState, } from '../../alerting/common'; import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; +import { RuleRegistry } from './rule_registry'; import { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; import { DefaultFieldMap } from './rule_registry/defaults/field_map'; @@ -93,3 +94,7 @@ export type RuleType< TAdditionalRuleExecutorServices >; }; + +export type FieldMapOf< + TRuleRegistry extends RuleRegistry +> = TRuleRegistry extends RuleRegistry ? TFieldMap : never; diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts index d87058ec28d73..ff4d36d9af72f 100644 --- a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -134,9 +134,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(apmIndexName.split('-').includes('observability')).to.be(true); expect(apmIndexName.split('-').includes('apm')).to.be(true); - expectSnapshot(indexNames[0]).toMatchInline( - `".kibana-alerts-observability-apm-8.0.0-000001"` - ); + expect(indexNames[0].startsWith('.kibana-alerts-observability-apm')).to.be(true); expect(get(apmIndex, 'index.mappings.properties.service.properties.environment.type')).to.be( 'keyword' From 11679efca4869b03029e751bc4f403c1fa4c0e85 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 9 Apr 2021 08:19:57 +0200 Subject: [PATCH 24/24] Use wildcard to prevent version mismatch in index target --- .../test/apm_api_integration/tests/alerts/rule_registry.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts index ff4d36d9af72f..97026d126d2a1 100644 --- a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -34,7 +34,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { const INDEXING_DELAY = 5000; const ALERTS_INDEX_TARGET = '.kibana-alerts-*-apm*'; - const ALERTS_INDEX_NAME = '.kibana-alerts-observability-apm-8.0.0-000001'; const APM_TRANSACTION_INDEX_NAME = 'apm-8.0.0-transaction'; const createTransactionEvent = (override: Record) => { @@ -94,7 +93,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { setTimeout(resolve, BULK_INDEX_DELAY); }); await es.indices.refresh({ - index: ALERTS_INDEX_NAME, + index: ALERTS_INDEX_TARGET, }); return nextAlert; @@ -250,7 +249,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { } await es.deleteByQuery({ - index: ALERTS_INDEX_NAME, + index: ALERTS_INDEX_TARGET, body: { query: { match_all: {},