[Security] Alert Telemetry for the Security app

x-pack/plugins/security_solution/kibana.json

@@ -27,7 +27,8 @@
     "spaces",
     "usageCollection",
     "lists",
-    "home"
+    "home",
+    "telemetry"
   ],
   "server": true,
   "ui": true,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -129,6 +129,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -254,6 +255,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -333,6 +335,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -393,6 +396,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -451,6 +455,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -534,6 +539,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -614,6 +620,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -663,6 +670,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleParams: sampleParams,
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -718,6 +726,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleParams: sampleParams,
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -789,6 +798,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleParams: sampleParams,
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts

@@ -17,6 +17,8 @@ import { SignalSearchResponse } from './types';
 import { filterEventsAgainstList } from './filter_events_with_list';
 import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
 import { getSignalTimeTuples } from './utils';
+import { TelemetryEventsSender } from '../../telemetry/sender';
+import { sendAlertTelemetryEvents } from './send_telemetry_events';
 
 interface SearchAfterAndBulkCreateParams {
   gap: moment.Duration | null;
@@ -26,6 +28,7 @@ interface SearchAfterAndBulkCreateParams {
   listClient: ListClient;
   exceptionsList: ExceptionListItemSchema[];
   logger: Logger;
+  eventsTelemetry: TelemetryEventsSender | undefined;
   id: string;
   inputIndexPattern: string[];
   signalsIndex: string;
@@ -62,6 +65,7 @@ export const searchAfterAndBulkCreate = async ({
   services,
   listClient,
   logger,
+  eventsTelemetry,
   id,
   inputIndexPattern,
   signalsIndex,
@@ -226,6 +230,16 @@ export const searchAfterAndBulkCreate = async ({
             toReturn.bulkCreateTimes.push(bulkDuration);
           }
 
+          if (eventsTelemetry !== undefined) {
+            sendAlertTelemetryEvents(
+              logger,
+              eventsTelemetry,
+              filteredEvents,
+              ruleParams,
+              buildRuleMessage
+            );
+          }
+
           logger.debug(
             buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`)
           );

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { selectEvents } from './send_telemetry_events';
+
+describe('sendAlertTelemetry', () => {
+  it('selectEvents', () => {
+    const filteredEvents = {
+      took: 0,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        failed: 0,
+        skipped: 0,
+      },
+      hits: {
+        total: 2,
+        max_score: 0,
+        hits: [
+          {
+            _index: 'x',
+            _type: 'x',
+            _id: 'x',
+            _score: 0,
+            _source: {
+              '@timestamp': 'x',
+              key1: 'hello',
+              datastream: {
+                dataset: 'endpoint.events',
+              },
+            },
+          },
+          {
+            _index: 'x',
+            _type: 'x',
+            _id: 'x',
+            _score: 0,
+            _source: {
+              '@timestamp': 'x',
+              key2: 'hello',
+              datastream: {
+                dataset: 'endpoint.alerts',
+                other: 'x',
+              },
+            },
+          },
+          {
+            _index: 'x',
+            _type: 'x',
+            _id: 'x',
+            _score: 0,
+            _source: {
+              '@timestamp': 'x',
+              key3: 'hello',
+              datastream: {},
+            },
+          },
+        ],
+      },
+    };
+
+    const sources = selectEvents(filteredEvents);
+    expect(sources).toStrictEqual([
+      {
+        '@timestamp': 'x',
+        key2: 'hello',
+        datastream: {
+          dataset: 'endpoint.alerts',
+          other: 'x',
+        },
+      },
+    ]);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TelemetryEventsSender, TelemetryEvent } from '../../telemetry/sender';
+import { RuleTypeParams } from '../types';
+import { BuildRuleMessage } from './rule_messages';
+import { SignalSearchResponse, SignalSource } from './types';
+import { Logger } from '../../../../../../../src/core/server';
+
+export interface SearchResultWithSource {
+  _source: SignalSource;
+}
+
+export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEvent[] {
+  const sources = filteredEvents.hits.hits.map(function (
+    obj: SearchResultWithSource
+  ): TelemetryEvent {
+    return obj._source;
+  });
+
+  return sources.filter(function (obj: TelemetryEvent) {
+    // Filter out non-endpoint alerts
+    return obj.datastream?.dataset === 'endpoint.alerts';
+  });
+}
+
+export function sendAlertTelemetryEvents(
+  logger: Logger,
+  eventsTelemetry: TelemetryEventsSender,
+  filteredEvents: SignalSearchResponse,
+  ruleParams: RuleTypeParams,
+  buildRuleMessage: BuildRuleMessage
+) {
+  const sources = selectEvents(filteredEvents);
+
+  try {
+    eventsTelemetry.queueTelemetryEvents(sources);
+  } catch (exc) {
+    logger.error(buildRuleMessage(`[-] queing telemetry events failed ${exc}`));
+  }
+}

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts

@@ -129,6 +129,7 @@ describe('rules_notification_alert_type', () => {
 
     alert = signalRulesAlertType({
       logger,
+      eventsTelemetry: undefined,
       version,
       ml: mlMock,
       lists: listMock.createSetup(),
@@ -335,6 +336,7 @@ describe('rules_notification_alert_type', () => {
         payload = getPayload(ruleAlert, alertServices) as jest.Mocked<RuleExecutorOptions>;
         alert = signalRulesAlertType({
           logger,
+          eventsTelemetry: undefined,
           version,
           ml: undefined,
           lists: undefined,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -45,14 +45,17 @@ import { ruleStatusServiceFactory } from './rule_status_service';
 import { buildRuleMessageFactory } from './rule_messages';
 import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client';
 import { getNotificationResultsLink } from '../notifications/utils';
+import { TelemetryEventsSender } from '../../telemetry/sender';
 
 export const signalRulesAlertType = ({
   logger,
+  eventsTelemetry,
   version,
   ml,
   lists,
 }: {
   logger: Logger;
+  eventsTelemetry: TelemetryEventsSender | undefined;
   version: string;
   ml: SetupPlugins['ml'];
   lists: SetupPlugins['lists'] | undefined;
@@ -322,6 +325,7 @@ export const signalRulesAlertType = ({
             ruleParams: params,
             services,
             logger,
+            eventsTelemetry,
             id: alertId,
             inputIndexPattern: inputIndex,
             signalsIndex: outputIndex,