From 9de21e469b9234cd1ad334c47d0f2eed47a33646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 20 Nov 2023 18:18:01 +0100 Subject: [PATCH 01/12] Diagnostic timelines task --- .../server/lib/telemetry/receiver.ts | 23 +- .../lib/telemetry/tasks/detection_rule.ts | 4 +- .../server/lib/telemetry/tasks/index.ts | 2 + .../server/lib/telemetry/tasks/timelines.ts | 7 +- .../telemetry/tasks/timelines_diagnostic.ts | 211 ++++++++++++++++++ 5 files changed, 240 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index c699f6a1e9698..bd74e285f0d6f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -165,6 +165,10 @@ export interface ITelemetryReceiver { fetchTimelineEndpointAlerts(interval: number): Promise>>; + fetchDiagnosticTimelineEndpointAlerts( + interval: number + ): Promise>>; + buildProcessTree( entityId: string, resolverSchema: ResolverSchema, @@ -697,7 +701,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return { events: telemetryEvents, count: aggregations?.prebuilt_rule_alert_count.value ?? 0 }; } - public async fetchTimelineEndpointAlerts(interval: number) { + private async fetchTimelineAlerts(index: string | undefined, interval: number) { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); } @@ -708,7 +712,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { // create and assign an initial point in time let pitId: OpenPointInTimeResponse['id'] = ( await this.esClient.openPointInTime({ - index: `${this.alertsIndex}*`, + index: `${index}*`, keep_alive: keepAlive, }) ).id; @@ -807,7 +811,20 @@ export class TelemetryReceiver implements ITelemetryReceiver { } tlog(this.logger, `Timeline alerts to return: ${alertsToReturn.length}`); - return alertsToReturn; + return alertsToReturn || []; + } + + public async fetchTimelineEndpointAlerts(interval: number) { + return this.fetchTimelineAlerts(this.alertsIndex, interval); + } + + public fetchDiagnosticTimelineEndpointAlerts( + interval: number + ): Promise>> { + // TODO: confirm that this is the proper index and move to + // another place, either config or constant + const index = '.logs-endpoint.diagnostic.collection-*'; + return this.fetchTimelineAlerts(index, interval); } public async buildProcessTree( diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index 4562cbb725cb4..2de4703dbb4ae 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -25,11 +25,11 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n timeout: '10m', version: '1.0.0', runTask: async ( - taskId: string, + _taskId: string, logger: Logger, receiver: ITelemetryReceiver, sender: ITelemetryEventsSender, - taskExecutionPeriod: TaskExecutionPeriod + _taskExecutionPeriod: TaskExecutionPeriod ) => { const startTime = Date.now(); const taskName = 'Security Solution Detection Rule Lists Telemetry'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts index e25b3690ee88d..d237757616f3e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -12,6 +12,7 @@ import { createTelemetrySecurityListTaskConfig } from './security_lists'; import { createTelemetryDetectionRuleListsTaskConfig } from './detection_rule'; import { createTelemetryPrebuiltRuleAlertsTaskConfig } from './prebuilt_rule_alerts'; import { createTelemetryTimelineTaskConfig } from './timelines'; +import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnostic'; import { createTelemetryConfigurationTaskConfig } from './configuration'; import { telemetryConfiguration } from '../configuration'; import { createTelemetryFilterListArtifactTaskConfig } from './filterlists'; @@ -26,6 +27,7 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { ), createTelemetryPrebuiltRuleAlertsTaskConfig(telemetryConfiguration.max_detection_alerts_batch), createTelemetryTimelineTaskConfig(), + createTelemetryDiagnosticTimelineTaskConfig(), createTelemetryConfigurationTaskConfig(), createTelemetryFilterListArtifactTaskConfig(), ]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts index bd50ffcd92817..95028d01f5c0b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts @@ -22,9 +22,11 @@ import { resolverEntity } from '../../../endpoint/routes/resolver/entity/utils/b import { tlog, createTaskMetric } from '../helpers'; export function createTelemetryTimelineTaskConfig() { + const taskName = 'Security Solution Timeline telemetry'; + return { type: 'security:telemetry-timelines', - title: 'Security Solution Timeline telemetry', + title: taskName, interval: '3h', timeout: '10m', version: '1.0.0', @@ -36,7 +38,6 @@ export function createTelemetryTimelineTaskConfig() { taskExecutionPeriod: TaskExecutionPeriod ) => { const startTime = Date.now(); - const taskName = 'Security Solution Timeline telemetry'; try { let counter = 0; @@ -79,6 +80,8 @@ export function createTelemetryTimelineTaskConfig() { createTaskMetric(taskName, true, startTime), ]); return counter; + } else { + tlog(logger, `${endpointAlerts.length} alerts received. Processing...`); } // Build process tree for each EP Alert recieved diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts new file mode 100644 index 0000000000000..e7b97be57830d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { Logger } from '@kbn/core/server'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SafeEndpointEvent } from '../../../../common/endpoint/types'; +import type { ITelemetryEventsSender } from '../sender'; +import type { ITelemetryReceiver } from '../receiver'; +import type { TaskExecutionPeriod } from '../task'; +import type { + EnhancedAlertEvent, + ESClusterInfo, + ESLicense, + TimelineTelemetryTemplate, + TimelineTelemetryEvent, +} from '../types'; +import { TELEMETRY_CHANNEL_TIMELINE, TASK_METRICS_CHANNEL } from '../constants'; +import { resolverEntity } from '../../../endpoint/routes/resolver/entity/utils/build_resolver_entity'; +import { tlog, createTaskMetric } from '../helpers'; + +interface ExtraInfo { + clusterInfo: ESClusterInfo; + licenseInfo: ESLicense | undefined; +} +interface TimeFrame { + startOfDay: string; + endOfDay: string; +} +interface TimelineResult { + nodes: number; + events: number; + timeline: TimelineTelemetryTemplate | undefined; +} + +export function createTelemetryDiagnosticTimelineTaskConfig() { + const taskName = 'Security Solution Diagnostic Timeline telemetry'; + + return { + type: 'security:telemetry-diagnostic-timelines', + title: taskName, + interval: '3h', + timeout: '10m', + version: '1.0.0', + runTask: async ( + taskId: string, + logger: Logger, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, + taskExecutionPeriod: TaskExecutionPeriod + ) => { + tlog( + logger, + `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` + ); + + const fetcher = new TelemetryTimelineFetcher(receiver); + + try { + let counter = 0; + + const alerts = await receiver.fetchDiagnosticTimelineEndpointAlerts(3); + + tlog(logger, `found ${alerts.length} alerts to process`); + + for (const alert of alerts) { + const result = await fetcher.fetchTimeline(alert); + + sender.getTelemetryUsageCluster()?.incrementCounter({ + counterName: 'telemetry_timeline', + counterType: 'timeline_node_count', + incrementBy: result.nodes, + }); + + sender.getTelemetryUsageCluster()?.incrementCounter({ + counterName: 'telemetry_timeline', + counterType: 'timeline_event_count', + incrementBy: result.events, + }); + + if (result.timeline) { + sender.sendOnDemand(TELEMETRY_CHANNEL_TIMELINE, [result.timeline]); + counter += 1; + } else { + tlog(logger, 'no events in timeline'); + } + } + + tlog(logger, `sent ${counter} timelines. Concluding timeline task.`); + + await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ + createTaskMetric(taskName, true, fetcher.startTime), + ]); + + return counter; + } catch (err) { + await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ + createTaskMetric(taskName, false, fetcher.startTime, err.message), + ]); + return 0; + } + }, + }; +} + +class TelemetryTimelineFetcher { + startTime: number; + private receiver: ITelemetryReceiver; + private extraInfo: Promise; + private timeFrame: TimeFrame; + + constructor(receiver: ITelemetryReceiver) { + this.receiver = receiver; + this.startTime = Date.now(); + this.extraInfo = this.lookupExtraInfo(); + this.timeFrame = this.calculateTimeFrame(); + } + + async fetchTimeline(event: estypes.SearchHit): Promise { + const eventId = event._source ? event._source['event.id'] : 'unknown'; + const alertUUID = event._source ? event._source['kibana.alert.uuid'] : 'unknown'; + + const entities = resolverEntity([event]); + + // Build Tree + const tree = await this.receiver.buildProcessTree( + entities[0].id, + entities[0].schema, + this.timeFrame.startOfDay, + this.timeFrame.endOfDay + ); + + const nodeIds = Array.isArray(tree) ? tree.map((node) => node?.id.toString()) : []; + + const eventsStore = await this.fetchEventLineage(nodeIds); + + const telemetryTimeline: TimelineTelemetryEvent[] = Array.isArray(tree) + ? tree.map((node) => { + return { + ...node, + event: eventsStore.get(node.id.toString()), + }; + }) + : []; + + let record; + if (telemetryTimeline.length >= 1) { + const { clusterInfo, licenseInfo } = await this.extraInfo; + record = { + '@timestamp': moment().toISOString(), + version: clusterInfo.version?.number, + cluster_name: clusterInfo.cluster_name, + cluster_uuid: clusterInfo.cluster_uuid, + license_uuid: licenseInfo?.uid, + alert_id: alertUUID, + event_id: eventId, + timeline: telemetryTimeline, + }; + } + + const result: TimelineResult = { + nodes: nodeIds.length, + events: eventsStore.size, + timeline: record, + }; + + return result; + } + + private async fetchEventLineage(nodeIds: string[]): Promise> { + const timelineEvents = await this.receiver.fetchTimelineEvents(nodeIds); + const eventsStore = new Map(); + for (const event of timelineEvents.hits.hits) { + const doc = event._source; + + if (doc !== null && doc !== undefined) { + const entityId = doc?.process?.entity_id?.toString(); + if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc); + } + } + return eventsStore; + } + + private async lookupExtraInfo(): Promise { + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + this.receiver.fetchClusterInfo(), + this.receiver.fetchLicenseInfo(), + ]); + + const clusterInfo: ESClusterInfo | undefined = + clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo); + + const licenseInfo: ESLicense | undefined = + licenseInfoPromise.status === 'fulfilled' + ? licenseInfoPromise.value + : ({} as ESLicense | undefined); + + return { clusterInfo, licenseInfo }; + } + + private calculateTimeFrame(): TimeFrame { + const now = moment(); + const startOfDay = now.startOf('day').toISOString(); + const endOfDay = now.endOf('day').toISOString(); + return { startOfDay, endOfDay }; + } +} From de6cfde5e837f4a77dfec4da29cab17934e1fc65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 10:54:00 +0100 Subject: [PATCH 02/12] Fix test --- .../test_suites/task_manager/check_registered_task_types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 7e48895a9060f..2217310e4e6ae 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -145,6 +145,7 @@ export default function ({ getService }: FtrProviderContext) { 'security:endpoint-meta-telemetry', 'security:telemetry-configuration', 'security:telemetry-detection-rules', + 'security:telemetry-diagnostic-timelines', 'security:telemetry-filterlist-artifact', 'security:telemetry-lists', 'security:telemetry-prebuilt-rule-alerts', From e8da4467e4f76a5d2d7e096d357fd8ffec5f5936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 13:58:55 +0100 Subject: [PATCH 03/12] Add unit tests --- .../server/lib/telemetry/__mocks__/index.ts | 5 +- .../tasks/timelines_diagnostic.test.ts | 64 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index f5718895fff26..e3a034bf51c27 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -71,7 +71,7 @@ export const stubLicenseInfo: ESLicense = { export const createMockTelemetryReceiver = ( diagnosticsAlert?: unknown, emptyTimelineTree?: boolean -): jest.Mocked => { +): jest.Mocked => { const processTreeResponse = emptyTimelineTree ? Promise.resolve([]) : Promise.resolve(Promise.resolve(stubProcessTree())); @@ -94,6 +94,9 @@ export const createMockTelemetryReceiver = ( fetchTimelineEndpointAlerts: jest .fn() .mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), + fetchDiagnosticTimelineEndpointAlerts: jest + .fn() + .mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), buildProcessTree: jest.fn().mockReturnValue(processTreeResponse), fetchTimelineEvents: jest.fn().mockReturnValue(Promise.resolve(stubFetchTimelineEvents())), fetchValueListMetaData: jest.fn(), diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts new file mode 100644 index 0000000000000..87beb283b4e08 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnostic'; +import { createMockTelemetryEventsSender, createMockTelemetryReceiver } from '../__mocks__'; + +describe('timeline telemetry diagnostic task test', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + test('timeline telemetry task should be correctly set up', async () => { + const testTaskExecutionPeriod = { + last: undefined, + current: new Date().toISOString(), + }; + const mockTelemetryEventsSender = createMockTelemetryEventsSender(); + const mockTelemetryReceiver = createMockTelemetryReceiver(); + const telemetryTelemetryDiagnosticTaskConfig = createTelemetryDiagnosticTimelineTaskConfig(); + + await telemetryTelemetryDiagnosticTaskConfig.runTask( + 'test-timeline-diagnostic-task-id', + logger, + mockTelemetryReceiver, + mockTelemetryEventsSender, + testTaskExecutionPeriod + ); + + expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchDiagnosticTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryEventsSender.getTelemetryUsageCluster).toHaveBeenCalled(); + expect(mockTelemetryEventsSender.sendOnDemand).toHaveBeenCalled(); + }); + + test('if no timeline events received it should not send a telemetry record', async () => { + const testTaskExecutionPeriod = { + last: undefined, + current: new Date().toISOString(), + }; + const mockTelemetryEventsSender = createMockTelemetryEventsSender(); + const mockTelemetryReceiver = createMockTelemetryReceiver(null, true); + const telemetryTelemetryDiagnosticTaskConfig = createTelemetryDiagnosticTimelineTaskConfig(); + + await telemetryTelemetryDiagnosticTaskConfig.runTask( + 'test-timeline-diagnostic-task-id', + logger, + mockTelemetryReceiver, + mockTelemetryEventsSender, + testTaskExecutionPeriod + ); + + expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchDiagnosticTimelineEndpointAlerts).toHaveBeenCalled(); + }); +}); From 28ee06366ed188929a87efd0e46325d0e471a286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 14:01:17 +0100 Subject: [PATCH 04/12] Apply PR comments --- .../security_solution/common/constants.ts | 1 + .../server/lib/telemetry/receiver.ts | 31 +++++++++++-------- .../lib/telemetry/tasks/detection_rule.ts | 14 ++++++--- .../server/lib/telemetry/tasks/timelines.ts | 5 +-- .../telemetry/tasks/timelines_diagnostic.ts | 15 ++++----- 5 files changed, 40 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index a7460bcd70345..77f38264d23df 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -41,6 +41,7 @@ export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const; export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults' as const; export const DEFAULT_ALERTS_INDEX = '.alerts-security.alerts' as const; +export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const; export const DEFAULT_SIGNALS_INDEX = '.siem-signals' as const; export const DEFAULT_PREVIEW_INDEX = '.preview.alerts-security.alerts' as const; export const DEFAULT_LISTS_INDEX = '.lists' as const; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index bd74e285f0d6f..a1278f4266830 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -68,7 +68,7 @@ import type { ValueListIndicatorMatchResponseAggregation, } from './types'; import { telemetryConfiguration } from './configuration'; -import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; +import { DEFAULT_DIAGNOSTIC_INDEX, ENDPOINT_METRICS_INDEX } from '../../../common/constants'; import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants'; export interface ITelemetryReceiver { @@ -163,10 +163,14 @@ export interface ITelemetryReceiver { fetchPrebuiltRuleAlerts(): Promise<{ events: TelemetryEvent[]; count: number }>; - fetchTimelineEndpointAlerts(interval: number): Promise>>; + fetchTimelineEndpointAlerts( + rangeFrom: string, + rangeTo: string + ): Promise>>; fetchDiagnosticTimelineEndpointAlerts( - interval: number + rangeFrom: string, + rangeTo: string ): Promise>>; buildProcessTree( @@ -400,7 +404,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { const query = { expand_wildcards: ['open' as const, 'hidden' as const], - index: '.logs-endpoint.diagnostic.collection-*', + index: `${DEFAULT_DIAGNOSTIC_INDEX}-*`, ignore_unavailable: true, size: telemetryConfiguration.telemetry_max_buffer_size, body: { @@ -701,7 +705,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return { events: telemetryEvents, count: aggregations?.prebuilt_rule_alert_count.value ?? 0 }; } - private async fetchTimelineAlerts(index: string | undefined, interval: number) { + private async fetchTimelineAlerts(index: string, rangeFrom: string, rangeTo: string) { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); } @@ -750,8 +754,8 @@ export class TelemetryReceiver implements ITelemetryReceiver { { range: { '@timestamp': { - gte: `now-${interval}h`, - lte: 'now', + gte: rangeFrom, + lte: rangeTo, }, }, }, @@ -814,17 +818,18 @@ export class TelemetryReceiver implements ITelemetryReceiver { return alertsToReturn || []; } - public async fetchTimelineEndpointAlerts(interval: number) { - return this.fetchTimelineAlerts(this.alertsIndex, interval); + public async fetchTimelineEndpointAlerts(rangeFrom: string, rangeTo: string) { + if (this.alertsIndex === undefined) { + throw Error('alerts index is not ready yet, skipping telemetry task'); + } + + return this.fetchTimelineAlerts(this.alertsIndex, rangeFrom, rangeTo); } public fetchDiagnosticTimelineEndpointAlerts( interval: number ): Promise>> { - // TODO: confirm that this is the proper index and move to - // another place, either config or constant - const index = '.logs-endpoint.diagnostic.collection-*'; - return this.fetchTimelineAlerts(index, interval); + return this.fetchTimelineAlerts(DEFAULT_DIAGNOSTIC_INDEX, interval); } public async buildProcessTree( diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index 2de4703dbb4ae..685a2de9611aa 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -21,18 +21,24 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n return { type: 'security:telemetry-detection-rules', title: 'Security Solution Detection Rule Lists Telemetry', - interval: '24h', - timeout: '10m', + interval: '1m', + timeout: '15m', version: '1.0.0', runTask: async ( - _taskId: string, + taskId: string, logger: Logger, receiver: ITelemetryReceiver, sender: ITelemetryEventsSender, - _taskExecutionPeriod: TaskExecutionPeriod + taskExecutionPeriod: TaskExecutionPeriod ) => { const startTime = Date.now(); const taskName = 'Security Solution Detection Rule Lists Telemetry'; + + tlog( + logger, + `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` + ); + try { const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ receiver.fetchClusterInfo(), diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts index 95028d01f5c0b..47e04d66ce881 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts @@ -70,8 +70,9 @@ export function createTelemetryTimelineTaskConfig() { }; // Fetch EP Alerts - - const endpointAlerts = await receiver.fetchTimelineEndpointAlerts(3); + const rangeFrom = taskExecutionPeriod.last ?? 'now-3h'; + const rangeTo = taskExecutionPeriod.current; + const endpointAlerts = await receiver.fetchTimelineEndpointAlerts(rangeFrom, rangeTo); // No EP Alerts -> Nothing to do if (endpointAlerts.length === 0 || endpointAlerts.length === undefined) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts index e7b97be57830d..1f815b71c6ff5 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -43,8 +43,8 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { return { type: 'security:telemetry-diagnostic-timelines', title: taskName, - interval: '3h', - timeout: '10m', + interval: '1m', + timeout: '15m', version: '1.0.0', runTask: async ( taskId: string, @@ -63,7 +63,10 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { try { let counter = 0; - const alerts = await receiver.fetchDiagnosticTimelineEndpointAlerts(3); + const rangeFrom = taskExecutionPeriod.last ?? 'now-3h'; + const rangeTo = taskExecutionPeriod.current; + + const alerts = await receiver.fetchDiagnosticTimelineEndpointAlerts(rangeFrom, rangeTo); tlog(logger, `found ${alerts.length} alerts to process`); @@ -191,13 +194,11 @@ class TelemetryTimelineFetcher { this.receiver.fetchLicenseInfo(), ]); - const clusterInfo: ESClusterInfo | undefined = + const clusterInfo: ESClusterInfo = clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo); const licenseInfo: ESLicense | undefined = - licenseInfoPromise.status === 'fulfilled' - ? licenseInfoPromise.value - : ({} as ESLicense | undefined); + licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value : ({} as ESLicense); return { clusterInfo, licenseInfo }; } From 54ac7c67f5dc75e0f77bf8de2ba4e4921776b205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 14:06:19 +0100 Subject: [PATCH 05/12] Apply PR comments --- .../server/lib/telemetry/tasks/detection_rule.ts | 4 ++-- .../server/lib/telemetry/tasks/timelines_diagnostic.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index 685a2de9611aa..8ccde7cd1946e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -21,8 +21,8 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n return { type: 'security:telemetry-detection-rules', title: 'Security Solution Detection Rule Lists Telemetry', - interval: '1m', - timeout: '15m', + interval: '24h', + timeout: '10m', version: '1.0.0', runTask: async ( taskId: string, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts index 1f815b71c6ff5..b7cdf06eaa336 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -43,7 +43,7 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { return { type: 'security:telemetry-diagnostic-timelines', title: taskName, - interval: '1m', + interval: '1h', timeout: '15m', version: '1.0.0', runTask: async ( From 66cf1c120c2c8f198f56e5bb4b2151b695ac4099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 14:52:08 +0100 Subject: [PATCH 06/12] Fix validation --- .../security_solution/server/lib/telemetry/receiver.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index a1278f4266830..d5150ccfaa07d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -827,9 +827,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public fetchDiagnosticTimelineEndpointAlerts( - interval: number + rangeFrom: string, + rangeTo: string ): Promise>> { - return this.fetchTimelineAlerts(DEFAULT_DIAGNOSTIC_INDEX, interval); + return this.fetchTimelineAlerts(DEFAULT_DIAGNOSTIC_INDEX, rangeFrom, rangeTo); } public async buildProcessTree( From b89d3c86fe90f83df8f6bac63d0262d48bc3ba30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 15:02:36 +0100 Subject: [PATCH 07/12] Fix validation --- .../security_solution/server/lib/telemetry/__mocks__/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index e3a034bf51c27..7088c8770d648 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -9,7 +9,7 @@ import moment from 'moment'; import type { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server'; import { TaskStatus } from '@kbn/task-manager-plugin/server'; import type { TelemetryEventsSender } from '../sender'; -import type { TelemetryReceiver } from '../receiver'; +import type { ITelemetryReceiver, TelemetryReceiver } from '../receiver'; import type { SecurityTelemetryTaskConfig } from '../task'; import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/package_policy'; import { stubEndpointAlertResponse, stubProcessTree, stubFetchTimelineEvents } from './timeline'; From f4ab9000ebe64f45345c56a2f056180afc3fe238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 21 Nov 2023 15:32:26 +0100 Subject: [PATCH 08/12] Refactoring to reuse timelines diagnostic logic --- .../server/lib/telemetry/helpers.ts | 126 ++++++++++++++- .../server/lib/telemetry/tasks/timelines.ts | 150 +++--------------- .../telemetry/tasks/timelines_diagnostic.ts | 131 +-------------- .../server/lib/telemetry/types.ts | 16 ++ 4 files changed, 167 insertions(+), 256 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index f5d6bc41ee349..3bd6a0d595a79 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -11,17 +11,25 @@ import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/packag import { merge, set } from 'lodash'; import type { Logger } from '@kbn/core/server'; import { sha256 } from 'js-sha256'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { copyAllowlistedFields, filterList } from './filterlists'; -import type { PolicyConfig, PolicyData } from '../../../common/endpoint/types'; +import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types'; +import type { ITelemetryReceiver } from './receiver'; import type { - ExceptionListItem, + EnhancedAlertEvent, ESClusterInfo, ESLicense, + ExceptionListItem, + ExtraInfo, ListTemplate, + TaskMetric, TelemetryEvent, + TimeFrame, + TimelineResult, + TimelineTelemetryEvent, ValueListResponse, - TaskMetric, } from './types'; +import type { TaskExecutionPeriod } from './task'; import { LIST_DETECTION_RULE_EXCEPTION, LIST_ENDPOINT_EXCEPTION, @@ -30,6 +38,7 @@ import { DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS, } from './constants'; import { tagsToEffectScope } from '../../../common/endpoint/service/trusted_apps/mapping'; +import { resolverEntity } from '../../endpoint/routes/resolver/entity/utils/build_resolver_entity'; /** * Determines the when the last run was in order to execute to. @@ -345,3 +354,114 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T return event; }; + +export const ranges = ( + taskExecutionPeriod: TaskExecutionPeriod, + defaultIntervalInHours: number = 3 +) => { + const rangeFrom = taskExecutionPeriod.last ?? `now-${defaultIntervalInHours}h`; + const rangeTo = taskExecutionPeriod.current; + + return { rangeFrom, rangeTo }; +}; + +export class TelemetryTimelineFetcher { + startTime: number; + private receiver: ITelemetryReceiver; + private extraInfo: Promise; + private timeFrame: TimeFrame; + + constructor(receiver: ITelemetryReceiver) { + this.receiver = receiver; + this.startTime = Date.now(); + this.extraInfo = this.lookupExtraInfo(); + this.timeFrame = this.calculateTimeFrame(); + } + + async fetchTimeline(event: estypes.SearchHit): Promise { + const eventId = event._source ? event._source['event.id'] : 'unknown'; + const alertUUID = event._source ? event._source['kibana.alert.uuid'] : 'unknown'; + + const entities = resolverEntity([event]); + + // Build Tree + const tree = await this.receiver.buildProcessTree( + entities[0].id, + entities[0].schema, + this.timeFrame.startOfDay, + this.timeFrame.endOfDay + ); + + const nodeIds = Array.isArray(tree) ? tree.map((node) => node?.id.toString()) : []; + + const eventsStore = await this.fetchEventLineage(nodeIds); + + const telemetryTimeline: TimelineTelemetryEvent[] = Array.isArray(tree) + ? tree.map((node) => { + return { + ...node, + event: eventsStore.get(node.id.toString()), + }; + }) + : []; + + let record; + if (telemetryTimeline.length >= 1) { + const { clusterInfo, licenseInfo } = await this.extraInfo; + record = { + '@timestamp': moment().toISOString(), + version: clusterInfo.version?.number, + cluster_name: clusterInfo.cluster_name, + cluster_uuid: clusterInfo.cluster_uuid, + license_uuid: licenseInfo?.uid, + alert_id: alertUUID, + event_id: eventId, + timeline: telemetryTimeline, + }; + } + + const result: TimelineResult = { + nodes: nodeIds.length, + events: eventsStore.size, + timeline: record, + }; + + return result; + } + + private async fetchEventLineage(nodeIds: string[]): Promise> { + const timelineEvents = await this.receiver.fetchTimelineEvents(nodeIds); + const eventsStore = new Map(); + for (const event of timelineEvents.hits.hits) { + const doc = event._source; + + if (doc !== null && doc !== undefined) { + const entityId = doc?.process?.entity_id?.toString(); + if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc); + } + } + return eventsStore; + } + + private async lookupExtraInfo(): Promise { + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + this.receiver.fetchClusterInfo(), + this.receiver.fetchLicenseInfo(), + ]); + + const clusterInfo: ESClusterInfo = + clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo); + + const licenseInfo: ESLicense | undefined = + licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value : ({} as ESLicense); + + return { clusterInfo, licenseInfo }; + } + + private calculateTimeFrame(): TimeFrame { + const now = moment(); + const startOfDay = now.startOf('day').toISOString(); + const endOfDay = now.endOf('day').toISOString(); + return { startOfDay, endOfDay }; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts index 47e04d66ce881..d1fafe15c94c5 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts @@ -5,21 +5,12 @@ * 2.0. */ -import moment from 'moment'; import type { Logger } from '@kbn/core/server'; -import type { SafeEndpointEvent } from '../../../../common/endpoint/types'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; -import type { - ESClusterInfo, - ESLicense, - TimelineTelemetryTemplate, - TimelineTelemetryEvent, -} from '../types'; import { TELEMETRY_CHANNEL_TIMELINE, TASK_METRICS_CHANNEL } from '../constants'; -import { resolverEntity } from '../../../endpoint/routes/resolver/entity/utils/build_resolver_entity'; -import { tlog, createTaskMetric } from '../helpers'; +import { createTaskMetric, ranges, TelemetryTimelineFetcher, tlog } from '../helpers'; export function createTelemetryTimelineTaskConfig() { const taskName = 'Security Solution Timeline telemetry'; @@ -27,9 +18,9 @@ export function createTelemetryTimelineTaskConfig() { return { type: 'security:telemetry-timelines', title: taskName, - interval: '3h', - timeout: '10m', - version: '1.0.0', + interval: '1h', + timeout: '15m', + version: '1.0.1', runTask: async ( taskId: string, logger: Logger, @@ -37,144 +28,55 @@ export function createTelemetryTimelineTaskConfig() { sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { - const startTime = Date.now(); - try { - let counter = 0; - - tlog(logger, `Running task: ${taskId}`); - - const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ - receiver.fetchClusterInfo(), - receiver.fetchLicenseInfo(), - ]); - - const clusterInfo = - clusterInfoPromise.status === 'fulfilled' - ? clusterInfoPromise.value - : ({} as ESClusterInfo); - - const licenseInfo = - licenseInfoPromise.status === 'fulfilled' - ? licenseInfoPromise.value - : ({} as ESLicense | undefined); + tlog( + logger, + `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` + ); - const now = moment(); - const startOfDay = now.startOf('day').toISOString(); - const endOfDay = now.endOf('day').toISOString(); + const fetcher = new TelemetryTimelineFetcher(receiver); - const baseDocument = { - version: clusterInfo.version?.number, - cluster_name: clusterInfo.cluster_name, - cluster_uuid: clusterInfo.cluster_uuid, - license_uuid: licenseInfo?.uid, - }; - - // Fetch EP Alerts - const rangeFrom = taskExecutionPeriod.last ?? 'now-3h'; - const rangeTo = taskExecutionPeriod.current; - const endpointAlerts = await receiver.fetchTimelineEndpointAlerts(rangeFrom, rangeTo); - - // No EP Alerts -> Nothing to do - if (endpointAlerts.length === 0 || endpointAlerts.length === undefined) { - tlog(logger, 'no endpoint alerts received. exiting telemetry task.'); - await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ - createTaskMetric(taskName, true, startTime), - ]); - return counter; - } else { - tlog(logger, `${endpointAlerts.length} alerts received. Processing...`); - } - - // Build process tree for each EP Alert recieved - - for (const alert of endpointAlerts) { - const eventId = alert._source ? alert._source['event.id'] : 'unknown'; - const alertUUID = alert._source ? alert._source['kibana.alert.uuid'] : 'unknown'; + try { + let counter = 0; - const entities = resolverEntity([alert]); + const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); - // Build Tree + const alerts = await receiver.fetchTimelineEndpointAlerts(rangeFrom, rangeTo); - const tree = await receiver.buildProcessTree( - entities[0].id, - entities[0].schema, - startOfDay, - endOfDay - ); + tlog(logger, `found ${alerts.length} alerts to process`); - const nodeIds = [] as string[]; - if (Array.isArray(tree)) { - for (const node of tree) { - const nodeId = node?.id.toString(); - nodeIds.push(nodeId); - } - } + for (const alert of alerts) { + const result = await fetcher.fetchTimeline(alert); sender.getTelemetryUsageCluster()?.incrementCounter({ counterName: 'telemetry_timeline', counterType: 'timeline_node_count', - incrementBy: nodeIds.length, + incrementBy: result.nodes, }); - // Fetch event lineage - - const timelineEvents = await receiver.fetchTimelineEvents(nodeIds); - const eventsStore = new Map(); - for (const event of timelineEvents.hits.hits) { - const doc = event._source; - - if (doc !== null && doc !== undefined) { - const entityId = doc?.process?.entity_id?.toString(); - if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc); - } - } - sender.getTelemetryUsageCluster()?.incrementCounter({ counterName: 'telemetry_timeline', counterType: 'timeline_event_count', - incrementBy: eventsStore.size, + incrementBy: result.events, }); - // Create telemetry record - - const telemetryTimeline: TimelineTelemetryEvent[] = []; - if (Array.isArray(tree)) { - for (const node of tree) { - const id = node.id.toString(); - const event = eventsStore.get(id); - - const timelineTelemetryEvent: TimelineTelemetryEvent = { - ...node, - event, - }; - - telemetryTimeline.push(timelineTelemetryEvent); - } - } - - if (telemetryTimeline.length >= 1) { - const record: TimelineTelemetryTemplate = { - '@timestamp': moment().toISOString(), - ...baseDocument, - alert_id: alertUUID, - event_id: eventId, - timeline: telemetryTimeline, - }; - - sender.sendOnDemand(TELEMETRY_CHANNEL_TIMELINE, [record]); + if (result.timeline) { + sender.sendOnDemand(TELEMETRY_CHANNEL_TIMELINE, [result.timeline]); counter += 1; } else { tlog(logger, 'no events in timeline'); } } - tlog(logger, `sent ${counter} timelines. concluding timeline task.`); + + tlog(logger, `sent ${counter} timelines. Concluding timeline task.`); + await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ - createTaskMetric(taskName, true, startTime), + createTaskMetric(taskName, true, fetcher.startTime), ]); + return counter; } catch (err) { await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ - createTaskMetric(taskName, false, startTime, err.message), + createTaskMetric(taskName, false, fetcher.startTime, err.message), ]); return 0; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts index b7cdf06eaa336..7ec083a8181c5 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -5,37 +5,12 @@ * 2.0. */ -import moment from 'moment'; import type { Logger } from '@kbn/core/server'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { SafeEndpointEvent } from '../../../../common/endpoint/types'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; -import type { - EnhancedAlertEvent, - ESClusterInfo, - ESLicense, - TimelineTelemetryTemplate, - TimelineTelemetryEvent, -} from '../types'; import { TELEMETRY_CHANNEL_TIMELINE, TASK_METRICS_CHANNEL } from '../constants'; -import { resolverEntity } from '../../../endpoint/routes/resolver/entity/utils/build_resolver_entity'; -import { tlog, createTaskMetric } from '../helpers'; - -interface ExtraInfo { - clusterInfo: ESClusterInfo; - licenseInfo: ESLicense | undefined; -} -interface TimeFrame { - startOfDay: string; - endOfDay: string; -} -interface TimelineResult { - nodes: number; - events: number; - timeline: TimelineTelemetryTemplate | undefined; -} +import { createTaskMetric, ranges, TelemetryTimelineFetcher, tlog } from '../helpers'; export function createTelemetryDiagnosticTimelineTaskConfig() { const taskName = 'Security Solution Diagnostic Timeline telemetry'; @@ -63,8 +38,7 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { try { let counter = 0; - const rangeFrom = taskExecutionPeriod.last ?? 'now-3h'; - const rangeTo = taskExecutionPeriod.current; + const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); const alerts = await receiver.fetchDiagnosticTimelineEndpointAlerts(rangeFrom, rangeTo); @@ -109,104 +83,3 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { }, }; } - -class TelemetryTimelineFetcher { - startTime: number; - private receiver: ITelemetryReceiver; - private extraInfo: Promise; - private timeFrame: TimeFrame; - - constructor(receiver: ITelemetryReceiver) { - this.receiver = receiver; - this.startTime = Date.now(); - this.extraInfo = this.lookupExtraInfo(); - this.timeFrame = this.calculateTimeFrame(); - } - - async fetchTimeline(event: estypes.SearchHit): Promise { - const eventId = event._source ? event._source['event.id'] : 'unknown'; - const alertUUID = event._source ? event._source['kibana.alert.uuid'] : 'unknown'; - - const entities = resolverEntity([event]); - - // Build Tree - const tree = await this.receiver.buildProcessTree( - entities[0].id, - entities[0].schema, - this.timeFrame.startOfDay, - this.timeFrame.endOfDay - ); - - const nodeIds = Array.isArray(tree) ? tree.map((node) => node?.id.toString()) : []; - - const eventsStore = await this.fetchEventLineage(nodeIds); - - const telemetryTimeline: TimelineTelemetryEvent[] = Array.isArray(tree) - ? tree.map((node) => { - return { - ...node, - event: eventsStore.get(node.id.toString()), - }; - }) - : []; - - let record; - if (telemetryTimeline.length >= 1) { - const { clusterInfo, licenseInfo } = await this.extraInfo; - record = { - '@timestamp': moment().toISOString(), - version: clusterInfo.version?.number, - cluster_name: clusterInfo.cluster_name, - cluster_uuid: clusterInfo.cluster_uuid, - license_uuid: licenseInfo?.uid, - alert_id: alertUUID, - event_id: eventId, - timeline: telemetryTimeline, - }; - } - - const result: TimelineResult = { - nodes: nodeIds.length, - events: eventsStore.size, - timeline: record, - }; - - return result; - } - - private async fetchEventLineage(nodeIds: string[]): Promise> { - const timelineEvents = await this.receiver.fetchTimelineEvents(nodeIds); - const eventsStore = new Map(); - for (const event of timelineEvents.hits.hits) { - const doc = event._source; - - if (doc !== null && doc !== undefined) { - const entityId = doc?.process?.entity_id?.toString(); - if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc); - } - } - return eventsStore; - } - - private async lookupExtraInfo(): Promise { - const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ - this.receiver.fetchClusterInfo(), - this.receiver.fetchLicenseInfo(), - ]); - - const clusterInfo: ESClusterInfo = - clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo); - - const licenseInfo: ESLicense | undefined = - licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value : ({} as ESLicense); - - return { clusterInfo, licenseInfo }; - } - - private calculateTimeFrame(): TimeFrame { - const now = moment(); - const startOfDay = now.startOf('day').toISOString(); - const endOfDay = now.endOf('day').toISOString(); - return { startOfDay, endOfDay }; - } -} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index df3b571714b29..433c1c01cf0bc 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -457,3 +457,19 @@ export interface ValueListResponse { exceptionListMetricsResponse: ValueListExceptionListResponseAggregation; indicatorMatchMetricsResponse: ValueListIndicatorMatchResponseAggregation; } + +export interface ExtraInfo { + clusterInfo: ESClusterInfo; + licenseInfo: ESLicense | undefined; +} + +export interface TimeFrame { + startOfDay: string; + endOfDay: string; +} + +export interface TimelineResult { + nodes: number; + events: number; + timeline: TimelineTelemetryTemplate | undefined; +} From 0e7fa97e47f0c3d88a5548ad5105d32cea538571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 22 Nov 2023 09:05:52 +0100 Subject: [PATCH 09/12] Fix test --- .../server/lib/telemetry/tasks/tasks.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts index f91de7fe5b194..75da7e5084ab2 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts @@ -52,8 +52,8 @@ describe('security telemetry - ', () => { expect(taskConfig.interval).toEqual('24h'); }); - test('timelines task is set to 3h', async () => { + test('timelines task is set to 1h', async () => { const taskConfig = createTelemetryTimelineTaskConfig(); - expect(taskConfig.interval).toEqual('3h'); + expect(taskConfig.interval).toEqual('1h'); }); }); From 954a31326727c46316063318c404caccb5ff2f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 22 Nov 2023 09:39:42 +0100 Subject: [PATCH 10/12] Apply PR comments --- .../server/lib/telemetry/__mocks__/index.ts | 8 ++--- .../server/lib/telemetry/receiver.ts | 31 ++++++------------- .../lib/telemetry/tasks/timelines.test.ts | 4 +-- .../server/lib/telemetry/tasks/timelines.ts | 6 +++- .../tasks/timelines_diagnostic.test.ts | 4 +-- .../telemetry/tasks/timelines_diagnostic.ts | 15 ++++++--- 6 files changed, 30 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index 7088c8770d648..3260a3e188242 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -91,15 +91,11 @@ export const createMockTelemetryReceiver = ( fetchEndpointList: jest.fn(), fetchDetectionRules: jest.fn().mockReturnValue({ body: null }), fetchEndpointMetadata: jest.fn(), - fetchTimelineEndpointAlerts: jest - .fn() - .mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), - fetchDiagnosticTimelineEndpointAlerts: jest - .fn() - .mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), + fetchTimelineAlerts: jest.fn().mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), buildProcessTree: jest.fn().mockReturnValue(processTreeResponse), fetchTimelineEvents: jest.fn().mockReturnValue(Promise.resolve(stubFetchTimelineEvents())), fetchValueListMetaData: jest.fn(), + getAlertsIndex: jest.fn().mockReturnValue('test-alerts-index'), } as unknown as jest.Mocked; }; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index d5150ccfaa07d..afa1f130c8a60 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -163,12 +163,8 @@ export interface ITelemetryReceiver { fetchPrebuiltRuleAlerts(): Promise<{ events: TelemetryEvent[]; count: number }>; - fetchTimelineEndpointAlerts( - rangeFrom: string, - rangeTo: string - ): Promise>>; - - fetchDiagnosticTimelineEndpointAlerts( + fetchTimelineAlerts( + index: string, rangeFrom: string, rangeTo: string ): Promise>>; @@ -185,6 +181,8 @@ export interface ITelemetryReceiver { ): Promise>>; fetchValueListMetaData(interval: number): Promise; + + getAlertsIndex(): string | undefined; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -232,6 +230,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.clusterInfo; } + public getAlertsIndex(): string | undefined { + return this.alertsIndex; + } + public async fetchDetectionRulesPackageVersion(): Promise { return this.packageService?.asInternalUser.getInstallation(PREBUILT_RULES_PACKAGE_NAME); } @@ -705,7 +707,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return { events: telemetryEvents, count: aggregations?.prebuilt_rule_alert_count.value ?? 0 }; } - private async fetchTimelineAlerts(index: string, rangeFrom: string, rangeTo: string) { + async fetchTimelineAlerts(index: string, rangeFrom: string, rangeTo: string) { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); } @@ -818,21 +820,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { return alertsToReturn || []; } - public async fetchTimelineEndpointAlerts(rangeFrom: string, rangeTo: string) { - if (this.alertsIndex === undefined) { - throw Error('alerts index is not ready yet, skipping telemetry task'); - } - - return this.fetchTimelineAlerts(this.alertsIndex, rangeFrom, rangeTo); - } - - public fetchDiagnosticTimelineEndpointAlerts( - rangeFrom: string, - rangeTo: string - ): Promise>> { - return this.fetchTimelineAlerts(DEFAULT_DIAGNOSTIC_INDEX, rangeFrom, rangeTo); - } - public async buildProcessTree( entityId: string, resolverSchema: ResolverSchema, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts index 16794dfa3f68f..930ef076edd3f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts @@ -35,7 +35,7 @@ describe('timeline telemetry task test', () => { expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); expect(mockTelemetryEventsSender.getTelemetryUsageCluster).toHaveBeenCalled(); expect(mockTelemetryEventsSender.sendOnDemand).toHaveBeenCalled(); }); @@ -59,6 +59,6 @@ describe('timeline telemetry task test', () => { expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts index d1fafe15c94c5..7e0c41e57eec4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts @@ -40,7 +40,11 @@ export function createTelemetryTimelineTaskConfig() { const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); - const alerts = await receiver.fetchTimelineEndpointAlerts(rangeFrom, rangeTo); + const alertsIndex = receiver.getAlertsIndex(); + if (!alertsIndex) { + throw Error('alerts index is not ready yet, skipping telemetry task'); + } + const alerts = await receiver.fetchTimelineAlerts(alertsIndex, rangeFrom, rangeTo); tlog(logger, `found ${alerts.length} alerts to process`); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts index 87beb283b4e08..887e0789e7754 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts @@ -35,7 +35,7 @@ describe('timeline telemetry diagnostic task test', () => { expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchDiagnosticTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); expect(mockTelemetryEventsSender.getTelemetryUsageCluster).toHaveBeenCalled(); expect(mockTelemetryEventsSender.sendOnDemand).toHaveBeenCalled(); }); @@ -59,6 +59,6 @@ describe('timeline telemetry diagnostic task test', () => { expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchDiagnosticTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts index 7ec083a8181c5..66ed39e3211e4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -10,6 +10,7 @@ import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; import { TELEMETRY_CHANNEL_TIMELINE, TASK_METRICS_CHANNEL } from '../constants'; +import { DEFAULT_DIAGNOSTIC_INDEX } from '../../../../common/constants'; import { createTaskMetric, ranges, TelemetryTimelineFetcher, tlog } from '../helpers'; export function createTelemetryDiagnosticTimelineTaskConfig() { @@ -40,7 +41,11 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); - const alerts = await receiver.fetchDiagnosticTimelineEndpointAlerts(rangeFrom, rangeTo); + const alerts = await receiver.fetchTimelineAlerts( + DEFAULT_DIAGNOSTIC_INDEX, + rangeFrom, + rangeTo + ); tlog(logger, `found ${alerts.length} alerts to process`); @@ -48,14 +53,14 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { const result = await fetcher.fetchTimeline(alert); sender.getTelemetryUsageCluster()?.incrementCounter({ - counterName: 'telemetry_timeline', - counterType: 'timeline_node_count', + counterName: 'telemetry_timeline_diagnostic', + counterType: 'timeline_diagnostic_node_count', incrementBy: result.nodes, }); sender.getTelemetryUsageCluster()?.incrementCounter({ - counterName: 'telemetry_timeline', - counterType: 'timeline_event_count', + counterName: 'telemetry_timeline_diagnostic', + counterType: 'timeline_diagnostic_event_count', incrementBy: result.events, }); From 93d5c4826773b7da516d025d2c5dfada3c9cc9d9 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 23 Nov 2023 10:21:37 +0000 Subject: [PATCH 11/12] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../public/application/components/connectors_ingestion.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/serverless_search/public/application/components/connectors_ingestion.tsx b/x-pack/plugins/serverless_search/public/application/components/connectors_ingestion.tsx index 6be9b28707dff..ede574abba12b 100644 --- a/x-pack/plugins/serverless_search/public/application/components/connectors_ingestion.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/connectors_ingestion.tsx @@ -50,7 +50,10 @@ export const ConnectorIngestionPanel: React.FC<{ assetBasePath: string }> = ({ a - createConnector()}> + createConnector()} + > {i18n.translate( 'xpack.serverlessSearch.ingestData.alternativeOptions.setupConnectorLabel', { @@ -74,6 +77,7 @@ export const ConnectorIngestionPanel: React.FC<{ assetBasePath: string }> = ({ a From 51b256959c3698d33cb71cdac1eeae53df89adf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 27 Nov 2023 15:39:14 +0100 Subject: [PATCH 12/12] Move constants to telemetry code --- x-pack/plugins/security_solution/common/constants.ts | 1 - .../security_solution/server/lib/telemetry/constants.ts | 2 ++ .../security_solution/server/lib/telemetry/receiver.ts | 3 ++- .../server/lib/telemetry/tasks/timelines_diagnostic.ts | 7 +++++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index e5d7b313ef524..75eda07fa185e 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -41,7 +41,6 @@ export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const; export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults' as const; export const DEFAULT_ALERTS_INDEX = '.alerts-security.alerts' as const; -export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const; export const DEFAULT_SIGNALS_INDEX = '.siem-signals' as const; export const DEFAULT_PREVIEW_INDEX = '.preview.alerts-security.alerts' as const; export const DEFAULT_LISTS_INDEX = '.lists' as const; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts index 50e0e0be47cdd..8e50e4590a72f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts @@ -27,6 +27,8 @@ export const INSIGHTS_CHANNEL = 'security-insights-v1'; export const TASK_METRICS_CHANNEL = 'task-metrics'; +export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const; + export const DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS = { linux: { advanced: { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index afa1f130c8a60..e416ae83652da 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -68,8 +68,9 @@ import type { ValueListIndicatorMatchResponseAggregation, } from './types'; import { telemetryConfiguration } from './configuration'; -import { DEFAULT_DIAGNOSTIC_INDEX, ENDPOINT_METRICS_INDEX } from '../../../common/constants'; +import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants'; +import { DEFAULT_DIAGNOSTIC_INDEX } from './constants'; export interface ITelemetryReceiver { start( diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts index 66ed39e3211e4..7148848984b0a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -9,8 +9,11 @@ import type { Logger } from '@kbn/core/server'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; -import { TELEMETRY_CHANNEL_TIMELINE, TASK_METRICS_CHANNEL } from '../constants'; -import { DEFAULT_DIAGNOSTIC_INDEX } from '../../../../common/constants'; +import { + DEFAULT_DIAGNOSTIC_INDEX, + TELEMETRY_CHANNEL_TIMELINE, + TASK_METRICS_CHANNEL, +} from '../constants'; import { createTaskMetric, ranges, TelemetryTimelineFetcher, tlog } from '../helpers'; export function createTelemetryDiagnosticTimelineTaskConfig() {