diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index f5718895fff26..3260a3e188242 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -9,7 +9,7 @@ import moment from 'moment'; import type { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server'; import { TaskStatus } from '@kbn/task-manager-plugin/server'; import type { TelemetryEventsSender } from '../sender'; -import type { TelemetryReceiver } from '../receiver'; +import type { ITelemetryReceiver, TelemetryReceiver } from '../receiver'; import type { SecurityTelemetryTaskConfig } from '../task'; import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/package_policy'; import { stubEndpointAlertResponse, stubProcessTree, stubFetchTimelineEvents } from './timeline'; @@ -71,7 +71,7 @@ export const stubLicenseInfo: ESLicense = { export const createMockTelemetryReceiver = ( diagnosticsAlert?: unknown, emptyTimelineTree?: boolean -): jest.Mocked => { +): jest.Mocked => { const processTreeResponse = emptyTimelineTree ? Promise.resolve([]) : Promise.resolve(Promise.resolve(stubProcessTree())); @@ -91,12 +91,11 @@ export const createMockTelemetryReceiver = ( fetchEndpointList: jest.fn(), fetchDetectionRules: jest.fn().mockReturnValue({ body: null }), fetchEndpointMetadata: jest.fn(), - fetchTimelineEndpointAlerts: jest - .fn() - .mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), + fetchTimelineAlerts: jest.fn().mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), buildProcessTree: jest.fn().mockReturnValue(processTreeResponse), fetchTimelineEvents: jest.fn().mockReturnValue(Promise.resolve(stubFetchTimelineEvents())), fetchValueListMetaData: jest.fn(), + getAlertsIndex: jest.fn().mockReturnValue('test-alerts-index'), } as unknown as jest.Mocked; }; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts index 50e0e0be47cdd..8e50e4590a72f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts @@ -27,6 +27,8 @@ export const INSIGHTS_CHANNEL = 'security-insights-v1'; export const TASK_METRICS_CHANNEL = 'task-metrics'; +export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const; + export const DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS = { linux: { advanced: { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index f5d6bc41ee349..3bd6a0d595a79 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -11,17 +11,25 @@ import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/packag import { merge, set } from 'lodash'; import type { Logger } from '@kbn/core/server'; import { sha256 } from 'js-sha256'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { copyAllowlistedFields, filterList } from './filterlists'; -import type { PolicyConfig, PolicyData } from '../../../common/endpoint/types'; +import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types'; +import type { ITelemetryReceiver } from './receiver'; import type { - ExceptionListItem, + EnhancedAlertEvent, ESClusterInfo, ESLicense, + ExceptionListItem, + ExtraInfo, ListTemplate, + TaskMetric, TelemetryEvent, + TimeFrame, + TimelineResult, + TimelineTelemetryEvent, ValueListResponse, - TaskMetric, } from './types'; +import type { TaskExecutionPeriod } from './task'; import { LIST_DETECTION_RULE_EXCEPTION, LIST_ENDPOINT_EXCEPTION, @@ -30,6 +38,7 @@ import { DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS, } from './constants'; import { tagsToEffectScope } from '../../../common/endpoint/service/trusted_apps/mapping'; +import { resolverEntity } from '../../endpoint/routes/resolver/entity/utils/build_resolver_entity'; /** * Determines the when the last run was in order to execute to. @@ -345,3 +354,114 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T return event; }; + +export const ranges = ( + taskExecutionPeriod: TaskExecutionPeriod, + defaultIntervalInHours: number = 3 +) => { + const rangeFrom = taskExecutionPeriod.last ?? `now-${defaultIntervalInHours}h`; + const rangeTo = taskExecutionPeriod.current; + + return { rangeFrom, rangeTo }; +}; + +export class TelemetryTimelineFetcher { + startTime: number; + private receiver: ITelemetryReceiver; + private extraInfo: Promise; + private timeFrame: TimeFrame; + + constructor(receiver: ITelemetryReceiver) { + this.receiver = receiver; + this.startTime = Date.now(); + this.extraInfo = this.lookupExtraInfo(); + this.timeFrame = this.calculateTimeFrame(); + } + + async fetchTimeline(event: estypes.SearchHit): Promise { + const eventId = event._source ? event._source['event.id'] : 'unknown'; + const alertUUID = event._source ? event._source['kibana.alert.uuid'] : 'unknown'; + + const entities = resolverEntity([event]); + + // Build Tree + const tree = await this.receiver.buildProcessTree( + entities[0].id, + entities[0].schema, + this.timeFrame.startOfDay, + this.timeFrame.endOfDay + ); + + const nodeIds = Array.isArray(tree) ? tree.map((node) => node?.id.toString()) : []; + + const eventsStore = await this.fetchEventLineage(nodeIds); + + const telemetryTimeline: TimelineTelemetryEvent[] = Array.isArray(tree) + ? tree.map((node) => { + return { + ...node, + event: eventsStore.get(node.id.toString()), + }; + }) + : []; + + let record; + if (telemetryTimeline.length >= 1) { + const { clusterInfo, licenseInfo } = await this.extraInfo; + record = { + '@timestamp': moment().toISOString(), + version: clusterInfo.version?.number, + cluster_name: clusterInfo.cluster_name, + cluster_uuid: clusterInfo.cluster_uuid, + license_uuid: licenseInfo?.uid, + alert_id: alertUUID, + event_id: eventId, + timeline: telemetryTimeline, + }; + } + + const result: TimelineResult = { + nodes: nodeIds.length, + events: eventsStore.size, + timeline: record, + }; + + return result; + } + + private async fetchEventLineage(nodeIds: string[]): Promise> { + const timelineEvents = await this.receiver.fetchTimelineEvents(nodeIds); + const eventsStore = new Map(); + for (const event of timelineEvents.hits.hits) { + const doc = event._source; + + if (doc !== null && doc !== undefined) { + const entityId = doc?.process?.entity_id?.toString(); + if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc); + } + } + return eventsStore; + } + + private async lookupExtraInfo(): Promise { + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + this.receiver.fetchClusterInfo(), + this.receiver.fetchLicenseInfo(), + ]); + + const clusterInfo: ESClusterInfo = + clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo); + + const licenseInfo: ESLicense | undefined = + licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value : ({} as ESLicense); + + return { clusterInfo, licenseInfo }; + } + + private calculateTimeFrame(): TimeFrame { + const now = moment(); + const startOfDay = now.startOf('day').toISOString(); + const endOfDay = now.endOf('day').toISOString(); + return { startOfDay, endOfDay }; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index c699f6a1e9698..e416ae83652da 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -70,6 +70,7 @@ import type { import { telemetryConfiguration } from './configuration'; import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants'; +import { DEFAULT_DIAGNOSTIC_INDEX } from './constants'; export interface ITelemetryReceiver { start( @@ -163,7 +164,11 @@ export interface ITelemetryReceiver { fetchPrebuiltRuleAlerts(): Promise<{ events: TelemetryEvent[]; count: number }>; - fetchTimelineEndpointAlerts(interval: number): Promise>>; + fetchTimelineAlerts( + index: string, + rangeFrom: string, + rangeTo: string + ): Promise>>; buildProcessTree( entityId: string, @@ -177,6 +182,8 @@ export interface ITelemetryReceiver { ): Promise>>; fetchValueListMetaData(interval: number): Promise; + + getAlertsIndex(): string | undefined; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -224,6 +231,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.clusterInfo; } + public getAlertsIndex(): string | undefined { + return this.alertsIndex; + } + public async fetchDetectionRulesPackageVersion(): Promise { return this.packageService?.asInternalUser.getInstallation(PREBUILT_RULES_PACKAGE_NAME); } @@ -396,7 +407,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { const query = { expand_wildcards: ['open' as const, 'hidden' as const], - index: '.logs-endpoint.diagnostic.collection-*', + index: `${DEFAULT_DIAGNOSTIC_INDEX}-*`, ignore_unavailable: true, size: telemetryConfiguration.telemetry_max_buffer_size, body: { @@ -697,7 +708,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return { events: telemetryEvents, count: aggregations?.prebuilt_rule_alert_count.value ?? 0 }; } - public async fetchTimelineEndpointAlerts(interval: number) { + async fetchTimelineAlerts(index: string, rangeFrom: string, rangeTo: string) { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); } @@ -708,7 +719,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { // create and assign an initial point in time let pitId: OpenPointInTimeResponse['id'] = ( await this.esClient.openPointInTime({ - index: `${this.alertsIndex}*`, + index: `${index}*`, keep_alive: keepAlive, }) ).id; @@ -746,8 +757,8 @@ export class TelemetryReceiver implements ITelemetryReceiver { { range: { '@timestamp': { - gte: `now-${interval}h`, - lte: 'now', + gte: rangeFrom, + lte: rangeTo, }, }, }, @@ -807,7 +818,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } tlog(this.logger, `Timeline alerts to return: ${alertsToReturn.length}`); - return alertsToReturn; + return alertsToReturn || []; } public async buildProcessTree( diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index 4562cbb725cb4..8ccde7cd1946e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -33,6 +33,12 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n ) => { const startTime = Date.now(); const taskName = 'Security Solution Detection Rule Lists Telemetry'; + + tlog( + logger, + `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` + ); + try { const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ receiver.fetchClusterInfo(), diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts index e25b3690ee88d..d237757616f3e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -12,6 +12,7 @@ import { createTelemetrySecurityListTaskConfig } from './security_lists'; import { createTelemetryDetectionRuleListsTaskConfig } from './detection_rule'; import { createTelemetryPrebuiltRuleAlertsTaskConfig } from './prebuilt_rule_alerts'; import { createTelemetryTimelineTaskConfig } from './timelines'; +import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnostic'; import { createTelemetryConfigurationTaskConfig } from './configuration'; import { telemetryConfiguration } from '../configuration'; import { createTelemetryFilterListArtifactTaskConfig } from './filterlists'; @@ -26,6 +27,7 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { ), createTelemetryPrebuiltRuleAlertsTaskConfig(telemetryConfiguration.max_detection_alerts_batch), createTelemetryTimelineTaskConfig(), + createTelemetryDiagnosticTimelineTaskConfig(), createTelemetryConfigurationTaskConfig(), createTelemetryFilterListArtifactTaskConfig(), ]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts index f91de7fe5b194..75da7e5084ab2 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts @@ -52,8 +52,8 @@ describe('security telemetry - ', () => { expect(taskConfig.interval).toEqual('24h'); }); - test('timelines task is set to 3h', async () => { + test('timelines task is set to 1h', async () => { const taskConfig = createTelemetryTimelineTaskConfig(); - expect(taskConfig.interval).toEqual('3h'); + expect(taskConfig.interval).toEqual('1h'); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts index 16794dfa3f68f..930ef076edd3f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts @@ -35,7 +35,7 @@ describe('timeline telemetry task test', () => { expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); expect(mockTelemetryEventsSender.getTelemetryUsageCluster).toHaveBeenCalled(); expect(mockTelemetryEventsSender.sendOnDemand).toHaveBeenCalled(); }); @@ -59,6 +59,6 @@ describe('timeline telemetry task test', () => { expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchTimelineEndpointAlerts).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts index bd50ffcd92817..7e0c41e57eec4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.ts @@ -5,29 +5,22 @@ * 2.0. */ -import moment from 'moment'; import type { Logger } from '@kbn/core/server'; -import type { SafeEndpointEvent } from '../../../../common/endpoint/types'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; -import type { - ESClusterInfo, - ESLicense, - TimelineTelemetryTemplate, - TimelineTelemetryEvent, -} from '../types'; import { TELEMETRY_CHANNEL_TIMELINE, TASK_METRICS_CHANNEL } from '../constants'; -import { resolverEntity } from '../../../endpoint/routes/resolver/entity/utils/build_resolver_entity'; -import { tlog, createTaskMetric } from '../helpers'; +import { createTaskMetric, ranges, TelemetryTimelineFetcher, tlog } from '../helpers'; export function createTelemetryTimelineTaskConfig() { + const taskName = 'Security Solution Timeline telemetry'; + return { type: 'security:telemetry-timelines', - title: 'Security Solution Timeline telemetry', - interval: '3h', - timeout: '10m', - version: '1.0.0', + title: taskName, + interval: '1h', + timeout: '15m', + version: '1.0.1', runTask: async ( taskId: string, logger: Logger, @@ -35,142 +28,59 @@ export function createTelemetryTimelineTaskConfig() { sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { - const startTime = Date.now(); - const taskName = 'Security Solution Timeline telemetry'; - try { - let counter = 0; - - tlog(logger, `Running task: ${taskId}`); - - const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ - receiver.fetchClusterInfo(), - receiver.fetchLicenseInfo(), - ]); - - const clusterInfo = - clusterInfoPromise.status === 'fulfilled' - ? clusterInfoPromise.value - : ({} as ESClusterInfo); - - const licenseInfo = - licenseInfoPromise.status === 'fulfilled' - ? licenseInfoPromise.value - : ({} as ESLicense | undefined); + tlog( + logger, + `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` + ); - const now = moment(); - const startOfDay = now.startOf('day').toISOString(); - const endOfDay = now.endOf('day').toISOString(); + const fetcher = new TelemetryTimelineFetcher(receiver); - const baseDocument = { - version: clusterInfo.version?.number, - cluster_name: clusterInfo.cluster_name, - cluster_uuid: clusterInfo.cluster_uuid, - license_uuid: licenseInfo?.uid, - }; - - // Fetch EP Alerts + try { + let counter = 0; - const endpointAlerts = await receiver.fetchTimelineEndpointAlerts(3); + const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); - // No EP Alerts -> Nothing to do - if (endpointAlerts.length === 0 || endpointAlerts.length === undefined) { - tlog(logger, 'no endpoint alerts received. exiting telemetry task.'); - await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ - createTaskMetric(taskName, true, startTime), - ]); - return counter; + const alertsIndex = receiver.getAlertsIndex(); + if (!alertsIndex) { + throw Error('alerts index is not ready yet, skipping telemetry task'); } + const alerts = await receiver.fetchTimelineAlerts(alertsIndex, rangeFrom, rangeTo); - // Build process tree for each EP Alert recieved - - for (const alert of endpointAlerts) { - const eventId = alert._source ? alert._source['event.id'] : 'unknown'; - const alertUUID = alert._source ? alert._source['kibana.alert.uuid'] : 'unknown'; + tlog(logger, `found ${alerts.length} alerts to process`); - const entities = resolverEntity([alert]); - - // Build Tree - - const tree = await receiver.buildProcessTree( - entities[0].id, - entities[0].schema, - startOfDay, - endOfDay - ); - - const nodeIds = [] as string[]; - if (Array.isArray(tree)) { - for (const node of tree) { - const nodeId = node?.id.toString(); - nodeIds.push(nodeId); - } - } + for (const alert of alerts) { + const result = await fetcher.fetchTimeline(alert); sender.getTelemetryUsageCluster()?.incrementCounter({ counterName: 'telemetry_timeline', counterType: 'timeline_node_count', - incrementBy: nodeIds.length, + incrementBy: result.nodes, }); - // Fetch event lineage - - const timelineEvents = await receiver.fetchTimelineEvents(nodeIds); - const eventsStore = new Map(); - for (const event of timelineEvents.hits.hits) { - const doc = event._source; - - if (doc !== null && doc !== undefined) { - const entityId = doc?.process?.entity_id?.toString(); - if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc); - } - } - sender.getTelemetryUsageCluster()?.incrementCounter({ counterName: 'telemetry_timeline', counterType: 'timeline_event_count', - incrementBy: eventsStore.size, + incrementBy: result.events, }); - // Create telemetry record - - const telemetryTimeline: TimelineTelemetryEvent[] = []; - if (Array.isArray(tree)) { - for (const node of tree) { - const id = node.id.toString(); - const event = eventsStore.get(id); - - const timelineTelemetryEvent: TimelineTelemetryEvent = { - ...node, - event, - }; - - telemetryTimeline.push(timelineTelemetryEvent); - } - } - - if (telemetryTimeline.length >= 1) { - const record: TimelineTelemetryTemplate = { - '@timestamp': moment().toISOString(), - ...baseDocument, - alert_id: alertUUID, - event_id: eventId, - timeline: telemetryTimeline, - }; - - sender.sendOnDemand(TELEMETRY_CHANNEL_TIMELINE, [record]); + if (result.timeline) { + sender.sendOnDemand(TELEMETRY_CHANNEL_TIMELINE, [result.timeline]); counter += 1; } else { tlog(logger, 'no events in timeline'); } } - tlog(logger, `sent ${counter} timelines. concluding timeline task.`); + + tlog(logger, `sent ${counter} timelines. Concluding timeline task.`); + await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ - createTaskMetric(taskName, true, startTime), + createTaskMetric(taskName, true, fetcher.startTime), ]); + return counter; } catch (err) { await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ - createTaskMetric(taskName, false, startTime, err.message), + createTaskMetric(taskName, false, fetcher.startTime, err.message), ]); return 0; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts new file mode 100644 index 0000000000000..887e0789e7754 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnostic'; +import { createMockTelemetryEventsSender, createMockTelemetryReceiver } from '../__mocks__'; + +describe('timeline telemetry diagnostic task test', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + test('timeline telemetry task should be correctly set up', async () => { + const testTaskExecutionPeriod = { + last: undefined, + current: new Date().toISOString(), + }; + const mockTelemetryEventsSender = createMockTelemetryEventsSender(); + const mockTelemetryReceiver = createMockTelemetryReceiver(); + const telemetryTelemetryDiagnosticTaskConfig = createTelemetryDiagnosticTimelineTaskConfig(); + + await telemetryTelemetryDiagnosticTaskConfig.runTask( + 'test-timeline-diagnostic-task-id', + logger, + mockTelemetryReceiver, + mockTelemetryEventsSender, + testTaskExecutionPeriod + ); + + expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); + expect(mockTelemetryEventsSender.getTelemetryUsageCluster).toHaveBeenCalled(); + expect(mockTelemetryEventsSender.sendOnDemand).toHaveBeenCalled(); + }); + + test('if no timeline events received it should not send a telemetry record', async () => { + const testTaskExecutionPeriod = { + last: undefined, + current: new Date().toISOString(), + }; + const mockTelemetryEventsSender = createMockTelemetryEventsSender(); + const mockTelemetryReceiver = createMockTelemetryReceiver(null, true); + const telemetryTelemetryDiagnosticTaskConfig = createTelemetryDiagnosticTimelineTaskConfig(); + + await telemetryTelemetryDiagnosticTaskConfig.runTask( + 'test-timeline-diagnostic-task-id', + logger, + mockTelemetryReceiver, + mockTelemetryEventsSender, + testTaskExecutionPeriod + ); + + expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled(); + expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts new file mode 100644 index 0000000000000..7148848984b0a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { ITelemetryEventsSender } from '../sender'; +import type { ITelemetryReceiver } from '../receiver'; +import type { TaskExecutionPeriod } from '../task'; +import { + DEFAULT_DIAGNOSTIC_INDEX, + TELEMETRY_CHANNEL_TIMELINE, + TASK_METRICS_CHANNEL, +} from '../constants'; +import { createTaskMetric, ranges, TelemetryTimelineFetcher, tlog } from '../helpers'; + +export function createTelemetryDiagnosticTimelineTaskConfig() { + const taskName = 'Security Solution Diagnostic Timeline telemetry'; + + return { + type: 'security:telemetry-diagnostic-timelines', + title: taskName, + interval: '1h', + timeout: '15m', + version: '1.0.0', + runTask: async ( + taskId: string, + logger: Logger, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, + taskExecutionPeriod: TaskExecutionPeriod + ) => { + tlog( + logger, + `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` + ); + + const fetcher = new TelemetryTimelineFetcher(receiver); + + try { + let counter = 0; + + const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); + + const alerts = await receiver.fetchTimelineAlerts( + DEFAULT_DIAGNOSTIC_INDEX, + rangeFrom, + rangeTo + ); + + tlog(logger, `found ${alerts.length} alerts to process`); + + for (const alert of alerts) { + const result = await fetcher.fetchTimeline(alert); + + sender.getTelemetryUsageCluster()?.incrementCounter({ + counterName: 'telemetry_timeline_diagnostic', + counterType: 'timeline_diagnostic_node_count', + incrementBy: result.nodes, + }); + + sender.getTelemetryUsageCluster()?.incrementCounter({ + counterName: 'telemetry_timeline_diagnostic', + counterType: 'timeline_diagnostic_event_count', + incrementBy: result.events, + }); + + if (result.timeline) { + sender.sendOnDemand(TELEMETRY_CHANNEL_TIMELINE, [result.timeline]); + counter += 1; + } else { + tlog(logger, 'no events in timeline'); + } + } + + tlog(logger, `sent ${counter} timelines. Concluding timeline task.`); + + await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ + createTaskMetric(taskName, true, fetcher.startTime), + ]); + + return counter; + } catch (err) { + await sender.sendOnDemand(TASK_METRICS_CHANNEL, [ + createTaskMetric(taskName, false, fetcher.startTime, err.message), + ]); + return 0; + } + }, + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index df3b571714b29..433c1c01cf0bc 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -457,3 +457,19 @@ export interface ValueListResponse { exceptionListMetricsResponse: ValueListExceptionListResponseAggregation; indicatorMatchMetricsResponse: ValueListIndicatorMatchResponseAggregation; } + +export interface ExtraInfo { + clusterInfo: ESClusterInfo; + licenseInfo: ESLicense | undefined; +} + +export interface TimeFrame { + startOfDay: string; + endOfDay: string; +} + +export interface TimelineResult { + nodes: number; + events: number; + timeline: TimelineTelemetryTemplate | undefined; +} diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 7e48895a9060f..2217310e4e6ae 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -145,6 +145,7 @@ export default function ({ getService }: FtrProviderContext) { 'security:endpoint-meta-telemetry', 'security:telemetry-configuration', 'security:telemetry-detection-rules', + 'security:telemetry-diagnostic-timelines', 'security:telemetry-filterlist-artifact', 'security:telemetry-lists', 'security:telemetry-prebuilt-rule-alerts',