diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts index 771e3e059c336..91f83d5e7cb37 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts @@ -7,12 +7,14 @@ export const TELEMETRY_MAX_BUFFER_SIZE = 100; -export const TELEMETRY_CHANNEL_LISTS = 'security-lists'; +export const TELEMETRY_CHANNEL_LISTS = 'security-lists-v2'; export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata'; -export const LIST_TRUSTED_APPLICATION = 'trusted_application'; +export const LIST_DETECTION_RULE_EXCEPTION = 'detection_rule_exception'; export const LIST_ENDPOINT_EXCEPTION = 'endpoint_exception'; export const LIST_ENDPOINT_EVENT_FILTER = 'endpoint_event_filter'; + +export const LIST_TRUSTED_APPLICATION = 'trusted_application'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts index 61172fac511f7..a29f195ed5ecc 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts @@ -129,13 +129,12 @@ export const allowlistEventFields: AllowlistFields = { export const exceptionListEventFields: AllowlistFields = { created_at: true, - description: true, effectScope: true, entries: true, id: true, name: true, - os: true, os_types: true, + rule_version: true, }; /** @@ -143,7 +142,7 @@ export const exceptionListEventFields: AllowlistFields = { * * @param allowlist * @param event - * @returns + * @returns TelemetryEvent with explicitly required fields */ export function copyAllowlistedFields( allowlist: AllowlistFields, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts index 647219e8c5585..528082d8cb5b1 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -8,13 +8,14 @@ import moment from 'moment'; import { createMockPackagePolicy } from './mocks'; import { + LIST_DETECTION_RULE_EXCEPTION, LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER, LIST_TRUSTED_APPLICATION, } from './constants'; import { getPreviousDiagTaskTimestamp, - getPreviousEpMetaTaskTimestamp, + getPreviousDailyTaskTimestamp, batchTelemetryRecords, isPackagePolicyList, templateExceptionList, @@ -53,7 +54,7 @@ describe('test endpoint meta telemetry scheduled task timing helper', () => { test('test -24 hours is returned when there is no previous task run', async () => { const executeTo = moment().utc().toISOString(); const executeFrom = undefined; - const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom); + const newExecuteFrom = getPreviousDailyTaskTimestamp(executeTo, executeFrom); expect(newExecuteFrom).toEqual(moment(executeTo).subtract(24, 'hours').toISOString()); }); @@ -61,7 +62,7 @@ describe('test endpoint meta telemetry scheduled task timing helper', () => { test('test -24 hours is returned when there was a previous task run', async () => { const executeTo = moment().utc().toISOString(); const executeFrom = moment(executeTo).subtract(24, 'hours').toISOString(); - const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom); + const newExecuteFrom = getPreviousDailyTaskTimestamp(executeTo, executeFrom); expect(newExecuteFrom).toEqual(executeFrom); }); @@ -71,7 +72,7 @@ describe('test endpoint meta telemetry scheduled task timing helper', () => { test('test 24 hours is returned when previous task run took longer than 24 hours', async () => { const executeTo = moment().utc().toISOString(); const executeFrom = moment(executeTo).subtract(72, 'hours').toISOString(); // down 3 days - const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom); + const newExecuteFrom = getPreviousDailyTaskTimestamp(executeTo, executeFrom); expect(newExecuteFrom).toEqual(moment(executeTo).subtract(24, 'hours').toISOString()); }); @@ -134,61 +135,88 @@ describe('test package policy type guard', () => { }); describe('list telemetry schema', () => { + test('detection rules document is correctly formed', () => { + const data = [{ id: 'test_1' }] as ExceptionListItem[]; + const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION); + + expect(templatedItems[0]?.detection_rule).not.toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).toBeUndefined(); + }); + + test('detection rules document is correctly formed with multiple entries', () => { + const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[]; + const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION); + + expect(templatedItems[0]?.detection_rule).not.toBeUndefined(); + expect(templatedItems[1]?.detection_rule).not.toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).toBeUndefined(); + }); + test('trusted apps document is correctly formed', () => { const data = [{ id: 'test_1' }] as ExceptionListItem[]; const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION); - expect(templatedItems[0]?.trusted_application.length).toEqual(1); - expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); - expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + expect(templatedItems[0]?.detection_rule).toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).not.toBeUndefined(); }); test('trusted apps document is correctly formed with multiple entries', () => { const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[]; const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION); - expect(templatedItems[0]?.trusted_application.length).toEqual(1); - expect(templatedItems[1]?.trusted_application.length).toEqual(1); - expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); - expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + expect(templatedItems[0]?.detection_rule).toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).not.toBeUndefined(); + expect(templatedItems[1]?.trusted_application).not.toBeUndefined(); }); test('endpoint exception document is correctly formed', () => { const data = [{ id: 'test_3' }] as ExceptionListItem[]; const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION); - expect(templatedItems[0]?.trusted_application.length).toEqual(0); - expect(templatedItems[0]?.endpoint_exception.length).toEqual(1); - expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + expect(templatedItems[0]?.detection_rule).toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).toBeUndefined(); }); test('endpoint exception document is correctly formed with multiple entries', () => { const data = [{ id: 'test_4' }, { id: 'test_4' }, { id: 'test_4' }] as ExceptionListItem[]; const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION); - expect(templatedItems[0]?.trusted_application.length).toEqual(0); - expect(templatedItems[0]?.endpoint_exception.length).toEqual(1); - expect(templatedItems[1]?.endpoint_exception.length).toEqual(1); - expect(templatedItems[2]?.endpoint_exception.length).toEqual(1); - expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + expect(templatedItems[0]?.detection_rule).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined(); + expect(templatedItems[1]?.endpoint_exception).not.toBeUndefined(); + expect(templatedItems[2]?.endpoint_exception).not.toBeUndefined(); + expect(templatedItems[0]?.trusted_application).toBeUndefined(); }); test('endpoint event filters document is correctly formed', () => { const data = [{ id: 'test_5' }] as ExceptionListItem[]; const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER); - expect(templatedItems[0]?.trusted_application.length).toEqual(0); - expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); - expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1); + expect(templatedItems[0]?.detection_rule).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).toBeUndefined(); }); test('endpoint event filters document is correctly formed with multiple entries', () => { const data = [{ id: 'test_6' }, { id: 'test_6' }] as ExceptionListItem[]; const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER); - expect(templatedItems[0]?.trusted_application.length).toEqual(0); - expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); - expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1); - expect(templatedItems[1]?.endpoint_event_filter.length).toEqual(1); + expect(templatedItems[0]?.detection_rule).toBeUndefined(); + expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined(); + expect(templatedItems[1]?.endpoint_event_filter).not.toBeUndefined(); + expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); + expect(templatedItems[0]?.trusted_application).toBeUndefined(); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index a9eaef3ce6edc..e72b0ba7d16fe 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -11,6 +11,7 @@ import { PackagePolicy } from '../../../../fleet/common/types/models/package_pol import { copyAllowlistedFields, exceptionListEventFields } from './filters'; import { ExceptionListItem, ListTemplate, TelemetryEvent } from './types'; import { + LIST_DETECTION_RULE_EXCEPTION, LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER, LIST_TRUSTED_APPLICATION, @@ -46,7 +47,7 @@ export const getPreviousDiagTaskTimestamp = ( * @param lastExecutionTimestamp * @returns the timestamp to search from */ -export const getPreviousEpMetaTaskTimestamp = ( +export const getPreviousDailyTaskTimestamp = ( executeTo: string, lastExecutionTimestamp?: string ) => { @@ -97,18 +98,16 @@ export function isPackagePolicyList( * Maps trusted application to shared telemetry object * * @param exceptionListItem - * @returns collection of endpoint exceptions + * @returns collection of trusted applications */ export const trustedApplicationToTelemetryEntry = (trustedApplication: TrustedApp) => { return { id: trustedApplication.id, - version: trustedApplication.version || '', name: trustedApplication.name, - description: trustedApplication.description, created_at: trustedApplication.created_at, updated_at: trustedApplication.updated_at, entries: trustedApplication.entries, - os: trustedApplication.os, + os_types: [trustedApplication.os], } as ExceptionListItem; }; @@ -121,9 +120,29 @@ export const trustedApplicationToTelemetryEntry = (trustedApplication: TrustedAp export const exceptionListItemToTelemetryEntry = (exceptionListItem: ExceptionListItemSchema) => { return { id: exceptionListItem.id, - version: exceptionListItem._version || '', name: exceptionListItem.name, - description: exceptionListItem.description, + created_at: exceptionListItem.created_at, + updated_at: exceptionListItem.updated_at, + entries: exceptionListItem.entries, + os_types: exceptionListItem.os_types, + } as ExceptionListItem; +}; + +/** + * Maps detection rule exception list items to shared telemetry object + * + * @param exceptionListItem + * @param ruleVersion + * @returns collection of detection rule exceptions + */ +export const ruleExceptionListItemToTelemetryEvent = ( + exceptionListItem: ExceptionListItemSchema, + ruleVersion: number +) => { + return { + id: exceptionListItem.item_id, + name: exceptionListItem.description, + rule_version: ruleVersion, created_at: exceptionListItem.created_at, updated_at: exceptionListItem.updated_at, entries: exceptionListItem.entries, @@ -141,9 +160,7 @@ export const exceptionListItemToTelemetryEntry = (exceptionListItem: ExceptionLi export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => { return listData.map((item) => { const template: ListTemplate = { - trusted_application: [], - endpoint_exception: [], - endpoint_event_filter: [], + '@timestamp': new Date().getTime(), }; // cast exception list type to a TelemetryEvent for allowlist filtering @@ -152,18 +169,23 @@ export const templateExceptionList = (listData: ExceptionListItem[], listType: s item as unknown as TelemetryEvent ); + if (listType === LIST_DETECTION_RULE_EXCEPTION) { + template.detection_rule = filteredListItem; + return template; + } + if (listType === LIST_TRUSTED_APPLICATION) { - template.trusted_application.push(filteredListItem); + template.trusted_application = filteredListItem; return template; } if (listType === LIST_ENDPOINT_EXCEPTION) { - template.endpoint_exception.push(filteredListItem); + template.endpoint_exception = filteredListItem; return template; } if (listType === LIST_ENDPOINT_EVENT_FILTER) { - template.endpoint_event_filter.push(filteredListItem); + template.endpoint_event_filter = filteredListItem; return template; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts index 20a71657b2ffe..9168683141e48 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -8,7 +8,7 @@ // eslint-disable-next-line max-classes-per-file import { TelemetryEventsSender } from './sender'; import { TelemetryReceiver } from './receiver'; -import { DiagnosticTask, EndpointTask, ExceptionListsTask } from './tasks'; +import { DiagnosticTask, EndpointTask, ExceptionListsTask, DetectionRulesTask } from './tasks'; import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; /** @@ -40,6 +40,8 @@ export const createMockTelemetryReceiver = (): jest.Mocked => fetchEndpointMetrics: jest.fn(), fetchEndpointPolicyResponses: jest.fn(), fetchTrustedApplications: jest.fn(), + fetchDetectionRules: jest.fn(), + fetchDetectionExceptionList: jest.fn(), } as unknown as jest.Mocked; }; @@ -79,3 +81,10 @@ export class MockTelemetryEndpointTask extends EndpointTask { export class MockExceptionListsTask extends ExceptionListsTask { public runTask = jest.fn(); } + +/** + * Creates a mocked Telemetry detection rules lists Task + */ +export class MockDetectionRuleListsTask extends DetectionRulesTask { + public runTask = jest.fn(); +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 038b7687784f4..94aa6c867304f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -17,8 +17,18 @@ import { AgentService, AgentPolicyServiceInterface } from '../../../../fleet/ser import { ExceptionListClient } from '../../../../lists/server'; import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; import { TELEMETRY_MAX_BUFFER_SIZE } from './constants'; -import { exceptionListItemToTelemetryEntry, trustedApplicationToTelemetryEntry } from './helpers'; -import { TelemetryEvent, ESLicense, ESClusterInfo, GetEndpointListResponse } from './types'; +import { + exceptionListItemToTelemetryEntry, + trustedApplicationToTelemetryEntry, + ruleExceptionListItemToTelemetryEvent, +} from './helpers'; +import { + TelemetryEvent, + ESLicense, + ESClusterInfo, + GetEndpointListResponse, + RuleSearchResult, +} from './types'; export class TelemetryReceiver { private readonly logger: Logger; @@ -27,6 +37,7 @@ export class TelemetryReceiver { private esClient?: ElasticsearchClient; private exceptionListClient?: ExceptionListClient; private soClient?: SavedObjectsClientContract; + private kibanaIndex?: string; private readonly max_records = 10_000; constructor(logger: Logger) { @@ -35,9 +46,11 @@ export class TelemetryReceiver { public async start( core?: CoreStart, + kibanaIndex?: string, endpointContextService?: EndpointAppContextService, exceptionListClient?: ExceptionListClient ) { + this.kibanaIndex = kibanaIndex; this.agentService = endpointContextService?.getAgentService(); this.agentPolicyService = endpointContextService?.getAgentPolicyService(); this.esClient = core?.elasticsearch.client.asInternalUser; @@ -240,6 +253,57 @@ export class TelemetryReceiver { }; } + public async fetchDetectionRules() { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve diagnostic alerts'); + } + + const query: SearchRequest = { + expand_wildcards: 'open,hidden', + index: `${this.kibanaIndex}*`, + ignore_unavailable: true, + size: this.max_records, + body: { + query: { + bool: { + filter: [ + { term: { 'alert.alertTypeId': 'siem.signals' } }, + { term: { 'alert.params.immutable': true } }, + ], + }, + }, + }, + }; + + return this.esClient.search(query); + } + + public async fetchDetectionExceptionList(listId: string, ruleVersion: number) { + if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { + throw Error('exception list client is unavailable: could not retrieve trusted applications'); + } + + // Ensure list is created if it does not exist + await this.exceptionListClient.createTrustedAppsList(); + + const results = await this.exceptionListClient?.findExceptionListsItem({ + listId: [listId], + filter: [], + perPage: this.max_records, + page: 1, + sortField: 'exception-list.created_at', + sortOrder: 'desc', + namespaceType: ['single'], + }); + + return { + data: results?.data.map((r) => ruleExceptionListItemToTelemetryEvent(r, ruleVersion)) ?? [], + total: results?.total ?? 0, + page: results?.page ?? 1, + per_page: results?.per_page ?? this.max_records, + }; + } + public async fetchClusterInfo(): Promise { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 0037aaa28fee3..b0792ed7b4610 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -18,7 +18,7 @@ import { } from '../../../../task_manager/server'; import { TelemetryReceiver } from './receiver'; import { allowlistEventFields, copyAllowlistedFields } from './filters'; -import { DiagnosticTask, EndpointTask, ExceptionListsTask } from './tasks'; +import { DiagnosticTask, EndpointTask, ExceptionListsTask, DetectionRulesTask } from './tasks'; import { createUsageCounterLabel } from './helpers'; import { TelemetryEvent } from './types'; import { TELEMETRY_MAX_BUFFER_SIZE } from './constants'; @@ -42,6 +42,7 @@ export class TelemetryEventsSender { private diagnosticTask?: DiagnosticTask; private endpointTask?: EndpointTask; private exceptionListsTask?: ExceptionListsTask; + private detectionRulesTask?: DetectionRulesTask; constructor(logger: Logger) { this.logger = logger.get('telemetry_events'); @@ -59,6 +60,12 @@ export class TelemetryEventsSender { if (taskManager) { this.diagnosticTask = new DiagnosticTask(this.logger, taskManager, this, telemetryReceiver); this.endpointTask = new EndpointTask(this.logger, taskManager, this, telemetryReceiver); + this.detectionRulesTask = new DetectionRulesTask( + this.logger, + taskManager, + this, + telemetryReceiver + ); this.exceptionListsTask = new ExceptionListsTask( this.logger, taskManager, @@ -80,6 +87,7 @@ export class TelemetryEventsSender { this.logger.debug(`starting security telemetry tasks`); this.diagnosticTask.start(taskManager); this.endpointTask.start(taskManager); + this.detectionRulesTask?.start(taskManager); this.exceptionListsTask?.start(taskManager); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.test.ts new file mode 100644 index 0000000000000..0a05afb8a6535 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.test.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from 'src/core/server/mocks'; +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../../../task_manager/server'; +import { + TelemetryDetectionRulesTask, + TelemetryDetectionRuleListsTaskConstants, +} from './detection_rule'; +import { + createMockTelemetryEventsSender, + MockDetectionRuleListsTask, + createMockTelemetryReceiver, +} from '../mocks'; + +describe('test detection rule exception lists telemetry', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + describe('basic telemetry sanity checks', () => { + test('detection rule lists task can register', () => { + const telemetryDiagTask = new TelemetryDetectionRulesTask( + logger, + taskManagerMock.createSetup(), + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() + ); + + expect(telemetryDiagTask).toBeInstanceOf(TelemetryDetectionRulesTask); + }); + }); + + test('detection rule task should be registered', () => { + const mockTaskManager = taskManagerMock.createSetup(); + new TelemetryDetectionRulesTask( + logger, + mockTaskManager, + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() + ); + + expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); + }); + + test('detection rule task should be scheduled', async () => { + const mockTaskManagerSetup = taskManagerMock.createSetup(); + const telemetryDiagTask = new TelemetryDetectionRulesTask( + logger, + mockTaskManagerSetup, + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() + ); + + const mockTaskManagerStart = taskManagerMock.createStart(); + await telemetryDiagTask.start(mockTaskManagerStart); + expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); + }); + + test('detection rule task should run', async () => { + const mockContext = createMockTelemetryEventsSender(true); + const mockTaskManager = taskManagerMock.createSetup(); + const mockReceiver = createMockTelemetryReceiver(); + const telemetryDiagTask = new MockDetectionRuleListsTask( + logger, + mockTaskManager, + mockContext, + mockReceiver + ); + + const mockTaskInstance = { + id: TelemetryDetectionRuleListsTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryDetectionRuleListsTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ + TelemetryDetectionRuleListsTaskConstants.TYPE + ].createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(telemetryDiagTask.runTask).toHaveBeenCalled(); + }); + + test('detection rule task should not query elastic if telemetry is not opted in', async () => { + const mockSender = createMockTelemetryEventsSender(false); + const mockTaskManager = taskManagerMock.createSetup(); + const mockReceiver = createMockTelemetryReceiver(); + new MockDetectionRuleListsTask(logger, mockTaskManager, mockSender, mockReceiver); + + const mockTaskInstance = { + id: TelemetryDetectionRuleListsTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryDetectionRuleListsTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ + TelemetryDetectionRuleListsTaskConstants.TYPE + ].createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockReceiver.fetchDiagnosticAlerts).not.toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts new file mode 100644 index 0000000000000..a362be187921d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { Logger } from 'src/core/server'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../../task_manager/server'; +import { LIST_DETECTION_RULE_EXCEPTION, TELEMETRY_CHANNEL_LISTS } from '../constants'; +import { batchTelemetryRecords, templateExceptionList } from '../helpers'; +import { TelemetryEventsSender } from '../sender'; +import { TelemetryReceiver } from '../receiver'; +import { ExceptionListItem, RuleSearchResult } from '../types'; + +export const TelemetryDetectionRuleListsTaskConstants = { + TIMEOUT: '10m', + TYPE: 'security:telemetry-detection-rules', + INTERVAL: '24h', + VERSION: '1.0.0', +}; + +const MAX_TELEMETRY_BATCH = 1_000; + +export class TelemetryDetectionRulesTask { + private readonly logger: Logger; + private readonly sender: TelemetryEventsSender; + private readonly receiver: TelemetryReceiver; + + constructor( + logger: Logger, + taskManager: TaskManagerSetupContract, + sender: TelemetryEventsSender, + receiver: TelemetryReceiver + ) { + this.logger = logger; + this.sender = sender; + this.receiver = receiver; + + taskManager.registerTaskDefinitions({ + [TelemetryDetectionRuleListsTaskConstants.TYPE]: { + title: 'Security Solution Detection Rule Lists Telemetry', + timeout: TelemetryDetectionRuleListsTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + const { state } = taskInstance; + + return { + run: async () => { + const taskExecutionTime = moment().utc().toISOString(); + const hits = await this.runTask(taskInstance.id); + + return { + state: { + lastExecutionTimestamp: taskExecutionTime, + runs: (state.runs || 0) + 1, + hits, + }, + }; + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + public start = async (taskManager: TaskManagerStartContract) => { + try { + await taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: TelemetryDetectionRuleListsTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + interval: TelemetryDetectionRuleListsTaskConstants.INTERVAL, + }, + state: { runs: 0 }, + params: { version: TelemetryDetectionRuleListsTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.error(`Error scheduling task, received ${e.message}`); + } + }; + + private getTaskId = (): string => { + return `${TelemetryDetectionRuleListsTaskConstants.TYPE}:${TelemetryDetectionRuleListsTaskConstants.VERSION}`; + }; + + public runTask = async (taskId: string) => { + if (taskId !== this.getTaskId()) { + return 0; + } + + const isOptedIn = await this.sender.isTelemetryOptedIn(); + if (!isOptedIn) { + return 0; + } + + // Lists Telemetry: Detection Rules + + const { body: prebuiltRules } = await this.receiver.fetchDetectionRules(); + + const cacheArray = prebuiltRules.hits.hits.reduce((cache, searchHit) => { + const rule = searchHit._source as RuleSearchResult; + const ruleId = rule.alert.params.ruleId; + + const shouldNotProcess = + rule === null || + rule === undefined || + ruleId === null || + ruleId === undefined || + searchHit._source?.alert.params.exceptionsList.length === 0; + + if (shouldNotProcess) { + return cache; + } + + cache.push(rule); + return cache; + }, [] as RuleSearchResult[]); + + const detectionRuleExceptions = [] as ExceptionListItem[]; + for (const item of cacheArray) { + const ruleVersion = item.alert.params.version; + + for (const ex of item.alert.params.exceptionsList) { + const listItem = await this.receiver.fetchDetectionExceptionList(ex.list_id, ruleVersion); + for (const exceptionItem of listItem.data) { + detectionRuleExceptions.push(exceptionItem); + } + } + } + + const detectionRuleExceptionsJson = templateExceptionList( + detectionRuleExceptions, + LIST_DETECTION_RULE_EXCEPTION + ); + + batchTelemetryRecords(detectionRuleExceptionsJson, MAX_TELEMETRY_BATCH).forEach((batch) => { + this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + }); + + return detectionRuleExceptions.length; + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 0c066deea17d9..c6bf4b06e70f0 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -14,7 +14,7 @@ import { } from '../../../../../task_manager/server'; import { batchTelemetryRecords, - getPreviousEpMetaTaskTimestamp, + getPreviousDailyTaskTimestamp, isPackagePolicyList, } from '../helpers'; import { TelemetryEventsSender } from '../sender'; @@ -76,7 +76,7 @@ export class TelemetryEndpointTask { return { run: async () => { const taskExecutionTime = moment().utc().toISOString(); - const lastExecutionTimestamp = getPreviousEpMetaTaskTimestamp( + const lastExecutionTimestamp = getPreviousDailyTaskTimestamp( taskExecutionTime, taskInstance.state?.lastExecutionTimestamp ); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts index e090252b88d8f..a850f848567cb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -8,3 +8,4 @@ export { TelemetryDiagTask as DiagnosticTask } from './diagnostic'; export { TelemetryEndpointTask as EndpointTask } from './endpoint'; export { TelemetryExceptionListsTask as ExceptionListsTask } from './security_lists'; +export { TelemetryDetectionRulesTask as DetectionRulesTask } from './detection_rule'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index abcad26ed000c..6aaf6f4371475 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -217,18 +217,45 @@ export interface GetEndpointListResponse { export interface ExceptionListItem { id: string; - version: string; + rule_version?: number; name: string; - description: string; created_at: string; updated_at: string; entries: object; - os: string; os_types: object; } export interface ListTemplate { - trusted_application: TelemetryEvent[]; - endpoint_exception: TelemetryEvent[]; - endpoint_event_filter: TelemetryEvent[]; + '@timestamp': number; + detection_rule?: TelemetryEvent; + endpoint_exception?: TelemetryEvent; + endpoint_event_filter?: TelemetryEvent; + trusted_application?: TelemetryEvent; +} + +// Detection Rule types + +interface ExceptionListEntry { + id: string; + list_id: string; + type: string; + namespace_type: string; +} + +interface DetectionRuleParms { + ruleId: string; + version: number; + type: string; + exceptionsList: ExceptionListEntry[]; +} + +export interface RuleSearchResult { + alert: { + name: string; + enabled: boolean; + tags: string[]; + createdAt: string; + updatedAt: string; + params: DetectionRuleParms; + }; } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 413b5c37bb06d..679e13a749a7d 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -452,7 +452,13 @@ export class Plugin implements IPlugin