From b5d78e85fed28f463fc062c3ff6628ef223f757c Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Mon, 26 May 2025 15:50:41 +0200 Subject: [PATCH 01/19] add snapshot telemtery for response actions rules closes https://github.com/elastic/security-team/issues/10384t --- .../server/lib/telemetry/helpers.ts | 43 ++++++ .../server/lib/telemetry/receiver.ts | 33 +++++ .../server/lib/telemetry/tasks/custom_rule.ts | 127 ++++++++++++++++++ .../server/lib/telemetry/tasks/index.ts | 4 + .../server/lib/telemetry/types.ts | 20 +++ 5 files changed, 227 insertions(+) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index 9eb0547535841..830fca4d757be 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -24,6 +24,8 @@ import type { ExtraInfo, ListTemplate, Nullable, + ResponseActionsRuleTemplate, + RulesParamsResponseActionsEntry, TelemetryEvent, TimeFrame, TimelineResult, @@ -236,6 +238,47 @@ export const templateExceptionList = ( }); }; +/** + * Constructs the response actions custom rule telemetry schema from a list of rule params + * */ +export const templateResponseActionsCustomRule = ( + responseActionsEntries: RulesParamsResponseActionsEntry[], + clusterInfo: ESClusterInfo, + licenseInfo: Nullable +): ResponseActionsRuleTemplate[] => { + return responseActionsEntries.map((item) => { + const telemetryTemplate: ResponseActionsRuleTemplate = { + '@timestamp': moment().toISOString(), + cluster_uuid: clusterInfo.cluster_uuid, + cluster_name: clusterInfo.cluster_name, + license_id: licenseInfo?.uid, + }; + + const { endpointActionCount, osqueryActionCount } = responseActionsEntries.reduce<{ + endpointActionCount: number; + osqueryActionCount: number; + }>( + (acc, entry) => { + if (entry.actionTypeId === '.endpoint') { + acc.endpointActionCount += 1; + } else if (entry.actionTypeId === '.osquery') { + acc.osqueryActionCount += 1; + } + return acc; + }, + { endpointActionCount: 0, osqueryActionCount: 0 } + ); + + return { + ...telemetryTemplate, + response_actions: { + endpoint_action_count: endpointActionCount, + osquery_actions_count: osqueryActionCount, + }, + }; + }); +}; + /** * Convert counter label list to kebab case * diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 5414677af5924..173316ae26326 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -15,6 +15,7 @@ import type { IScopedClusterClient, ElasticsearchClient, SavedObjectsClientContract, + SavedObjectsFindResponse, } from '@kbn/core/server'; import type { AggregationsAggregate, @@ -54,6 +55,8 @@ import type { } from '@kbn/fleet-plugin/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import moment from 'moment'; +import { findRulesSo } from '@kbn/alerting-plugin/server/data/rule'; +import type { RawRule } from '@kbn/alerting-plugin/server/types'; import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import type { ExperimentalFeatures } from '../../../common'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; @@ -219,6 +222,8 @@ export interface ITelemetryReceiver { > >; + fetchResponseActionsRules(): Promise>>; + fetchDetectionExceptionList( listId: string, ruleVersion: number @@ -749,6 +754,34 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.esClient().search(query, { meta: true }); } + /** + * Find elastic rules SOs which are the rules that have immutable set to true and are of a particular rule type + * @returns custom elastic rules SOs with response actions enabled + */ + public async fetchResponseActionsRules() { + if (this.soClient === undefined || this.soClient === null) { + throw Error( + 'saved object client is unavailable: cannot retrieve custom detection rules with response actions enabled' + ); + } + + const timeFrom = `alert.updated_at >= ${moment.utc().subtract(24, 'hours').valueOf()}`; + const enabledCustomRules = `alert.attributes.params.immutable: false AND alert.attributes.enabled: true`; + const responseActionsRules = `alert.attributes.params.responseActions: *`; + const combinedFilters = [enabledCustomRules, responseActionsRules, timeFrom].join(' AND '); + + return findRulesSo({ + savedObjectsClient: this.soClient, + savedObjectsFindOptions: { + filter: combinedFilters, + perPage: this.maxRecords, + page: 1, + sortField: 'updatedAt', + sortOrder: 'desc', + }, + }); + } + public async fetchDetectionExceptionList(listId: string, ruleVersion: number) { if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { throw Error('exception list client is unavailable: could not retrieve trusted applications'); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts new file mode 100644 index 0000000000000..f0d6ec592dcbe --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep } from 'lodash'; +import type { Logger } from '@kbn/core/server'; +import { TELEMETRY_CHANNEL_LISTS } from '../constants'; +import { + batchTelemetryRecords, + templateResponseActionsCustomRule, + newTelemetryLogger, + createUsageCounterLabel, + safeValue, +} from '../helpers'; +import type { ITelemetryEventsSender } from '../sender'; +import type { ITelemetryReceiver } from '../receiver'; +import type { RulesParamsResponseActionsEntry, RuleSearchResult } from '../types'; +import type { TaskExecutionPeriod } from '../task'; +import type { ITaskMetricsService } from '../task_metrics.types'; + +export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryBatch: number) { + const taskName = 'Security Solution Response Actions Rules Telemetry'; + const taskType = 'security:telemetry-response-actions-rules'; + return { + type: taskType, + title: taskName, + interval: '24h', + timeout: '10m', + version: '1.0.0', + runTask: async ( + taskId: string, + logger: Logger, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, + taskMetricsService: ITaskMetricsService, + taskExecutionPeriod: TaskExecutionPeriod + ) => { + const mdc = { task_id: taskId, task_execution_period: taskExecutionPeriod }; + const log = newTelemetryLogger(logger.get('response_actions_rules'), mdc); + const usageCollector = sender.getTelemetryUsageCluster(); + const usageLabelPrefix: string[] = ['security_telemetry', 'response-actions-rules']; + const trace = taskMetricsService.start(taskType); + + log.l('Running response actions rules telemetry task'); + + try { + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + receiver.fetchClusterInfo(), + receiver.fetchLicenseInfo(), + ]); + + const clusterInfo = safeValue(clusterInfoPromise); + const licenseInfo = safeValue(licenseInfoPromise); + + const { body: responseActionRules } = await receiver.fetchResponseActionsRules(); + + if (!responseActionRules) { + log.debug('no prebuilt rules found'); + await taskMetricsService.end(trace); + return 0; + } + + const cacheArray = responseActionRules.hits.hits.reduce((cache, searchHit) => { + const rule = searchHit._source as RuleSearchResult; + const ruleId = rule.alert.params.ruleId; + + const shouldNotProcess = + rule === null || + rule === undefined || + ruleId === null || + ruleId === undefined || + searchHit._source?.alert.params.responseActions.length === 0; + + if (shouldNotProcess) { + return cache; + } + + cache.push(rule); + return cache; + }, [] as RuleSearchResult[]); + + const rulesParamsResponseActionsRulesEntries = [] as RulesParamsResponseActionsEntry[]; + for (const item of cacheArray) { + for (const el of item.alert.params.responseActions) { + rulesParamsResponseActionsRulesEntries.push({ + ...el, + }); + } + } + + const responseActionsRulesJson = templateResponseActionsCustomRule( + rulesParamsResponseActionsRulesEntries, + clusterInfo, + licenseInfo + ); + log.l('Custom response actions rule json length', { + length: responseActionsRulesJson.length, + }); + + usageCollector?.incrementCounter({ + counterName: createUsageCounterLabel(usageLabelPrefix), + counterType: 'response_actions_rules_count', + incrementBy: responseActionsRulesJson.length, + }); + + const batches = batchTelemetryRecords( + cloneDeep(responseActionsRulesJson), + maxTelemetryBatch + ); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + } + await taskMetricsService.end(trace); + + log.l('Task executed', { length: responseActionsRulesJson.length }); + + return responseActionsRulesJson.length; + } catch (err) { + await taskMetricsService.end(trace, err); + return 0; + } + }, + }; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts index b5496373937c2..3a6ab44873779 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -18,6 +18,7 @@ import { telemetryConfiguration } from '../configuration'; import { createTelemetryFilterListArtifactTaskConfig } from './filterlists'; import { createTelemetryIndicesMetadataTaskConfig } from './indices.metadata'; import { createIngestStatsTaskConfig } from './ingest_pipelines_stats'; +import { createTelemetryCustomResponseActionRulesTaskConfig } from './custom_rule'; export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { return [ @@ -27,6 +28,9 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { createTelemetryDetectionRuleListsTaskConfig( telemetryConfiguration.max_detection_rule_telemetry_batch ), + createTelemetryCustomResponseActionRulesTaskConfig( + telemetryConfiguration.max_detection_rule_telemetry_batch + ), createTelemetryPrebuiltRuleAlertsTaskConfig(telemetryConfiguration.max_detection_alerts_batch), createTelemetryTimelineTaskConfig(), createTelemetryDiagnosticTimelineTaskConfig(), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index e21d60c03a1a6..dbd4be41442a1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -386,11 +386,31 @@ interface ExceptionListEntry { namespace_type: string; } +export interface ResponseActionsRuleTemplate { + '@timestamp': string; + cluster_uuid: string; + cluster_name: string; + license_id: string | undefined; + response_actions?: TelemetryEvent; +} +export interface RulesParamsResponseActionsEntry { + actionTypeId: '.endpoint' | '.osquery'; + params: + | { + command: string; + comment?: string; + } + | { + query: string; + }; +} + interface DetectionRuleParms { ruleId: string; version: number; type: string; exceptionsList: ExceptionListEntry[]; + responseActions: RulesParamsResponseActionsEntry[]; } export interface RuleSearchResult { From ec5e4d38dea78ec50f184e2761835d66ecd4745b Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 28 May 2025 14:45:02 +0200 Subject: [PATCH 02/19] fix KQL filter to select response action rules SOs --- .../server/lib/telemetry/receiver.ts | 14 +++++-- .../server/lib/telemetry/tasks/custom_rule.ts | 40 +++++++++++++------ .../server/lib/telemetry/types.ts | 18 ++++++++- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 173316ae26326..941f5fd1a2c16 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -222,7 +222,7 @@ export interface ITelemetryReceiver { > >; - fetchResponseActionsRules(): Promise>>; + fetchResponseActionsRules(): ReturnType; fetchDetectionExceptionList( listId: string, @@ -766,13 +766,21 @@ export class TelemetryReceiver implements ITelemetryReceiver { } const timeFrom = `alert.updated_at >= ${moment.utc().subtract(24, 'hours').valueOf()}`; - const enabledCustomRules = `alert.attributes.params.immutable: false AND alert.attributes.enabled: true`; - const responseActionsRules = `alert.attributes.params.responseActions: *`; + const enabledCustomRules = + 'alert.attributes.params.immutable: false AND alert.attributes.consumer: "siem"'; + const responseActionsRules = + 'alert.attributes.params.responseActions.actionTypeId: .endpoint OR ' + + 'alert.attributes.params.responseActions.actionTypeId: .osquery'; const combinedFilters = [enabledCustomRules, responseActionsRules, timeFrom].join(' AND '); return findRulesSo({ savedObjectsClient: this.soClient, savedObjectsFindOptions: { + fields: [ + 'alert.attributes.consumer', + 'alert.attributes.params.immutable', + 'alert.attributes.params.responseActions.actionTypeId', + ], filter: combinedFilters, perPage: this.maxRecords, page: 1, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts index f0d6ec592dcbe..e2a97a9eae3ab 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts @@ -17,7 +17,7 @@ import { } from '../helpers'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; -import type { RulesParamsResponseActionsEntry, RuleSearchResult } from '../types'; +import type { RulesParamsResponseActionsEntry, ResponseActionRules } from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; @@ -55,36 +55,50 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const clusterInfo = safeValue(clusterInfoPromise); const licenseInfo = safeValue(licenseInfoPromise); - const { body: responseActionRules } = await receiver.fetchResponseActionsRules(); + const { saved_objects: customRules } = await receiver.fetchResponseActionsRules(); - if (!responseActionRules) { - log.debug('no prebuilt rules found'); + if (!customRules.length) { + log.debug('no custom response action rules found'); await taskMetricsService.end(trace); return 0; } - const cacheArray = responseActionRules.hits.hits.reduce((cache, searchHit) => { - const rule = searchHit._source as RuleSearchResult; - const ruleId = rule.alert.params.ruleId; + const cacheArray = customRules.reduce((acc, rule) => { + const ruleId = rule.id; const shouldNotProcess = rule === null || rule === undefined || ruleId === null || ruleId === undefined || - searchHit._source?.alert.params.responseActions.length === 0; + rule.attributes.params.responseActions.length === 0; if (shouldNotProcess) { - return cache; + return acc; } - cache.push(rule); - return cache; - }, [] as RuleSearchResult[]); + acc.push({ + id: ruleId, + namespaces: rule.namespaces ?? [], + attributes: { + consumer: rule.attributes.consumer, + createdAt: rule.attributes.createdAt, + name: rule.attributes.name, + enabled: rule.attributes.enabled, + immutable: rule.attributes.params.immutable, + params: { + responseActions: rule.attributes.params.responseActions, + }, + tags: rule.attributes.tags, + updatedAt: rule.attributes.updatedAt, + }, + }); + return acc; + }, []); const rulesParamsResponseActionsRulesEntries = [] as RulesParamsResponseActionsEntry[]; for (const item of cacheArray) { - for (const el of item.alert.params.responseActions) { + for (const el of item.attributes.params.responseActions) { rulesParamsResponseActionsRulesEntries.push({ ...el, }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index dbd4be41442a1..bf69c10c72d1a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -405,12 +405,28 @@ export interface RulesParamsResponseActionsEntry { }; } +export interface ResponseActionRules { + id: string; + namespaces: string[]; + attributes: { + consumer: string; + createdAt: string; + name: string; + enabled: boolean; + immutable: boolean; + params: { + responseActions: RulesParamsResponseActionsEntry[]; + }; + tags: string[]; + updatedAt: string; + }; +} + interface DetectionRuleParms { ruleId: string; version: number; type: string; exceptionsList: ExceptionListEntry[]; - responseActions: RulesParamsResponseActionsEntry[]; } export interface RuleSearchResult { From 68556676ab0fb3775efb3f9ffec8fd3886ced145 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 3 Jun 2025 14:26:01 +0200 Subject: [PATCH 03/19] cleanup event data and types --- .../server/lib/telemetry/helpers.ts | 49 +++++++++---------- .../server/lib/telemetry/receiver.ts | 2 - .../server/lib/telemetry/tasks/custom_rule.ts | 35 +++++-------- .../server/lib/telemetry/types.ts | 14 +++--- 4 files changed, 44 insertions(+), 56 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index 830fca4d757be..388f9bcd3ab7a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -24,13 +24,14 @@ import type { ExtraInfo, ListTemplate, Nullable, - ResponseActionsRuleTemplate, - RulesParamsResponseActionsEntry, + ResponseActionsRuleTelemetryTemplate, + ResponseActionRules, TelemetryEvent, TimeFrame, TimelineResult, TimelineTelemetryEvent, ValueListResponse, + RulesParamsResponseActionsEntry, } from './types'; import type { TaskExecutionPeriod } from './task'; import { @@ -241,44 +242,42 @@ export const templateExceptionList = ( /** * Constructs the response actions custom rule telemetry schema from a list of rule params * */ -export const templateResponseActionsCustomRule = ( - responseActionsEntries: RulesParamsResponseActionsEntry[], +export const responseActionsCustomRuleTelemetryData = ( + responseActionsRules: ResponseActionRules, clusterInfo: ESClusterInfo, licenseInfo: Nullable -): ResponseActionsRuleTemplate[] => { - return responseActionsEntries.map((item) => { - const telemetryTemplate: ResponseActionsRuleTemplate = { +): ResponseActionsRuleTelemetryTemplate[] => { + return responseActionsRules.map((item) => { + const baseTelemetryData: ResponseActionsRuleTelemetryTemplate = { '@timestamp': moment().toISOString(), cluster_uuid: clusterInfo.cluster_uuid, cluster_name: clusterInfo.cluster_name, license_id: licenseInfo?.uid, }; - const { endpointActionCount, osqueryActionCount } = responseActionsEntries.reduce<{ - endpointActionCount: number; - osqueryActionCount: number; - }>( - (acc, entry) => { - if (entry.actionTypeId === '.endpoint') { - acc.endpointActionCount += 1; - } else if (entry.actionTypeId === '.osquery') { - acc.osqueryActionCount += 1; - } - return acc; - }, - { endpointActionCount: 0, osqueryActionCount: 0 } - ); - return { - ...telemetryTemplate, + ...baseTelemetryData, response_actions: { - endpoint_action_count: endpointActionCount, - osquery_actions_count: osqueryActionCount, + rules: responseActionsRules, + endpoint_rules_count: getRulesCountForActionType(responseActionsRules, '.endpoint'), + osquery_rules_count: getRulesCountForActionType(responseActionsRules, '.osquery'), }, }; }); }; +const getRulesCountForActionType = ( + responseActionsRules: ResponseActionRules, + actionTypeId: RulesParamsResponseActionsEntry['actionTypeId'] +): number => { + return responseActionsRules.filter( + (rule) => + rule.attributes.params.responseActions.filter( + (action) => action.actionTypeId === actionTypeId + ).length + ).length; +}; + /** * Convert counter label list to kebab case * diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 941f5fd1a2c16..98e290a5b2782 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -15,7 +15,6 @@ import type { IScopedClusterClient, ElasticsearchClient, SavedObjectsClientContract, - SavedObjectsFindResponse, } from '@kbn/core/server'; import type { AggregationsAggregate, @@ -56,7 +55,6 @@ import type { import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import moment from 'moment'; import { findRulesSo } from '@kbn/alerting-plugin/server/data/rule'; -import type { RawRule } from '@kbn/alerting-plugin/server/types'; import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import type { ExperimentalFeatures } from '../../../common'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts index e2a97a9eae3ab..cc200487f27b7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts @@ -10,14 +10,14 @@ import type { Logger } from '@kbn/core/server'; import { TELEMETRY_CHANNEL_LISTS } from '../constants'; import { batchTelemetryRecords, - templateResponseActionsCustomRule, + responseActionsCustomRuleTelemetryData, newTelemetryLogger, createUsageCounterLabel, safeValue, } from '../helpers'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; -import type { RulesParamsResponseActionsEntry, ResponseActionRules } from '../types'; +import type { ResponseActionRules } from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; @@ -63,23 +63,22 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB return 0; } - const cacheArray = customRules.reduce((acc, rule) => { + const responseActionRulesArray = customRules.reduce((acc, rule) => { const ruleId = rule.id; - const shouldNotProcess = + const shouldNotProcessTelemetry = rule === null || rule === undefined || ruleId === null || ruleId === undefined || rule.attributes.params.responseActions.length === 0; - if (shouldNotProcess) { + if (shouldNotProcessTelemetry) { return acc; } acc.push({ id: ruleId, - namespaces: rule.namespaces ?? [], attributes: { consumer: rule.attributes.consumer, createdAt: rule.attributes.createdAt, @@ -89,39 +88,29 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB params: { responseActions: rule.attributes.params.responseActions, }, - tags: rule.attributes.tags, updatedAt: rule.attributes.updatedAt, }, }); return acc; }, []); - const rulesParamsResponseActionsRulesEntries = [] as RulesParamsResponseActionsEntry[]; - for (const item of cacheArray) { - for (const el of item.attributes.params.responseActions) { - rulesParamsResponseActionsRulesEntries.push({ - ...el, - }); - } - } - - const responseActionsRulesJson = templateResponseActionsCustomRule( - rulesParamsResponseActionsRulesEntries, + const responseActionsRulesTelemetryEvent = responseActionsCustomRuleTelemetryData( + responseActionRulesArray, clusterInfo, licenseInfo ); log.l('Custom response actions rule json length', { - length: responseActionsRulesJson.length, + length: responseActionsRulesTelemetryEvent.length, }); usageCollector?.incrementCounter({ counterName: createUsageCounterLabel(usageLabelPrefix), counterType: 'response_actions_rules_count', - incrementBy: responseActionsRulesJson.length, + incrementBy: responseActionsRulesTelemetryEvent.length, }); const batches = batchTelemetryRecords( - cloneDeep(responseActionsRulesJson), + cloneDeep(responseActionsRulesTelemetryEvent), maxTelemetryBatch ); for (const batch of batches) { @@ -129,9 +118,9 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB } await taskMetricsService.end(trace); - log.l('Task executed', { length: responseActionsRulesJson.length }); + log.l('Task executed', { length: responseActionsRulesTelemetryEvent.length }); - return responseActionsRulesJson.length; + return responseActionsRulesTelemetryEvent.length; } catch (err) { await taskMetricsService.end(trace, err); return 0; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index bf69c10c72d1a..fe579d268998b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -386,12 +386,16 @@ interface ExceptionListEntry { namespace_type: string; } -export interface ResponseActionsRuleTemplate { +export interface ResponseActionsRuleTelemetryTemplate { '@timestamp': string; cluster_uuid: string; cluster_name: string; license_id: string | undefined; - response_actions?: TelemetryEvent; + response_actions?: { + rules: ResponseActionRules; + endpoint_rules_count: number; + osquery_rules_count: number; + }; } export interface RulesParamsResponseActionsEntry { actionTypeId: '.endpoint' | '.osquery'; @@ -405,9 +409,8 @@ export interface RulesParamsResponseActionsEntry { }; } -export interface ResponseActionRules { +export type ResponseActionRules = Array<{ id: string; - namespaces: string[]; attributes: { consumer: string; createdAt: string; @@ -417,10 +420,9 @@ export interface ResponseActionRules { params: { responseActions: RulesParamsResponseActionsEntry[]; }; - tags: string[]; updatedAt: string; }; -} +}>; interface DetectionRuleParms { ruleId: string; From cff4aa66fe09ef14446b74b37c336d6a58eb0a90 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 3 Jun 2025 20:31:10 +0200 Subject: [PATCH 04/19] update registered task test --- .../test_suites/task_manager/check_registered_task_types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index a1ab089fdf847..3c94e0cb17774 100644 --- a/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -182,6 +182,7 @@ export default function ({ getService }: FtrProviderContext) { 'security:telemetry-filterlist-artifact', 'security:telemetry-lists', 'security:telemetry-prebuilt-rule-alerts', + 'security:telemetry-response-actions-rules', 'security:telemetry-timelines', 'session_cleanup', 'slo:bulk-delete-task', From 0dd238594f4df09f5f7007bc1e5026ca765219d9 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 4 Jun 2025 10:36:42 +0200 Subject: [PATCH 05/19] fix --- .../security_solution/server/lib/telemetry/receiver.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 98e290a5b2782..2219f16b3d84c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -765,16 +765,20 @@ export class TelemetryReceiver implements ITelemetryReceiver { const timeFrom = `alert.updated_at >= ${moment.utc().subtract(24, 'hours').valueOf()}`; const enabledCustomRules = - 'alert.attributes.params.immutable: false AND alert.attributes.consumer: "siem"'; + 'alert.attributes.enabled: true AND ' + + 'alert.attributes.params.immutable: false AND ' + + 'alert.attributes.consumer: "siem"'; const responseActionsRules = - 'alert.attributes.params.responseActions.actionTypeId: .endpoint OR ' + - 'alert.attributes.params.responseActions.actionTypeId: .osquery'; + '(alert.attributes.params.responseActions.actionTypeId: .endpoint OR ' + + 'alert.attributes.params.responseActions.actionTypeId: .osquery)'; const combinedFilters = [enabledCustomRules, responseActionsRules, timeFrom].join(' AND '); return findRulesSo({ savedObjectsClient: this.soClient, savedObjectsFindOptions: { fields: [ + 'alert.updated_at', + 'alert.attributes.enabled', 'alert.attributes.consumer', 'alert.attributes.params.immutable', 'alert.attributes.params.responseActions.actionTypeId', From c7d63e3d65ff475957e04adc5250e623574a2091 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 4 Jun 2025 14:39:19 +0200 Subject: [PATCH 06/19] fix KQL --- .../server/lib/telemetry/helpers.ts | 9 +++++ .../server/lib/telemetry/receiver.ts | 37 +++++++++++++------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index 388f9bcd3ab7a..5024f49c64a2c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -13,6 +13,7 @@ import { set } from '@kbn/safer-lodash-set'; import type { Logger, LogMeta } from '@kbn/core/server'; import { sha256 } from 'js-sha256'; import type { estypes } from '@elastic/elasticsearch'; +import { fromKueryExpression, type KueryNode } from '@kbn/es-query'; import { copyAllowlistedFields, filterList } from './filterlists'; import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types'; import type { ITelemetryReceiver } from './receiver'; @@ -420,6 +421,14 @@ export const copyLicenseFields = (lic: ESLicense) => { }; }; +export function stringToKueryNode(expression?: string): KueryNode | undefined { + if (!expression) { + return; + } + + return fromKueryExpression(expression); +} + export class TelemetryTimelineFetcher { private receiver: ITelemetryReceiver; private extraInfo: Promise; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 2219f16b3d84c..0690d389097b6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -55,6 +55,11 @@ import type { import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import moment from 'moment'; import { findRulesSo } from '@kbn/alerting-plugin/server/data/rule'; +import { + buildFilter, + buildConsumersFilter, + combineFilters, +} from '@kbn/alerting-plugin/server/rules_client/common/filters'; import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import type { ExperimentalFeatures } from '../../../common'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; @@ -64,6 +69,7 @@ import { ruleExceptionListItemToTelemetryEvent, setClusterInfo, newTelemetryLogger, + stringToKueryNode, } from './helpers'; import { Fetcher } from '../../endpoint/routes/resolver/tree/utils/fetch'; import type { TreeOptions, TreeResponse } from '../../endpoint/routes/resolver/tree/utils/fetch'; @@ -763,30 +769,37 @@ export class TelemetryReceiver implements ITelemetryReceiver { ); } - const timeFrom = `alert.updated_at >= ${moment.utc().subtract(24, 'hours').valueOf()}`; - const enabledCustomRules = - 'alert.attributes.enabled: true AND ' + - 'alert.attributes.params.immutable: false AND ' + - 'alert.attributes.consumer: "siem"'; - const responseActionsRules = - '(alert.attributes.params.responseActions.actionTypeId: .endpoint OR ' + - 'alert.attributes.params.responseActions.actionTypeId: .osquery)'; - const combinedFilters = [enabledCustomRules, responseActionsRules, timeFrom].join(' AND '); + const timeFrom = stringToKueryNode( + `alert.updated_at >= ${moment.utc().subtract(24, 'hours').valueOf()}` + ); + const consumersFilter = buildConsumersFilter(['siem', 'securitySolution']); + const enabledFilter = stringToKueryNode('alert.attributes.enabled: true'); + const customRulesFilter = stringToKueryNode('alert.attributes.params.immutable: false'); + const responseActionsFilter = buildFilter({ + filters: ['.endpoint', '.osquery'], + field: 'params.responseActions.actionTypeId', + operator: 'or', + }); + + const combinedFilter = combineFilters( + [consumersFilter, enabledFilter, customRulesFilter, responseActionsFilter, timeFrom], + 'and' + ); return findRulesSo({ savedObjectsClient: this.soClient, savedObjectsFindOptions: { + filter: combinedFilter, fields: [ 'alert.updated_at', - 'alert.attributes.enabled', 'alert.attributes.consumer', + 'alert.attributes.enabled', 'alert.attributes.params.immutable', 'alert.attributes.params.responseActions.actionTypeId', ], - filter: combinedFilters, perPage: this.maxRecords, page: 1, - sortField: 'updatedAt', + sortField: 'updated_at', sortOrder: 'desc', }, }); From cef14d9789ed8b17cac7fcd52231c4eaba4cae24 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 11 Jun 2025 13:18:58 +0200 Subject: [PATCH 07/19] use ES Client instead of rulesSO client --- .../server/lib/telemetry/helpers.ts | 67 +++++----- .../server/lib/telemetry/receiver.ts | 116 +++++++++++------- .../server/lib/telemetry/tasks/custom_rule.ts | 94 +++++++------- .../server/lib/telemetry/types.ts | 59 +++++---- 4 files changed, 179 insertions(+), 157 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index 5024f49c64a2c..5aa5efaf90306 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -13,7 +13,6 @@ import { set } from '@kbn/safer-lodash-set'; import type { Logger, LogMeta } from '@kbn/core/server'; import { sha256 } from 'js-sha256'; import type { estypes } from '@elastic/elasticsearch'; -import { fromKueryExpression, type KueryNode } from '@kbn/es-query'; import { copyAllowlistedFields, filterList } from './filterlists'; import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types'; import type { ITelemetryReceiver } from './receiver'; @@ -32,7 +31,6 @@ import type { TimelineResult, TimelineTelemetryEvent, ValueListResponse, - RulesParamsResponseActionsEntry, } from './types'; import type { TaskExecutionPeriod } from './task'; import { @@ -247,36 +245,37 @@ export const responseActionsCustomRuleTelemetryData = ( responseActionsRules: ResponseActionRules, clusterInfo: ESClusterInfo, licenseInfo: Nullable -): ResponseActionsRuleTelemetryTemplate[] => { - return responseActionsRules.map((item) => { - const baseTelemetryData: ResponseActionsRuleTelemetryTemplate = { - '@timestamp': moment().toISOString(), - cluster_uuid: clusterInfo.cluster_uuid, - cluster_name: clusterInfo.cluster_name, - license_id: licenseInfo?.uid, - }; - - return { - ...baseTelemetryData, - response_actions: { - rules: responseActionsRules, - endpoint_rules_count: getRulesCountForActionType(responseActionsRules, '.endpoint'), - osquery_rules_count: getRulesCountForActionType(responseActionsRules, '.osquery'), +): ResponseActionsRuleTelemetryTemplate => { + const baseTelemetryData: ResponseActionsRuleTelemetryTemplate = { + '@timestamp': moment().toISOString(), + cluster_uuid: clusterInfo.cluster_uuid, + cluster_name: clusterInfo.cluster_name, + license_id: licenseInfo?.uid, + response_actions: { + endpoint: { + ids: [], + count: 0, }, - }; - }); -}; + osquery: { + ids: [], + count: 0, + }, + }, + }; -const getRulesCountForActionType = ( - responseActionsRules: ResponseActionRules, - actionTypeId: RulesParamsResponseActionsEntry['actionTypeId'] -): number => { - return responseActionsRules.filter( - (rule) => - rule.attributes.params.responseActions.filter( - (action) => action.actionTypeId === actionTypeId - ).length - ).length; + return { + ...baseTelemetryData, + response_actions: { + endpoint: { + ids: responseActionsRules.endpoint, + count: responseActionsRules.endpoint.length, + }, + osquery: { + ids: responseActionsRules.osquery, + count: responseActionsRules.osquery.length, + }, + }, + }; }; /** @@ -421,14 +420,6 @@ export const copyLicenseFields = (lic: ESLicense) => { }; }; -export function stringToKueryNode(expression?: string): KueryNode | undefined { - if (!expression) { - return; - } - - return fromKueryExpression(expression); -} - export class TelemetryTimelineFetcher { private receiver: ITelemetryReceiver; private extraInfo: Promise; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 0690d389097b6..b0887abb4423b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -54,12 +54,7 @@ import type { } from '@kbn/fleet-plugin/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import moment from 'moment'; -import { findRulesSo } from '@kbn/alerting-plugin/server/data/rule'; -import { - buildFilter, - buildConsumersFilter, - combineFilters, -} from '@kbn/alerting-plugin/server/rules_client/common/filters'; + import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import type { ExperimentalFeatures } from '../../../common'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; @@ -69,7 +64,6 @@ import { ruleExceptionListItemToTelemetryEvent, setClusterInfo, newTelemetryLogger, - stringToKueryNode, } from './helpers'; import { Fetcher } from '../../endpoint/routes/resolver/tree/utils/fetch'; import type { TreeOptions, TreeResponse } from '../../endpoint/routes/resolver/tree/utils/fetch'; @@ -226,7 +220,9 @@ export interface ITelemetryReceiver { > >; - fetchResponseActionsRules(): ReturnType; + fetchResponseActionsRules(): Promise< + TransportResult>, unknown> + >; fetchDetectionExceptionList( listId: string, @@ -763,46 +759,72 @@ export class TelemetryReceiver implements ITelemetryReceiver { * @returns custom elastic rules SOs with response actions enabled */ public async fetchResponseActionsRules() { - if (this.soClient === undefined || this.soClient === null) { - throw Error( - 'saved object client is unavailable: cannot retrieve custom detection rules with response actions enabled' - ); - } - - const timeFrom = stringToKueryNode( - `alert.updated_at >= ${moment.utc().subtract(24, 'hours').valueOf()}` - ); - const consumersFilter = buildConsumersFilter(['siem', 'securitySolution']); - const enabledFilter = stringToKueryNode('alert.attributes.enabled: true'); - const customRulesFilter = stringToKueryNode('alert.attributes.params.immutable: false'); - const responseActionsFilter = buildFilter({ - filters: ['.endpoint', '.osquery'], - field: 'params.responseActions.actionTypeId', - operator: 'or', - }); - - const combinedFilter = combineFilters( - [consumersFilter, enabledFilter, customRulesFilter, responseActionsFilter, timeFrom], - 'and' - ); - - return findRulesSo({ - savedObjectsClient: this.soClient, - savedObjectsFindOptions: { - filter: combinedFilter, - fields: [ - 'alert.updated_at', - 'alert.attributes.consumer', - 'alert.attributes.enabled', - 'alert.attributes.params.immutable', - 'alert.attributes.params.responseActions.actionTypeId', - ], - perPage: this.maxRecords, - page: 1, - sortField: 'updated_at', - sortOrder: 'desc', + const query: SearchRequest = { + index: `${this.getAlertsIndex()}*`, + ignore_unavailable: true, + size: 0, // no query results required - only aggregation quantity + from: 0, + query: { + bool: { + must: [ + { + term: { + 'kibana.alert.rule.immutable': false, + }, + }, + { + term: { + 'kibana.alert.rule.enabled': true, + }, + }, + { + terms: { + 'kibana.alert.rule.consumer': ['siem', 'securitySolution'], + }, + }, + { + terms: { + 'kibana.alert.rule.parameters.response_actions.action_type_id': [ + '.endpoint', + '.osquery', + ], + }, + }, + { + range: { + 'kibana.alert.rule.updated_at': { + gte: 'now-24h/h', + lte: 'now', + }, + }, + }, + ], + }, }, - }); + sort: [ + { + 'kibana.alert.rule.updated_at': { + order: 'desc', + }, + }, + ], + aggs: { + actionTypes: { + terms: { + field: 'kibana.alert.rule.parameters.response_actions.action_type_id', + }, + aggs: { + rulesInfo: { + terms: { + field: 'kibana.alert.rule.uuid', + }, + }, + }, + }, + }, + }; + + return this.esClient().search(query, { meta: true }); } public async fetchDetectionExceptionList(listId: string, ruleVersion: number) { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts index cc200487f27b7..6a4f36632ea08 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts @@ -17,7 +17,7 @@ import { } from '../helpers'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; -import type { ResponseActionRules } from '../types'; +import type { ResponseActionRules, ResponseActionsRuleResponseAggregations } from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; @@ -41,7 +41,14 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const mdc = { task_id: taskId, task_execution_period: taskExecutionPeriod }; const log = newTelemetryLogger(logger.get('response_actions_rules'), mdc); const usageCollector = sender.getTelemetryUsageCluster(); - const usageLabelPrefix: string[] = ['security_telemetry', 'response-actions-rules']; + const usageLabelEndpointPrefix: string[] = [ + 'security_telemetry', + 'endpoint-response-actions-rules', + ]; + const usageLabelOsqueryPrefix: string[] = [ + 'security_telemetry', + 'osquery-response-actions-rules', + ]; const trace = taskMetricsService.start(taskType); log.l('Running response actions rules telemetry task'); @@ -55,72 +62,77 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const clusterInfo = safeValue(clusterInfoPromise); const licenseInfo = safeValue(licenseInfoPromise); - const { saved_objects: customRules } = await receiver.fetchResponseActionsRules(); + const { + body: { aggregations }, + } = (await receiver.fetchResponseActionsRules()) ?? {}; - if (!customRules.length) { + if (!aggregations || !aggregations.actionTypes) { log.debug('no custom response action rules found'); await taskMetricsService.end(trace); return 0; } - const responseActionRulesArray = customRules.reduce((acc, rule) => { - const ruleId = rule.id; - - const shouldNotProcessTelemetry = - rule === null || - rule === undefined || - ruleId === null || - ruleId === undefined || - rule.attributes.params.responseActions.length === 0; - - if (shouldNotProcessTelemetry) { - return acc; + const responseActionRules = ( + aggregations as unknown as ResponseActionsRuleResponseAggregations + ).actionTypes.buckets.reduce((acc, agg) => { + if (agg.key === '.endpoint') { + acc.endpoint = agg.rulesInfo.buckets.map((rule) => rule.key); + } else if (agg.key === '.osquery') { + acc.osquery = agg.rulesInfo.buckets.map((rule) => rule.key); } - - acc.push({ - id: ruleId, - attributes: { - consumer: rule.attributes.consumer, - createdAt: rule.attributes.createdAt, - name: rule.attributes.name, - enabled: rule.attributes.enabled, - immutable: rule.attributes.params.immutable, - params: { - responseActions: rule.attributes.params.responseActions, - }, - updatedAt: rule.attributes.updatedAt, - }, - }); return acc; - }, []); + }, {} as ResponseActionRules); + + const shouldNotProcessTelemetry = + responseActionRules.endpoint === undefined || + responseActionRules.osquery === undefined || + responseActionRules.endpoint.length === 0 || + responseActionRules.osquery.length === 0; + + if (shouldNotProcessTelemetry) { + return 0; + } - const responseActionsRulesTelemetryEvent = responseActionsCustomRuleTelemetryData( - responseActionRulesArray, + const responseActionsRulesTelemetryData = responseActionsCustomRuleTelemetryData( + responseActionRules, clusterInfo, licenseInfo ); - log.l('Custom response actions rule json length', { - length: responseActionsRulesTelemetryEvent.length, + + log.l('Custom response actions rules data', { + data: JSON.stringify(responseActionsRulesTelemetryData), }); usageCollector?.incrementCounter({ - counterName: createUsageCounterLabel(usageLabelPrefix), + counterName: createUsageCounterLabel(usageLabelEndpointPrefix), counterType: 'response_actions_rules_count', - incrementBy: responseActionsRulesTelemetryEvent.length, + incrementBy: responseActionsRulesTelemetryData.response_actions.endpoint.count, + }); + + usageCollector?.incrementCounter({ + counterName: createUsageCounterLabel(usageLabelOsqueryPrefix), + counterType: 'response_actions_rules_count', + incrementBy: responseActionsRulesTelemetryData.response_actions.osquery.count, }); const batches = batchTelemetryRecords( - cloneDeep(responseActionsRulesTelemetryEvent), + cloneDeep(Object.values(responseActionsRulesTelemetryData)), maxTelemetryBatch ); + for (const batch of batches) { await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); } await taskMetricsService.end(trace); - log.l('Task executed', { length: responseActionsRulesTelemetryEvent.length }); + const totalCount = Object.values(responseActionsRulesTelemetryData.response_actions) + .map((r) => r.count) + .reduce((a, b) => a + b, 0); + log.l('Response actions rules telemetry task executed', { + totalCount, + }); - return responseActionsRulesTelemetryEvent.length; + return totalCount; } catch (err) { await taskMetricsService.end(trace, err); return 0; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index fe579d268998b..cd0ebe3c84daa 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -386,44 +386,41 @@ interface ExceptionListEntry { namespace_type: string; } +export interface ResponseActionsRuleResponseAggregations { + actionTypes: { + buckets: Array<{ + key: '.endpoint' | '.osquery'; + doc_count: number; + rulesInfo: { + buckets: Array<{ + key: string; // rule ID + doc_count: number; + }>; + }; + }>; + }; +} + +interface ResponseActionsRuleTelemetry { + ids: string[]; + count: number; +} + export interface ResponseActionsRuleTelemetryTemplate { '@timestamp': string; cluster_uuid: string; cluster_name: string; license_id: string | undefined; - response_actions?: { - rules: ResponseActionRules; - endpoint_rules_count: number; - osquery_rules_count: number; - }; -} -export interface RulesParamsResponseActionsEntry { - actionTypeId: '.endpoint' | '.osquery'; - params: - | { - command: string; - comment?: string; - } - | { - query: string; - }; -} - -export type ResponseActionRules = Array<{ - id: string; - attributes: { - consumer: string; - createdAt: string; - name: string; - enabled: boolean; - immutable: boolean; - params: { - responseActions: RulesParamsResponseActionsEntry[]; - }; - updatedAt: string; + response_actions: { + endpoint: ResponseActionsRuleTelemetry; + osquery: ResponseActionsRuleTelemetry; }; -}>; +} +export interface ResponseActionRules { + endpoint: string[]; + osquery: string[]; +} interface DetectionRuleParms { ruleId: string; version: number; From f74d61137cf7fc21551f8839d48fe852a7b7bd56 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 11 Jun 2025 13:55:32 +0200 Subject: [PATCH 08/19] rename --- .../tasks/{custom_rule.ts => custom_response_actions_rule.ts} | 0 .../security_solution/server/lib/telemetry/tasks/index.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/{custom_rule.ts => custom_response_actions_rule.ts} (100%) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_rule.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts index 3a6ab44873779..6e6c4457cf956 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -18,7 +18,7 @@ import { telemetryConfiguration } from '../configuration'; import { createTelemetryFilterListArtifactTaskConfig } from './filterlists'; import { createTelemetryIndicesMetadataTaskConfig } from './indices.metadata'; import { createIngestStatsTaskConfig } from './ingest_pipelines_stats'; -import { createTelemetryCustomResponseActionRulesTaskConfig } from './custom_rule'; +import { createTelemetryCustomResponseActionRulesTaskConfig } from './custom_response_actions_rule'; export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { return [ From ebd1d20f1207a2ecd67fd5778590d8b5e47937d6 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 11 Jun 2025 13:57:08 +0200 Subject: [PATCH 09/19] test --- .../server/lib/telemetry/__mocks__/index.ts | 3 ++ .../custom_response_actions_rule.test.ts | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.test.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index e738815ac2553..bfc3b6b2bff6d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -131,6 +131,9 @@ export const createMockTelemetryReceiver = ( fetchTrustedApplications: jest.fn(), fetchEndpointList: jest.fn(), fetchDetectionRules: jest.fn().mockReturnValue({ body: null }), + fetchResponseActionsRules: jest + .fn() + .mockReturnValue({ body: { aggregations: { actionTypes: {} } } }), fetchEndpointMetadata: jest.fn().mockReturnValue(Promise.resolve(new Map())), fetchTimelineAlerts: jest.fn().mockReturnValue(Promise.resolve(stubEndpointAlertResponse())), buildProcessTree: jest.fn().mockReturnValue(processTreeResponse), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.test.ts new file mode 100644 index 0000000000000..7d7d56374f2ce --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { createTelemetryCustomResponseActionRulesTaskConfig } from './custom_response_actions_rule'; +import { + createMockTelemetryEventsSender, + createMockTelemetryReceiver, + createMockTaskMetrics, +} from '../__mocks__'; + +describe('security response actions rule task test', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + test('security response actions rule task should fetch response actions rules data', async () => { + const testTaskExecutionPeriod = { + last: undefined, + current: new Date().toISOString(), + }; + const mockTelemetryEventsSender = createMockTelemetryEventsSender(); + const mockTelemetryReceiver = createMockTelemetryReceiver(); + const telemetryCustomResponseActionsRulesTaskConfig = + createTelemetryCustomResponseActionRulesTaskConfig(1); + const mockTaskMetrics = createMockTaskMetrics(); + + await telemetryCustomResponseActionsRulesTaskConfig.runTask( + 'test-id', + logger, + mockTelemetryReceiver, + mockTelemetryEventsSender, + mockTaskMetrics, + testTaskExecutionPeriod + ); + + expect(mockTelemetryReceiver.fetchResponseActionsRules).toHaveBeenCalled(); + }); +}); From 6cbb4dbbb16c02366b5e4324051f281de72ee83f Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Thu, 12 Jun 2025 13:26:00 +0200 Subject: [PATCH 10/19] move task behind telemetry feature flag --- .../server/lib/telemetry/tasks/index.ts | 20 ++++++++++++++----- .../security_solution/server/plugin.ts | 5 ++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts index 6e6c4457cf956..5f2fa7f955e66 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { ExperimentalFeatures } from '../../../../common'; import type { SecurityTelemetryTaskConfig } from '../task'; import { createTelemetryDiagnosticsTaskConfig } from './diagnostic'; import { createTelemetryEndpointTaskConfig } from './endpoint'; @@ -20,17 +21,16 @@ import { createTelemetryIndicesMetadataTaskConfig } from './indices.metadata'; import { createIngestStatsTaskConfig } from './ingest_pipelines_stats'; import { createTelemetryCustomResponseActionRulesTaskConfig } from './custom_response_actions_rule'; -export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { - return [ +export function createTelemetryTaskConfigs( + experimentalFeatures: ExperimentalFeatures +): SecurityTelemetryTaskConfig[] { + const tasks = [ createTelemetryDiagnosticsTaskConfig(), createTelemetryEndpointTaskConfig(telemetryConfiguration.max_security_list_telemetry_batch), createTelemetrySecurityListTaskConfig(telemetryConfiguration.max_endpoint_telemetry_batch), createTelemetryDetectionRuleListsTaskConfig( telemetryConfiguration.max_detection_rule_telemetry_batch ), - createTelemetryCustomResponseActionRulesTaskConfig( - telemetryConfiguration.max_detection_rule_telemetry_batch - ), createTelemetryPrebuiltRuleAlertsTaskConfig(telemetryConfiguration.max_detection_alerts_batch), createTelemetryTimelineTaskConfig(), createTelemetryDiagnosticTimelineTaskConfig(), @@ -39,4 +39,14 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { createTelemetryIndicesMetadataTaskConfig(), createIngestStatsTaskConfig(), ]; + + if (experimentalFeatures.responseActionsTelemetryEnabled) { + tasks.push( + createTelemetryCustomResponseActionRulesTaskConfig( + telemetryConfiguration.max_detection_rule_telemetry_batch + ) + ); + } + + return tasks; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts index da5c1d7d79f71..1cf7fa943e563 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts @@ -186,7 +186,10 @@ export class Plugin implements ISecuritySolutionPlugin { ); this.ruleMonitoringService = createRuleMonitoringService(this.config, this.logger); - this.telemetryEventsSender = new TelemetryEventsSender(this.logger); + this.telemetryEventsSender = new TelemetryEventsSender( + this.logger, + this.config.experimentalFeatures + ); this.asyncTelemetryEventsSender = new AsyncTelemetryEventsSender(this.logger); this.telemetryReceiver = new TelemetryReceiver(this.logger); From 462bdef7fe5665fe1b8ced3b7c0ead7f2a817c20 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Thu, 12 Jun 2025 14:04:52 +0200 Subject: [PATCH 11/19] fix types --- .../server/lib/telemetry/async_sender.test.ts | 16 ++++++++++--- .../server/lib/telemetry/sender.test.ts | 23 ++++++++++--------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/async_sender.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/async_sender.test.ts index 5545983ec9c17..64d9c2c8962f6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/async_sender.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/async_sender.test.ts @@ -21,6 +21,7 @@ import { createMockUsageCounter, } from './__mocks__'; import { TelemetryEventsSender } from './sender'; +import type { ExperimentalFeatures } from '../../../common'; jest.mock('axios'); jest.mock('./receiver'); @@ -1006,7 +1007,10 @@ describe('AsyncTelemetryEventsSender', () => { describe('ITelemetryEventsSender integration', () => { it('should send events using the async service', async () => { - const serviceV1 = new TelemetryEventsSender(loggingSystemMock.createLogger()); + const serviceV1 = new TelemetryEventsSender( + loggingSystemMock.createLogger(), + {} as ExperimentalFeatures + ); service.setup(DEFAULT_RETRY_CONFIG, DEFAULT_QUEUE_CONFIG, receiver, telemetryPluginSetup); service.start(telemetryPluginStart); @@ -1038,7 +1042,10 @@ describe('AsyncTelemetryEventsSender', () => { const bufferTimeSpanMillis = initialTimeSpan * 10; const events = ['e1', 'e2', 'e3']; const expectedBody = events.map((e) => JSON.stringify(e)).join('\n'); - const serviceV1 = new TelemetryEventsSender(loggingSystemMock.createLogger()); + const serviceV1 = new TelemetryEventsSender( + loggingSystemMock.createLogger(), + {} as ExperimentalFeatures + ); serviceV1.setup(receiver, telemetryPluginSetup, undefined, telemetryUsageCounter, service); @@ -1077,7 +1084,10 @@ describe('AsyncTelemetryEventsSender', () => { ...detectionAlertsBefore, bufferTimeSpanMillis: 5001, }; - const serviceV1 = new TelemetryEventsSender(loggingSystemMock.createLogger()); + const serviceV1 = new TelemetryEventsSender( + loggingSystemMock.createLogger(), + {} as ExperimentalFeatures + ); serviceV1.setup(receiver, telemetryPluginSetup, undefined, telemetryUsageCounter, service); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.test.ts index 648bb8358e1d2..c3ef4c7fbc92a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.test.ts @@ -6,6 +6,7 @@ */ /* eslint-disable dot-notation */ +import type { ExperimentalFeatures } from '../../../common'; import { TelemetryEventsSender } from './sender'; import { loggingSystemMock } from '@kbn/core/server/mocks'; import { usageCountersServiceMock } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counters_service.mock'; @@ -25,13 +26,13 @@ describe('TelemetryEventsSender', () => { describe('processEvents', () => { it('returns empty array when empty array is passed', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); const result = sender.processEvents([]); expect(result).toStrictEqual([]); }); it('applies the allowlist', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); const input = [ { credential_access: { @@ -465,13 +466,13 @@ describe('TelemetryEventsSender', () => { describe('queueTelemetryEvents', () => { it('queues two events', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); expect(sender['queue'].length).toBe(2); }); it('queues more than maxQueueSize events', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); sender['maxQueueSize'] = 5; sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); @@ -481,7 +482,7 @@ describe('TelemetryEventsSender', () => { }); it('empties the queue when sending', async () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); sender['telemetryStart'] = { getIsOptedIn: jest.fn(async () => true), isOptedIn$: new Observable(), @@ -514,7 +515,7 @@ describe('TelemetryEventsSender', () => { }); it("shouldn't send when telemetry is disabled", async () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); sender['sendEvents'] = jest.fn(); const telemetryStart = { getIsOptedIn: jest.fn(async () => false), @@ -531,7 +532,7 @@ describe('TelemetryEventsSender', () => { }); it("shouldn't send when telemetry when opted in but cannot connect to elastic telemetry services", async () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); sender['sendEvents'] = jest.fn(); const telemetryStart = { getIsOptedIn: jest.fn(async () => true), @@ -558,28 +559,28 @@ describe('getV3UrlFromV2', () => { }); it('should return prod url', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); expect( sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint') ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint'); }); it('should work when receiving a V3 URL', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); expect( sender.getV3UrlFromV2('https://telemetry.elastic.co/v3/send/channel', 'alerts-endpoint') ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint'); }); it('should return staging url', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); expect( sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') ).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint'); }); it('should support ports and auth', () => { - const sender = new TelemetryEventsSender(logger); + const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures); expect( sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint') ).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint'); From 156c4989d5ca008bc36c0b18c6c490c65ec8cd90 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Thu, 12 Jun 2025 14:32:01 +0200 Subject: [PATCH 12/19] fix test --- x-pack/platform/test/plugin_api_integration/config.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/platform/test/plugin_api_integration/config.ts b/x-pack/platform/test/plugin_api_integration/config.ts index a13369b607ff3..7a894a92d2a25 100644 --- a/x-pack/platform/test/plugin_api_integration/config.ts +++ b/x-pack/platform/test/plugin_api_integration/config.ts @@ -36,6 +36,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.eventLog.logEntries=true', '--xpack.eventLog.indexEntries=true', '--xpack.task_manager.monitored_aggregated_stats_refresh_rate=5000', + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'responseActionsTelemetryEnabled', + ])}`, `--xpack.stack_connectors.enableExperimental=${JSON.stringify([ 'crowdstrikeConnectorOn', 'microsoftDefenderEndpointOn', From 5bb1e8409d6c1134a2cf2cbdf0c850d8ff65b7ca Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 17 Jun 2025 21:13:08 +0200 Subject: [PATCH 13/19] fix types --- .../security_solution/server/lib/telemetry/sender.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.ts index 8e99a9e12981c..74a19afec617e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.ts @@ -19,6 +19,7 @@ import type { TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; import { exhaustMap, Subject, takeUntil, timer } from 'rxjs'; +import type { ExperimentalFeatures } from '../../../common'; import type { ITelemetryReceiver } from './receiver'; import { copyAllowlistedFields, filterList } from './filterlists'; import { createTelemetryTaskConfigs } from './tasks'; @@ -99,6 +100,7 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { private readonly initialCheckDelayMs = 10 * 1000; private readonly checkIntervalMs = 60 * 1000; private readonly logger: TelemetryLogger; + private readonly experimentalFeatures: ExperimentalFeatures; private readonly stop$ = new Subject(); private maxQueueSize = telemetryConfiguration.telemetry_max_buffer_size; private telemetryStart?: TelemetryPluginStart; @@ -116,7 +118,8 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { private asyncTelemetrySender?: IAsyncTelemetryEventsSender; - constructor(logger: Logger) { + constructor(logger: Logger, experimentalFeatures: ExperimentalFeatures) { + this.experimentalFeatures = experimentalFeatures; this.logger = newTelemetryLogger(logger.get('telemetry_events.sender')); } @@ -131,7 +134,7 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { this.telemetryUsageCounter = telemetryUsageCounter; if (taskManager) { const taskMetricsService = new TaskMetricsService(this.logger, this); - this.telemetryTasks = createTelemetryTaskConfigs().map( + this.telemetryTasks = createTelemetryTaskConfigs(this.experimentalFeatures).map( (config: SecurityTelemetryTaskConfig) => { const task = new SecurityTelemetryTask( config, From 2cd4c87c9694e445b6dcbd0e6cb4c4099677dfb7 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 17 Jun 2025 21:13:51 +0200 Subject: [PATCH 14/19] review changes --- .../server/lib/telemetry/receiver.ts | 11 +++++---- .../tasks/custom_response_actions_rule.ts | 23 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index b0887abb4423b..062d32c59a1fe 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -220,7 +220,10 @@ export interface ITelemetryReceiver { > >; - fetchResponseActionsRules(): Promise< + fetchResponseActionsRules( + executeFrom: string, + executeTo: string + ): Promise< TransportResult>, unknown> >; @@ -758,7 +761,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { * Find elastic rules SOs which are the rules that have immutable set to true and are of a particular rule type * @returns custom elastic rules SOs with response actions enabled */ - public async fetchResponseActionsRules() { + public async fetchResponseActionsRules(executeFrom: string, executeTo: string) { const query: SearchRequest = { index: `${this.getAlertsIndex()}*`, ignore_unavailable: true, @@ -793,8 +796,8 @@ export class TelemetryReceiver implements ITelemetryReceiver { { range: { 'kibana.alert.rule.updated_at': { - gte: 'now-24h/h', - lte: 'now', + gte: executeFrom, + lte: executeTo, }, }, }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts index 6a4f36632ea08..6a6b960e87373 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts @@ -20,6 +20,7 @@ import type { ITelemetryReceiver } from '../receiver'; import type { ResponseActionRules, ResponseActionsRuleResponseAggregations } from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; +import { telemetryConfiguration } from '../configuration'; export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryBatch: number) { const taskName = 'Security Solution Response Actions Rules Telemetry'; @@ -64,7 +65,10 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const { body: { aggregations }, - } = (await receiver.fetchResponseActionsRules()) ?? {}; + } = await receiver.fetchResponseActionsRules( + taskExecutionPeriod.last ?? 'now-24h', + taskExecutionPeriod.current + ); if (!aggregations || !aggregations.actionTypes) { log.debug('no custom response action rules found'); @@ -90,6 +94,8 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB responseActionRules.osquery.length === 0; if (shouldNotProcessTelemetry) { + log.debug('no new custom response action rules found'); + await taskMetricsService.end(trace); return 0; } @@ -115,14 +121,17 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB incrementBy: responseActionsRulesTelemetryData.response_actions.osquery.count, }); - const batches = batchTelemetryRecords( - cloneDeep(Object.values(responseActionsRulesTelemetryData)), - maxTelemetryBatch - ); + const documents = cloneDeep(Object.values(responseActionsRulesTelemetryData)); - for (const batch of batches) { - await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + if (telemetryConfiguration.use_async_sender) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, documents); + } else { + const batches = batchTelemetryRecords(documents, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + } } + await taskMetricsService.end(trace); const totalCount = Object.values(responseActionsRulesTelemetryData.response_actions) From 72453fdada73988cde43682186ca6879976a1fa6 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 18 Jun 2025 16:29:05 +0200 Subject: [PATCH 15/19] fetch SOs from the correct index --- .../server/lib/telemetry/helpers.ts | 24 ++++--------- .../server/lib/telemetry/receiver.ts | 36 +++++++++---------- .../tasks/custom_response_actions_rule.ts | 19 +++++----- 3 files changed, 34 insertions(+), 45 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index 5aa5efaf90306..0f132c9a67989 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -251,29 +251,17 @@ export const responseActionsCustomRuleTelemetryData = ( cluster_uuid: clusterInfo.cluster_uuid, cluster_name: clusterInfo.cluster_name, license_id: licenseInfo?.uid, - response_actions: { - endpoint: { - ids: [], - count: 0, - }, - osquery: { - ids: [], - count: 0, - }, + response_actions_rules: { + endpoint: 0, + osquery: 0, }, }; return { ...baseTelemetryData, - response_actions: { - endpoint: { - ids: responseActionsRules.endpoint, - count: responseActionsRules.endpoint.length, - }, - osquery: { - ids: responseActionsRules.osquery, - count: responseActionsRules.osquery.length, - }, + response_actions_rules: { + endpoint: responseActionsRules.endpoint, + osquery: responseActionsRules.osquery, }, }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 062d32c59a1fe..d6d1590d0e9b0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -55,6 +55,7 @@ import type { import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import moment from 'moment'; +import { RULE_SAVED_OBJECT_TYPE } from '@kbn/alerting-plugin/server'; import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import type { ExperimentalFeatures } from '../../../common'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; @@ -763,7 +764,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { */ public async fetchResponseActionsRules(executeFrom: string, executeTo: string) { const query: SearchRequest = { - index: `${this.getAlertsIndex()}*`, + index: `${this.getIndexForType?.(RULE_SAVED_OBJECT_TYPE)}`, ignore_unavailable: true, size: 0, // no query results required - only aggregation quantity from: 0, @@ -772,30 +773,36 @@ export class TelemetryReceiver implements ITelemetryReceiver { must: [ { term: { - 'kibana.alert.rule.immutable': false, + type: 'alert', }, }, { term: { - 'kibana.alert.rule.enabled': true, + 'alert.params.immutable': { + value: false, + }, + }, + }, + { + term: { + 'alert.enabled': { + value: true, + }, }, }, { terms: { - 'kibana.alert.rule.consumer': ['siem', 'securitySolution'], + 'alert.consumer': ['siem', 'securitySolution'], }, }, { terms: { - 'kibana.alert.rule.parameters.response_actions.action_type_id': [ - '.endpoint', - '.osquery', - ], + 'alert.params.responseActions.actionTypeId': ['.endpoint', '.osquery'], }, }, { range: { - 'kibana.alert.rule.updated_at': { + 'alert.updatedAt': { gte: executeFrom, lte: executeTo, }, @@ -806,7 +813,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, sort: [ { - 'kibana.alert.rule.updated_at': { + 'alert.updatedAt': { order: 'desc', }, }, @@ -814,14 +821,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { aggs: { actionTypes: { terms: { - field: 'kibana.alert.rule.parameters.response_actions.action_type_id', - }, - aggs: { - rulesInfo: { - terms: { - field: 'kibana.alert.rule.uuid', - }, - }, + field: 'alert.params.responseActions.actionTypeId', }, }, }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts index 6a6b960e87373..cbe6e678503e5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts @@ -80,9 +80,9 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB aggregations as unknown as ResponseActionsRuleResponseAggregations ).actionTypes.buckets.reduce((acc, agg) => { if (agg.key === '.endpoint') { - acc.endpoint = agg.rulesInfo.buckets.map((rule) => rule.key); + acc.endpoint = agg.doc_count; } else if (agg.key === '.osquery') { - acc.osquery = agg.rulesInfo.buckets.map((rule) => rule.key); + acc.osquery = agg.doc_count; } return acc; }, {} as ResponseActionRules); @@ -90,8 +90,8 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const shouldNotProcessTelemetry = responseActionRules.endpoint === undefined || responseActionRules.osquery === undefined || - responseActionRules.endpoint.length === 0 || - responseActionRules.osquery.length === 0; + responseActionRules.endpoint === 0 || + responseActionRules.osquery === 0; if (shouldNotProcessTelemetry) { log.debug('no new custom response action rules found'); @@ -112,13 +112,13 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB usageCollector?.incrementCounter({ counterName: createUsageCounterLabel(usageLabelEndpointPrefix), counterType: 'response_actions_rules_count', - incrementBy: responseActionsRulesTelemetryData.response_actions.endpoint.count, + incrementBy: responseActionsRulesTelemetryData.response_actions_rules.endpoint, }); usageCollector?.incrementCounter({ counterName: createUsageCounterLabel(usageLabelOsqueryPrefix), counterType: 'response_actions_rules_count', - incrementBy: responseActionsRulesTelemetryData.response_actions.osquery.count, + incrementBy: responseActionsRulesTelemetryData.response_actions_rules.osquery, }); const documents = cloneDeep(Object.values(responseActionsRulesTelemetryData)); @@ -134,9 +134,10 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB await taskMetricsService.end(trace); - const totalCount = Object.values(responseActionsRulesTelemetryData.response_actions) - .map((r) => r.count) - .reduce((a, b) => a + b, 0); + const totalCount = Object.values( + responseActionsRulesTelemetryData.response_actions_rules + ).reduce((acc, count) => acc + count, 0); + log.l('Response actions rules telemetry task executed', { totalCount, }); From a454389843ede07d93c48029a3eb742432c08bed Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 18 Jun 2025 16:57:12 +0200 Subject: [PATCH 16/19] update counter types --- .../lib/telemetry/tasks/custom_response_actions_rule.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts index cbe6e678503e5..1de7cbc74c148 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts @@ -111,13 +111,13 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB usageCollector?.incrementCounter({ counterName: createUsageCounterLabel(usageLabelEndpointPrefix), - counterType: 'response_actions_rules_count', + counterType: 'response_actions_endpoint_rules_count', incrementBy: responseActionsRulesTelemetryData.response_actions_rules.endpoint, }); usageCollector?.incrementCounter({ counterName: createUsageCounterLabel(usageLabelOsqueryPrefix), - counterType: 'response_actions_rules_count', + counterType: 'response_actions_osquery_rules_count', incrementBy: responseActionsRulesTelemetryData.response_actions_rules.osquery, }); From ac8a3f39707f57604242e6c4175ce020dd77b864 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Thu, 19 Jun 2025 09:52:00 +0200 Subject: [PATCH 17/19] fix types --- .../tasks/custom_response_actions_rule.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts index 1de7cbc74c148..a50a79d5e921c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts @@ -78,20 +78,20 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const responseActionRules = ( aggregations as unknown as ResponseActionsRuleResponseAggregations - ).actionTypes.buckets.reduce((acc, agg) => { - if (agg.key === '.endpoint') { - acc.endpoint = agg.doc_count; - } else if (agg.key === '.osquery') { - acc.osquery = agg.doc_count; - } - return acc; - }, {} as ResponseActionRules); + ).actionTypes.buckets.reduce( + (acc, agg) => { + if (agg.key === '.endpoint') { + acc.endpoint = agg.doc_count; + } else if (agg.key === '.osquery') { + acc.osquery = agg.doc_count; + } + return acc; + }, + { endpoint: 0, osquery: 0 } + ); const shouldNotProcessTelemetry = - responseActionRules.endpoint === undefined || - responseActionRules.osquery === undefined || - responseActionRules.endpoint === 0 || - responseActionRules.osquery === 0; + responseActionRules.endpoint === 0 || responseActionRules.osquery === 0; if (shouldNotProcessTelemetry) { log.debug('no new custom response action rules found'); From 450c7a68900fc452f3b9e31a32f32acf9b239784 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Thu, 19 Jun 2025 10:50:33 +0200 Subject: [PATCH 18/19] update missing type changes refs ac8a3f39707f57604242e6c4175ce020dd77b864 --- .../server/lib/telemetry/types.ts | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index cd0ebe3c84daa..3995d25fce6d8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -391,35 +391,21 @@ export interface ResponseActionsRuleResponseAggregations { buckets: Array<{ key: '.endpoint' | '.osquery'; doc_count: number; - rulesInfo: { - buckets: Array<{ - key: string; // rule ID - doc_count: number; - }>; - }; }>; }; } -interface ResponseActionsRuleTelemetry { - ids: string[]; - count: number; -} - export interface ResponseActionsRuleTelemetryTemplate { '@timestamp': string; cluster_uuid: string; cluster_name: string; license_id: string | undefined; - response_actions: { - endpoint: ResponseActionsRuleTelemetry; - osquery: ResponseActionsRuleTelemetry; - }; + response_actions_rules: ResponseActionRules; } export interface ResponseActionRules { - endpoint: string[]; - osquery: string[]; + endpoint: number; + osquery: number; } interface DetectionRuleParms { ruleId: string; From d30fd64fcc41a32fca411977e139488a05903e49 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Fri, 20 Jun 2025 11:59:03 +0200 Subject: [PATCH 19/19] use `sendAsync` review changes --- .../telemetry/tasks/custom_response_actions_rule.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts index a50a79d5e921c..946f3b8813238 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.ts @@ -7,7 +7,6 @@ import { cloneDeep } from 'lodash'; import type { Logger } from '@kbn/core/server'; -import { TELEMETRY_CHANNEL_LISTS } from '../constants'; import { batchTelemetryRecords, responseActionsCustomRuleTelemetryData, @@ -17,7 +16,11 @@ import { } from '../helpers'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; -import type { ResponseActionRules, ResponseActionsRuleResponseAggregations } from '../types'; +import { + TelemetryChannel, + type ResponseActionRules, + type ResponseActionsRuleResponseAggregations, +} from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; import { telemetryConfiguration } from '../configuration'; @@ -124,11 +127,11 @@ export function createTelemetryCustomResponseActionRulesTaskConfig(maxTelemetryB const documents = cloneDeep(Object.values(responseActionsRulesTelemetryData)); if (telemetryConfiguration.use_async_sender) { - await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, documents); + await sender.sendAsync(TelemetryChannel.LISTS, documents); } else { const batches = batchTelemetryRecords(documents, maxTelemetryBatch); for (const batch of batches) { - await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + await sender.sendOnDemand(TelemetryChannel.LISTS, batch); } }