[8.19] [SecruitySolution][Endpoint][ResponseActions] Add usage telemtery for response actions rules (#221518)

x-pack/platform/test/plugin_api_integration/config.ts

@@ -38,6 +38,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         '--xpack.task_manager.monitored_aggregated_stats_refresh_rate=5000',
         '--xpack.task_manager.ephemeral_tasks.enabled=false',
         '--xpack.task_manager.ephemeral_tasks.request_capacity=100',
+        `--xpack.securitySolution.enableExperimental=${JSON.stringify([
+          'responseActionsTelemetryEnabled',
+        ])}`,
         `--xpack.stack_connectors.enableExperimental=${JSON.stringify([
           'crowdstrikeConnectorOn',
           'microsoftDefenderEndpointOn',

x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts

@@ -186,6 +186,7 @@ export default function ({ getService }: FtrProviderContext) {
         'security:telemetry-filterlist-artifact',
         'security:telemetry-lists',
         'security:telemetry-prebuilt-rule-alerts',
+        'security:telemetry-response-actions-rules',
         'security:telemetry-timelines',
         'session_cleanup',
         'slo:bulk-delete-task',

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts

@@ -131,6 +131,9 @@ export const createMockTelemetryReceiver = (
     fetchTrustedApplications: jest.fn(),
     fetchEndpointList: jest.fn(),
     fetchDetectionRules: jest.fn().mockReturnValue({ body: null }),
+    fetchResponseActionsRules: jest
+      .fn()
+      .mockReturnValue({ body: { aggregations: { actionTypes: {} } } }),
     fetchEndpointMetadata: jest.fn().mockReturnValue(Promise.resolve(new Map())),
     fetchTimelineAlerts: jest.fn().mockReturnValue(Promise.resolve(stubEndpointAlertResponse())),
     buildProcessTree: jest.fn().mockReturnValue(processTreeResponse),

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/async_sender.test.ts

@@ -21,6 +21,7 @@ import {
   createMockUsageCounter,
 } from './__mocks__';
 import { TelemetryEventsSender } from './sender';
+import type { ExperimentalFeatures } from '../../../common';
 
 jest.mock('axios');
 jest.mock('./receiver');
@@ -1006,7 +1007,10 @@ describe('AsyncTelemetryEventsSender', () => {
 
   describe('ITelemetryEventsSender integration', () => {
     it('should send events using the async service', async () => {
-      const serviceV1 = new TelemetryEventsSender(loggingSystemMock.createLogger());
+      const serviceV1 = new TelemetryEventsSender(
+        loggingSystemMock.createLogger(),
+        {} as ExperimentalFeatures
+      );
 
       service.setup(DEFAULT_RETRY_CONFIG, DEFAULT_QUEUE_CONFIG, receiver, telemetryPluginSetup);
       service.start(telemetryPluginStart);
@@ -1038,7 +1042,10 @@ describe('AsyncTelemetryEventsSender', () => {
       const bufferTimeSpanMillis = initialTimeSpan * 10;
       const events = ['e1', 'e2', 'e3'];
       const expectedBody = events.map((e) => JSON.stringify(e)).join('\n');
-      const serviceV1 = new TelemetryEventsSender(loggingSystemMock.createLogger());
+      const serviceV1 = new TelemetryEventsSender(
+        loggingSystemMock.createLogger(),
+        {} as ExperimentalFeatures
+      );
 
       serviceV1.setup(receiver, telemetryPluginSetup, undefined, telemetryUsageCounter, service);
 
@@ -1077,7 +1084,10 @@ describe('AsyncTelemetryEventsSender', () => {
         ...detectionAlertsBefore,
         bufferTimeSpanMillis: 5001,
       };
-      const serviceV1 = new TelemetryEventsSender(loggingSystemMock.createLogger());
+      const serviceV1 = new TelemetryEventsSender(
+        loggingSystemMock.createLogger(),
+        {} as ExperimentalFeatures
+      );
 
       serviceV1.setup(receiver, telemetryPluginSetup, undefined, telemetryUsageCounter, service);
 

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts

@@ -24,6 +24,8 @@ import type {
   ExtraInfo,
   ListTemplate,
   Nullable,
+  ResponseActionsRuleTelemetryTemplate,
+  ResponseActionRules,
   TelemetryEvent,
   TimeFrame,
   TimelineResult,
@@ -236,6 +238,34 @@ export const templateExceptionList = (
   });
 };
 
+/**
+ * Constructs the response actions custom rule telemetry schema from a list of rule params
+ * */
+export const responseActionsCustomRuleTelemetryData = (
+  responseActionsRules: ResponseActionRules,
+  clusterInfo: ESClusterInfo,
+  licenseInfo: Nullable<ESLicense>
+): ResponseActionsRuleTelemetryTemplate => {
+  const baseTelemetryData: ResponseActionsRuleTelemetryTemplate = {
+    '@timestamp': moment().toISOString(),
+    cluster_uuid: clusterInfo.cluster_uuid,
+    cluster_name: clusterInfo.cluster_name,
+    license_id: licenseInfo?.uid,
+    response_actions_rules: {
+      endpoint: 0,
+      osquery: 0,
+    },
+  };
+
+  return {
+    ...baseTelemetryData,
+    response_actions_rules: {
+      endpoint: responseActionsRules.endpoint,
+      osquery: responseActionsRules.osquery,
+    },
+  };
+};
+
 /**
  * Convert counter label list to kebab case
  *

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -54,6 +54,8 @@ import type {
 } from '@kbn/fleet-plugin/server';
 import type { ExceptionListClient } from '@kbn/lists-plugin/server';
 import moment from 'moment';
+
+import { RULE_SAVED_OBJECT_TYPE } from '@kbn/alerting-plugin/server';
 import type { ExperimentalFeatures } from '../../../common';
 import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
 import {
@@ -219,6 +221,13 @@ export interface ITelemetryReceiver {
     >
   >;
 
+  fetchResponseActionsRules(
+    executeFrom: string,
+    executeTo: string
+  ): Promise<
+    TransportResult<SearchResponse<unknown, Record<string, AggregationsAggregate>>, unknown>
+  >;
+
   fetchDetectionExceptionList(
     listId: string,
     ruleVersion: number
@@ -749,6 +758,78 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     return this.esClient().search<RuleSearchResult>(query, { meta: true });
   }
 
+  /**
+   * Find elastic rules SOs which are the rules that have immutable set to true and are of a particular rule type
+   * @returns custom elastic rules SOs with response actions enabled
+   */
+  public async fetchResponseActionsRules(executeFrom: string, executeTo: string) {
+    const query: SearchRequest = {
+      index: `${this.getIndexForType?.(RULE_SAVED_OBJECT_TYPE)}`,
+      ignore_unavailable: true,
+      size: 0, // no query results required - only aggregation quantity
+      from: 0,
+      query: {
+        bool: {
+          must: [
+            {
+              term: {
+                type: 'alert',
+              },
+            },
+            {
+              term: {
+                'alert.params.immutable': {
+                  value: false,
+                },
+              },
+            },
+            {
+              term: {
+                'alert.enabled': {
+                  value: true,
+                },
+              },
+            },
+            {
+              terms: {
+                'alert.consumer': ['siem', 'securitySolution'],
+              },
+            },
+            {
+              terms: {
+                'alert.params.responseActions.actionTypeId': ['.endpoint', '.osquery'],
+              },
+            },
+            {
+              range: {
+                'alert.updatedAt': {
+                  gte: executeFrom,
+                  lte: executeTo,
+                },
+              },
+            },
+          ],
+        },
+      },
+      sort: [
+        {
+          'alert.updatedAt': {
+            order: 'desc',
+          },
+        },
+      ],
+      aggs: {
+        actionTypes: {
+          terms: {
+            field: 'alert.params.responseActions.actionTypeId',
+          },
+        },
+      },
+    };
+
+    return this.esClient().search<unknown>(query, { meta: true });
+  }
+
   public async fetchDetectionExceptionList(listId: string, ruleVersion: number) {
     if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) {
       throw Error('exception list client is unavailable: could not retrieve trusted applications');

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.test.ts

@@ -6,6 +6,7 @@
  */
 
 /* eslint-disable dot-notation */
+import type { ExperimentalFeatures } from '../../../common';
 import { TelemetryEventsSender } from './sender';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 import { usageCountersServiceMock } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counters_service.mock';
@@ -25,13 +26,13 @@ describe('TelemetryEventsSender', () => {
 
   describe('processEvents', () => {
     it('returns empty array when empty array is passed', () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       const result = sender.processEvents([]);
       expect(result).toStrictEqual([]);
     });
 
     it('applies the allowlist', () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       const input = [
         {
           credential_access: {
@@ -465,13 +466,13 @@ describe('TelemetryEventsSender', () => {
 
   describe('queueTelemetryEvents', () => {
     it('queues two events', () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]);
       expect(sender['queue'].length).toBe(2);
     });
 
     it('queues more than maxQueueSize events', () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       sender['maxQueueSize'] = 5;
       sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]);
       sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]);
@@ -481,7 +482,7 @@ describe('TelemetryEventsSender', () => {
     });
 
     it('empties the queue when sending', async () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       sender['telemetryStart'] = {
         getIsOptedIn: jest.fn(async () => true),
         isOptedIn$: new Observable<boolean>(),
@@ -514,7 +515,7 @@ describe('TelemetryEventsSender', () => {
     });
 
     it("shouldn't send when telemetry is disabled", async () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       sender['sendEvents'] = jest.fn();
       const telemetryStart = {
         getIsOptedIn: jest.fn(async () => false),
@@ -531,7 +532,7 @@ describe('TelemetryEventsSender', () => {
     });
 
     it("shouldn't send when telemetry when opted in but cannot connect to elastic telemetry services", async () => {
-      const sender = new TelemetryEventsSender(logger);
+      const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
       sender['sendEvents'] = jest.fn();
       const telemetryStart = {
         getIsOptedIn: jest.fn(async () => true),
@@ -558,28 +559,28 @@ describe('getV3UrlFromV2', () => {
   });
 
   it('should return prod url', () => {
-    const sender = new TelemetryEventsSender(logger);
+    const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
     expect(
       sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint')
     ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint');
   });
 
   it('should work when receiving a V3 URL', () => {
-    const sender = new TelemetryEventsSender(logger);
+    const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
     expect(
       sender.getV3UrlFromV2('https://telemetry.elastic.co/v3/send/channel', 'alerts-endpoint')
     ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint');
   });
 
   it('should return staging url', () => {
-    const sender = new TelemetryEventsSender(logger);
+    const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
     expect(
       sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint')
     ).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint');
   });
 
   it('should support ports and auth', () => {
-    const sender = new TelemetryEventsSender(logger);
+    const sender = new TelemetryEventsSender(logger, {} as ExperimentalFeatures);
     expect(
       sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint')
     ).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint');

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/sender.ts

@@ -19,6 +19,7 @@ import type {
   TaskManagerStartContract,
 } from '@kbn/task-manager-plugin/server';
 import { exhaustMap, Subject, takeUntil, timer } from 'rxjs';
+import type { ExperimentalFeatures } from '../../../common';
 import type { ITelemetryReceiver } from './receiver';
 import { copyAllowlistedFields, filterList } from './filterlists';
 import { createTelemetryTaskConfigs } from './tasks';
@@ -99,6 +100,7 @@ export class TelemetryEventsSender implements ITelemetryEventsSender {
   private readonly initialCheckDelayMs = 10 * 1000;
   private readonly checkIntervalMs = 60 * 1000;
   private readonly logger: TelemetryLogger;
+  private readonly experimentalFeatures: ExperimentalFeatures;
   private readonly stop$ = new Subject<void>();
   private maxQueueSize = telemetryConfiguration.telemetry_max_buffer_size;
   private telemetryStart?: TelemetryPluginStart;
@@ -116,7 +118,8 @@ export class TelemetryEventsSender implements ITelemetryEventsSender {
 
   private asyncTelemetrySender?: IAsyncTelemetryEventsSender;
 
-  constructor(logger: Logger) {
+  constructor(logger: Logger, experimentalFeatures: ExperimentalFeatures) {
+    this.experimentalFeatures = experimentalFeatures;
     this.logger = newTelemetryLogger(logger.get('telemetry_events.sender'));
   }
 
@@ -131,7 +134,7 @@ export class TelemetryEventsSender implements ITelemetryEventsSender {
     this.telemetryUsageCounter = telemetryUsageCounter;
     if (taskManager) {
       const taskMetricsService = new TaskMetricsService(this.logger, this);
-      this.telemetryTasks = createTelemetryTaskConfigs().map(
+      this.telemetryTasks = createTelemetryTaskConfigs(this.experimentalFeatures).map(
         (config: SecurityTelemetryTaskConfig) => {
           const task = new SecurityTelemetryTask(
             config,

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/custom_response_actions_rule.test.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from '@kbn/core/server/mocks';
+import { createTelemetryCustomResponseActionRulesTaskConfig } from './custom_response_actions_rule';
+import {
+  createMockTelemetryEventsSender,
+  createMockTelemetryReceiver,
+  createMockTaskMetrics,
+} from '../__mocks__';
+
+describe('security response actions rule task test', () => {
+  let logger: ReturnType<typeof loggingSystemMock.createLogger>;
+
+  beforeEach(() => {
+    logger = loggingSystemMock.createLogger();
+  });
+
+  test('security response actions rule task should fetch response actions rules data', async () => {
+    const testTaskExecutionPeriod = {
+      last: undefined,
+      current: new Date().toISOString(),
+    };
+    const mockTelemetryEventsSender = createMockTelemetryEventsSender();
+    const mockTelemetryReceiver = createMockTelemetryReceiver();
+    const telemetryCustomResponseActionsRulesTaskConfig =
+      createTelemetryCustomResponseActionRulesTaskConfig(1);
+    const mockTaskMetrics = createMockTaskMetrics();
+
+    await telemetryCustomResponseActionsRulesTaskConfig.runTask(
+      'test-id',
+      logger,
+      mockTelemetryReceiver,
+      mockTelemetryEventsSender,
+      mockTaskMetrics,
+      testTaskExecutionPeriod
+    );
+
+    expect(mockTelemetryReceiver.fetchResponseActionsRules).toHaveBeenCalled();
+  });
+});