[7.x] Detection Rule Exception List telemetry (#113239)

x-pack/plugins/security_solution/server/lib/telemetry/constants.ts

@@ -7,12 +7,14 @@
 
 export const TELEMETRY_MAX_BUFFER_SIZE = 100;
 
-export const TELEMETRY_CHANNEL_LISTS = 'security-lists';
+export const TELEMETRY_CHANNEL_LISTS = 'security-lists-v2';
 
 export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata';
 
-export const LIST_TRUSTED_APPLICATION = 'trusted_application';
+export const LIST_DETECTION_RULE_EXCEPTION = 'detection_rule_exception';
 
 export const LIST_ENDPOINT_EXCEPTION = 'endpoint_exception';
 
 export const LIST_ENDPOINT_EVENT_FILTER = 'endpoint_event_filter';
+
+export const LIST_TRUSTED_APPLICATION = 'trusted_application';

x-pack/plugins/security_solution/server/lib/telemetry/filters.ts

@@ -129,21 +129,20 @@ export const allowlistEventFields: AllowlistFields = {
 
 export const exceptionListEventFields: AllowlistFields = {
   created_at: true,
-  description: true,
   effectScope: true,
   entries: true,
   id: true,
   name: true,
-  os: true,
   os_types: true,
+  rule_version: true,
 };
 
 /**
  * Filters out information not required for downstream analysis
  *
  * @param allowlist
  * @param event
- * @returns
+ * @returns TelemetryEvent with explicitly required fields
  */
 export function copyAllowlistedFields(
   allowlist: AllowlistFields,

x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts

@@ -8,13 +8,14 @@
 import moment from 'moment';
 import { createMockPackagePolicy } from './mocks';
 import {
+  LIST_DETECTION_RULE_EXCEPTION,
   LIST_ENDPOINT_EXCEPTION,
   LIST_ENDPOINT_EVENT_FILTER,
   LIST_TRUSTED_APPLICATION,
 } from './constants';
 import {
   getPreviousDiagTaskTimestamp,
-  getPreviousEpMetaTaskTimestamp,
+  getPreviousDailyTaskTimestamp,
   batchTelemetryRecords,
   isPackagePolicyList,
   templateExceptionList,
@@ -53,15 +54,15 @@ describe('test endpoint meta telemetry scheduled task timing helper', () => {
   test('test -24 hours is returned when there is no previous task run', async () => {
     const executeTo = moment().utc().toISOString();
     const executeFrom = undefined;
-    const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom);
+    const newExecuteFrom = getPreviousDailyTaskTimestamp(executeTo, executeFrom);
 
     expect(newExecuteFrom).toEqual(moment(executeTo).subtract(24, 'hours').toISOString());
   });
 
   test('test -24 hours is returned when there was a previous task run', async () => {
     const executeTo = moment().utc().toISOString();
     const executeFrom = moment(executeTo).subtract(24, 'hours').toISOString();
-    const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom);
+    const newExecuteFrom = getPreviousDailyTaskTimestamp(executeTo, executeFrom);
 
     expect(newExecuteFrom).toEqual(executeFrom);
   });
@@ -71,7 +72,7 @@ describe('test endpoint meta telemetry scheduled task timing helper', () => {
   test('test 24 hours is returned when previous task run took longer than 24 hours', async () => {
     const executeTo = moment().utc().toISOString();
     const executeFrom = moment(executeTo).subtract(72, 'hours').toISOString(); // down 3 days
-    const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom);
+    const newExecuteFrom = getPreviousDailyTaskTimestamp(executeTo, executeFrom);
 
     expect(newExecuteFrom).toEqual(moment(executeTo).subtract(24, 'hours').toISOString());
   });
@@ -134,61 +135,88 @@ describe('test package policy type guard', () => {
 });
 
 describe('list telemetry schema', () => {
+  test('detection rules document is correctly formed', () => {
+    const data = [{ id: 'test_1' }] as ExceptionListItem[];
+    const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION);
+
+    expect(templatedItems[0]?.detection_rule).not.toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).toBeUndefined();
+  });
+
+  test('detection rules document is correctly formed with multiple entries', () => {
+    const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
+    const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION);
+
+    expect(templatedItems[0]?.detection_rule).not.toBeUndefined();
+    expect(templatedItems[1]?.detection_rule).not.toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).toBeUndefined();
+  });
+
   test('trusted apps document is correctly formed', () => {
     const data = [{ id: 'test_1' }] as ExceptionListItem[];
     const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);
 
-    expect(templatedItems[0]?.trusted_application.length).toEqual(1);
-    expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
+    expect(templatedItems[0]?.detection_rule).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).not.toBeUndefined();
   });
 
   test('trusted apps document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
     const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);
 
-    expect(templatedItems[0]?.trusted_application.length).toEqual(1);
-    expect(templatedItems[1]?.trusted_application.length).toEqual(1);
-    expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
+    expect(templatedItems[0]?.detection_rule).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).not.toBeUndefined();
+    expect(templatedItems[1]?.trusted_application).not.toBeUndefined();
   });
 
   test('endpoint exception document is correctly formed', () => {
     const data = [{ id: 'test_3' }] as ExceptionListItem[];
     const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);
 
-    expect(templatedItems[0]?.trusted_application.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_exception.length).toEqual(1);
-    expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
+    expect(templatedItems[0]?.detection_rule).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).toBeUndefined();
   });
 
   test('endpoint exception document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_4' }, { id: 'test_4' }, { id: 'test_4' }] as ExceptionListItem[];
     const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);
 
-    expect(templatedItems[0]?.trusted_application.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_exception.length).toEqual(1);
-    expect(templatedItems[1]?.endpoint_exception.length).toEqual(1);
-    expect(templatedItems[2]?.endpoint_exception.length).toEqual(1);
-    expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
+    expect(templatedItems[0]?.detection_rule).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined();
+    expect(templatedItems[1]?.endpoint_exception).not.toBeUndefined();
+    expect(templatedItems[2]?.endpoint_exception).not.toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).toBeUndefined();
   });
 
   test('endpoint event filters document is correctly formed', () => {
     const data = [{ id: 'test_5' }] as ExceptionListItem[];
     const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);
 
-    expect(templatedItems[0]?.trusted_application.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1);
+    expect(templatedItems[0]?.detection_rule).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).toBeUndefined();
   });
 
   test('endpoint event filters document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_6' }, { id: 'test_6' }] as ExceptionListItem[];
     const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);
 
-    expect(templatedItems[0]?.trusted_application.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
-    expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1);
-    expect(templatedItems[1]?.endpoint_event_filter.length).toEqual(1);
+    expect(templatedItems[0]?.detection_rule).toBeUndefined();
+    expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined();
+    expect(templatedItems[1]?.endpoint_event_filter).not.toBeUndefined();
+    expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
+    expect(templatedItems[0]?.trusted_application).toBeUndefined();
   });
 });

x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts

@@ -11,6 +11,7 @@ import { PackagePolicy } from '../../../../fleet/common/types/models/package_pol
 import { copyAllowlistedFields, exceptionListEventFields } from './filters';
 import { ExceptionListItem, ListTemplate, TelemetryEvent } from './types';
 import {
+  LIST_DETECTION_RULE_EXCEPTION,
   LIST_ENDPOINT_EXCEPTION,
   LIST_ENDPOINT_EVENT_FILTER,
   LIST_TRUSTED_APPLICATION,
@@ -46,7 +47,7 @@ export const getPreviousDiagTaskTimestamp = (
  * @param lastExecutionTimestamp
  * @returns the timestamp to search from
  */
-export const getPreviousEpMetaTaskTimestamp = (
+export const getPreviousDailyTaskTimestamp = (
   executeTo: string,
   lastExecutionTimestamp?: string
 ) => {
@@ -97,18 +98,16 @@ export function isPackagePolicyList(
  * Maps trusted application to shared telemetry object
  *
  * @param exceptionListItem
- * @returns collection of endpoint exceptions
+ * @returns collection of trusted applications
  */
 export const trustedApplicationToTelemetryEntry = (trustedApplication: TrustedApp) => {
   return {
     id: trustedApplication.id,
-    version: trustedApplication.version || '',
     name: trustedApplication.name,
-    description: trustedApplication.description,
     created_at: trustedApplication.created_at,
     updated_at: trustedApplication.updated_at,
     entries: trustedApplication.entries,
-    os: trustedApplication.os,
+    os_types: [trustedApplication.os],
   } as ExceptionListItem;
 };
 
@@ -121,9 +120,29 @@ export const trustedApplicationToTelemetryEntry = (trustedApplication: TrustedAp
 export const exceptionListItemToTelemetryEntry = (exceptionListItem: ExceptionListItemSchema) => {
   return {
     id: exceptionListItem.id,
-    version: exceptionListItem._version || '',
     name: exceptionListItem.name,
-    description: exceptionListItem.description,
+    created_at: exceptionListItem.created_at,
+    updated_at: exceptionListItem.updated_at,
+    entries: exceptionListItem.entries,
+    os_types: exceptionListItem.os_types,
+  } as ExceptionListItem;
+};
+
+/**
+ * Maps detection rule exception list items to shared telemetry object
+ *
+ * @param exceptionListItem
+ * @param ruleVersion
+ * @returns collection of detection rule exceptions
+ */
+export const ruleExceptionListItemToTelemetryEvent = (
+  exceptionListItem: ExceptionListItemSchema,
+  ruleVersion: number
+) => {
+  return {
+    id: exceptionListItem.item_id,
+    name: exceptionListItem.description,
+    rule_version: ruleVersion,
     created_at: exceptionListItem.created_at,
     updated_at: exceptionListItem.updated_at,
     entries: exceptionListItem.entries,
@@ -141,9 +160,7 @@ export const exceptionListItemToTelemetryEntry = (exceptionListItem: ExceptionLi
 export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => {
   return listData.map((item) => {
     const template: ListTemplate = {
-      trusted_application: [],
-      endpoint_exception: [],
-      endpoint_event_filter: [],
+      '@timestamp': new Date().getTime(),
     };
 
     // cast exception list type to a TelemetryEvent for allowlist filtering
@@ -152,18 +169,23 @@ export const templateExceptionList = (listData: ExceptionListItem[], listType: s
       item as unknown as TelemetryEvent
     );
 
+    if (listType === LIST_DETECTION_RULE_EXCEPTION) {
+      template.detection_rule = filteredListItem;
+      return template;
+    }
+
     if (listType === LIST_TRUSTED_APPLICATION) {
-      template.trusted_application.push(filteredListItem);
+      template.trusted_application = filteredListItem;
       return template;
     }
 
     if (listType === LIST_ENDPOINT_EXCEPTION) {
-      template.endpoint_exception.push(filteredListItem);
+      template.endpoint_exception = filteredListItem;
       return template;
     }
 
     if (listType === LIST_ENDPOINT_EVENT_FILTER) {
-      template.endpoint_event_filter.push(filteredListItem);
+      template.endpoint_event_filter = filteredListItem;
       return template;
     }
 

x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts

@@ -8,7 +8,7 @@
 // eslint-disable-next-line max-classes-per-file
 import { TelemetryEventsSender } from './sender';
 import { TelemetryReceiver } from './receiver';
-import { DiagnosticTask, EndpointTask, ExceptionListsTask } from './tasks';
+import { DiagnosticTask, EndpointTask, ExceptionListsTask, DetectionRulesTask } from './tasks';
 import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy';
 
 /**
@@ -40,6 +40,8 @@ export const createMockTelemetryReceiver = (): jest.Mocked<TelemetryReceiver> =>
     fetchEndpointMetrics: jest.fn(),
     fetchEndpointPolicyResponses: jest.fn(),
     fetchTrustedApplications: jest.fn(),
+    fetchDetectionRules: jest.fn(),
+    fetchDetectionExceptionList: jest.fn(),
   } as unknown as jest.Mocked<TelemetryReceiver>;
 };
 
@@ -79,3 +81,10 @@ export class MockTelemetryEndpointTask extends EndpointTask {
 export class MockExceptionListsTask extends ExceptionListsTask {
   public runTask = jest.fn();
 }
+
+/**
+ * Creates a mocked Telemetry detection rules lists Task
+ */
+export class MockDetectionRuleListsTask extends DetectionRulesTask {
+  public runTask = jest.fn();
+}