Adding Usage Telemetry for Detection Rules & Security Lists Tasks

x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -18,7 +18,7 @@ import type {
   SearchRequest,
   SearchResponse,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '@kbn/securitysolution-list-constants';
+import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import {
   EQL_RULE_TYPE_ID,
   INDICATOR_RULE_TYPE_ID,
@@ -43,6 +43,7 @@ import type {
   PackageService,
 } from '@kbn/fleet-plugin/server';
 import type { ExceptionListClient } from '@kbn/lists-plugin/server';
+import moment from 'moment';
 import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
 import {
   exceptionListItemToTelemetryEntry,
@@ -439,11 +440,12 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     // Ensure list is created if it does not exist
     await this.exceptionListClient.createTrustedAppsList();
 
+    const timeFrom = moment.utc().subtract(1, 'day').valueOf();
     const results = await this.exceptionListClient.findExceptionListItem({
-      listId: ENDPOINT_TRUSTED_APPS_LIST_ID,
+      listId: ENDPOINT_ARTIFACT_LISTS.trustedApps.id,
       page: 1,
       perPage: 10_000,
-      filter: undefined,
+      filter: `exception-list-agnostic.attributes.created_at >= ${timeFrom}`,
       namespaceType: 'agnostic',
       sortField: 'name',
       sortOrder: 'asc',
@@ -465,11 +467,12 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     // Ensure list is created if it does not exist
     await this.exceptionListClient.createEndpointList();
 
+    const timeFrom = moment.utc().subtract(1, 'day').valueOf();
     const results = await this.exceptionListClient.findExceptionListItem({
       listId,
       page: 1,
       perPage: this.maxRecords,
-      filter: undefined,
+      filter: `exception-list-agnostic.attributes.created_at >= ${timeFrom}`,
       namespaceType: 'agnostic',
       sortField: 'name',
       sortOrder: 'asc',
@@ -545,9 +548,14 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     // Ensure list is created if it does not exist
     await this.exceptionListClient.createTrustedAppsList();
 
+    const timeFrom = `exception-list.attributes.created_at >= ${moment
+      .utc()
+      .subtract(24, 'hours')
+      .valueOf()}`;
+
     const results = await this.exceptionListClient?.findExceptionListsItem({
       listId: [listId],
-      filter: [],
+      filter: [timeFrom],
       perPage: this.maxRecords,
       page: 1,
       sortField: 'exception-list.created_at',

x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts

@@ -11,7 +11,13 @@ import {
   TELEMETRY_CHANNEL_LISTS,
   TASK_METRICS_CHANNEL,
 } from '../constants';
-import { batchTelemetryRecords, templateExceptionList, tlog, createTaskMetric } from '../helpers';
+import {
+  batchTelemetryRecords,
+  templateExceptionList,
+  tlog,
+  createTaskMetric,
+  createUsageCounterLabel,
+} from '../helpers';
 import type { ITelemetryEventsSender } from '../sender';
 import type { ITelemetryReceiver } from '../receiver';
 import type { ExceptionListItem, ESClusterInfo, ESLicense, RuleSearchResult } from '../types';
@@ -31,6 +37,10 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
       sender: ITelemetryEventsSender,
       taskExecutionPeriod: TaskExecutionPeriod
     ) => {
+      const usageCollector = sender.getTelemetryUsageCluster();
+
+      const usageLabelPrefix: string[] = ['security_telemetry', 'detection-rules'];
+
       const startTime = Date.now();
       const taskName = 'Security Solution Detection Rule Lists Telemetry';
       try {
@@ -98,14 +108,21 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
           LIST_DETECTION_RULE_EXCEPTION
         );
         tlog(logger, `Detection rule exception json length ${detectionRuleExceptionsJson.length}`);
+
+        usageCollector?.incrementCounter({
+          counterName: createUsageCounterLabel(usageLabelPrefix),
+          counterType: 'detection_rule_count',
+          incrementBy: detectionRuleExceptionsJson.length,
+        });
+
         const batches = batchTelemetryRecords(detectionRuleExceptionsJson, maxTelemetryBatch);
         for (const batch of batches) {
           await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch);
         }
         await sender.sendOnDemand(TASK_METRICS_CHANNEL, [
           createTaskMetric(taskName, true, startTime),
         ]);
-        return detectionRuleExceptions.length;
+        return detectionRuleExceptionsJson.length;
       } catch (err) {
         await sender.sendOnDemand(TASK_METRICS_CHANNEL, [
           createTaskMetric(taskName, false, startTime, err.message),

x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts

@@ -6,10 +6,7 @@
  */
 
 import type { Logger } from '@kbn/core/server';
-import {
-  ENDPOINT_LIST_ID,
-  ENDPOINT_EVENT_FILTERS_LIST_ID,
-} from '@kbn/securitysolution-list-constants';
+import { ENDPOINT_LIST_ID, ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import {
   LIST_ENDPOINT_EXCEPTION,
   LIST_ENDPOINT_EVENT_FILTER,
@@ -23,6 +20,8 @@ import {
   templateExceptionList,
   createTaskMetric,
   formatValueListMetaData,
+  createUsageCounterLabel,
+  tlog,
 } from '../helpers';
 import type { ITelemetryEventsSender } from '../sender';
 import type { ITelemetryReceiver } from '../receiver';
@@ -42,10 +41,16 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
       sender: ITelemetryEventsSender,
       taskExecutionPeriod: TaskExecutionPeriod
     ) => {
+      const usageCollector = sender.getTelemetryUsageCluster();
+
+      const usageLabelPrefix: string[] = ['security_telemetry', 'lists'];
+
       const startTime = Date.now();
       const taskName = 'Security Solution Lists Telemetry';
       try {
-        let count = 0;
+        let trustedApplicationsCount = 0;
+        let endpointExceptionsCount = 0;
+        let endpointEventFiltersCount = 0;
 
         const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
           receiver.fetchClusterInfo(),
@@ -71,7 +76,14 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
             licenseInfo,
             LIST_TRUSTED_APPLICATION
           );
-          count += trustedAppsJson.length;
+          trustedApplicationsCount = trustedAppsJson.length;
+          tlog(logger, `Trusted Apps: ${trustedApplicationsCount}`);
+
+          usageCollector?.incrementCounter({
+            counterName: createUsageCounterLabel(usageLabelPrefix),
+            counterType: 'trusted_apps_count',
+            incrementBy: trustedApplicationsCount,
+          });
 
           const batches = batchTelemetryRecords(trustedAppsJson, maxTelemetryBatch);
           for (const batch of batches) {
@@ -89,7 +101,14 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
             licenseInfo,
             LIST_ENDPOINT_EXCEPTION
           );
-          count += epExceptionsJson.length;
+          endpointExceptionsCount = epExceptionsJson.length;
+          tlog(logger, `EP Exceptions: ${endpointExceptionsCount}`);
+
+          usageCollector?.incrementCounter({
+            counterName: createUsageCounterLabel(usageLabelPrefix),
+            counterType: 'endpoint_exceptions_count',
+            incrementBy: endpointExceptionsCount,
+          });
 
           const batches = batchTelemetryRecords(epExceptionsJson, maxTelemetryBatch);
           for (const batch of batches) {
@@ -99,15 +118,22 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
 
         // Lists Telemetry: Endpoint Event Filters
 
-        const epFilters = await receiver.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID);
+        const epFilters = await receiver.fetchEndpointList(ENDPOINT_ARTIFACT_LISTS.eventFilters.id);
         if (epFilters?.data) {
           const epFiltersJson = templateExceptionList(
             epFilters.data,
             clusterInfo,
             licenseInfo,
             LIST_ENDPOINT_EVENT_FILTER
           );
-          count += epFiltersJson.length;
+          endpointEventFiltersCount = epFiltersJson.length;
+          tlog(logger, `EP Event Filters: ${endpointEventFiltersCount}`);
+
+          usageCollector?.incrementCounter({
+            counterName: createUsageCounterLabel(usageLabelPrefix),
+            counterType: 'endpoint_event_filters_count',
+            incrementBy: endpointEventFiltersCount,
+          });
 
           const batches = batchTelemetryRecords(epFiltersJson, maxTelemetryBatch);
           for (const batch of batches) {
@@ -130,7 +156,7 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
         await sender.sendOnDemand(TASK_METRICS_CHANNEL, [
           createTaskMetric(taskName, true, startTime),
         ]);
-        return count;
+        return trustedApplicationsCount + endpointExceptionsCount + endpointEventFiltersCount;
       } catch (err) {
         await sender.sendOnDemand(TASK_METRICS_CHANNEL, [
           createTaskMetric(taskName, false, startTime, err.message),