Diagnostic timelines task

x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts

@@ -9,7 +9,7 @@ import moment from 'moment';
 import type { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server';
 import { TaskStatus } from '@kbn/task-manager-plugin/server';
 import type { TelemetryEventsSender } from '../sender';
-import type { TelemetryReceiver } from '../receiver';
+import type { ITelemetryReceiver, TelemetryReceiver } from '../receiver';
 import type { SecurityTelemetryTaskConfig } from '../task';
 import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/package_policy';
 import { stubEndpointAlertResponse, stubProcessTree, stubFetchTimelineEvents } from './timeline';
@@ -71,7 +71,7 @@ export const stubLicenseInfo: ESLicense = {
 export const createMockTelemetryReceiver = (
   diagnosticsAlert?: unknown,
   emptyTimelineTree?: boolean
-): jest.Mocked<TelemetryReceiver> => {
+): jest.Mocked<ITelemetryReceiver> => {
   const processTreeResponse = emptyTimelineTree
     ? Promise.resolve([])
     : Promise.resolve(Promise.resolve(stubProcessTree()));
@@ -91,12 +91,11 @@ export const createMockTelemetryReceiver = (
     fetchEndpointList: jest.fn(),
     fetchDetectionRules: jest.fn().mockReturnValue({ body: null }),
     fetchEndpointMetadata: jest.fn(),
-    fetchTimelineEndpointAlerts: jest
-      .fn()
-      .mockReturnValue(Promise.resolve(stubEndpointAlertResponse())),
+    fetchTimelineAlerts: jest.fn().mockReturnValue(Promise.resolve(stubEndpointAlertResponse())),
     buildProcessTree: jest.fn().mockReturnValue(processTreeResponse),
     fetchTimelineEvents: jest.fn().mockReturnValue(Promise.resolve(stubFetchTimelineEvents())),
     fetchValueListMetaData: jest.fn(),
+    getAlertsIndex: jest.fn().mockReturnValue('test-alerts-index'),
   } as unknown as jest.Mocked<TelemetryReceiver>;
 };
 

x-pack/plugins/security_solution/server/lib/telemetry/constants.ts

@@ -27,6 +27,8 @@ export const INSIGHTS_CHANNEL = 'security-insights-v1';
 
 export const TASK_METRICS_CHANNEL = 'task-metrics';
 
+export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const;
+
 export const DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS = {
   linux: {
     advanced: {

x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts

@@ -11,17 +11,25 @@ import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/packag
 import { merge, set } from 'lodash';
 import type { Logger } from '@kbn/core/server';
 import { sha256 } from 'js-sha256';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { copyAllowlistedFields, filterList } from './filterlists';
-import type { PolicyConfig, PolicyData } from '../../../common/endpoint/types';
+import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types';
+import type { ITelemetryReceiver } from './receiver';
 import type {
-  ExceptionListItem,
+  EnhancedAlertEvent,
   ESClusterInfo,
   ESLicense,
+  ExceptionListItem,
+  ExtraInfo,
   ListTemplate,
+  TaskMetric,
   TelemetryEvent,
+  TimeFrame,
+  TimelineResult,
+  TimelineTelemetryEvent,
   ValueListResponse,
-  TaskMetric,
 } from './types';
+import type { TaskExecutionPeriod } from './task';
 import {
   LIST_DETECTION_RULE_EXCEPTION,
   LIST_ENDPOINT_EXCEPTION,
@@ -30,6 +38,7 @@ import {
   DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS,
 } from './constants';
 import { tagsToEffectScope } from '../../../common/endpoint/service/trusted_apps/mapping';
+import { resolverEntity } from '../../endpoint/routes/resolver/entity/utils/build_resolver_entity';
 
 /**
  * Determines the when the last run was in order to execute to.
@@ -345,3 +354,114 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T
 
   return event;
 };
+
+export const ranges = (
+  taskExecutionPeriod: TaskExecutionPeriod,
+  defaultIntervalInHours: number = 3
+) => {
+  const rangeFrom = taskExecutionPeriod.last ?? `now-${defaultIntervalInHours}h`;
+  const rangeTo = taskExecutionPeriod.current;
+
+  return { rangeFrom, rangeTo };
+};
+
+export class TelemetryTimelineFetcher {
+  startTime: number;
+  private receiver: ITelemetryReceiver;
+  private extraInfo: Promise<ExtraInfo>;
+  private timeFrame: TimeFrame;
+
+  constructor(receiver: ITelemetryReceiver) {
+    this.receiver = receiver;
+    this.startTime = Date.now();
+    this.extraInfo = this.lookupExtraInfo();
+    this.timeFrame = this.calculateTimeFrame();
+  }
+
+  async fetchTimeline(event: estypes.SearchHit<EnhancedAlertEvent>): Promise<TimelineResult> {
+    const eventId = event._source ? event._source['event.id'] : 'unknown';
+    const alertUUID = event._source ? event._source['kibana.alert.uuid'] : 'unknown';
+
+    const entities = resolverEntity([event]);
+
+    // Build Tree
+    const tree = await this.receiver.buildProcessTree(
+      entities[0].id,
+      entities[0].schema,
+      this.timeFrame.startOfDay,
+      this.timeFrame.endOfDay
+    );
+
+    const nodeIds = Array.isArray(tree) ? tree.map((node) => node?.id.toString()) : [];
+
+    const eventsStore = await this.fetchEventLineage(nodeIds);
+
+    const telemetryTimeline: TimelineTelemetryEvent[] = Array.isArray(tree)
+      ? tree.map((node) => {
+          return {
+            ...node,
+            event: eventsStore.get(node.id.toString()),
+          };
+        })
+      : [];
+
+    let record;
+    if (telemetryTimeline.length >= 1) {
+      const { clusterInfo, licenseInfo } = await this.extraInfo;
+      record = {
+        '@timestamp': moment().toISOString(),
+        version: clusterInfo.version?.number,
+        cluster_name: clusterInfo.cluster_name,
+        cluster_uuid: clusterInfo.cluster_uuid,
+        license_uuid: licenseInfo?.uid,
+        alert_id: alertUUID,
+        event_id: eventId,
+        timeline: telemetryTimeline,
+      };
+    }
+
+    const result: TimelineResult = {
+      nodes: nodeIds.length,
+      events: eventsStore.size,
+      timeline: record,
+    };
+
+    return result;
+  }
+
+  private async fetchEventLineage(nodeIds: string[]): Promise<Map<string, SafeEndpointEvent>> {
+    const timelineEvents = await this.receiver.fetchTimelineEvents(nodeIds);
+    const eventsStore = new Map<string, SafeEndpointEvent>();
+    for (const event of timelineEvents.hits.hits) {
+      const doc = event._source;
+
+      if (doc !== null && doc !== undefined) {
+        const entityId = doc?.process?.entity_id?.toString();
+        if (entityId !== null && entityId !== undefined) eventsStore.set(entityId, doc);
+      }
+    }
+    return eventsStore;
+  }
+
+  private async lookupExtraInfo(): Promise<ExtraInfo> {
+    const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
+      this.receiver.fetchClusterInfo(),
+      this.receiver.fetchLicenseInfo(),
+    ]);
+
+    const clusterInfo: ESClusterInfo =
+      clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo);
+
+    const licenseInfo: ESLicense | undefined =
+      licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value : ({} as ESLicense);
+
+    return { clusterInfo, licenseInfo };
+  }
+
+  private calculateTimeFrame(): TimeFrame {
+    const now = moment();
+    const startOfDay = now.startOf('day').toISOString();
+    const endOfDay = now.endOf('day').toISOString();
+    return { startOfDay, endOfDay };
+  }
+}

x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -70,6 +70,7 @@ import type {
 import { telemetryConfiguration } from './configuration';
 import { ENDPOINT_METRICS_INDEX } from '../../../common/constants';
 import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants';
+import { DEFAULT_DIAGNOSTIC_INDEX } from './constants';
 
 export interface ITelemetryReceiver {
   start(
@@ -163,7 +164,11 @@ export interface ITelemetryReceiver {
 
   fetchPrebuiltRuleAlerts(): Promise<{ events: TelemetryEvent[]; count: number }>;
 
-  fetchTimelineEndpointAlerts(interval: number): Promise<Array<SearchHit<EnhancedAlertEvent>>>;
+  fetchTimelineAlerts(
+    index: string,
+    rangeFrom: string,
+    rangeTo: string
+  ): Promise<Array<SearchHit<EnhancedAlertEvent>>>;
 
   buildProcessTree(
     entityId: string,
@@ -177,6 +182,8 @@ export interface ITelemetryReceiver {
   ): Promise<SearchResponse<SafeEndpointEvent, Record<string, AggregationsAggregate>>>;
 
   fetchValueListMetaData(interval: number): Promise<ValueListResponse>;
+
+  getAlertsIndex(): string | undefined;
 }
 
 export class TelemetryReceiver implements ITelemetryReceiver {
@@ -224,6 +231,10 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     return this.clusterInfo;
   }
 
+  public getAlertsIndex(): string | undefined {
+    return this.alertsIndex;
+  }
+
   public async fetchDetectionRulesPackageVersion(): Promise<Installation | undefined> {
     return this.packageService?.asInternalUser.getInstallation(PREBUILT_RULES_PACKAGE_NAME);
   }
@@ -396,7 +407,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
 
     const query = {
       expand_wildcards: ['open' as const, 'hidden' as const],
-      index: '.logs-endpoint.diagnostic.collection-*',
+      index: `${DEFAULT_DIAGNOSTIC_INDEX}-*`,
       ignore_unavailable: true,
       size: telemetryConfiguration.telemetry_max_buffer_size,
       body: {
@@ -697,7 +708,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     return { events: telemetryEvents, count: aggregations?.prebuilt_rule_alert_count.value ?? 0 };
   }
 
-  public async fetchTimelineEndpointAlerts(interval: number) {
+  async fetchTimelineAlerts(index: string, rangeFrom: string, rangeTo: string) {
     if (this.esClient === undefined || this.esClient === null) {
       throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation');
     }
@@ -708,7 +719,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     // create and assign an initial point in time
     let pitId: OpenPointInTimeResponse['id'] = (
       await this.esClient.openPointInTime({
-        index: `${this.alertsIndex}*`,
+        index: `${index}*`,
         keep_alive: keepAlive,
       })
     ).id;
@@ -746,8 +757,8 @@ export class TelemetryReceiver implements ITelemetryReceiver {
               {
                 range: {
                   '@timestamp': {
-                    gte: `now-${interval}h`,
-                    lte: 'now',
+                    gte: rangeFrom,
+                    lte: rangeTo,
                   },
                 },
               },
@@ -807,7 +818,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
     }
 
     tlog(this.logger, `Timeline alerts to return: ${alertsToReturn.length}`);
-    return alertsToReturn;
+    return alertsToReturn || [];
   }
 
   public async buildProcessTree(

x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts

@@ -33,6 +33,12 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
     ) => {
       const startTime = Date.now();
       const taskName = 'Security Solution Detection Rule Lists Telemetry';
+
+      tlog(
+        logger,
+        `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]`
+      );
+
       try {
         const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
           receiver.fetchClusterInfo(),

x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts

@@ -12,6 +12,7 @@ import { createTelemetrySecurityListTaskConfig } from './security_lists';
 import { createTelemetryDetectionRuleListsTaskConfig } from './detection_rule';
 import { createTelemetryPrebuiltRuleAlertsTaskConfig } from './prebuilt_rule_alerts';
 import { createTelemetryTimelineTaskConfig } from './timelines';
+import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnostic';
 import { createTelemetryConfigurationTaskConfig } from './configuration';
 import { telemetryConfiguration } from '../configuration';
 import { createTelemetryFilterListArtifactTaskConfig } from './filterlists';
@@ -26,6 +27,7 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] {
     ),
     createTelemetryPrebuiltRuleAlertsTaskConfig(telemetryConfiguration.max_detection_alerts_batch),
     createTelemetryTimelineTaskConfig(),
+    createTelemetryDiagnosticTimelineTaskConfig(),
     createTelemetryConfigurationTaskConfig(),
     createTelemetryFilterListArtifactTaskConfig(),
   ];

x-pack/plugins/security_solution/server/lib/telemetry/tasks/tasks.test.ts

@@ -52,8 +52,8 @@ describe('security telemetry - ', () => {
     expect(taskConfig.interval).toEqual('24h');
   });
 
-  test('timelines task is set to 3h', async () => {
+  test('timelines task is set to 1h', async () => {
     const taskConfig = createTelemetryTimelineTaskConfig();
-    expect(taskConfig.interval).toEqual('3h');
+    expect(taskConfig.interval).toEqual('1h');
   });
 });

x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines.test.ts

@@ -35,7 +35,7 @@ describe('timeline telemetry task test', () => {
 
     expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled();
     expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled();
-    expect(mockTelemetryReceiver.fetchTimelineEndpointAlerts).toHaveBeenCalled();
+    expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled();
     expect(mockTelemetryEventsSender.getTelemetryUsageCluster).toHaveBeenCalled();
     expect(mockTelemetryEventsSender.sendOnDemand).toHaveBeenCalled();
   });
@@ -59,6 +59,6 @@ describe('timeline telemetry task test', () => {
 
     expect(mockTelemetryReceiver.buildProcessTree).toHaveBeenCalled();
     expect(mockTelemetryReceiver.fetchTimelineEvents).toHaveBeenCalled();
-    expect(mockTelemetryReceiver.fetchTimelineEndpointAlerts).toHaveBeenCalled();
+    expect(mockTelemetryReceiver.fetchTimelineAlerts).toHaveBeenCalled();
   });
 });