[Security Solution] Exclude cold and frozen tiers in analyzer queries

packages/kbn-management/settings/setting_ids/index.ts

@@ -167,6 +167,9 @@ export const SECURITY_SOLUTION_DEFAULT_ALERT_TAGS_KEY = 'securitySolution:alertT
 /** This Kibana Advanced Setting allows users to enable/disable the Expandable Flyout */
 export const SECURITY_SOLUTION_ENABLE_EXPANDABLE_FLYOUT_SETTING =
   'securitySolution:enableExpandableFlyout' as const;
+/** This Kibana Advanced Setting allows users to enable/disable querying cold and frozen data tiers in analyzer */
+export const SECURITY_SOLUTION_EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER =
+  'securitySolution:excludeColdAndFrozenTiersInAnalyzer' as const;
 
 // Timelion settings
 export const TIMELION_ES_DEFAULT_INDEX_ID = 'timelion:es.default_index';

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -114,6 +114,10 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'securitySolution:excludeColdAndFrozenTiersInAnalyzer': {
+    type: 'boolean',
+    _meta: { description: 'Non-default value of setting.' },
+  },
   'securitySolution:enableCcsWarning': {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -64,6 +64,7 @@ export interface UsageStats {
   'securitySolution:refreshIntervalDefaults': string;
   'securitySolution:enableNewsFeed': boolean;
   'securitySolution:enableExpandableFlyout': boolean;
+  'securitySolution:excludeColdAndFrozenTiersInAnalyzer': boolean;
   'securitySolution:enableCcsWarning': boolean;
   'search:includeFrozen': boolean;
   'courier:maxConcurrentShardRequests': number;

src/plugins/telemetry/schema/oss_plugins.json

@@ -9356,6 +9356,12 @@
             "description": "Non-default value of setting."
           }
         },
+        "securitySolution:excludeColdAndFrozenTiersInAnalyzer": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
+        },
         "securitySolution:enableCcsWarning": {
           "type": "boolean",
           "_meta": {

x-pack/plugins/security_solution/common/constants.ts

@@ -165,6 +165,10 @@ export const ENABLE_NEWS_FEED_SETTING = 'securitySolution:enableNewsFeed' as con
 /** This Kibana Advanced Setting allows users to enable/disable the Expandable Flyout */
 export const ENABLE_EXPANDABLE_FLYOUT_SETTING = 'securitySolution:enableExpandableFlyout' as const;
 
+/** This Kibana Advanced Setting allows users to enable/disable querying cold and frozen data tiers in analyzer */
+export const EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER =
+  'securitySolution:excludeColdAndFrozenTiersInAnalyzer' as const;
+
 /** This Kibana Advanced Setting enables the warnings for CCS read permissions */
 export const ENABLE_CCS_READ_WARNING_SETTING = 'securitySolution:enableCcsWarning' as const;
 

x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity/handler.ts

@@ -7,9 +7,11 @@
 
 import type { RequestHandler } from '@kbn/core/server';
 import type { TypeOf } from '@kbn/config-schema';
+import { EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER } from '../../../../../common/constants';
 import type { validateEntities } from '../../../../../common/endpoint/schema/resolver';
 import type { ResolverEntityIndex } from '../../../../../common/endpoint/types';
 import { resolverEntity } from './utils/build_resolver_entity';
+import { createSharedFilters } from '../utils/shared_filters';
 
 /**
  * This is used to get an 'entity_id' which is an internal-to-Resolver concept, from an `_id`, which
@@ -22,6 +24,10 @@ export function handleEntities(): RequestHandler<unknown, TypeOf<typeof validate
     } = request;
 
     const esClient = (await context.core).elasticsearch.client;
+    const excludeColdAndFrozenTiers = await (
+      await context.core
+    ).uiSettings.client.get<boolean>(EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER);
+
     const queryResponse = await esClient.asCurrentUser.search({
       ignore_unavailable: true,
       index: indices,
@@ -31,6 +37,7 @@ export function handleEntities(): RequestHandler<unknown, TypeOf<typeof validate
         query: {
           bool: {
             filter: [
+              ...createSharedFilters({ excludeColdAndFrozenTiers }),
               {
                 // only return documents with the matching _id
                 ids: {

x-pack/plugins/security_solution/server/endpoint/routes/resolver/events.ts

@@ -8,6 +8,7 @@
 import type { TypeOf } from '@kbn/config-schema';
 import type { RequestHandler } from '@kbn/core/server';
 import type { RuleRegistryPluginStartContract } from '@kbn/rule-registry-plugin/server';
+import { EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER } from '../../../../common/constants';
 import type { ResolverPaginatedEvents, SafeResolverEvent } from '../../../../common/endpoint/types';
 import type { validateEvents } from '../../../../common/endpoint/schema/resolver';
 import { EventsQuery } from './queries/events';
@@ -44,11 +45,15 @@ export function handleEvents(
     } = req;
     const eventsClient = (await context.core).elasticsearch.client;
     const alertsClient = await ruleRegistry.getRacClientWithRequest(req);
+    const shouldExcludeColdAndFrozenTiers = await (
+      await context.core
+    ).uiSettings.client.get<boolean>(EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER);
 
     const eventsQuery = new EventsQuery({
       pagination: PaginationBuilder.createBuilder(limit, afterEvent),
       indexPatterns: body.indexPatterns,
       timeRange: body.timeRange,
+      shouldExcludeColdAndFrozenTiers,
     });
     const results = await eventsQuery.search(eventsClient, body, alertsClient);
     return res.ok({

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts

@@ -24,8 +24,9 @@ export class EventsQuery extends BaseResolverQuery {
     timeRange,
     isInternalRequest,
     pagination,
+    shouldExcludeColdAndFrozenTiers,
   }: ResolverQueryParams & { pagination: PaginationBuilder }) {
-    super({ indexPatterns, timeRange, isInternalRequest });
+    super({ indexPatterns, timeRange, isInternalRequest, shouldExcludeColdAndFrozenTiers });
     this.pagination = pagination;
   }
 
@@ -36,6 +37,7 @@ export class EventsQuery extends BaseResolverQuery {
           filter: [
             ...filters,
             ...this.getRangeFilter(),
+            ...this.getColdAndFrozenTierFilter(),
             {
               term: { 'event.kind': 'event' },
             },

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts

@@ -11,6 +11,7 @@ import type { RequestHandler } from '@kbn/core/server';
 import type { TypeOf } from '@kbn/config-schema';
 import type { RuleRegistryPluginStartContract } from '@kbn/rule-registry-plugin/server';
 import type { LicensingPluginStart } from '@kbn/licensing-plugin/server';
+import { EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER } from '../../../../../common/constants';
 import type { ConfigType } from '../../../../config';
 
 import type { validateTree } from '../../../../../common/endpoint/schema/resolver';
@@ -30,6 +31,9 @@ export function handleTree(
     const license = await firstValueFrom(licensing.license$);
     const hasAccessToInsightsRelatedByProcessAncestry =
       insightsRelatedAlertsByProcessAncestry && license.hasAtLeast('platinum');
+    const shouldExcludeColdAndFrozenTiers = await (
+      await context.core
+    ).uiSettings.client.get<boolean>(EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER);
 
     if (hasAccessToInsightsRelatedByProcessAncestry) {
       featureUsageService.notifyUsage('ALERTS_BY_PROCESS_ANCESTRY');
@@ -39,7 +43,7 @@ export function handleTree(
       ? await ruleRegistry.getRacClientWithRequest(req)
       : undefined;
     const fetcher = new Fetcher(client, alertsClient);
-    const body = await fetcher.tree(req.body);
+    const body = await fetcher.tree({ ...req.body, shouldExcludeColdAndFrozenTiers });
     return res.ok({
       body,
     });

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts

@@ -7,14 +7,16 @@
 
 import type { JsonValue } from '@kbn/utility-types';
 import type { ResolverSchema } from '../../../../../../common/endpoint/types';
+import { createSharedFilters } from '../../utils/shared_filters';
 import type { TimeRange } from '../utils';
 import { resolverFields } from '../utils';
 
 export interface ResolverQueryParams {
   readonly schema?: ResolverSchema;
   readonly indexPatterns: string | string[];
-  readonly timeRange: TimeRange | undefined;
   readonly isInternalRequest?: boolean;
+  readonly shouldExcludeColdAndFrozenTiers?: boolean;
+  readonly timeRange: TimeRange | undefined;
   readonly resolverFields?: JsonValue[];
   getRangeFilter?: () => Array<{
     range: { '@timestamp': { gte: string; lte: string; format: string } };
@@ -24,11 +26,18 @@ export interface ResolverQueryParams {
 export class BaseResolverQuery implements ResolverQueryParams {
   readonly schema: ResolverSchema;
   readonly indexPatterns: string | string[];
-  readonly timeRange: TimeRange | undefined;
   readonly isInternalRequest?: boolean;
+  readonly shouldExcludeColdAndFrozenTiers?: boolean;
+  readonly timeRange: TimeRange | undefined;
   readonly resolverFields?: JsonValue[];
 
-  constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) {
+  constructor({
+    schema,
+    indexPatterns,
+    timeRange,
+    isInternalRequest,
+    shouldExcludeColdAndFrozenTiers,
+  }: ResolverQueryParams) {
     const schemaOrDefault = schema
       ? schema
       : {
@@ -40,6 +49,13 @@ export class BaseResolverQuery implements ResolverQueryParams {
     this.indexPatterns = indexPatterns;
     this.timeRange = timeRange;
     this.isInternalRequest = isInternalRequest;
+    this.shouldExcludeColdAndFrozenTiers = shouldExcludeColdAndFrozenTiers;
+  }
+
+  getColdAndFrozenTierFilter() {
+    return createSharedFilters({
+      excludeColdAndFrozenTiers: !!this.shouldExcludeColdAndFrozenTiers,
+    });
   }
 
   getRangeFilter() {

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts

@@ -20,8 +20,14 @@ import { BaseResolverQuery } from './base';
 export class DescendantsQuery extends BaseResolverQuery {
   declare readonly resolverFields: JsonValue[];
 
-  constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) {
-    super({ schema, indexPatterns, timeRange, isInternalRequest });
+  constructor({
+    schema,
+    indexPatterns,
+    timeRange,
+    isInternalRequest,
+    shouldExcludeColdAndFrozenTiers,
+  }: ResolverQueryParams) {
+    super({ schema, indexPatterns, timeRange, isInternalRequest, shouldExcludeColdAndFrozenTiers });
   }
 
   private query(nodes: NodeID[], size: number): JsonObject {
@@ -37,6 +43,7 @@ export class DescendantsQuery extends BaseResolverQuery {
         bool: {
           filter: [
             ...this.getRangeFilter(),
+            ...this.getColdAndFrozenTierFilter(),
             {
               terms: { [this.schema.parent]: nodes },
             },

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts

@@ -18,8 +18,14 @@ import { BaseResolverQuery } from './base';
  */
 export class LifecycleQuery extends BaseResolverQuery {
   declare readonly resolverFields: JsonValue[];
-  constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) {
-    super({ schema, indexPatterns, timeRange, isInternalRequest });
+  constructor({
+    schema,
+    indexPatterns,
+    timeRange,
+    isInternalRequest,
+    shouldExcludeColdAndFrozenTiers,
+  }: ResolverQueryParams) {
+    super({ schema, indexPatterns, timeRange, isInternalRequest, shouldExcludeColdAndFrozenTiers });
   }
 
   private query(nodes: NodeID[]): JsonObject {
@@ -34,6 +40,7 @@ export class LifecycleQuery extends BaseResolverQuery {
       query: {
         bool: {
           filter: [
+            ...this.getColdAndFrozenTierFilter(),
             ...this.getRangeFilter(),
             {
               terms: { [this.schema.id]: nodes },

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts

@@ -38,6 +38,7 @@ export interface TreeOptions {
   nodes: NodeID[];
   indexPatterns: string[];
   includeHits?: boolean;
+  shouldExcludeColdAndFrozenTiers?: boolean;
 }
 
 export type TreeResponse = Promise<
@@ -53,6 +54,7 @@ export type TreeResponse = Promise<
  */
 export class Fetcher {
   private alertsClient?: AlertsClient;
+
   constructor(private readonly client: IScopedClusterClient, alertsClient?: AlertsClient) {
     this.alertsClient = alertsClient;
   }
@@ -168,6 +170,7 @@ export class Fetcher {
       indexPatterns: options.indexPatterns,
       timeRange: options.timeRange,
       isInternalRequest,
+      shouldExcludeColdAndFrozenTiers: !!options.shouldExcludeColdAndFrozenTiers,
     });
 
     let nodes = options.nodes;
@@ -218,6 +221,7 @@ export class Fetcher {
       indexPatterns: options.indexPatterns,
       timeRange: options.timeRange,
       isInternalRequest,
+      shouldExcludeColdAndFrozenTiers: !!options.shouldExcludeColdAndFrozenTiers,
     });
 
     let nodes: NodeID[] = options.nodes;

x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/shared_filters.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface SharedFiltersOptions {
+  excludeColdAndFrozenTiers: boolean;
+}
+
+export const createSharedFilters = ({ excludeColdAndFrozenTiers }: SharedFiltersOptions) => {
+  const filters = [];
+
+  if (excludeColdAndFrozenTiers) {
+    filters.push({
+      bool: {
+        must_not: {
+          terms: {
+            _tier: ['data_frozen', 'data_cold'],
+          },
+        },
+      },
+    });
+  }
+
+  return filters;
+};

x-pack/plugins/security_solution/server/ui_settings.ts

@@ -37,6 +37,7 @@ import {
   DEFAULT_ALERT_TAGS_KEY,
   DEFAULT_ALERT_TAGS_VALUE,
   ENABLE_EXPANDABLE_FLYOUT_SETTING,
+  EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER,
 } from '../common/constants';
 import type { ExperimentalFeatures } from '../common/experimental_features';
 import { LogLevelSetting } from '../common/api/detection_engine/rule_monitoring';
@@ -185,6 +186,26 @@ export const initUiSettings = (
       requiresPageReload: true,
       schema: schema.boolean(),
     },
+    [EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER]: {
+      name: i18n.translate(
+        'xpack.securitySolution.uiSettings.excludeColdAndFrozenTiersInAnalyzer',
+        {
+          defaultMessage: 'Exclude cold and frozen tiers in Analyzer',
+        }
+      ),
+      value: false,
+      description: i18n.translate(
+        'xpack.securitySolution.uiSettings.excludeColdAndFrozenTiersInAnalyzerDescription',
+        {
+          defaultMessage:
+            '<p>When enabled, cold and frozen tiers will be skipped in analyzer queries</p>',
+        }
+      ),
+      type: 'boolean',
+      category: [APP_ID],
+      requiresPageReload: true,
+      schema: schema.boolean(),
+    },
     [DEFAULT_RULES_TABLE_REFRESH_SETTING]: {
       name: i18n.translate('xpack.securitySolution.uiSettings.rulesTableRefresh', {
         defaultMessage: 'Rules auto refresh',