diff --git a/packages/kbn-management/settings/setting_ids/index.ts b/packages/kbn-management/settings/setting_ids/index.ts index 31db42d321d2d..c176890334dc1 100644 --- a/packages/kbn-management/settings/setting_ids/index.ts +++ b/packages/kbn-management/settings/setting_ids/index.ts @@ -167,6 +167,9 @@ export const SECURITY_SOLUTION_DEFAULT_ALERT_TAGS_KEY = 'securitySolution:alertT /** This Kibana Advanced Setting allows users to enable/disable the Expandable Flyout */ export const SECURITY_SOLUTION_ENABLE_EXPANDABLE_FLYOUT_SETTING = 'securitySolution:enableExpandableFlyout' as const; +/** This Kibana Advanced Setting allows users to enable/disable querying cold and frozen data tiers in analyzer */ +export const SECURITY_SOLUTION_EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER = + 'securitySolution:excludeColdAndFrozenTiersInAnalyzer' as const; // Timelion settings export const TIMELION_ES_DEFAULT_INDEX_ID = 'timelion:es.default_index'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 0862d6ece004a..358e05f611a22 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -114,6 +114,10 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'securitySolution:excludeColdAndFrozenTiersInAnalyzer': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, 'securitySolution:enableCcsWarning': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 3499471e0d5a8..bbe0ae928d718 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -64,6 +64,7 @@ export interface UsageStats { 'securitySolution:refreshIntervalDefaults': string; 'securitySolution:enableNewsFeed': boolean; 'securitySolution:enableExpandableFlyout': boolean; + 'securitySolution:excludeColdAndFrozenTiersInAnalyzer': boolean; 'securitySolution:enableCcsWarning': boolean; 'search:includeFrozen': boolean; 'courier:maxConcurrentShardRequests': number; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 0854944f39404..88c22da35eb66 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -9356,6 +9356,12 @@ "description": "Non-default value of setting." } }, + "securitySolution:excludeColdAndFrozenTiersInAnalyzer": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } + }, "securitySolution:enableCcsWarning": { "type": "boolean", "_meta": { diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index e50533f223928..d79a446c75dcf 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -165,6 +165,10 @@ export const ENABLE_NEWS_FEED_SETTING = 'securitySolution:enableNewsFeed' as con /** This Kibana Advanced Setting allows users to enable/disable the Expandable Flyout */ export const ENABLE_EXPANDABLE_FLYOUT_SETTING = 'securitySolution:enableExpandableFlyout' as const; +/** This Kibana Advanced Setting allows users to enable/disable querying cold and frozen data tiers in analyzer */ +export const EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER = + 'securitySolution:excludeColdAndFrozenTiersInAnalyzer' as const; + /** This Kibana Advanced Setting enables the warnings for CCS read permissions */ export const ENABLE_CCS_READ_WARNING_SETTING = 'securitySolution:enableCcsWarning' as const; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity/handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity/handler.ts index 33e655b49d21a..a605226ddfb95 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity/handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity/handler.ts @@ -7,9 +7,11 @@ import type { RequestHandler } from '@kbn/core/server'; import type { TypeOf } from '@kbn/config-schema'; +import { EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER } from '../../../../../common/constants'; import type { validateEntities } from '../../../../../common/endpoint/schema/resolver'; import type { ResolverEntityIndex } from '../../../../../common/endpoint/types'; import { resolverEntity } from './utils/build_resolver_entity'; +import { createSharedFilters } from '../utils/shared_filters'; /** * This is used to get an 'entity_id' which is an internal-to-Resolver concept, from an `_id`, which @@ -22,6 +24,10 @@ export function handleEntities(): RequestHandler(EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER); + const queryResponse = await esClient.asCurrentUser.search({ ignore_unavailable: true, index: indices, @@ -31,6 +37,7 @@ export function handleEntities(): RequestHandler(EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER); const eventsQuery = new EventsQuery({ pagination: PaginationBuilder.createBuilder(limit, afterEvent), indexPatterns: body.indexPatterns, timeRange: body.timeRange, + shouldExcludeColdAndFrozenTiers, }); const results = await eventsQuery.search(eventsClient, body, alertsClient); return res.ok({ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts index bd0642702fc11..3b8d95fa1aa9e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts @@ -24,8 +24,9 @@ export class EventsQuery extends BaseResolverQuery { timeRange, isInternalRequest, pagination, + shouldExcludeColdAndFrozenTiers, }: ResolverQueryParams & { pagination: PaginationBuilder }) { - super({ indexPatterns, timeRange, isInternalRequest }); + super({ indexPatterns, timeRange, isInternalRequest, shouldExcludeColdAndFrozenTiers }); this.pagination = pagination; } @@ -36,6 +37,7 @@ export class EventsQuery extends BaseResolverQuery { filter: [ ...filters, ...this.getRangeFilter(), + ...this.getColdAndFrozenTierFilter(), { term: { 'event.kind': 'event' }, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts index b4931da5ae16d..48c99cd8ed112 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts @@ -11,6 +11,7 @@ import type { RequestHandler } from '@kbn/core/server'; import type { TypeOf } from '@kbn/config-schema'; import type { RuleRegistryPluginStartContract } from '@kbn/rule-registry-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import { EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER } from '../../../../../common/constants'; import type { ConfigType } from '../../../../config'; import type { validateTree } from '../../../../../common/endpoint/schema/resolver'; @@ -30,6 +31,9 @@ export function handleTree( const license = await firstValueFrom(licensing.license$); const hasAccessToInsightsRelatedByProcessAncestry = insightsRelatedAlertsByProcessAncestry && license.hasAtLeast('platinum'); + const shouldExcludeColdAndFrozenTiers = await ( + await context.core + ).uiSettings.client.get(EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER); if (hasAccessToInsightsRelatedByProcessAncestry) { featureUsageService.notifyUsage('ALERTS_BY_PROCESS_ANCESTRY'); @@ -39,7 +43,7 @@ export function handleTree( ? await ruleRegistry.getRacClientWithRequest(req) : undefined; const fetcher = new Fetcher(client, alertsClient); - const body = await fetcher.tree(req.body); + const body = await fetcher.tree({ ...req.body, shouldExcludeColdAndFrozenTiers }); return res.ok({ body, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts index 256f2b58b6864..34e8c463dc779 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts @@ -7,14 +7,16 @@ import type { JsonValue } from '@kbn/utility-types'; import type { ResolverSchema } from '../../../../../../common/endpoint/types'; +import { createSharedFilters } from '../../utils/shared_filters'; import type { TimeRange } from '../utils'; import { resolverFields } from '../utils'; export interface ResolverQueryParams { readonly schema?: ResolverSchema; readonly indexPatterns: string | string[]; - readonly timeRange: TimeRange | undefined; readonly isInternalRequest?: boolean; + readonly shouldExcludeColdAndFrozenTiers?: boolean; + readonly timeRange: TimeRange | undefined; readonly resolverFields?: JsonValue[]; getRangeFilter?: () => Array<{ range: { '@timestamp': { gte: string; lte: string; format: string } }; @@ -24,11 +26,18 @@ export interface ResolverQueryParams { export class BaseResolverQuery implements ResolverQueryParams { readonly schema: ResolverSchema; readonly indexPatterns: string | string[]; - readonly timeRange: TimeRange | undefined; readonly isInternalRequest?: boolean; + readonly shouldExcludeColdAndFrozenTiers?: boolean; + readonly timeRange: TimeRange | undefined; readonly resolverFields?: JsonValue[]; - constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { + constructor({ + schema, + indexPatterns, + timeRange, + isInternalRequest, + shouldExcludeColdAndFrozenTiers, + }: ResolverQueryParams) { const schemaOrDefault = schema ? schema : { @@ -40,6 +49,13 @@ export class BaseResolverQuery implements ResolverQueryParams { this.indexPatterns = indexPatterns; this.timeRange = timeRange; this.isInternalRequest = isInternalRequest; + this.shouldExcludeColdAndFrozenTiers = shouldExcludeColdAndFrozenTiers; + } + + getColdAndFrozenTierFilter() { + return createSharedFilters({ + excludeColdAndFrozenTiers: !!this.shouldExcludeColdAndFrozenTiers, + }); } getRangeFilter() { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts index daa3a513821d1..aa6547506b366 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts @@ -20,8 +20,14 @@ import { BaseResolverQuery } from './base'; export class DescendantsQuery extends BaseResolverQuery { declare readonly resolverFields: JsonValue[]; - constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { - super({ schema, indexPatterns, timeRange, isInternalRequest }); + constructor({ + schema, + indexPatterns, + timeRange, + isInternalRequest, + shouldExcludeColdAndFrozenTiers, + }: ResolverQueryParams) { + super({ schema, indexPatterns, timeRange, isInternalRequest, shouldExcludeColdAndFrozenTiers }); } private query(nodes: NodeID[], size: number): JsonObject { @@ -37,6 +43,7 @@ export class DescendantsQuery extends BaseResolverQuery { bool: { filter: [ ...this.getRangeFilter(), + ...this.getColdAndFrozenTierFilter(), { terms: { [this.schema.parent]: nodes }, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts index 26f917a3008d2..84ebd8b4ae532 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts @@ -18,8 +18,14 @@ import { BaseResolverQuery } from './base'; */ export class LifecycleQuery extends BaseResolverQuery { declare readonly resolverFields: JsonValue[]; - constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { - super({ schema, indexPatterns, timeRange, isInternalRequest }); + constructor({ + schema, + indexPatterns, + timeRange, + isInternalRequest, + shouldExcludeColdAndFrozenTiers, + }: ResolverQueryParams) { + super({ schema, indexPatterns, timeRange, isInternalRequest, shouldExcludeColdAndFrozenTiers }); } private query(nodes: NodeID[]): JsonObject { @@ -34,6 +40,7 @@ export class LifecycleQuery extends BaseResolverQuery { query: { bool: { filter: [ + ...this.getColdAndFrozenTierFilter(), ...this.getRangeFilter(), { terms: { [this.schema.id]: nodes }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts index 85b5bbd8a277d..42757767b944e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts @@ -38,6 +38,7 @@ export interface TreeOptions { nodes: NodeID[]; indexPatterns: string[]; includeHits?: boolean; + shouldExcludeColdAndFrozenTiers?: boolean; } export type TreeResponse = Promise< @@ -53,6 +54,7 @@ export type TreeResponse = Promise< */ export class Fetcher { private alertsClient?: AlertsClient; + constructor(private readonly client: IScopedClusterClient, alertsClient?: AlertsClient) { this.alertsClient = alertsClient; } @@ -168,6 +170,7 @@ export class Fetcher { indexPatterns: options.indexPatterns, timeRange: options.timeRange, isInternalRequest, + shouldExcludeColdAndFrozenTiers: !!options.shouldExcludeColdAndFrozenTiers, }); let nodes = options.nodes; @@ -218,6 +221,7 @@ export class Fetcher { indexPatterns: options.indexPatterns, timeRange: options.timeRange, isInternalRequest, + shouldExcludeColdAndFrozenTiers: !!options.shouldExcludeColdAndFrozenTiers, }); let nodes: NodeID[] = options.nodes; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/shared_filters.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/shared_filters.ts new file mode 100644 index 0000000000000..105e706082589 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/shared_filters.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface SharedFiltersOptions { + excludeColdAndFrozenTiers: boolean; +} + +export const createSharedFilters = ({ excludeColdAndFrozenTiers }: SharedFiltersOptions) => { + const filters = []; + + if (excludeColdAndFrozenTiers) { + filters.push({ + bool: { + must_not: { + terms: { + _tier: ['data_frozen', 'data_cold'], + }, + }, + }, + }); + } + + return filters; +}; diff --git a/x-pack/plugins/security_solution/server/ui_settings.ts b/x-pack/plugins/security_solution/server/ui_settings.ts index 18b5a01b2ad30..ac610c14d7f8f 100644 --- a/x-pack/plugins/security_solution/server/ui_settings.ts +++ b/x-pack/plugins/security_solution/server/ui_settings.ts @@ -37,6 +37,7 @@ import { DEFAULT_ALERT_TAGS_KEY, DEFAULT_ALERT_TAGS_VALUE, ENABLE_EXPANDABLE_FLYOUT_SETTING, + EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER, } from '../common/constants'; import type { ExperimentalFeatures } from '../common/experimental_features'; import { LogLevelSetting } from '../common/api/detection_engine/rule_monitoring'; @@ -185,6 +186,26 @@ export const initUiSettings = ( requiresPageReload: true, schema: schema.boolean(), }, + [EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER]: { + name: i18n.translate( + 'xpack.securitySolution.uiSettings.excludeColdAndFrozenTiersInAnalyzer', + { + defaultMessage: 'Exclude cold and frozen tiers in Analyzer', + } + ), + value: false, + description: i18n.translate( + 'xpack.securitySolution.uiSettings.excludeColdAndFrozenTiersInAnalyzerDescription', + { + defaultMessage: + '

When enabled, cold and frozen tiers will be skipped in analyzer queries

', + } + ), + type: 'boolean', + category: [APP_ID], + requiresPageReload: true, + schema: schema.boolean(), + }, [DEFAULT_RULES_TABLE_REFRESH_SETTING]: { name: i18n.translate('xpack.securitySolution.uiSettings.rulesTableRefresh', { defaultMessage: 'Rules auto refresh',