diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index 697b993d64539..7c3ded7fdc921 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -534,6 +534,61 @@ describe('create_signals', () => { }); }); + test('it respects overriderBody params', () => { + const query = buildEventsSearchQuery({ + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortIds: undefined, + primaryTimestamp: '@timestamp', + secondaryTimestamp: undefined, + runtimeMappings: undefined, + overrideBody: { + _source: false, + fields: ['@timestamp'], + }, + }); + expect(query).toEqual({ + allow_no_indices: true, + index: ['auditbeat-*'], + size: 100, + runtime_mappings: undefined, + track_total_hits: undefined, + ignore_unavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + ], + }, + }, + _source: false, + fields: ['@timestamp'], + runtime_mappings: undefined, + sort: [ + { + '@timestamp': { + order: 'asc', + unmapped_type: 'date', + }, + }, + ], + }, + }); + }); + describe('buildEqlSearchRequest', () => { test('should build a basic request with time range', () => { const request = buildEqlSearchRequest({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 001550ec3671e..a04e27c1f8837 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { isEmpty } from 'lodash'; import type { Filter } from '@kbn/es-query'; +import type { OverrideBodyQuery } from './types'; import type { RuleFilterArray, TimestampOverride, @@ -27,6 +28,7 @@ interface BuildEventsSearchQuery { secondaryTimestamp: TimestampOverride | undefined; trackTotalHits?: boolean; additionalFilters?: estypes.QueryDslQueryContainer[]; + overrideBody?: OverrideBodyQuery; } interface BuildEqlSearchRequestParams { @@ -132,6 +134,7 @@ export const buildEventsSearchQuery = ({ secondaryTimestamp, trackTotalHits, additionalFilters, + overrideBody, }: BuildEventsSearchQuery) => { const timestamps = secondaryTimestamp ? [primaryTimestamp, secondaryTimestamp] @@ -193,6 +196,7 @@ export const buildEventsSearchQuery = ({ ...(aggregations ? { aggregations } : {}), runtime_mappings: runtimeMappings, sort, + ...overrideBody, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index 28b00e45dd5a3..7b4e1b8ecf00a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -14,6 +14,10 @@ import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { ruleExecutionLogMock } from '../rule_monitoring/mocks'; +import { buildEventsSearchQuery } from './build_events_query'; + +jest.mock('./build_events_query'); +const mockBuildEventsSearchQuery = buildEventsSearchQuery as jest.Mock; describe('singleSearchAfter', () => { const mockService: RuleExecutorServicesMock = alertsMock.createRuleExecutorServices(); @@ -157,4 +161,47 @@ describe('singleSearchAfter', () => { }) ).rejects.toThrow('Fake Error'); }); + + test('singleSearchAfter passes overrideBody to buildEventsSearchQuery', async () => { + mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId()) + ); + await singleSearchAfter({ + searchAfterSortIds: undefined, + index: [], + from: 'now-360s', + to: 'now', + services: mockService, + ruleExecutionLogger, + pageSize: 1, + filter: {}, + primaryTimestamp: '@timestamp', + secondaryTimestamp: undefined, + runtimeMappings: undefined, + overrideBody: { + _source: false, + fields: ['@timestamp'], + }, + }); + + expect(mockBuildEventsSearchQuery).toHaveBeenCalledWith({ + additionalFilters: undefined, + aggregations: undefined, + filter: {}, + from: 'now-360s', + index: [], + primaryTimestamp: '@timestamp', + runtimeMappings: undefined, + searchAfterSortIds: undefined, + secondaryTimestamp: undefined, + size: 1, + sortOrder: undefined, + to: 'now', + trackTotalHits: undefined, + overrideBody: { + _source: false, + fields: ['@timestamp'], + }, + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index e1ef6e5867859..84f8fcf94af64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -11,7 +11,7 @@ import type { AlertInstanceState, RuleExecutorServices, } from '@kbn/alerting-plugin/server'; -import type { SignalSearchResponse, SignalSource } from './types'; +import type { SignalSearchResponse, SignalSource, OverrideBodyQuery } from './types'; import { buildEventsSearchQuery } from './build_events_query'; import { createErrorsFromShard, makeFloatString } from './utils'; import type { TimestampOverride } from '../../../../common/detection_engine/rule_schema'; @@ -34,6 +34,7 @@ interface SingleSearchAfterParams { trackTotalHits?: boolean; runtimeMappings: estypes.MappingRuntimeFields | undefined; additionalFilters?: estypes.QueryDslQueryContainer[]; + overrideBody?: OverrideBodyQuery; } // utilize search_after for paging results into bulk. @@ -55,6 +56,7 @@ export const singleSearchAfter = async < secondaryTimestamp, trackTotalHits, additionalFilters, + overrideBody, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; @@ -76,6 +78,11 @@ export const singleSearchAfter = async < secondaryTimestamp, trackTotalHits, additionalFilters, + /** + * overrideBody allows the search after to ignore the _source property of the result, + * thus reducing the size of the response and increasing the performance of the query. + */ + overrideBody, }); const start = performance.now(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 0ab6ec2a01dda..2bdd9533f1ea0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -124,6 +124,11 @@ export const createThreatSignals = async ({ _source: false, }; + const eventListConfig = { + fields: threatMapping.map((mapping) => mapping.entries.map((item) => item.field)).flat(), + _source: false, + }; + const threatEnrichment = buildThreatEnrichment({ ruleExecutionLogger, services, @@ -197,6 +202,7 @@ export const createThreatSignals = async ({ primaryTimestamp, secondaryTimestamp, exceptionFilter, + eventListConfig, }), createSignal: (slicedChunk) => diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts index 21d13ec421858..32c2f00af64ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts @@ -27,6 +27,7 @@ export const getEventList = async ({ secondaryTimestamp, runtimeMappings, exceptionFilter, + eventListConfig, }: EventsOptions): Promise> => { const calculatedPerPage = perPage ?? MAX_PER_PAGE; if (calculatedPerPage > 10000) { @@ -59,6 +60,7 @@ export const getEventList = async ({ sortOrder: 'desc', trackTotalHits: false, runtimeMappings, + overrideBody: eventListConfig, }); ruleExecutionLogger.debug(`Retrieved events items of size: ${searchResult.hits.hits.length}`); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 03175fa0dda11..84cf1142e9e9c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -34,6 +34,7 @@ import type { SearchAfterAndBulkCreateReturnType, SignalsEnrichment, WrapHits, + OverrideBodyQuery, } from '../types'; import type { CompleteRule, ThreatRuleParams } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; @@ -265,6 +266,7 @@ export interface EventsOptions { tuple: RuleRangeTuple; runtimeMappings: estypes.MappingRuntimeFields | undefined; exceptionFilter: Filter | undefined; + eventListConfig?: OverrideBodyQuery; } export interface EventDoc { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index af354baa67903..877239af49a9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -342,3 +342,9 @@ export type EventGroupingMultiBucketAggregationResult = ESSearchResponse< }; } >; + +// the new fields can be added later if needed +export interface OverrideBodyQuery { + _source?: estypes.SearchSourceConfig; + fields?: estypes.Fields; +}