diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts index 923954b298806..e4f14e17dc7a0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts @@ -436,7 +436,7 @@ export const sampleDocRiskScore = (riskScore?: unknown): SignalSourceHit => ({ sort: [], }); -export const sampleEmptyDocSearchResults = (): SignalSearchResponse => ({ +export const sampleEmptyDocSearchResults = () => ({ took: 10, timed_out: false, _shards: { @@ -446,7 +446,10 @@ export const sampleEmptyDocSearchResults = (): SignalSearchResponse => ({ skipped: 0, }, hits: { - total: 0, + total: { + value: 0, + relation: 'eq' as const, + }, max_score: 100, hits: [], }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts index 71c013e9b5e7e..990620e47d841 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts @@ -22,7 +22,7 @@ export const getEventList = async ({ eventListConfig, indexFields, sortOrder = 'desc', -}: EventsOptions): Promise> => { +}: EventsOptions): Promise> => { const { inputIndex, ruleExecutionLogger, @@ -53,14 +53,13 @@ export const getEventList = async ({ fields: indexFields, }); - const { searchResult } = await singleSearchAfter({ + const searchRequest = buildEventsSearchQuery({ + aggregations: undefined, searchAfterSortIds: searchAfter, index: inputIndex, from: tuple.from.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, - pageSize: calculatedPerPage, + size: calculatedPerPage, filter: queryFilter, primaryTimestamp, secondaryTimestamp, @@ -70,6 +69,12 @@ export const getEventList = async ({ overrideBody: eventListConfig, }); + const { searchResult } = await singleSearchAfter({ + searchRequest, + services, + ruleExecutionLogger, + }); + ruleExecutionLogger.debug(`Retrieved events items of size: ${searchResult.hits.hits.length}`); return searchResult; }; @@ -98,6 +103,7 @@ export const getEventCount = async ({ fields: indexFields, }); const eventSearchQueryBodyQuery = buildEventsSearchQuery({ + aggregations: undefined, index, from: tuple.from.toISOString(), to: tuple.to.toISOString(), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts index db4926ccb2a40..ef6eeee742cd1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts @@ -238,7 +238,7 @@ export interface SignalMatch { export type GetDocumentListInterface = (params: { searchAfter: estypes.SortResults | undefined; -}) => Promise>; +}) => Promise>; export type CreateSignalInterface = ( params: EventItem[] | ThreatListItem[] diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts index 6df36881cd7de..ce914eb3b78bb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts @@ -11,16 +11,6 @@ import type { SignalSource } from '../types'; import type { GenericBulkCreateResponse } from '../factories/bulk_create_factory'; import type { NewTermsFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; -export type RecentTermsAggResult = ESSearchResponse< - SignalSource, - { aggregations: ReturnType } ->; - -export type NewTermsAggResult = ESSearchResponse< - SignalSource, - { aggregations: ReturnType } ->; - export type CompositeDocFetchAggResult = ESSearchResponse< SignalSource, { aggregations: ReturnType } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 87e40bd519f7a..5d5939a6e1bae 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -14,16 +14,12 @@ import { SERVER_APP_ID } from '../../../../../common/constants'; import { NewTermsRuleParams } from '../../rule_schema'; import type { SecurityAlertType } from '../types'; import { singleSearchAfter } from '../utils/single_search_after'; +import { buildEventsSearchQuery } from '../utils/build_events_query'; import { getFilter } from '../utils/get_filter'; import { wrapNewTermsAlerts } from './wrap_new_terms_alerts'; import { bulkCreateSuppressedNewTermsAlertsInMemory } from './bulk_create_suppressed_alerts_in_memory'; import type { EventsAndTerms } from './types'; -import type { - RecentTermsAggResult, - DocFetchAggResult, - NewTermsAggResult, - CreateAlertsHook, -} from './build_new_terms_aggregation'; +import type { CreateAlertsHook } from './build_new_terms_aggregation'; import type { NewTermsFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; import { buildRecentTermsAgg, @@ -149,7 +145,7 @@ export const createNewTermsAlertType = (): SecurityAlertType< alertSuppression: params.alertSuppression, licensing, }); - let afterKey; + let afterKey: Record | undefined; const result = createSearchAfterReturnType(); @@ -170,12 +166,7 @@ export const createNewTermsAlertType = (): SecurityAlertType< // PHASE 1: Fetch a page of terms using a composite aggregation. This will collect a page from // all of the terms seen over the last rule interval. In the next phase we'll determine which // ones are new. - const { - searchResult, - searchDuration, - searchErrors, - loggedRequests: firstPhaseLoggedRequests = [], - } = await singleSearchAfter({ + const searchRequest = buildEventsSearchQuery({ aggregations: buildRecentTermsAgg({ fields: params.newTermsFields, after: afterKey, @@ -185,13 +176,22 @@ export const createNewTermsAlertType = (): SecurityAlertType< // The time range for the initial composite aggregation is the rule interval, `from` and `to` from: tuple.from.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, filter: esFilter, - pageSize: 0, + size: 0, primaryTimestamp, secondaryTimestamp, runtimeMappings, + }); + + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: firstPhaseLoggedRequests = [], + } = await singleSearchAfter({ + searchRequest, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findAllTerms', @@ -203,8 +203,7 @@ export const createNewTermsAlertType = (): SecurityAlertType< : undefined, }); loggedRequests.push(...firstPhaseLoggedRequests); - const searchResultWithAggs = searchResult as RecentTermsAggResult; - if (!searchResultWithAggs.aggregations) { + if (!searchResult.aggregations) { throw new Error('Aggregations were missing on recent terms search result'); } logger.debug(`Time spent on composite agg: ${searchDuration}`); @@ -214,10 +213,10 @@ export const createNewTermsAlertType = (): SecurityAlertType< // If the aggregation returns no after_key it signals that we've paged through all results // and the current page is empty so we can immediately break. - if (searchResultWithAggs.aggregations.new_terms.after_key == null) { + if (searchResult.aggregations.new_terms.after_key == null) { break; } - const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; + const bucketsForField = searchResult.aggregations.new_terms.buckets; const createAlertsHook: CreateAlertsHook = async (aggResult) => { const eventsAndTerms: EventsAndTerms[] = ( @@ -311,12 +310,7 @@ export const createNewTermsAlertType = (): SecurityAlertType< // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); - const { - searchResult: pageSearchResult, - searchDuration: pageSearchDuration, - searchErrors: pageSearchErrors, - loggedRequests: pageSearchLoggedRequests = [], - } = await singleSearchAfter({ + const pageSearchRequest = buildEventsSearchQuery({ aggregations: buildNewTermsAgg({ newValueWindowStart: tuple.from, timestampField: aggregatableTimestampField, @@ -330,12 +324,20 @@ export const createNewTermsAlertType = (): SecurityAlertType< // in addition to the rule interval from: parsedHistoryWindowSize.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, filter: esFilter, - pageSize: 0, + size: 0, primaryTimestamp, secondaryTimestamp, + }); + const { + searchResult: pageSearchResult, + searchDuration: pageSearchDuration, + searchErrors: pageSearchErrors, + loggedRequests: pageSearchLoggedRequests = [], + } = await singleSearchAfter({ + searchRequest: pageSearchRequest, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findNewTerms', @@ -350,8 +352,7 @@ export const createNewTermsAlertType = (): SecurityAlertType< logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`); - const pageSearchResultWithAggs = pageSearchResult as NewTermsAggResult; - if (!pageSearchResultWithAggs.aggregations) { + if (!pageSearchResult.aggregations) { throw new Error('Aggregations were missing on new terms search result'); } @@ -359,17 +360,12 @@ export const createNewTermsAlertType = (): SecurityAlertType< // the rule interval for that term. This is the first document to contain the new term, and will // become the basis of the resulting alert. // One document could become multiple alerts if the document contains an array with multiple new terms. - if (pageSearchResultWithAggs.aggregations.new_terms.buckets.length > 0) { - const actualNewTerms = pageSearchResultWithAggs.aggregations.new_terms.buckets.map( + if (pageSearchResult.aggregations.new_terms.buckets.length > 0) { + const actualNewTerms = pageSearchResult.aggregations.new_terms.buckets.map( (bucket) => bucket.key ); - const { - searchResult: docFetchSearchResult, - searchDuration: docFetchSearchDuration, - searchErrors: docFetchSearchErrors, - loggedRequests: docFetchLoggedRequests = [], - } = await singleSearchAfter({ + const docFetchSearchRequest = buildEventsSearchQuery({ aggregations: buildDocFetchAgg({ timestampField: aggregatableTimestampField, field: params.newTermsFields[0], @@ -381,12 +377,20 @@ export const createNewTermsAlertType = (): SecurityAlertType< // For phase 3, we go back to aggregating only over the rule interval - excluding the history window from: tuple.from.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, filter: esFilter, - pageSize: 0, + size: 0, primaryTimestamp, secondaryTimestamp, + }); + const { + searchResult: docFetchSearchResult, + searchDuration: docFetchSearchDuration, + searchErrors: docFetchSearchErrors, + loggedRequests: docFetchLoggedRequests = [], + } = await singleSearchAfter({ + searchRequest: docFetchSearchRequest, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findDocuments', @@ -401,13 +405,11 @@ export const createNewTermsAlertType = (): SecurityAlertType< result.errors.push(...docFetchSearchErrors); loggedRequests.push(...docFetchLoggedRequests); - const docFetchResultWithAggs = docFetchSearchResult as DocFetchAggResult; - - if (!docFetchResultWithAggs.aggregations) { + if (!docFetchSearchResult.aggregations) { throw new Error('Aggregations were missing on document fetch search result'); } - const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs); + const bulkCreateResult = await createAlertsHook(docFetchSearchResult); if (bulkCreateResult.alertsWereTruncated) { result.warningMessages.push( @@ -420,7 +422,7 @@ export const createNewTermsAlertType = (): SecurityAlertType< } } - afterKey = searchResultWithAggs.aggregations.new_terms.after_key; + afterKey = searchResult.aggregations.new_terms.after_key; } scheduleNotificationResponseActionsService({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts index a521776d52fe1..4c13a713f050e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -28,6 +28,7 @@ import { stringifyAfterKey, } from '../utils/utils'; import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression'; +import { buildEventsSearchQuery } from '../utils/build_events_query'; import type { SecurityRuleServices, @@ -135,12 +136,7 @@ const multiTermsCompositeNonRetryable = async ({ // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. - const { - searchResult: pageSearchResult, - searchDuration: pageSearchDuration, - searchErrors: pageSearchErrors, - loggedRequests: pageSearchLoggedRequests = [], - } = await singleSearchAfter({ + const searchRequest = buildEventsSearchQuery({ aggregations: buildCompositeNewTermsAgg({ newValueWindowStart: tuple.from, timestampField: aggregatableTimestampField, @@ -155,12 +151,20 @@ const multiTermsCompositeNonRetryable = async ({ // in addition to the rule interval from: parsedHistoryWindowSize.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, filter: esFilterForBatch, - pageSize: 0, + size: 0, primaryTimestamp, secondaryTimestamp, + }); + const { + searchResult: pageSearchResult, + searchDuration: pageSearchDuration, + searchErrors: pageSearchErrors, + loggedRequests: pageSearchLoggedRequests = [], + } = await singleSearchAfter({ + searchRequest, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findNewTerms', @@ -187,12 +191,7 @@ const multiTermsCompositeNonRetryable = async ({ // become the basis of the resulting alert. // One document could become multiple alerts if the document contains an array with multiple new terms. if (pageSearchResultWithAggs.aggregations.new_terms.buckets.length > 0) { - const { - searchResult: docFetchSearchResult, - searchDuration: docFetchSearchDuration, - searchErrors: docFetchSearchErrors, - loggedRequests: docFetchLoggedRequests = [], - } = await singleSearchAfter({ + const searchRequestPhase3 = buildEventsSearchQuery({ aggregations: buildCompositeDocFetchAgg({ newValueWindowStart: tuple.from, timestampField: aggregatableTimestampField, @@ -205,12 +204,20 @@ const multiTermsCompositeNonRetryable = async ({ index: inputIndex, from: parsedHistoryWindowSize.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, filter: esFilterForBatch, - pageSize: 0, + size: 0, primaryTimestamp, secondaryTimestamp, + }); + const { + searchResult: docFetchSearchResult, + searchDuration: docFetchSearchDuration, + searchErrors: docFetchSearchErrors, + loggedRequests: docFetchLoggedRequests = [], + } = await singleSearchAfter({ + searchRequest: searchRequestPhase3, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findDocuments', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index 0e527216c9313..4ce9fff66356a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -10,7 +10,7 @@ import type moment from 'moment'; import type { estypes } from '@elastic/elasticsearch'; import { withSecuritySpan } from '../../../../../utils/with_security_span'; -import { buildTimeRangeFilter } from '../../utils/build_events_query'; +import { buildEventsSearchQuery, buildTimeRangeFilter } from '../../utils/build_events_query'; import type { SecurityRuleServices, SecuritySharedParams, @@ -186,29 +186,31 @@ export const groupAndBulkCreate = async ({ missingBucket: suppressOnMissingFields, }); - const eventsSearchParams = { + const searchRequest = buildEventsSearchQuery({ aggregations: groupingAggregation, searchAfterSortIds: undefined, index: sharedParams.inputIndex, from: tuple.from.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger: sharedParams.ruleExecutionLogger, filter, - pageSize: 0, + size: 0, primaryTimestamp: sharedParams.primaryTimestamp, secondaryTimestamp: sharedParams.secondaryTimestamp, runtimeMappings: sharedParams.runtimeMappings, additionalFilters: bucketHistoryFilter, - loggedRequestsConfig: isLoggedRequestsEnabled - ? { - type: 'findDocuments', - description: i18n.FIND_EVENTS_DESCRIPTION, - } - : undefined, - }; + }); const { searchResult, searchDuration, searchErrors, loggedRequests } = - await singleSearchAfter(eventsSearchParams); + await singleSearchAfter({ + searchRequest, + services, + ruleExecutionLogger: sharedParams.ruleExecutionLogger, + loggedRequestsConfig: isLoggedRequestsEnabled + ? { + type: 'findDocuments', + description: i18n.FIND_EVENTS_DESCRIPTION, + } + : undefined, + }); if (isLoggedRequestsEnabled) { toReturn.loggedRequests = loggedRequests; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index 3161616b85653..8438ce14d7f83 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; - import { allowedExperimentalValues } from '../../../../../common/experimental_features'; import { createQueryAlertType } from './create_query_alert_type'; import { createRuleTypeMocks } from '../__mocks__/rule_type'; @@ -83,27 +81,23 @@ describe('Custom Query Alerts', () => { alerting.registerType(queryAlertType); - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - sequences: [], - events: [], - total: { - relation: 'eq', - value: 0, - }, + services.scopedClusterClient.asCurrentUser.search.mockResolvedValue({ + hits: { + hits: [], + total: { + relation: 'eq', + value: 0, }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); + }, + took: 0, + timed_out: false, + _shards: { + failed: 0, + skipped: 0, + successful: 1, + total: 1, + }, + }); const params = getQueryRuleParams(); @@ -126,27 +120,23 @@ describe('Custom Query Alerts', () => { alerting.registerType(queryAlertType); - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [sampleDocNoSortId()], - sequences: [], - events: [], - total: { - relation: 'eq', - value: 1, - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, + services.scopedClusterClient.asCurrentUser.search.mockResolvedValue({ + hits: { + hits: [sampleDocNoSortId()], + total: { + relation: 'eq', + value: 1, }, - }) - ); + }, + took: 0, + timed_out: false, + _shards: { + failed: 0, + skipped: 0, + successful: 1, + total: 1, + }, + }); const params = getQueryRuleParams(); @@ -193,27 +183,23 @@ describe('Custom Query Alerts', () => { }, }); - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [sampleDocNoSortId()], - sequences: [], - events: [], - total: { - relation: 'eq', - value: 1, - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, + services.scopedClusterClient.asCurrentUser.search.mockResolvedValue({ + hits: { + hits: [sampleDocNoSortId()], + total: { + relation: 'eq', + value: 1, }, - }) - ); + }, + took: 0, + timed_out: false, + _shards: { + failed: 0, + skipped: 0, + successful: 1, + total: 1, + }, + }); await executor({ params }); @@ -234,27 +220,23 @@ describe('Custom Query Alerts', () => { alerting.registerType(queryAlertType); - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [sampleDocNoSortId()], - sequences: [], - events: [], - total: { - relation: 'eq', - value: 1, - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, + services.scopedClusterClient.asCurrentUser.search.mockResolvedValue({ + hits: { + hits: [sampleDocNoSortId()], + total: { + relation: 'eq', + value: 1, }, - }) - ); + }, + took: 0, + timed_out: false, + _shards: { + failed: 0, + skipped: 0, + successful: 1, + total: 1, + }, + }); const params = getQueryRuleParams(); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts index 7379ba6d066a8..932251ad7b413 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts @@ -9,7 +9,7 @@ import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/s import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { buildReasonMessageForThresholdAlert } from '../utils/reason_formatters'; -import type { ThresholdBucket } from './types'; +import type { ThresholdCompositeBucket } from './types'; import type { SecurityRuleServices, SecuritySharedParams } from '../types'; import type { ThresholdRuleParams } from '../../rule_schema'; import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; @@ -20,7 +20,7 @@ import { transformBulkCreatedItemsToHits } from './utils'; interface BulkCreateSuppressedThresholdAlertsParams { sharedParams: SecuritySharedParams; - buckets: ThresholdBucket[]; + buckets: ThresholdCompositeBucket[]; services: SecurityRuleServices; startedAt: Date; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts index 3ffe818c9c909..47f634918f834 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts @@ -11,7 +11,7 @@ import type { ThresholdNormalized } from '../../../../../common/api/detection_en import type { GenericBulkCreateResponse } from '../factories/bulk_create_factory'; import { calculateThresholdSignalUuid } from './utils'; import { buildReasonMessageForThresholdAlert } from '../utils/reason_formatters'; -import type { ThresholdBucket } from './types'; +import type { ThresholdCompositeBucket } from './types'; import type { SecurityRuleServices, SecuritySharedParams } from '../types'; import type { ThresholdRuleParams } from '../../rule_schema'; import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; @@ -19,13 +19,13 @@ import { bulkCreate, wrapHits } from '../factories'; interface BulkCreateThresholdSignalsParams { sharedParams: SecuritySharedParams; - buckets: ThresholdBucket[]; + buckets: ThresholdCompositeBucket[]; services: SecurityRuleServices; startedAt: Date; } export const transformBucketIntoHit = ( - bucket: ThresholdBucket, + bucket: ThresholdCompositeBucket, inputIndex: string, startedAt: Date, from: Date, @@ -65,7 +65,7 @@ export const transformBucketIntoHit = ( }; export const getTransformedHits = ( - buckets: ThresholdBucket[], + buckets: ThresholdCompositeBucket[], inputIndex: string, startedAt: Date, from: Date, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.test.ts deleted file mode 100644 index ab19edade87f7..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.test.ts +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; -import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; -import { sampleEmptyDocSearchResults } from '../__mocks__/es_results'; -import * as single_search_after from '../utils/single_search_after'; -import { findThresholdSignals } from './find_threshold_signals'; -import { TIMESTAMP } from '@kbn/rule-data-utils'; -import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; -import { buildTimestampRuntimeMapping } from '../utils'; -import { TIMESTAMP_RUNTIME_FIELD } from '../constants'; -import { getQueryFilter } from '../utils/get_query_filter'; -import type { ESBoolQuery } from '../../../../../common/typed_json'; - -const mockSingleSearchAfter = jest.fn(async () => ({ - searchResult: { - ...sampleEmptyDocSearchResults(), - aggregations: { - thresholdTerms: { - buckets: [], - }, - }, - }, - searchDuration: '0.0', - searchErrors: [], -})); - -let filter: ESBoolQuery; - -describe('findThresholdSignals', () => { - let mockService: RuleExecutorServicesMock; - const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); - - beforeEach(() => { - jest.clearAllMocks(); - jest.spyOn(single_search_after, 'singleSearchAfter').mockImplementation(mockSingleSearchAfter); - mockService = alertsMock.createRuleExecutorServices(); - const queryFilter = getQueryFilter({ - query: '', - language: 'kuery', - filters: [], - index: ['*'], - exceptionFilter: undefined, - }); - filter = queryFilter; - }); - - it('should generate a threshold signal query when only a value is provided', async () => { - await findThresholdSignals({ - from: 'now-6m', - to: 'now', - maxSignals: 100, - inputIndexPattern: ['*'], - services: mockService, - ruleExecutionLogger, - filter, - threshold: { - field: [], - value: 100, - }, - runtimeMappings: undefined, - primaryTimestamp: TIMESTAMP, - secondaryTimestamp: undefined, - aggregatableTimestampField: TIMESTAMP, - }); - expect(mockSingleSearchAfter).toHaveBeenCalledWith( - expect.objectContaining({ - aggregations: { - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - min_timestamp: { - min: { - field: '@timestamp', - }, - }, - }, - }) - ); - }); - - it('should generate a threshold signal query when a field and value are provided', async () => { - await findThresholdSignals({ - from: 'now-6m', - to: 'now', - maxSignals: 100, - inputIndexPattern: ['*'], - services: mockService, - ruleExecutionLogger, - filter, - threshold: { - field: ['host.name'], - value: 100, - }, - runtimeMappings: undefined, - primaryTimestamp: TIMESTAMP, - secondaryTimestamp: undefined, - aggregatableTimestampField: TIMESTAMP, - }); - expect(mockSingleSearchAfter).toHaveBeenCalledWith( - expect.objectContaining({ - aggregations: { - thresholdTerms: { - composite: { - size: 10000, - after: undefined, - sources: [ - { - 'host.name': { - terms: { - field: 'host.name', - }, - }, - }, - ], - }, - aggs: { - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - min_timestamp: { - min: { - field: '@timestamp', - }, - }, - count_check: { - bucket_selector: { - buckets_path: { - docCount: '_count', - }, - script: `params.docCount >= 100`, - }, - }, - }, - }, - }, - }) - ); - }); - - it('should generate a threshold signal query when multiple fields and a value are provided', async () => { - await findThresholdSignals({ - from: 'now-6m', - to: 'now', - maxSignals: 100, - inputIndexPattern: ['*'], - services: mockService, - ruleExecutionLogger, - filter, - threshold: { - field: ['host.name', 'user.name'], - value: 100, - cardinality: [], - }, - runtimeMappings: undefined, - primaryTimestamp: TIMESTAMP, - secondaryTimestamp: undefined, - aggregatableTimestampField: TIMESTAMP, - }); - expect(mockSingleSearchAfter).toHaveBeenCalledWith( - expect.objectContaining({ - aggregations: { - thresholdTerms: { - composite: { - size: 10000, - after: undefined, - sources: [ - { - 'host.name': { - terms: { - field: 'host.name', - }, - }, - }, - { - 'user.name': { - terms: { - field: 'user.name', - }, - }, - }, - ], - }, - aggs: { - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - min_timestamp: { - min: { - field: '@timestamp', - }, - }, - count_check: { - bucket_selector: { - buckets_path: { - docCount: '_count', - }, - script: `params.docCount >= 100`, - }, - }, - }, - }, - }, - }) - ); - }); - - it('should generate a threshold signal query when multiple fields, a value, and cardinality field/value are provided', async () => { - await findThresholdSignals({ - from: 'now-6m', - to: 'now', - maxSignals: 100, - inputIndexPattern: ['*'], - services: mockService, - ruleExecutionLogger, - filter, - threshold: { - field: ['host.name', 'user.name'], - value: 100, - cardinality: [ - { - field: 'destination.ip', - value: 2, - }, - ], - }, - runtimeMappings: undefined, - primaryTimestamp: TIMESTAMP, - secondaryTimestamp: undefined, - aggregatableTimestampField: TIMESTAMP, - }); - expect(mockSingleSearchAfter).toHaveBeenCalledWith( - expect.objectContaining({ - aggregations: { - thresholdTerms: { - composite: { - size: 10000, - after: undefined, - sources: [ - { - 'host.name': { - terms: { - field: 'host.name', - }, - }, - }, - { - 'user.name': { - terms: { - field: 'user.name', - }, - }, - }, - ], - }, - aggs: { - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - min_timestamp: { - min: { - field: '@timestamp', - }, - }, - count_check: { - bucket_selector: { - buckets_path: { - docCount: '_count', - }, - script: `params.docCount >= 100`, - }, - }, - cardinality_count: { - cardinality: { - field: 'destination.ip', - }, - }, - cardinality_check: { - bucket_selector: { - buckets_path: { - cardinalityCount: 'cardinality_count', - }, - script: 'params.cardinalityCount >= 2', - }, - }, - }, - }, - }, - }) - ); - }); - - it('should generate a threshold signal query when only a value and a cardinality field/value are provided', async () => { - await findThresholdSignals({ - from: 'now-6m', - to: 'now', - maxSignals: 100, - inputIndexPattern: ['*'], - services: mockService, - ruleExecutionLogger, - filter, - threshold: { - cardinality: [ - { - field: 'source.ip', - value: 5, - }, - ], - field: [], - value: 200, - }, - runtimeMappings: undefined, - primaryTimestamp: TIMESTAMP, - secondaryTimestamp: undefined, - aggregatableTimestampField: TIMESTAMP, - }); - expect(mockSingleSearchAfter).toHaveBeenCalledWith( - expect.objectContaining({ - aggregations: { - cardinality_count: { - cardinality: { - field: 'source.ip', - }, - }, - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - min_timestamp: { - min: { - field: '@timestamp', - }, - }, - }, - }) - ); - }); - - it('should generate a threshold signal query with timestamp override', async () => { - const timestampOverride = 'event.ingested'; - const { aggregatableTimestampField, timestampRuntimeMappings } = { - aggregatableTimestampField: TIMESTAMP_RUNTIME_FIELD, - timestampRuntimeMappings: buildTimestampRuntimeMapping({ - timestampOverride, - }), - }; - - await findThresholdSignals({ - from: 'now-6m', - to: 'now', - maxSignals: 100, - inputIndexPattern: ['*'], - services: mockService, - ruleExecutionLogger, - filter, - threshold: { - cardinality: [ - { - field: 'source.ip', - value: 5, - }, - ], - field: [], - value: 200, - }, - runtimeMappings: timestampRuntimeMappings, - primaryTimestamp: timestampOverride, - secondaryTimestamp: TIMESTAMP, - aggregatableTimestampField, - }); - expect(mockSingleSearchAfter).toHaveBeenCalledWith( - expect.objectContaining({ - primaryTimestamp: timestampOverride, - secondaryTimestamp: TIMESTAMP, - runtimeMappings: buildTimestampRuntimeMapping({ timestampOverride }), - aggregations: { - cardinality_count: { - cardinality: { - field: 'source.ip', - }, - }, - max_timestamp: { - max: { - field: TIMESTAMP_RUNTIME_FIELD, - }, - }, - min_timestamp: { - min: { - field: TIMESTAMP_RUNTIME_FIELD, - }, - }, - }, - }) - ); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts index 9d6320d95fef7..4a222b3e89a73 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts @@ -20,16 +20,13 @@ import type { TimestampOverride, } from '../../../../../common/api/detection_engine/model/rule_schema'; import { singleSearchAfter } from '../utils/single_search_after'; +import { buildEventsSearchQuery } from '../utils/build_events_query'; import { buildThresholdMultiBucketAggregation, buildThresholdSingleBucketAggregation, } from './build_threshold_aggregation'; -import type { - ThresholdMultiBucketAggregationResult, - ThresholdBucket, - ThresholdSingleBucketAggregationResult, -} from './types'; -import { shouldFilterByCardinality, searchResultHasAggs } from './utils'; +import type { ThresholdCompositeBucket } from './types'; +import { shouldFilterByCardinality } from './utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { getMaxSignalsWarning, stringifyAfterKey } from '../utils/utils'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; @@ -58,7 +55,6 @@ interface SearchAfterResults { searchErrors: string[]; } -// eslint-disable-next-line complexity export const findThresholdSignals = async ({ from, to, @@ -74,14 +70,14 @@ export const findThresholdSignals = async ({ aggregatableTimestampField, isLoggedRequestsEnabled, }: FindThresholdSignalsParams): Promise<{ - buckets: ThresholdBucket[]; + buckets: ThresholdCompositeBucket[]; searchDurations: string[]; searchErrors: string[]; warnings: string[]; loggedRequests?: RulePreviewLoggedRequest[]; }> => { // Leaf aggregations used below - const buckets: ThresholdBucket[] = []; + const buckets: ThresholdCompositeBucket[] = []; const searchAfterResults: SearchAfterResults = { searchDurations: [], searchErrors: [], @@ -94,29 +90,33 @@ export const findThresholdSignals = async ({ if (hasThresholdFields(threshold)) { let sortKeys: Record | undefined; do { - const { - searchResult, - searchDuration, - searchErrors, - loggedRequests: thresholdLoggedRequests, - } = await singleSearchAfter({ + const searchRequest = buildEventsSearchQuery({ aggregations: buildThresholdMultiBucketAggregation({ threshold, aggregatableTimestampField, sortKeys, }), index: inputIndexPattern, - searchAfterSortIds: undefined, from, to, - services, - ruleExecutionLogger, + runtimeMappings, filter, - pageSize: 0, + size: 0, sortOrder: 'desc', - runtimeMappings, + searchAfterSortIds: undefined, primaryTimestamp, secondaryTimestamp, + }); + + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: thresholdLoggedRequests, + } = await singleSearchAfter({ + searchRequest, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findThresholdBuckets', @@ -134,40 +134,40 @@ export const findThresholdSignals = async ({ sortKeys = undefined; // this will eject us out of the loop // if a search failure occurs on a secondary iteration, // we will return early. - } else if (searchResultHasAggs(searchResult)) { - const thresholdTerms = searchResult.aggregations?.thresholdTerms; - sortKeys = thresholdTerms?.after_key; - buckets.push( - ...((searchResult.aggregations?.thresholdTerms.buckets as ThresholdBucket[]) ?? []) - ); + } else if (searchResult.aggregations != null) { + const thresholdTerms = searchResult.aggregations.thresholdTerms; + sortKeys = thresholdTerms.after_key; + buckets.push(...thresholdTerms.buckets); } else { throw new Error('Aggregations were missing on threshold rule search result'); } } while (sortKeys && buckets.length <= maxSignals); } else { - const { - searchResult, - searchDuration, - searchErrors, - loggedRequests: thresholdLoggedRequests, - } = await singleSearchAfter({ + const searchRequest = buildEventsSearchQuery({ aggregations: buildThresholdSingleBucketAggregation({ threshold, aggregatableTimestampField, }), - searchAfterSortIds: undefined, index: inputIndexPattern, from, to, - services, - ruleExecutionLogger, + runtimeMappings, filter, - pageSize: 0, + size: 0, sortOrder: 'desc', - trackTotalHits: true, - runtimeMappings, + searchAfterSortIds: undefined, primaryTimestamp, secondaryTimestamp, + }); + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: thresholdLoggedRequests, + } = await singleSearchAfter({ + searchRequest, + services, + ruleExecutionLogger, loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findThresholdBuckets', @@ -180,12 +180,9 @@ export const findThresholdSignals = async ({ searchAfterResults.searchErrors.push(...searchErrors); loggedRequests.push(...(thresholdLoggedRequests ?? [])); - if ( - !searchResultHasAggs(searchResult) && - isEmpty(searchErrors) - ) { - throw new Error('Aggregations were missing on threshold rule search result'); - } else if (searchResultHasAggs(searchResult)) { + if (isEmpty(searchErrors)) { + searchAfterResults.searchErrors.push(...searchErrors); + } else if (searchResult.aggregations != null) { const docCount = searchResult.hits.total.value; if ( docCount >= threshold.value && @@ -196,13 +193,13 @@ export const findThresholdSignals = async ({ buckets.push({ doc_count: docCount, key: {}, - max_timestamp: searchResult.aggregations?.max_timestamp ?? { value: null }, - min_timestamp: searchResult.aggregations?.min_timestamp ?? { value: null }, - ...(includeCardinalityFilter - ? { cardinality_count: searchResult.aggregations?.cardinality_count } - : {}), + max_timestamp: searchResult.aggregations.max_timestamp, + min_timestamp: searchResult.aggregations.min_timestamp, + cardinality_count: searchResult.aggregations.cardinality_count, }); } + } else { + throw new Error('Aggregations were missing on threshold rule search result'); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts index 1383840ec276a..136e1b7d379f0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts @@ -6,7 +6,6 @@ */ import dateMath from '@kbn/datemath'; -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock'; import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { thresholdExecutor } from './threshold'; @@ -48,14 +47,12 @@ describe('threshold_executor', () => { sharedParams.ruleExecutionLogger = ruleExecutionLogger; beforeEach(() => { ruleServices = createPersistenceExecutorOptionsMock(); - ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - ...sampleEmptyAggsSearchResults(), - aggregations: { - thresholdTerms: { buckets: [] }, - }, - }) - ); + ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValue({ + ...sampleEmptyAggsSearchResults(), + aggregations: { + thresholdTerms: { buckets: [] }, + }, + }); mockScheduledNotificationResponseAction = jest.fn(); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/types.ts index 1f79a5c505078..3489bda7c21c0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/types.ts @@ -7,7 +7,7 @@ import type { AggregationsCardinalityAggregate, - AggregationsCompositeBucket, + AggregationsCompositeBucketKeys, AggregationsMaxAggregate, AggregationsMinAggregate, } from '@elastic/elasticsearch/lib/api/types'; @@ -40,8 +40,7 @@ export type ThresholdSingleBucketAggregationResult = ESSearchResponse< } >; -export type ThresholdCompositeBucket = AggregationsCompositeBucket & ThresholdLeafAggregates; -export type ThresholdBucket = ThresholdCompositeBucket; +export type ThresholdCompositeBucket = AggregationsCompositeBucketKeys & ThresholdLeafAggregates; export interface ThresholdResult { terms?: Array<{ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts index d85801de7e5d2..dc8955891a4d8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts @@ -24,7 +24,7 @@ import type { } from '../../../../../common/api/detection_engine/model/alerts'; import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; -import type { ThresholdBucket } from './types'; +import type { ThresholdCompositeBucket } from './types'; import type { BuildReasonMessage } from '../utils/reason_formatters'; import { transformBucketIntoHit } from './bulk_create_threshold_signals'; import type { SecuritySharedParams } from '../types'; @@ -43,7 +43,7 @@ export const wrapSuppressedThresholdALerts = ({ startedAt, }: { sharedParams: SecuritySharedParams; - buckets: ThresholdBucket[]; + buckets: ThresholdCompositeBucket[]; buildReasonMessage: BuildReasonMessage; startedAt: Date; }): Array> => { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts index ac91e20c577af..5e284ef938a84 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts @@ -10,6 +10,7 @@ import { buildEventsSearchQuery } from './build_events_query'; describe('create_signals', () => { test('it builds a now-5m up to today filter', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -64,6 +65,7 @@ describe('create_signals', () => { test('it builds a now-5m up to today filter with timestampOverride', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -159,6 +161,7 @@ describe('create_signals', () => { test('it builds a filter without @timestamp fallback if `secondaryTimestamp` is undefined', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -214,6 +217,7 @@ describe('create_signals', () => { test('if searchAfterSortIds is a valid sortId string', () => { const fakeSortId = '123456789012'; const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -269,6 +273,7 @@ describe('create_signals', () => { test('if searchAfterSortIds is a valid sortId number', () => { const fakeSortIdNumber = 123456789012; const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -323,6 +328,7 @@ describe('create_signals', () => { }); test('if aggregations is not provided it should not be included', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -445,6 +451,7 @@ describe('create_signals', () => { test('if trackTotalHits is provided it should be included', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -461,6 +468,7 @@ describe('create_signals', () => { test('if sortOrder is provided it should be included', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -485,6 +493,7 @@ describe('create_signals', () => { test('it respects sort order for timestampOverride', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', @@ -514,6 +523,7 @@ describe('create_signals', () => { test('it respects overriderBody params', () => { const query = buildEventsSearchQuery({ + aggregations: undefined, index: ['auditbeat-*'], from: 'now-5m', to: 'today', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts index 5bdf1dddf76cb..93e938ab31101 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts @@ -9,8 +9,10 @@ import { isEmpty } from 'lodash'; import type { OverrideBodyQuery } from '../types'; import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; -interface BuildEventsSearchQuery { - aggregations?: Record; +interface BuildEventsSearchQuery< + TAggs extends Record | undefined = undefined +> { + aggregations: TAggs; index: string[]; from: string; to: string; @@ -99,7 +101,9 @@ export const buildTimeRangeFilter = ({ }; }; -export const buildEventsSearchQuery = ({ +export const buildEventsSearchQuery = < + TAggs extends Record | undefined +>({ aggregations, index, from, @@ -114,7 +118,7 @@ export const buildEventsSearchQuery = ({ trackTotalHits, additionalFilters, overrideBody, -}: BuildEventsSearchQuery): estypes.SearchRequest => { +}: BuildEventsSearchQuery) => { const timestamps = secondaryTimestamp ? [primaryTimestamp, secondaryTimestamp] : [primaryTimestamp]; @@ -152,7 +156,7 @@ export const buildEventsSearchQuery = ({ }); } - const searchQuery: estypes.SearchRequest = { + const searchQuery = { allow_no_indices: true, index, ignore_unavailable: true, @@ -170,7 +174,7 @@ export const buildEventsSearchQuery = ({ }, ...docFields, ], - ...(aggregations ? { aggregations } : {}), + aggregations, runtime_mappings: runtimeMappings, sort, ...overrideBody, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts index 4cbf79f97f7c4..79ed617c4d2af 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts @@ -18,8 +18,6 @@ import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas import type { SearchListItemArraySchema } from '@kbn/securitysolution-io-ts-list-types'; import { getSearchListItemResponseMock } from '@kbn/lists-plugin/common/schemas/response/search_list_item_schema.mock'; -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; - import type { CommonAlertFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import { ALERT_RULE_CATEGORY, @@ -88,9 +86,7 @@ describe('searchAfterAndBulkCreate', () => { test('should return success with number of searches less than max signals', async () => { ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -106,9 +102,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -124,9 +118,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -142,9 +134,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(9, 12)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(9, 12)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -160,9 +150,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) + sampleDocSearchResultsNoSortIdNoHits() ); const exceptionItem = getExceptionListItemSchemaMock(); @@ -195,9 +183,7 @@ describe('searchAfterAndBulkCreate', () => { test('should return success with number of searches less than max signals with gap', async () => { ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ createdAlerts: [ @@ -212,9 +198,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -230,9 +214,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -248,9 +230,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) + sampleDocSearchResultsNoSortIdNoHits() ); const exceptionItem = getExceptionListItemSchemaMock(); @@ -283,9 +263,7 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when no search results are in the allowlist', async () => { ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -316,9 +294,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) + sampleDocSearchResultsNoSortIdNoHits() ); const exceptionItem = getExceptionListItemSchemaMock(); @@ -357,20 +333,14 @@ describe('searchAfterAndBulkCreate', () => { listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); ruleServices.scopedClusterClient.asCurrentUser.search .mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ - '1.1.1.1', - '2.2.2.2', - '2.2.2.2', - '2.2.2.2', - ]) - ) + repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '2.2.2.2', + '2.2.2.2', + ]) ) - .mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) - ); + .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits()); const exceptionItem = getExceptionListItemSchemaMock(); exceptionItem.entries = [ @@ -428,22 +398,16 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search .mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId( - 4, - 4, - someGuids.slice(0, 3), - ['1.1.1.1', '2.2.2.2', '2.2.2.2', '2.2.2.2'], - // this is the case we are testing, if we receive an empty string for one of the sort ids. - ['', '2222222222222'] - ) + repeatedSearchResultsWithSortId( + 4, + 4, + someGuids.slice(0, 3), + ['1.1.1.1', '2.2.2.2', '2.2.2.2', '2.2.2.2'], + // this is the case we are testing, if we receive an empty string for one of the sort ids. + ['', '2222222222222'] ) ) - .mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) - ); + .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits()); const { success, createdSignalsCount } = await searchAfterAndBulkCreate({ sharedParams, @@ -467,14 +431,12 @@ describe('searchAfterAndBulkCreate', () => { listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3), [ - '1.1.1.1', - '2.2.2.2', - '2.2.2.2', - '2.2.2.2', - ]) - ) + repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '2.2.2.2', + '2.2.2.2', + ]) ); const exceptionItem = getExceptionListItemSchemaMock(); @@ -506,9 +468,7 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when no sortId present but search results are in the allowlist', async () => { ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -567,9 +527,7 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when no exceptions list provided', async () => { ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -600,9 +558,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) + sampleDocSearchResultsNoSortIdNoHits() ); listClient.searchListItemByValues = jest.fn(({ value }) => @@ -639,7 +595,7 @@ describe('searchAfterAndBulkCreate', () => { }, ]; ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleEmptyDocSearchResults()) + sampleEmptyDocSearchResults() ); listClient.searchListItemByValues = jest.fn(({ value }) => Promise.resolve( @@ -723,9 +679,7 @@ describe('searchAfterAndBulkCreate', () => { ], }; ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -748,9 +702,7 @@ describe('searchAfterAndBulkCreate', () => { ruleServices.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce(bulkItem); // adds the response with errors we are testing ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -766,9 +718,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -784,9 +734,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(9, 12)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(9, 12)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -802,9 +750,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) + sampleDocSearchResultsNoSortIdNoHits() ); const { success, createdSignalsCount, errors } = await searchAfterAndBulkCreate({ sharedParams, @@ -821,9 +767,7 @@ describe('searchAfterAndBulkCreate', () => { it('invokes the enrichment callback with signal search results', async () => { ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -839,9 +783,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -857,9 +799,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) - ) + repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)) ); ruleServices.alertWithPersistence.mockResolvedValueOnce({ @@ -875,9 +815,7 @@ describe('searchAfterAndBulkCreate', () => { }); ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsNoSortIdNoHits() - ) + sampleDocSearchResultsNoSortIdNoHits() ); const mockEnrichment = jest.fn((a) => a); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index 938031a93bc77..b4f7f3b72e2e7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -10,6 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import { singleSearchAfter } from './single_search_after'; import { filterEventsAgainstList } from './large_list_filters/filter_events_against_list'; import { sendAlertTelemetryEvents } from './send_telemetry_events'; +import { buildEventsSearchQuery } from './build_events_query'; import { createSearchAfterReturnType, createSearchAfterReturnTypeFromResponse, @@ -102,26 +103,30 @@ export const searchAfterAndBulkCreateFactory = async ({ } in index pattern "${inputIndexPattern}"` ); - const { - searchResult, - searchDuration, - searchErrors, - loggedRequests: singleSearchLoggedRequests = [], - } = await singleSearchAfter({ - searchAfterSortIds: sortIds, + const searchAfterQuery = buildEventsSearchQuery({ + aggregations: undefined, index: inputIndexPattern, - runtimeMappings, from: tuple.from.toISOString(), to: tuple.to.toISOString(), - services, - ruleExecutionLogger, + runtimeMappings, filter, - pageSize: Math.ceil(Math.min(maxSignals, pageSize)), + size: Math.ceil(Math.min(maxSignals, pageSize)), + sortOrder, + searchAfterSortIds: sortIds, primaryTimestamp, secondaryTimestamp, trackTotalHits, - sortOrder, additionalFilters, + }); + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: singleSearchLoggedRequests = [], + } = await singleSearchAfter({ + searchRequest: searchAfterQuery, + services, + ruleExecutionLogger, loggedRequestsConfig: createLoggedRequestsConfig( isLoggedRequestsEnabled, sortIds, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.test.ts index dd34e355f6995..537b15d1aab3f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.test.ts @@ -5,23 +5,17 @@ * 2.0. */ import type { estypes } from '@elastic/elasticsearch'; -import { - sampleDocSearchResultsNoSortId, - sampleDocSearchResultsWithSortId, -} from '../__mocks__/es_results'; +import { sampleDocSearchResultsNoSortId } from '../__mocks__/es_results'; import { singleSearchAfter } from './single_search_after'; import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; -import { buildEventsSearchQuery } from './build_events_query'; - -jest.mock('./build_events_query'); -const mockBuildEventsSearchQuery = buildEventsSearchQuery as jest.Mock; describe('singleSearchAfter', () => { const mockService: RuleExecutorServicesMock = alertsMock.createRuleExecutorServices(); const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + const mockSearchRequest = { query: { match_all: {} } }; beforeEach(() => { jest.clearAllMocks(); @@ -29,39 +23,23 @@ describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId()) + sampleDocSearchResultsNoSortId() ); const { searchResult } = await singleSearchAfter({ - searchAfterSortIds: undefined, - index: [], - from: 'now-360s', - to: 'now', + searchRequest: mockSearchRequest, services: mockService, ruleExecutionLogger, - pageSize: 1, - filter: {}, - primaryTimestamp: '@timestamp', - secondaryTimestamp: undefined, - runtimeMappings: undefined, }); expect(searchResult).toEqual(sampleDocSearchResultsNoSortId()); }); test('if singleSearchAfter returns an empty failure array', async () => { mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId()) + sampleDocSearchResultsNoSortId() ); const { searchErrors } = await singleSearchAfter({ - searchAfterSortIds: undefined, - index: [], - from: 'now-360s', - to: 'now', + searchRequest: mockSearchRequest, services: mockService, ruleExecutionLogger, - pageSize: 1, - filter: {}, - primaryTimestamp: '@timestamp', - secondaryTimestamp: undefined, - runtimeMappings: undefined, }); expect(searchErrors).toEqual([]); }); @@ -83,125 +61,41 @@ describe('singleSearchAfter', () => { }, }, ]; - mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - took: 10, - timed_out: false, - _shards: { - total: 10, - successful: 10, - failed: 1, - skipped: 0, - failures: errors, - }, - hits: { - total: 100, - max_score: 100, - hits: [], - }, - }) - ); + mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 1, + skipped: 0, + failures: errors, + }, + hits: { + total: 100, + max_score: 100, + hits: [], + }, + }); const { searchErrors } = await singleSearchAfter({ - searchAfterSortIds: undefined, - index: [], - from: 'now-360s', - to: 'now', + searchRequest: mockSearchRequest, services: mockService, ruleExecutionLogger, - pageSize: 1, - filter: {}, - primaryTimestamp: '@timestamp', - secondaryTimestamp: undefined, - runtimeMappings: undefined, }); expect(searchErrors).toEqual([ 'index: "index-123" reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', ]); }); - test('if singleSearchAfter works with a given sort id', async () => { - const searchAfterSortIds = ['1234567891111']; - mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - sampleDocSearchResultsWithSortId() - ) - ); - const { searchResult } = await singleSearchAfter({ - searchAfterSortIds, - index: [], - from: 'now-360s', - to: 'now', - services: mockService, - ruleExecutionLogger, - pageSize: 1, - filter: {}, - primaryTimestamp: '@timestamp', - secondaryTimestamp: undefined, - runtimeMappings: undefined, - }); - expect(searchResult).toEqual(sampleDocSearchResultsWithSortId()); - }); test('if singleSearchAfter throws error', async () => { - const searchAfterSortIds = ['1234567891111']; mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( elasticsearchClientMock.createErrorTransportRequestPromise(new Error('Fake Error')) ); await expect( singleSearchAfter({ - searchAfterSortIds, - index: [], - from: 'now-360s', - to: 'now', + searchRequest: mockSearchRequest, services: mockService, ruleExecutionLogger, - pageSize: 1, - filter: {}, - primaryTimestamp: '@timestamp', - secondaryTimestamp: undefined, - runtimeMappings: undefined, }) ).rejects.toThrow('Fake Error'); }); - - test('singleSearchAfter passes overrideBody to buildEventsSearchQuery', async () => { - mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId()) - ); - await singleSearchAfter({ - searchAfterSortIds: undefined, - index: [], - from: 'now-360s', - to: 'now', - services: mockService, - ruleExecutionLogger, - pageSize: 1, - filter: {}, - primaryTimestamp: '@timestamp', - secondaryTimestamp: undefined, - runtimeMappings: undefined, - overrideBody: { - _source: false, - fields: ['@timestamp'], - }, - }); - - expect(mockBuildEventsSearchQuery).toHaveBeenCalledWith({ - additionalFilters: undefined, - aggregations: undefined, - filter: {}, - from: 'now-360s', - index: [], - primaryTimestamp: '@timestamp', - runtimeMappings: undefined, - searchAfterSortIds: undefined, - secondaryTimestamp: undefined, - size: 1, - sortOrder: undefined, - to: 'now', - trackTotalHits: undefined, - overrideBody: { - _source: false, - fields: ['@timestamp'], - }, - }); - }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index b860367c242f2..3bd45d5ab85e8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -5,69 +5,37 @@ * 2.0. */ import type { estypes } from '@elastic/elasticsearch'; +import type { ESSearchResponse } from '@kbn/es-types'; import { performance } from 'perf_hooks'; import type { AlertInstanceContext, AlertInstanceState, RuleExecutorServices, } from '@kbn/alerting-plugin/server'; -import type { - SignalSearchResponse, - SignalSource, - OverrideBodyQuery, - LoggedRequestsConfig, -} from '../types'; -import { buildEventsSearchQuery } from './build_events_query'; +import type { SignalSource, LoggedRequestsConfig } from '../types'; import { createErrorsFromShard, makeFloatString } from './utils'; -import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { logSearchRequest } from './logged_requests'; -export interface SingleSearchAfterParams { - aggregations?: Record; - searchAfterSortIds: estypes.SortResults | undefined; - index: string[]; - from: string; - to: string; +export interface SingleSearchAfterParams { + searchRequest: TSearchRequest; services: RuleExecutorServices; ruleExecutionLogger: IRuleExecutionLogForExecutors; - pageSize: number; - sortOrder?: estypes.SortOrder; - filter: estypes.QueryDslQueryContainer; - primaryTimestamp: TimestampOverride; - secondaryTimestamp: TimestampOverride | undefined; - trackTotalHits?: boolean; - runtimeMappings: estypes.MappingRuntimeFields | undefined; - additionalFilters?: estypes.QueryDslQueryContainer[]; - overrideBody?: OverrideBodyQuery; loggedRequestsConfig?: LoggedRequestsConfig; } // utilize search_after for paging results into bulk. export const singleSearchAfter = async < - TAggregations = Record + TSearchRequest extends estypes.SearchRequest = estypes.SearchRequest >({ - aggregations, - searchAfterSortIds, - index, - runtimeMappings, - from, - to, + searchRequest, services, - filter, ruleExecutionLogger, - pageSize, - sortOrder, - primaryTimestamp, - secondaryTimestamp, - trackTotalHits, - additionalFilters, - overrideBody, loggedRequestsConfig, -}: SingleSearchAfterParams): Promise<{ - searchResult: SignalSearchResponse; +}: SingleSearchAfterParams): Promise<{ + searchResult: ESSearchResponse; searchDuration: string; searchErrors: string[]; loggedRequests?: RulePreviewLoggedRequest[]; @@ -76,33 +44,10 @@ export const singleSearchAfter = async < const loggedRequests: RulePreviewLoggedRequest[] = []; try { - const searchAfterQuery = buildEventsSearchQuery({ - aggregations, - index, - from, - to, - runtimeMappings, - filter, - size: pageSize, - sortOrder, - searchAfterSortIds, - primaryTimestamp, - secondaryTimestamp, - trackTotalHits, - additionalFilters, - /** - * overrideBody allows the search after to ignore the _source property of the result, - * thus reducing the size of the response and increasing the performance of the query. - */ - overrideBody, - }); - const start = performance.now(); - const { body: nextSearchAfterResult } = - await services.scopedClusterClient.asCurrentUser.search( - searchAfterQuery, - { meta: true } - ); + const nextSearchAfterResult = (await services.scopedClusterClient.asCurrentUser.search( + searchRequest + )) as ESSearchResponse; const end = performance.now(); @@ -114,7 +59,7 @@ export const singleSearchAfter = async < loggedRequests.push({ request: loggedRequestsConfig.skipRequestQuery ? undefined - : logSearchRequest(searchAfterQuery), + : logSearchRequest(searchRequest), description: loggedRequestsConfig.description, request_type: loggedRequestsConfig.type, duration: Math.round(end - start), @@ -129,34 +74,6 @@ export const singleSearchAfter = async < }; } catch (exc) { ruleExecutionLogger.error(`Searching events operation failed: ${exc}`); - if ( - exc.message.includes(`No mapping found for [${primaryTimestamp}] in order to sort on`) || - (secondaryTimestamp && - exc.message.includes(`No mapping found for [${secondaryTimestamp}] in order to sort on`)) - ) { - const searchRes: SignalSearchResponse = { - took: 0, - timed_out: false, - _shards: { - total: 1, - successful: 1, - failed: 0, - skipped: 0, - }, - hits: { - total: 0, - max_score: 0, - hits: [], - }, - }; - return { - searchResult: searchRes, - searchDuration: '-1.0', - searchErrors: exc.message, - loggedRequests, - }; - } - throw exc; } });