diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.test.ts index e287712104949..6fba7d76276e2 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.test.ts @@ -606,6 +606,66 @@ describe('wrapScopedClusterClient', () => { ); expect(logger.warn).not.toHaveBeenCalled(); }); + + test('throws error when es|ql async search throws abort error', async () => { + const { abortController, scopedClusterClient, childClient } = getMockClusterClients(); + + abortController.abort(); + childClient.transport.request.mockRejectedValueOnce( + new Error('Request has been aborted by the user') + ); + + const abortableSearchClient = createWrappedScopedClusterClientFactory({ + scopedClusterClient, + rule, + logger, + abortController, + }).client(); + + await expect( + abortableSearchClient.asInternalUser.transport.request({ + method: 'POST', + path: '/_query/async', + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"ES|QL search has been aborted due to cancelled execution"` + ); + + expect(loggingSystemMock.collect(logger).debug[0][0]).toEqual( + `executing ES|QL query for rule .test-rule-type:abcdefg in space my-space - {\"method\":\"POST\",\"path\":\"/_query/async\"} - with options {}` + ); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + test('throws error when es|ql async query poll throws abort error', async () => { + const { abortController, scopedClusterClient, childClient } = getMockClusterClients(); + + abortController.abort(); + childClient.transport.request.mockRejectedValueOnce( + new Error('Request has been aborted by the user') + ); + + const abortableSearchClient = createWrappedScopedClusterClientFactory({ + scopedClusterClient, + rule, + logger, + abortController, + }).client(); + + await expect( + abortableSearchClient.asInternalUser.transport.request({ + method: 'GET', + path: '/_query/async/FjhHTHlyRVltUm5xck1tV0RFN18wREEeOUxMcnkxZ3NTd0MzOTNabm1NQW9TUTozMjY1NjQ3', + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"ES|QL search has been aborted due to cancelled execution"` + ); + + expect(loggingSystemMock.collect(logger).debug[0][0]).toEqual( + `executing ES|QL query for rule .test-rule-type:abcdefg in space my-space - {\"method\":\"GET\",\"path\":\"/_query/async/FjhHTHlyRVltUm5xck1tV0RFN18wREEeOUxMcnkxZ3NTd0MzOTNabm1NQW9TUTozMjY1NjQ3\"} - with options {}` + ); + expect(logger.warn).not.toHaveBeenCalled(); + }); }); test('uses asInternalUser when specified', async () => { diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.ts b/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.ts index 3bf87b3653c5a..a3abae3aee203 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.ts @@ -162,7 +162,10 @@ function getWrappedTransportRequestFn(opts: WrapEsClientOpts) { options?: TransportRequestOptions ): Promise> { // Wrap ES|QL requests with an abort signal - if (params.method === 'POST' && params.path === '/_query') { + if ( + (params.method === 'POST' && ['/_query', '/_query/async'].includes(params.path)) || + (params.method === 'GET' && params.path.startsWith('/_query/async')) + ) { let requestOptions: TransportRequestOptions = {}; try { requestOptions = options ?? {}; diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.test.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.test.ts index 68c01770fd6f7..249e6c0a5cf71 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.test.ts @@ -211,6 +211,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -319,6 +320,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -492,6 +494,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -596,6 +599,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -738,6 +742,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -853,6 +858,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -969,6 +975,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -1075,6 +1082,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -1181,6 +1189,7 @@ describe('RuleTypeRunner', () => { expect(ruleType.executor).toHaveBeenCalledWith({ executionId: 'abc', + ruleExecutionTimeout: '5m', services: { alertFactory: alertsClient.factory(), alertsClient: alertsClient.client(), @@ -1247,6 +1256,47 @@ describe('RuleTypeRunner', () => { shouldLogAlerts: true, }); }); + + test('should call executor with custom ruleExecutionTimeout', async () => { + const mockRuleExecutionTimeout = '15m'; + + await ruleTypeRunner.run({ + context: { + alertingEventLogger, + flappingSettings: DEFAULT_FLAPPING_SETTINGS, + queryDelaySec: 0, + request: fakeRequest, + maintenanceWindowsService, + ruleId: RULE_ID, + ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`, + ruleRunMetricsStore, + spaceId: 'default', + isServerless: false, + }, + alertsClient, + executionId: 'abc', + executorServices: { + getDataViews, + ruleMonitoringService: publicRuleMonitoringService, + ruleResultService: publicRuleResultService, + savedObjectsClient, + uiSettingsClient, + wrappedScopedClusterClient, + getWrappedSearchSourceClient, + }, + rule: mockedRule, + ruleType: { ...ruleType, ruleTaskTimeout: mockRuleExecutionTimeout }, + startedAt: new Date(DATE_1970), + state: mockTaskInstance().state, + validatedParams: mockedRuleParams, + }); + + expect(ruleType.executor).toHaveBeenCalledWith( + expect.objectContaining({ + ruleExecutionTimeout: mockRuleExecutionTimeout, + }) + ); + }); }); }); diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.ts index 5a9a88d856744..61fb30db8a44a 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/rule_type_runner.ts @@ -286,6 +286,7 @@ export class RuleTypeRunner< ...(startedAtOverridden ? { forceNow: startedAt } : {}), }), isServerless: context.isServerless, + ruleExecutionTimeout: ruleType.ruleTaskTimeout, }) ) ); diff --git a/x-pack/platform/plugins/shared/alerting/server/types.ts b/x-pack/platform/plugins/shared/alerting/server/types.ts index f0db7c3f4ed43..3b8d9c717e64f 100644 --- a/x-pack/platform/plugins/shared/alerting/server/types.ts +++ b/x-pack/platform/plugins/shared/alerting/server/types.ts @@ -134,6 +134,7 @@ export interface RuleExecutorOptions< flappingSettings: RulesSettingsFlappingProperties; getTimeRange: (timeWindow?: string) => GetTimeRangeResult; isServerless: boolean; + ruleExecutionTimeout?: string; } export interface RuleParamsAndRefs { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts index 8e8093d96c3ac..91344c7c78dca 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts @@ -293,7 +293,7 @@ export const previewRulesRoute = ( rule, services: { shouldWriteAlerts, - shouldStopExecution: () => false, + shouldStopExecution: () => isAborted, alertsClient: null, alertFactory, savedObjectsClient: coreContext.savedObjects.client, @@ -322,6 +322,7 @@ export const previewRulesRoute = ( return { dateStart: date, dateEnd: date }; }, isServerless, + ruleExecutionTimeout: `${PREVIEW_TIMEOUT_SECONDS}s`, })) as { state: TState; loggedRequests: RulePreviewLoggedRequest[] }); const errors = loggedStatusChanges diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/build_esql_search_request.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/build_esql_search_request.ts index 54ddd9b818c23..27c77d30758c7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/build_esql_search_request.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/build_esql_search_request.ts @@ -23,6 +23,7 @@ export interface BuildEqlSearchRequestParams { primaryTimestamp: TimestampOverride; secondaryTimestamp: TimestampOverride | undefined; exceptionFilter: Filter | undefined; + ruleExecutionTimeout: string | undefined; } export const buildEsqlSearchRequest = ({ @@ -34,6 +35,7 @@ export const buildEsqlSearchRequest = ({ secondaryTimestamp, exceptionFilter, size, + ruleExecutionTimeout, }: BuildEqlSearchRequestParams) => { const esFilter = getQueryFilter({ query: '', @@ -61,5 +63,7 @@ export const buildEsqlSearchRequest = ({ filter: requestFilter, }, }, + wait_for_completion_timeout: '4m', // hard limit request timeout is 5m set by ES proxy and alerting framework. So, we should be fine to wait 4m for async query completion. If rule execution is shorter than 4m and query was not completed, it will be aborted. + ...(ruleExecutionTimeout ? { keep_alive: ruleExecutionTimeout } : {}), }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts index cf206beb9ea1c..e33e5bc76b6bc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts @@ -29,10 +29,8 @@ import { fetchSourceDocuments } from './fetch_source_documents'; import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import type { CreateRuleOptions, SecuritySharedParams, SignalSource } from '../types'; -import { logEsqlRequest } from '../utils/logged_requests'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; import { checkErrorDetails } from '../utils/check_error_details'; -import * as i18n from '../translations'; import { addToSearchAfterReturn, @@ -54,6 +52,7 @@ export const esqlExecutor = async ({ experimentalFeatures, licensing, scheduleNotificationResponseActionsService, + ruleExecutionTimeout, }: { sharedParams: SecuritySharedParams; services: RuleExecutorServices; @@ -61,6 +60,7 @@ export const esqlExecutor = async ({ experimentalFeatures: ExperimentalFeatures; licensing: LicensingPluginSetup; scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService']; + ruleExecutionTimeout?: string; }) => { const { completeRule, @@ -108,16 +108,10 @@ export const esqlExecutor = async ({ primaryTimestamp, secondaryTimestamp, exceptionFilter, + ruleExecutionTimeout, }); const esqlQueryString = { drop_null_columns: true }; - if (isLoggedRequestsEnabled) { - loggedRequests.push({ - request: logEsqlRequest(esqlRequest, esqlQueryString), - description: i18n.ESQL_SEARCH_REQUEST_DESCRIPTION, - }); - } - ruleExecutionLogger.debug(`ES|QL query request: ${JSON.stringify(esqlRequest)}`); const exceptionsWarning = getUnprocessedExceptionsWarnings(unprocessedExceptions); if (exceptionsWarning) { @@ -130,15 +124,14 @@ export const esqlExecutor = async ({ esClient: services.scopedClusterClient.asCurrentUser, requestBody: esqlRequest, requestQueryParams: esqlQueryString, + shouldStopExecution: services.shouldStopExecution, + ruleExecutionLogger, + loggedRequests: isLoggedRequestsEnabled ? loggedRequests : undefined, }); const esqlSearchDuration = performance.now() - esqlSignalSearchStart; result.searchAfterTimes.push(makeFloatString(esqlSearchDuration)); - if (isLoggedRequestsEnabled && loggedRequests[0]) { - loggedRequests[0].duration = Math.round(esqlSearchDuration); - } - ruleExecutionLogger.debug(`ES|QL query request took: ${esqlSearchDuration}ms`); const isRuleAggregating = computeIsESQLQueryAggregating(completeRule.ruleParams.query); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.test.ts new file mode 100644 index 0000000000000..745e20a84026c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.test.ts @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { performEsqlRequest } from './esql_request'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; + +const columns = [ + { name: '_id', type: 'keyword' as const }, + { name: 'agent.name', type: 'keyword' as const }, + { name: 'agent.version', type: 'keyword' as const }, + { name: 'agent.type', type: 'keyword' as const }, +]; +const values = [['doc-id', 'agent-name', '8.8.1', 'packetbeat']]; + +const requestBody = { + query: 'from test* METADATA _id | limit 101', + filter: { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: '2025-04-02T10:13:52.235Z', + gte: '2013-11-04T16:13:52.235Z', + format: 'strict_date_optional_time', + }, + }, + }, + { + bool: { + must: [], + filter: [], + should: [], + must_not: [], + }, + }, + ], + }, + }, + wait_for_completion_timeout: '4m', +}; +const requestQueryParams = { drop_null_columns: true }; + +describe('performEsqlRequest', () => { + const esClient = elasticsearchServiceMock.createElasticsearchClient(); + const shouldStopExecution: jest.Mock = jest.fn(); + shouldStopExecution.mockReturnValue(false); + + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + }); + + it('returns results immediately when the async query completed', async () => { + const mockResponse = { + id: 'QUERY-ID', + is_running: false, + columns, + values, + }; + + esClient.transport.request.mockResolvedValueOnce(mockResponse); + + const result = await performEsqlRequest({ + esClient, + requestBody, + requestQueryParams, + shouldStopExecution, + }); + + expect(result).toEqual(mockResponse); + expect(esClient.transport.request).toHaveBeenCalledTimes(2); + expect(esClient.transport.request).toHaveBeenCalledWith({ + method: 'POST', + path: '/_query/async', + body: requestBody, + querystring: requestQueryParams, + }); + expect(esClient.transport.request).toHaveBeenCalledWith({ + method: 'DELETE', + path: '/_query/async/QUERY-ID', + }); + }); + + it('polls until the query is completed', async () => { + const mockSubmitResponse = { + id: 'QUERY-ID', + is_running: true, + columns: [], + values: [], + }; + + const mockPollResponse = { + ...mockSubmitResponse, + is_running: false, + columns, + values, + }; + + esClient.transport.request + .mockResolvedValueOnce(mockSubmitResponse) + .mockResolvedValueOnce(mockPollResponse); + + const waitForPerformEsql = performEsqlRequest({ + esClient, + requestBody, + requestQueryParams, + shouldStopExecution, + }); + + await jest.advanceTimersByTimeAsync(15000); + + const result = await waitForPerformEsql; + + expect(result).toEqual(mockPollResponse); + expect(esClient.transport.request).toHaveBeenCalledTimes(3); + expect(esClient.transport.request).toHaveBeenNthCalledWith(1, { + method: 'POST', + path: '/_query/async', + body: requestBody, + querystring: requestQueryParams, + }); + expect(esClient.transport.request).toHaveBeenNthCalledWith(2, { + method: 'GET', + path: '/_query/async/QUERY-ID', + }); + expect(esClient.transport.request).toHaveBeenCalledWith({ + method: 'DELETE', + path: '/_query/async/QUERY-ID', + }); + }); + + it('throws an error if execution is cancelled', async () => { + const mockSubmitResponse = { + id: 'QUERY-ID', + is_running: true, + columns: [], + values: [], + }; + + esClient.transport.request.mockResolvedValue(mockSubmitResponse); + shouldStopExecution.mockReturnValue(true); + + const waitForPerformEsql = performEsqlRequest({ + esClient, + requestBody, + requestQueryParams, + shouldStopExecution, + }).catch((error) => { + expect(error.message).toBe('Rule execution cancelled due to timeout'); + }); + + await jest.advanceTimersByTimeAsync(15000); + await waitForPerformEsql; + expect.assertions(1); + }); + + it('deletes query if error happens during polling', async () => { + const mockSubmitResponse = { + id: 'QUERY-ID', + is_running: true, + columns: [], + values: [], + }; + + esClient.transport.request + .mockResolvedValueOnce(mockSubmitResponse) + .mockRejectedValueOnce(new Error('Test error')); + + const waitForPerformEsql = performEsqlRequest({ + esClient, + requestBody, + requestQueryParams: {}, + shouldStopExecution, + }).catch((error) => { + expect(error.message).toBe('Test error'); + }); + + await jest.advanceTimersByTimeAsync(15000); + await waitForPerformEsql; + + expect(esClient.transport.request).toHaveBeenCalledWith({ + method: 'DELETE', + path: '/_query/async/QUERY-ID', + }); + + expect.assertions(2); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.ts index 2415d3d142f72..c000762d2246f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.ts @@ -5,14 +5,35 @@ * 2.0. */ -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { performance } from 'perf_hooks'; +import type { ElasticsearchClient } from '@kbn/core/server'; import { getKbnServerError } from '@kbn/kibana-utils-plugin/server'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; +import { logEsqlRequest } from '../utils/logged_requests'; +import * as i18n from '../translations'; + +const setLatestRequestDuration = ( + startTime: number, + loggedRequests: RulePreviewLoggedRequest[] | undefined +) => { + const duration = performance.now() - startTime; + if (loggedRequests && loggedRequests?.[loggedRequests.length - 1]) { + loggedRequests[loggedRequests.length - 1].duration = Math.round(duration); + } +}; export interface EsqlResultColumn { name: string; type: 'date' | 'keyword'; } +type AsyncEsqlResponse = { + id: string; + is_running: boolean; +} & EsqlTable; + export type EsqlResultRow = Array; export interface EsqlTable { @@ -24,29 +45,95 @@ export const performEsqlRequest = async ({ esClient, requestBody, requestQueryParams, + ruleExecutionLogger, + shouldStopExecution, + loggedRequests, }: { - logger?: Logger; + ruleExecutionLogger?: IRuleExecutionLogForExecutors; esClient: ElasticsearchClient; - requestBody: Record; - requestQueryParams?: { drop_null_columns?: boolean }; + requestBody: { + query: string; + filter: QueryDslQueryContainer; + }; + requestQueryParams?: { + drop_null_columns?: boolean; + }; + shouldStopExecution: () => boolean; + loggedRequests?: RulePreviewLoggedRequest[]; }): Promise => { - const search = async () => { - try { - const rawResponse = await esClient.transport.request({ - method: 'POST', - path: '/_query', - body: requestBody, - querystring: requestQueryParams, - }); - return { - rawResponse, - isPartial: false, - isRunning: false, - }; - } catch (e) { - throw getKbnServerError(e); + let pollInterval = 10 * 1000; // Poll every 10 seconds + let pollCount = 0; + let queryId: string = ''; + + try { + loggedRequests?.push({ + request: logEsqlRequest(requestBody, requestQueryParams), + description: i18n.ESQL_SEARCH_REQUEST_DESCRIPTION, + }); + const asyncSearchStarted = performance.now(); + const asyncEsqlResponse = await esClient.transport.request({ + method: 'POST', + path: '/_query/async', + body: requestBody, + querystring: requestQueryParams, + }); + setLatestRequestDuration(asyncSearchStarted, loggedRequests); + + queryId = asyncEsqlResponse.id; + const isRunning = asyncEsqlResponse.is_running; + + if (!isRunning) { + return asyncEsqlResponse; } - }; - return (await search()).rawResponse; + // Poll for long-executing query + while (true) { + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + + loggedRequests?.push({ + request: `GET /_query/async/${queryId}`, + description: i18n.ESQL_POLL_REQUEST_DESCRIPTION, + }); + const pollStarted = performance.now(); + const pollResponse = await esClient.transport.request({ + method: 'GET', + path: `/_query/async/${queryId}`, + }); + setLatestRequestDuration(pollStarted, loggedRequests); + + if (!pollResponse.is_running) { + return pollResponse; + } + + pollCount++; + + if (pollCount > 60) { + pollInterval = 60 * 1000; // Increase the poll interval after 10m + } + + const isCancelled = shouldStopExecution(); // Execution will be cancelled if rule times out + ruleExecutionLogger?.debug(`Polling for query ID: ${queryId}, isCancelled: ${isCancelled}`); + + if (isCancelled) { + throw new Error('Rule execution cancelled due to timeout'); + } + ruleExecutionLogger?.debug(`Query is still running for query ID: ${queryId}`); + } + } catch (error) { + ruleExecutionLogger?.error(`Error while performing ES|QL search: ${error?.message}`); + throw getKbnServerError(error); + } finally { + if (queryId) { + loggedRequests?.push({ + request: `DELETE /_query/async/${queryId}`, + description: i18n.ESQL_DELETE_REQUEST_DESCRIPTION, + }); + const deleteStarted = performance.now(); + await esClient.transport.request({ + method: 'DELETE', + path: `/_query/async/${queryId}`, + }); + setLatestRequestDuration(deleteStarted, loggedRequests); + } + } }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index be04cba2df7f8..10901d0f5fcfc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -14,6 +14,20 @@ export const ESQL_SEARCH_REQUEST_DESCRIPTION = i18n.translate( } ); +export const ESQL_POLL_REQUEST_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlRuleType.esqlPollRequestDescription', + { + defaultMessage: 'ES|QL request to poll for async search results', + } +); + +export const ESQL_DELETE_REQUEST_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlRuleType.esqlDeleteRequestDescription', + { + defaultMessage: 'ES|QL request to delete async search query', + } +); + export const FIND_SOURCE_DOCUMENTS_REQUEST_DESCRIPTION = i18n.translate( 'xpack.securitySolution.detectionEngine.esqlRuleType.findSourceDocumentsRequestDescription', { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_esql.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_esql.ts index 8b03f563597b3..9ad0c2de82cb9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_esql.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_esql.ts @@ -24,5 +24,9 @@ export const logEsqlRequest = ( }, []) .join('&'); - return `POST _query${urlParams ? `?${urlParams}` : ''}\n${JSON.stringify(requestBody, null, 2)}`; + return `POST _query/async${urlParams ? `?${urlParams}` : ''}\n${JSON.stringify( + requestBody, + null, + 2 + )}`; };