From f5e30e6b97d072104f8b3aaa6f36aab8fbd31872 Mon Sep 17 00:00:00 2001 From: Nikita Indik Date: Thu, 12 Feb 2026 15:57:26 +0100 Subject: [PATCH] Revert accidentally merged commit --- .../create_security_rule_type_wrapper.ts | 16 ++++---- .../eql/build_alert_group_from_sequence.ts | 2 +- .../detection_engine/rule_types/eql/eql.ts | 2 +- .../detection_engine/rule_types/esql/esql.ts | 16 ++++---- .../factories/bulk_create_factory.ts | 6 +-- .../threat_mapping/create_event_signal.ts | 10 ++--- .../threat_mapping/create_threat_signal.ts | 12 +++--- .../threat_mapping/create_threat_signals.ts | 33 +++++++++------- .../get_allowed_fields_for_terms_query.ts | 2 +- .../threat_mapping/get_event_count.ts | 6 +-- .../threat_mapping/get_threat_list.ts | 4 +- .../detection_engine/rule_types/ml/ml.test.ts | 12 ++++-- .../lib/detection_engine/rule_types/ml/ml.ts | 18 ++++----- .../new_terms/multi_terms_composite.ts | 4 +- .../group_and_bulk_create.ts | 8 ++-- .../utils/bulk_create_with_suppression.ts | 6 +-- .../create_set_to_filter_against.ts | 6 +-- .../filter_events_against_list.test.ts | 2 +- .../filter_events_against_list.ts | 6 +-- .../rule_types/utils/log_shard_failure.ts | 2 +- .../utils/search_after_bulk_create_factory.ts | 39 +++++++++---------- .../rule_types/utils/send_telemetry_events.ts | 2 +- .../rule_types/utils/single_search_after.ts | 2 +- .../rule_types/utils/utils.ts | 4 +- 24 files changed, 111 insertions(+), 109 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index b45c47a729df3..5a50ed815a842 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -217,7 +217,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = const refresh = isPreview ? false : true; - ruleExecutionLogger.debug(`Starting execution with interval: ${interval}`); + ruleExecutionLogger.debug(`Starting Security Rule execution (interval: ${interval})`); await ruleExecutionLogger.logStatusChange({ newStatus: RuleExecutionStatusEnum.running, @@ -279,13 +279,13 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = if (SavedObjectsErrorHelpers.isNotFoundError(exc)) { await ruleExecutionLogger.logStatusChange({ newStatus: RuleExecutionStatusEnum.failed, - message: `Data view is not found.\nError: ${exc}`, + message: `Data View not found ${exc}`, userError: true, }); } else { await ruleExecutionLogger.logStatusChange({ newStatus: RuleExecutionStatusEnum.failed, - message: `Check for indices to search failed.\nError: ${exc}`, + message: `Check for indices to search failed ${exc}`, }); } @@ -589,12 +589,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = }); } else if (!(result.warningMessages.length > 0) && !(wrapperWarnings.length > 0)) { ruleExecutionLogger.debug('Security Rule execution completed'); - ruleExecutionLogger.info( - `Alerts created: ${createdSignalsCount}\nFinished indexing ${createdSignalsCount} alerts into "${ruleDataClient.indexNameWithNamespace( + ruleExecutionLogger.debug( + `Finished indexing ${createdSignalsCount} alerts into ${ruleDataClient.indexNameWithNamespace( spaceId - )}".${ + )} ${ !isEmpty(tuples) - ? ` Searched between date ranges: ${JSON.stringify(tuples, null, 2)}.` + ? `searched between date ranges ${JSON.stringify(tuples, null, 2)}` : '' }` ); @@ -614,7 +614,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = await ruleExecutionLogger.logStatusChange({ newStatus: RuleExecutionStatusEnum.failed, - message: `An error occurred during rule execution. ${errorMessage}`, + message: `An error occurred during rule execution: message: "${errorMessage}"`, userError: checkErrorDetails(errorMessage).isUserError, metrics: { searchDurations: result.searchAfterTimes, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 61c91560426e7..851635bf86bed 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -106,7 +106,7 @@ export const buildAlertGroupFromSequence = ({ }) ); } catch (error) { - ruleExecutionLogger.debug(`Error building alert group from sequence\nError: ${error}`); + ruleExecutionLogger.error(error); return { shellAlert: undefined, buildingBlocks: [] }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index d4543d443ef76..87496501ce356 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -98,7 +98,7 @@ export const eqlExecutor = async ({ tiebreakerField: ruleParams.tiebreakerField, }); - ruleExecutionLogger.trace(`EQL query to execute\n${JSON.stringify(request)}`); + ruleExecutionLogger.debug(`EQL query request: ${JSON.stringify(request)}`); const exceptionsWarning = getUnprocessedExceptionsWarnings(sharedParams.unprocessedExceptions); if (exceptionsWarning) { result.warningMessages.push(exceptionsWarning); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts index 64c2fff7939dd..cd72345409595 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts @@ -145,7 +145,7 @@ export const esqlExecutor = async ({ }; const hasLoggedRequestsReachedLimit = iteration >= 2; - ruleExecutionLogger.trace(`ES|QL query to execute\n${JSON.stringify(esqlRequest)}`); + ruleExecutionLogger.debug(`ES|QL query request: ${JSON.stringify(esqlRequest)}`); const exceptionsWarning = getUnprocessedExceptionsWarnings(unprocessedExceptions); if (exceptionsWarning) { result.warningMessages.push(exceptionsWarning); @@ -166,8 +166,8 @@ export const esqlExecutor = async ({ const esqlSearchDuration = performance.now() - esqlSignalSearchStart; result.searchAfterTimes.push(makeFloatString(esqlSearchDuration)); - ruleExecutionLogger.trace( - `ES|QL query iteration\nIteration: ${iteration}. Search took: ${esqlSearchDuration}ms.` + ruleExecutionLogger.debug( + `ES|QL query request for ${iteration} iteration took: ${esqlSearchDuration}ms` ); const results = response.values.map((row) => rowToDocument(response.columns, row)); @@ -235,8 +235,8 @@ export const esqlExecutor = async ({ maxNumberOfAlertsMultiplier: 1, }); - ruleExecutionLogger.info( - `Alerts created: ${bulkCreateResult.createdItemsCount}. Alerts suppressed: ${bulkCreateResult.suppressedItemsCount}.` + ruleExecutionLogger.debug( + `Created ${bulkCreateResult.createdItemsCount} alerts. Suppressed ${bulkCreateResult.suppressedItemsCount} alerts` ); updateExcludedDocuments({ @@ -268,7 +268,7 @@ export const esqlExecutor = async ({ }); addToSearchAfterReturn({ current: result, next: bulkCreateResult }); - ruleExecutionLogger.info(`Alerts created: ${bulkCreateResult.createdItemsCount}.`); + ruleExecutionLogger.debug(`Created ${bulkCreateResult.createdItemsCount} alerts`); updateExcludedDocuments({ excludedDocuments, @@ -293,8 +293,8 @@ export const esqlExecutor = async ({ // no more results will be found if (response.values.length < size) { - ruleExecutionLogger.trace( - `End of search. Found ${response.values.length} results\nPage size ${size}.` + ruleExecutionLogger.debug( + `End of search: Found ${response.values.length} results with page size ${size}` ); break; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index 6604973aa08e1..524671096230f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -71,7 +71,7 @@ export const bulkCreate = async ({ }); return enrichedAlerts; } catch (error) { - ruleExecutionLogger.error(`Error enriching alerts\nError: ${error}`); + ruleExecutionLogger.error(`Alerts enrichment failed: ${error}`); throw error; } finally { enrichmentsTimeFinish = performance.now(); @@ -91,10 +91,10 @@ export const bulkCreate = async ({ const end = performance.now(); - ruleExecutionLogger.debug(`Bulk processing alerts took ${makeFloatString(end - start)}ms.`); + ruleExecutionLogger.debug(`Alerts bulk process took ${makeFloatString(end - start)} ms`); if (!isEmpty(errors)) { - ruleExecutionLogger.warn(`Error bulk processing alerts\nError: ${JSON.stringify(errors)}`); + ruleExecutionLogger.warn(`Alerts bulk process finished with errors: ${JSON.stringify(errors)}`); return { errors: Object.keys(errors), success: false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts index 0343aaedc688d..eeb79fdd24ee5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts @@ -106,7 +106,7 @@ export const createEventSignal = async ({ loadFields: true, }); - ruleExecutionLogger.debug(`Matched signals found: ${ids?.length}`); + ruleExecutionLogger.debug(`${ids?.length} matched signals found`); const enrichment = threatEnrichmentFactory({ signalIdToMatchedQueriesMap, @@ -138,12 +138,12 @@ export const createEventSignal = async ({ } else { createResult = await searchAfterAndBulkCreate(searchAfterBulkCreateParams); } - ruleExecutionLogger.trace( - `Match checks completed\n${ + ruleExecutionLogger.debug( + `${ currentEventList.length - } items have completed match checks. Search times (ms): ${ + } items have completed match checks and the total times to search were ${ createResult.searchAfterTimes.length !== 0 ? createResult.searchAfterTimes : '(unknown) ' - }.` + }ms` ); return createResult; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts index d510e616c4e92..e5b20455e488f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts @@ -56,7 +56,7 @@ export const createThreatSignal = async ({ if (!threatFilter.query || threatFilter.query?.bool.should.length === 0) { // empty threat list and we do not want to return everything as being // a hit so opt to return the existing result. - ruleExecutionLogger.trace( + ruleExecutionLogger.debug( 'Indicator items are empty after filtering for missing data, returning without attempting a match' ); return currentResult; @@ -74,7 +74,7 @@ export const createThreatSignal = async ({ loadFields: true, }); - ruleExecutionLogger.trace( + ruleExecutionLogger.debug( `${threatFilter.query?.bool.should.length} indicator items are being checked for existence of matches` ); @@ -115,12 +115,12 @@ export const createThreatSignal = async ({ result = await searchAfterAndBulkCreate(searchAfterBulkCreateParams); } - ruleExecutionLogger.trace( - `Match checks completed\n${ + ruleExecutionLogger.debug( + `${ threatFilter.query?.bool.should.length - } items have completed match checks. Search times (ms): ${ + } items have completed match checks and the total times to search were ${ result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) ' - }.` + }ms` ); return result; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts index 7b60279d0ad2e..f04996132e0dc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts @@ -77,7 +77,7 @@ export const createThreatSignals = async ({ }); const params = completeRule.ruleParams; - ruleExecutionLogger.trace('Indicator matching rule starting'); + ruleExecutionLogger.debug('Indicator matching rule starting'); const perPage = concurrentSearches * itemsPerSearch; const verifyExecutionCanProceed = buildExecutionIntervalValidator( completeRule.ruleConfig.schedule.interval @@ -185,7 +185,7 @@ export const createThreatSignals = async ({ while (list.hits.hits.length !== 0) { verifyExecutionCanProceed(); const chunks = chunk(chunkPage, list.hits.hits); - ruleExecutionLogger.trace(`${chunks.length} concurrent indicator searches are starting.`); + ruleExecutionLogger.debug(`${chunks.length} concurrent indicator searches are starting.`); const concurrentSearchesPerformed = chunks.map>(createSignal); const searchesPerformed = await Promise.all(concurrentSearchesPerformed); @@ -205,8 +205,8 @@ export const createThreatSignals = async ({ // allowed by elasticsearch. The sliced chunk is used in createSignal to generate // threat filters. chunkPage = maxClauseCountValue; - ruleExecutionLogger.debug( - `Max clause count error received from Elasticsearch. Setting rule page size to ${maxClauseCountValue}.` + ruleExecutionLogger.warn( + `maxClauseCount error received from elasticsearch, setting IM rule page size to ${maxClauseCountValue}` ); // only store results + errors that are not related to maxClauseCount @@ -232,7 +232,10 @@ export const createThreatSignals = async ({ } documentCount -= list.hits.hits.length; ruleExecutionLogger.debug( - `Alert candidates found: ${results.createdSignalsCount}.\nConcurrent indicator match searches completed. Search took: ${results.searchAfterTimes}ms. Bulk create times (ms): ${results.bulkCreateTimes}. Are all operations successful: ${results.success}.` + `Concurrent indicator match searches completed with ${results.createdSignalsCount} signals found`, + `search times of ${results.searchAfterTimes}ms,`, + `bulk create times ${results.bulkCreateTimes}ms,`, + `all successes are ${results.success}` ); // if alerts suppressed it means suppression enabled, so suppression alert limit should be applied (5 * max_signals) @@ -243,7 +246,7 @@ export const createThreatSignals = async ({ results.warningMessages.push(getMaxSignalsWarning()); } ruleExecutionLogger.debug( - `Max alerts per run reached\n${params.maxSignals}. Additional ${documentCount} documents are not checked.` + `Indicator match has reached its max signals count ${params.maxSignals}. Additional documents not checked are ${documentCount}` ); break; } else if ( @@ -254,15 +257,15 @@ export const createThreatSignals = async ({ ) { // warning should be already set ruleExecutionLogger.debug( - `Max alerts per run reached\nIndicator match has reached its max signals count ${ + `Indicator match has reached its max signals count ${ MAX_SIGNALS_SUPPRESSION_MULTIPLIER * params.maxSignals - }. Additional ${documentCount} documents are not checked.` + }. Additional documents not checked are ${documentCount}` ); break; } - ruleExecutionLogger.trace(`Documents items left to check: ${documentCount}`); + ruleExecutionLogger.debug(`Documents items left to check are ${documentCount}`); if (maxClauseCountValue > Number.NEGATIVE_INFINITY) { - ruleExecutionLogger.trace(`Re-running search due to max clause count error`); + ruleExecutionLogger.debug(`Re-running search since we hit max clause count error`); // re-run search with smaller max clause count; list = await getDocumentList({ searchAfter: undefined }); @@ -277,8 +280,8 @@ export const createThreatSignals = async ({ const hasNegativeDateSort = sortIds?.some((val) => Number(val) < 0); if (hasNegativeDateSort) { - ruleExecutionLogger.trace( - `Negative date sort ID encountered\nValue: ${sortIds}. Threat search stopped.` + ruleExecutionLogger.debug( + `Negative date sort id value encountered: ${sortIds}. Threat search stopped.` ); break; @@ -379,8 +382,8 @@ export const createThreatSignals = async ({ await services.scopedClusterClient.asCurrentUser.closePointInTime({ id: threatPitId }); } catch (error) { // Don't fail due to a bad point in time closure. We have seen failures in e2e tests during nominal operations. - ruleExecutionLogger.debug( - `Error trying to close point in time\nPIT ID: "${threatPitId}". It will expire within "${THREAT_PIT_KEEP_ALIVE}". Error: "${error}".` + ruleExecutionLogger.warn( + `Error trying to close point in time: "${threatPitId}", it will expire within "${THREAT_PIT_KEEP_ALIVE}". Error is: "${error}"` ); } scheduleNotificationResponseActionsService({ @@ -388,6 +391,6 @@ export const createThreatSignals = async ({ signalsCount: results.createdSignalsCount, responseActions: completeRule.ruleParams.responseActions, }); - ruleExecutionLogger.trace('Indicator matching rule has completed'); + ruleExecutionLogger.debug('Indicator matching rule has completed'); return results; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts index c7eef7e36eb79..4e1c50c72745c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts @@ -69,7 +69,7 @@ export const getAllowedFieldsForTermQuery = async ({ ), }; } catch (e) { - ruleExecutionLogger.debug(`Error getting allowed fields for the terms query\nError: ${e}`); + ruleExecutionLogger.debug(`Can't get allowed fields for terms query: ${e}`); return allowedFieldsForTermsQuery; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts index 28c080feb944c..990620e47d841 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts @@ -40,8 +40,8 @@ export const getEventList = async ({ throw new TypeError('perPage cannot exceed the size of 10000'); } - ruleExecutionLogger.trace( - `Querying events\nIndex: "${sharedParams.inputIndex}", searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items.` + ruleExecutionLogger.debug( + `Querying the events items from the index: "${sharedParams.inputIndex}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items` ); const queryFilter = getQueryFilter({ @@ -75,7 +75,7 @@ export const getEventList = async ({ ruleExecutionLogger, }); - ruleExecutionLogger.debug(`Events retrieved: ${searchResult.hits.hits.length}`); + ruleExecutionLogger.debug(`Retrieved events items of size: ${searchResult.hits.hits.length}`); return searchResult; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts index d2c4b21a05bc5..ac27cd185dd5a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts @@ -54,7 +54,7 @@ export const getThreatList = async ({ }); ruleExecutionLogger.debug( - `Querying indicator items\nIndex: "${threatIndex}", searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items.` + `Querying the indicator items from the index: "${threatIndex}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items` ); const response = await esClient.search< @@ -74,7 +74,7 @@ export const getThreatList = async ({ pit: { id: pitId }, }); - ruleExecutionLogger.debug(`Indicator items retrieved: ${response.hits.hits.length}`); + ruleExecutionLogger.debug(`Retrieved indicator items of size: ${response.hits.hits.length}`); reassignPitId(response.pit_id); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts index 69e2a89a92184..fedfec6707fdf 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts @@ -98,8 +98,10 @@ describe('ml_executor', () => { isAlertSuppressionActive: true, scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction, }); - expect(ruleExecutionLogger.debug).toHaveBeenCalled(); - expect(ruleExecutionLogger.debug.mock.calls[0][0]).toContain('ML jobs are not started'); + expect(ruleExecutionLogger.warn).toHaveBeenCalled(); + expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain( + 'Machine learning job(s) are not started' + ); expect(result.warningMessages.length).toEqual(1); }); @@ -120,8 +122,10 @@ describe('ml_executor', () => { isAlertSuppressionActive: true, scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction, }); - expect(ruleExecutionLogger.debug).toHaveBeenCalled(); - expect(ruleExecutionLogger.debug.mock.calls[0][0]).toContain('ML jobs are not started'); + expect(ruleExecutionLogger.warn).toHaveBeenCalled(); + expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain( + 'Machine learning job(s) are not started' + ); expect(result.warningMessages.length).toEqual(1); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts index 00f6770d26b73..a38ea441c99f8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts @@ -81,19 +81,19 @@ export const mlExecutor = async ({ jobSummaries.some((job) => !isJobStarted(job.jobState, job.datafeedState)) ) { const warningMessage = [ - 'ML jobs are not started', + 'Machine learning job(s) are not started:', ...jobSummaries.map((job) => [ - `Job ID: "${job.id}"`, - `Job name: "${job?.customSettings?.security_app_display_name ?? job.id}"`, - `Job status: "${job.jobState}"`, - `Datafeed status: "${job.datafeedState}"`, - ].join('. ') + `job id: "${job.id}"`, + `job name: "${job?.customSettings?.security_app_display_name ?? job.id}"`, + `job status: "${job.jobState}"`, + `datafeed status: "${job.datafeedState}"`, + ].join(', ') ), - ].join('\n'); + ].join(' '); result.warningMessages.push(warningMessage); - ruleExecutionLogger.debug(warningMessage); + ruleExecutionLogger.warn(warningMessage); result.warning = true; } @@ -139,7 +139,7 @@ export const mlExecutor = async ({ const anomalyCount = filteredAnomalyHits.length; if (anomalyCount) { - ruleExecutionLogger.info(`Alerts from ML anomalies: ${anomalyCount}`); + ruleExecutionLogger.debug(`Found ${anomalyCount} signals from ML anomalies`); } if ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts index 7b5e2fae86b44..7750daed86073 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -284,8 +284,8 @@ export const multiTermsComposite = async ( } retryBatchSize = retryBatchSize / 2; - ruleExecutionLogger.debug( - `New terms query failed due to too many clauses\nError: ${e.message}. Retrying #${retryCount} with ${retryBatchSize} for composite aggregation.` + ruleExecutionLogger.warn( + `New terms query for multiple fields failed due to too many clauses in query: ${e.message}. Retrying #${retryCount} with ${retryBatchSize} for composite aggregation` ); throw e; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index 500795e90d442..4ce9fff66356a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -282,8 +282,8 @@ export const groupAndBulkCreate = async ({ ruleType: 'query', }); addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); - sharedParams.ruleExecutionLogger.info( - `Alerts created: ${bulkCreateResult.createdItemsCount}` + sharedParams.ruleExecutionLogger.debug( + `created ${bulkCreateResult.createdItemsCount} signals` ); } else { const bulkCreateResult = await bulkCreate({ @@ -298,8 +298,8 @@ export const groupAndBulkCreate = async ({ suppressedItemsCount: getNumberOfSuppressedAlerts(bulkCreateResult.createdItems, []), }, }); - sharedParams.ruleExecutionLogger.info( - `Alerts created: ${bulkCreateResult.createdItemsCount}` + sharedParams.ruleExecutionLogger.debug( + `created ${bulkCreateResult.createdItemsCount} signals` ); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index d8ef3eb59e925..2679f7bcf6d26 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -83,7 +83,7 @@ export const bulkCreateWithSuppression = async < }); return enrichedAlerts; } catch (error) { - ruleExecutionLogger.error(`Error enriching alerts\nError: ${error}.`); + ruleExecutionLogger.error(`Alerts enrichment failed: ${error}`); throw error; } finally { enrichmentsTimeFinish = performance.now(); @@ -112,7 +112,7 @@ export const bulkCreateWithSuppression = async < const end = performance.now(); - ruleExecutionLogger.debug(`Bulk processing alerts took ${makeFloatString(end - start)}ms.`); + ruleExecutionLogger.debug(`Alerts bulk process took ${makeFloatString(end - start)} ms`); // query rule type suppression does not happen in memory, so we can't just count createdAlerts and suppressedAlerts // for this rule type we need to look into alerts suppression properties, extract those values and sum up @@ -124,7 +124,7 @@ export const bulkCreateWithSuppression = async < : suppressedAlerts.length; if (!isEmpty(errors)) { - ruleExecutionLogger.warn(`Error bulk processing alerts\nError: ${JSON.stringify(errors)}.`); + ruleExecutionLogger.warn(`Alerts bulk process finished with errors: ${JSON.stringify(errors)}`); return { errors: Object.keys(errors), success: false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/create_set_to_filter_against.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/create_set_to_filter_against.ts index e5b518a57d8e8..e419d13589a57 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/create_set_to_filter_against.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/create_set_to_filter_against.ts @@ -36,8 +36,8 @@ export const createSetToFilterAgainst = async ({ return acc; }, new Set()); - ruleExecutionLogger.trace( - `Distinct values from field: ${[...valuesFromSearchResultField].length}` + ruleExecutionLogger.debug( + `number of distinct values from ${field}: ${[...valuesFromSearchResultField].length}` ); const matchedListItems = await listClient.searchListItemByValues({ @@ -47,7 +47,7 @@ export const createSetToFilterAgainst = async ({ }); ruleExecutionLogger.debug( - `Matched items from list: ${matchedListItems.length}\nList ID: "${listId}".` + `number of matched items from list with id ${listId}: ${matchedListItems.length}` ); return new Set( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.test.ts index ea79ac5b00a42..d5c566383ebba 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.test.ts @@ -62,7 +62,7 @@ describe('filterEventsAgainstList', () => { expect(included.length).toEqual(4); expect(excluded.length).toEqual(0); expect(ruleExecutionLogger.debug.mock.calls[0][0]).toContain( - 'No exception items of type list found' + 'No exception items of type list found - return unfiltered events' ); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.ts index f8629d8ba000b..8a8f7d34aa3a8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/large_list_filters/filter_events_against_list.ts @@ -47,9 +47,7 @@ export const filterEventsAgainstList = async ({ ); if (!atLeastOneLargeValueList) { - ruleExecutionLogger.debug( - 'No exception items of type list found\nReturning unfiltered events.' - ); + ruleExecutionLogger.debug('No exception items of type list found - return unfiltered events'); return [events, []]; } @@ -76,7 +74,7 @@ export const filterEventsAgainstList = async ({ fieldAndSetTuples, }); ruleExecutionLogger.debug( - `Events filtered by exception: ${nextExcludedEvents.length}\nException ID: "${exceptionItem.id}".` + `Exception with id ${exceptionItem.id} filtered out ${nextExcludedEvents.length} events` ); return [nextIncludedEvents, [...excludedEvents, ...nextExcludedEvents]]; }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/log_shard_failure.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/log_shard_failure.ts index 208883817813a..a5110f7ea11bb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/log_shard_failure.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/log_shard_failure.ts @@ -20,7 +20,7 @@ export const logShardFailures = ( isSequenceQuery, JSON.stringify(shardFailures) ); - ruleExecutionLogger.error(`Shard failure\nError: ${shardFailureMessage}.`); + ruleExecutionLogger.error(shardFailureMessage); if (isSequenceQuery) { result.errors.push(shardFailureMessage); } else { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index 219197224351b..14d6ec2931669 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -97,10 +97,10 @@ export const searchAfterAndBulkCreateFactory = async ({ while (toReturn.createdSignalsCount <= maxSignals) { const cycleNum = `cycle ${searchingIteration++}`; try { - ruleExecutionLogger.trace( - `Searching events\n${cycleNum}. Searching events after cursor ${JSON.stringify( - sortIds - )} in index pattern "${inputIndexPattern}".` + ruleExecutionLogger.debug( + `[${cycleNum}] Searching events${ + sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' + } in index pattern "${inputIndexPattern}"` ); const searchAfterQuery = buildEventsSearchQuery({ @@ -152,19 +152,17 @@ export const searchAfterAndBulkCreateFactory = async ({ ); if (totalHits === 0 || searchResult.hits.hits.length === 0) { - ruleExecutionLogger.trace( - `No results found in cycle\n${cycleNum}. Found 0 events after cursor ${JSON.stringify( - sortIds - )}.` + ruleExecutionLogger.debug( + `[${cycleNum}] Found 0 events ${ + sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' + }` ); break; } else { - ruleExecutionLogger.trace( - `Results found in cycle\n${cycleNum}. Found ${ - searchResult.hits.hits.length - } of total ${totalHits} events after cursor ${JSON.stringify( - sortIds - )}. Last cursor: ${JSON.stringify(lastSortIds)}.` + ruleExecutionLogger.debug( + `[${cycleNum}] Found ${searchResult.hits.hits.length} of total ${totalHits} events${ + sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' + }, last cursor ${JSON.stringify(lastSortIds)}` ); } @@ -189,8 +187,8 @@ export const searchAfterAndBulkCreateFactory = async ({ toReturn, }); - ruleExecutionLogger.trace( - `Created alerts from enriched events\n${cycleNum}. Created ${bulkCreateResult.createdItemsCount} alerts from ${enrichedEvents.length} events.` + ruleExecutionLogger.debug( + `[${cycleNum}] Created ${bulkCreateResult.createdItemsCount} alerts from ${enrichedEvents.length} events` ); sendAlertTelemetryEvents( @@ -214,12 +212,13 @@ export const searchAfterAndBulkCreateFactory = async ({ if (lastSortIds != null && lastSortIds.length !== 0 && !hasNegativeNumber) { sortIds = lastSortIds; } else { - ruleExecutionLogger.trace(`Failed to fetch last event cursor\n${cycleNum}.`); + ruleExecutionLogger.debug(`[${cycleNum}] Unable to fetch last event cursor`); break; } } catch (exc: unknown) { ruleExecutionLogger.error( - `Error extracting/processing events or creating alerts\nError: ${JSON.stringify(exc)}.` + 'Unable to extract/process events or create alerts', + JSON.stringify(exc) ); return mergeReturns([ toReturn, @@ -230,9 +229,7 @@ export const searchAfterAndBulkCreateFactory = async ({ ]); } } - ruleExecutionLogger.debug( - `Completed bulk indexing. Alerts created: ${toReturn.createdSignalsCount}.` - ); + ruleExecutionLogger.debug(`Completed bulk indexing of ${toReturn.createdSignalsCount} alert`); if (isLoggedRequestsEnabled) { toReturn.loggedRequests = loggedRequests; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/send_telemetry_events.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/send_telemetry_events.ts index 29a5160e0420e..c3c4f8c4d5f23 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/send_telemetry_events.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/send_telemetry_events.ts @@ -78,6 +78,6 @@ export function sendAlertTelemetryEvents( ); eventsTelemetry.sendAsync(TelemetryChannel.ENDPOINT_ALERTS, filtered); } catch (exc) { - ruleExecutionLogger.debug(`Error queuing telemetry events\nError: ${exc}.`); + ruleExecutionLogger.error(`Queuing telemetry events failed: ${exc}`); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index 834234fe76857..3bd45d5ab85e8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -73,7 +73,7 @@ export const singleSearchAfter = async < loggedRequests, }; } catch (exc) { - ruleExecutionLogger.error(`Error searching events\nError: ${exc}.`); + ruleExecutionLogger.error(`Searching events operation failed: ${exc}`); throw exc; } }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index d9587a50710d6..6389df5b5eb4b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -434,9 +434,9 @@ export const getRuleRangeTuples = async ({ const intervalDuration = parseInterval(interval); if (intervalDuration == null) { ruleExecutionLogger.error( - `Error computing gap between rule runs\nError: could not parse rule interval "${JSON.stringify( + `Failed to compute gap between rule runs: could not parse rule interval "${JSON.stringify( interval - )}".` + )}"` ); return { tuples,