From 87dd41718ae7ccb041bd13a6933a0e1b9964738a Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Fri, 27 Mar 2026 10:57:56 +0100 Subject: [PATCH 01/12] add matched indices count mapping --- .../platform/plugins/shared/event_log/generated/mappings.json | 3 +++ x-pack/platform/plugins/shared/event_log/generated/schemas.ts | 1 + x-pack/platform/plugins/shared/event_log/scripts/mappings.js | 3 +++ 3 files changed, 7 insertions(+) diff --git a/x-pack/platform/plugins/shared/event_log/generated/mappings.json b/x-pack/platform/plugins/shared/event_log/generated/mappings.json index bab32b1306272..33a8753ca3df8 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/mappings.json +++ b/x-pack/platform/plugins/shared/event_log/generated/mappings.json @@ -477,6 +477,9 @@ } } }, + "matched_indices_count": { + "type": "long" + }, "frozen_indices_queried_count": { "type": "long" }, diff --git a/x-pack/platform/plugins/shared/event_log/generated/schemas.ts b/x-pack/platform/plugins/shared/event_log/generated/schemas.ts index ae7c76d82d515..97987c1ebac1a 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/schemas.ts +++ b/x-pack/platform/plugins/shared/event_log/generated/schemas.ts @@ -209,6 +209,7 @@ export const EventSchema = schema.maybe( type: ecsString(), }) ), + matched_indices_count: ecsStringOrNumber(), frozen_indices_queried_count: ecsStringOrNumber(), rule_type_run_duration_ms: ecsStringOrNumber(), process_alerts_duration_ms: ecsStringOrNumber(), diff --git a/x-pack/platform/plugins/shared/event_log/scripts/mappings.js b/x-pack/platform/plugins/shared/event_log/scripts/mappings.js index 14360e7e23bb9..a967afe94f310 100644 --- a/x-pack/platform/plugins/shared/event_log/scripts/mappings.js +++ b/x-pack/platform/plugins/shared/event_log/scripts/mappings.js @@ -243,6 +243,9 @@ exports.EcsCustomPropertyMappings = { }, }, }, + matched_indices_count: { + type: 'long', + }, frozen_indices_queried_count: { type: 'long', }, From add9ca80b310090ea003444d0bacc506f16fdc04 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Fri, 27 Mar 2026 11:02:06 +0100 Subject: [PATCH 02/12] add matched_indices_count rule execution metric --- .../server/lib/alerting_event_logger/alerting_event_logger.ts | 1 + .../alerting/server/task_runner/ad_hoc_task_runner.test.ts | 1 + .../shared/alerting/server/task_runner/task_runner.test.ts | 3 ++- x-pack/platform/plugins/shared/alerting/server/types.ts | 1 + .../rule_types/validation/run_execution_validation.ts | 3 +++ 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index ff8403a24ad85..5aab5e8d7cbc8 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -801,6 +801,7 @@ export function updateEvent(event: IEvent, opts: UpdateEventOpts) { if (consumerMetrics) { set(event, 'kibana.alert.rule.execution.metrics', { ...event.kibana?.alert?.rule?.execution?.metrics, + matched_indices_count: consumerMetrics.matched_indices_count, alerts_candidate_count: consumerMetrics.alerts_candidate_count, alerts_suppressed_count: consumerMetrics.alerts_suppressed_count, frozen_indices_queried_count: consumerMetrics.frozen_indices_queried_count, diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts index 0d958321e0c32..d4c10475682d6 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts @@ -595,6 +595,7 @@ describe('Ad Hoc Task Runner', () => { test('passes consumer metrics to AlertingEventLogger', async () => { const consumerMetrics = { + matched_indices_count: 3, alerts_candidate_count: 100, total_enrichment_duration_ms: 50, }; diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts index 51a2a4689c77e..65c886c6459ee 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts @@ -395,9 +395,10 @@ describe('Task Runner', () => { test('passes consumer metrics to AlertingEventLogger', async () => { const consumerMetrics = { + matched_indices_count: 3, + frozen_indices_queried_count: 3, alerts_candidate_count: 42, alerts_suppressed_count: 7, - frozen_indices_queried_count: 3, }; ruleType.executor.mockImplementation(async ({ services: { ruleMonitoringService } }) => { ruleMonitoringService?.setMetrics(consumerMetrics); diff --git a/x-pack/platform/plugins/shared/alerting/server/types.ts b/x-pack/platform/plugins/shared/alerting/server/types.ts index bfbe693928151..a2956725e6e86 100644 --- a/x-pack/platform/plugins/shared/alerting/server/types.ts +++ b/x-pack/platform/plugins/shared/alerting/server/types.ts @@ -457,6 +457,7 @@ export interface ConsumerExecutionMetrics { total_enrichment_duration_ms: number; gap_duration_s: number; gap_range: { lte: string; gte: string }; + matched_indices_count: number; alerts_candidate_count: number; alerts_suppressed_count: number; frozen_indices_queried_count: number; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts index 3bc0d37ec6aca..9526440d5e09c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts @@ -69,6 +69,9 @@ export const runExecutionValidation = async ( try { const indexPatternsWithMatches = await indexPatterns.getIndexPatternsWithMatches(inputIndex); + // Collect rule execution metrics + ruleExecutionLogger.logMetric('matched_indices_count', indexPatternsWithMatches.length); + if (indexPatternsWithMatches.length === 0) { warnings.push( `Unable to find matching indices for rule ${ruleName}. This warning will persist until one of the following occurs: a matching index is created or the rule is disabled.` From b60dc1952499542b5aebb9ebe441fc8e909dd662 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 31 Mar 2026 11:01:57 +0200 Subject: [PATCH 03/12] add tests --- .../server/fetcher/index_patterns_fetcher.ts | 43 +++-- .../validation/run_execution_validation.ts | 14 +- .../eql_metrics.ts | 137 +++++++++++++-- .../esql_metrics.ts | 119 +++++++++++-- .../indicator_match_metrics.ts | 154 ++++++++++++++++- .../machine_learning_metrics.ts | 24 +++ .../new_terms_metrics.ts | 158 +++++++++++++++--- .../custom_query_metrics.ts | 131 +++++++++++++-- .../threshold_metrics.ts | 143 ++++++++++++++-- 9 files changed, 815 insertions(+), 108 deletions(-) diff --git a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts index ff78341b26ed5..bfee9fc9d5ba2 100644 --- a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts +++ b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts @@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; import { keyBy } from 'lodash'; -import { defer, from } from 'rxjs'; +import { catchError, defer, from, map, of } from 'rxjs'; import { rateLimitingForkJoin } from '../../common/data_views/utils'; import type { QueryDslQueryContainer } from '../../common/types'; @@ -163,7 +163,9 @@ export class IndexPatternsFetcher { * @param indexPatterns - index pattern list * @returns index pattern list of index patterns that match indices */ - async getIndexPatternsWithMatches(indexPatterns: string[]): Promise { + async getIndexPatternsWithMatches( + indexPatterns: string[] + ): Promise<{ matchedIndexPatterns: string[]; matchedIndices?: string[] }> { const indexPatternsObs = indexPatterns.map((indexPattern) => { // when checking a negative pattern, check if the positive pattern exists const indexToQuery = indexPattern.trim().startsWith('-') @@ -176,20 +178,37 @@ export class IndexPatternsFetcher { fields: ['_id'], pattern: indexToQuery, }) + ).pipe( + map((match) => ({ ...match, indexPattern })), + catchError(() => of({ fields: [], indices: [], indexPattern })) ) ); }); - return new Promise((resolve) => { - rateLimitingForkJoin(indexPatternsObs, 3, { fields: [], indices: [] }).subscribe((value) => { - resolve(value.map((v) => v.indices.length > 0)); + return new Promise<{ matchedIndexPatterns: string[]; matchedIndices?: string[] }>((resolve) => { + rateLimitingForkJoin(indexPatternsObs, 3, { + fields: [], + indices: [], + indexPattern: '', + }).subscribe((indexPatternMatches) => { + const matchedIndexPatterns = new Set(); + const uniqueMatchedIndices = new Set(); + + for (const indexPatternMatch of indexPatternMatches) { + if (indexPatternMatch.indices.length > 0) { + matchedIndexPatterns.add(indexPatternMatch.indexPattern); + } + + for (const index of indexPatternMatch.indices) { + uniqueMatchedIndices.add(index); + } + } + + resolve({ + matchedIndexPatterns: Array.from(matchedIndexPatterns), + matchedIndices: Array.from(uniqueMatchedIndices), + }); }); - }) - .then((allPatterns: boolean[]) => - indexPatterns.filter( - (indexPattern, i, self) => self.indexOf(indexPattern) === i && allPatterns[i] - ) - ) - .catch(() => indexPatterns); + }).catch(() => ({ matchedIndexPatterns: indexPatterns })); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts index 9526440d5e09c..33a129dd37a31 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts @@ -67,12 +67,13 @@ export const runExecutionValidation = async ( const indexPatterns = new IndexPatternsFetcher(scopedClusterClient.asCurrentUser); try { - const indexPatternsWithMatches = await indexPatterns.getIndexPatternsWithMatches(inputIndex); + const { matchedIndexPatterns, matchedIndices } = + await indexPatterns.getIndexPatternsWithMatches(inputIndex); // Collect rule execution metrics - ruleExecutionLogger.logMetric('matched_indices_count', indexPatternsWithMatches.length); + ruleExecutionLogger.logMetric('matched_indices_count', matchedIndices?.length); - if (indexPatternsWithMatches.length === 0) { + if (matchedIndexPatterns.length === 0) { warnings.push( `Unable to find matching indices for rule ${ruleName}. This warning will persist until one of the following occurs: a matching index is created or the rule is disabled.` ); @@ -84,11 +85,10 @@ export const runExecutionValidation = async ( if (isThreatParams(params)) { try { - const threatIndexPatternsWithMatches = await indexPatterns.getIndexPatternsWithMatches( - params.threatIndex - ); + const { matchedIndexPatterns: matchedThreatIndexPatterns } = + await indexPatterns.getIndexPatternsWithMatches(params.threatIndex); - if (threatIndexPatternsWithMatches.length === 0) { + if (matchedThreatIndexPatterns.length === 0) { warnings.push( `Unable to find matching threat indicator indices for rule ${ruleName}. This warning will persist until one of the following occurs: a matching threat index is created or the rule is disabled.` ); diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts index aa58f22ea93bc..079c744319cff 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts @@ -6,6 +6,7 @@ */ import expect from 'expect'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, deleteAllAlerts } from '@kbn/detections-response-ftr-services'; import { getEqlRuleParams, @@ -21,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-1', + index: 'logs-test-1', log, }); @@ -31,29 +32,133 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-1', + index: 'logs-test-1,logs-test-2', ignore_unavailable: true, }); - await es.indices.create({ - index: 'logs-1', - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - host: { - properties: { - name: { - type: 'keyword', - }, + + const mappings: MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + host: { + properties: { + name: { + type: 'keyword', }, }, }, }, + }; + await es.indices.create({ + index: 'logs-test-1', + mappings, + }); + await es.indices.create({ + index: 'logs-test-2', + mappings, }); }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('records matched_indices_count for one matching index pattern', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { + name: 'test', + }, + }; + const rule = getEqlRuleParams({ + index: ['logs-test-1'], + query: 'any where true', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(1); + }); + + it('records matched_indices_count for a single index pattern with wildcard', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { + name: 'test', + }, + }; + const rule = getEqlRuleParams({ + index: ['logs-test-*'], + query: 'any where true', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + + it('records matched_indices_count for multiple matching index patterns', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { + name: 'test', + }, + }; + const rule = getEqlRuleParams({ + index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + query: 'any where true', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const timestamp = new Date().toISOString(); @@ -64,7 +169,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: 'any where true', from: 'now-35m', interval: '30m', @@ -93,7 +198,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: 'any where true', alert_suppression: { group_by: ['host.name'], diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts index bddd5e4b5ad16..5a3a707302fc0 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts @@ -6,6 +6,7 @@ */ import expect from 'expect'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, deleteAllAlerts } from '@kbn/detections-response-ftr-services'; import { getEsqlRuleParams, @@ -21,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-1', + index: 'logs-test-1', log, }); @@ -31,29 +32,115 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-1', + index: 'logs-test-1,logs-test-2', ignore_unavailable: true, }); - await es.indices.create({ - index: 'logs-1', - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - host: { - properties: { - name: { - type: 'keyword', - }, + + const mappings: MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + host: { + properties: { + name: { + type: 'keyword', }, }, }, }, + }; + await es.indices.create({ + index: 'logs-test-1', + mappings, + }); + await es.indices.create({ + index: 'logs-test-2', + mappings, }); }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('records matched_indices_count for one source index in the ES|QL query', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getEsqlRuleParams({ + query: 'from logs-test-1 metadata _id, _index, _version', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(1); + }); + + it('records matched_indices_count for a single index pattern with wildcard in the ES|QL query', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getEsqlRuleParams({ + query: 'from logs-test-* metadata _id, _index, _version', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + + it('records matched_indices_count for multiple source indices in the ES|QL query', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getEsqlRuleParams({ + query: 'from logs-te*, logs-test-1, logs-test-2 metadata _id, _index, _version', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const timestamp = new Date().toISOString(); @@ -62,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-1 metadata _id, _index, _version', + query: 'from logs-test-1 metadata _id, _index, _version', from: 'now-35m', interval: '30m', enabled: true, @@ -85,7 +172,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-1 metadata _id, _index, _version', + query: 'from logs-test-1 metadata _id, _index, _version', alert_suppression: { group_by: ['host.name'], duration: { diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts index 065d006667c75..58b48bd87a7fb 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts @@ -7,6 +7,7 @@ import expect from 'expect'; import { v4 as uuidv4 } from 'uuid'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { deleteAllAlerts, deleteAllRules, createRule } from '@kbn/detections-response-ftr-services'; import { dataGeneratorFactory, @@ -22,12 +23,12 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexThreatIndicatorDocuments } = dataGeneratorFactory({ es, - index: 'logs-ti_1', + index: 'ti_1', log, }); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-1', + index: 'logs-test-1', log, }); @@ -35,9 +36,150 @@ export default ({ getService }: FtrProviderContext) => { beforeEach(async () => { await deleteAllAlerts(supertest, log, es); await deleteAllRules(supertest, log); + + await es.indices.delete({ + index: 'logs-test-1,logs-test-2', + ignore_unavailable: true, + }); + + const mappings: MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + id: { + type: 'keyword', + }, + host: { + properties: { + name: { + type: 'keyword', + }, + }, + }, + }, + }; + await es.indices.create({ + index: 'logs-test-1', + mappings, + }); + await es.indices.create({ + index: 'logs-test-2', + mappings, + }); }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('records matched_indices_count for one matching source index pattern', async () => { + const id = uuidv4(); + const timestamp = new Date().toISOString(); + const threatIndicatorDocument = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const document = { + id, + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getThreatMatchRuleParams({ + threat_query: '*:*', + threat_index: ['ti_1'], + query: `id : "${id}"`, + index: ['logs-test-1'], + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexThreatIndicatorDocuments([threatIndicatorDocument]); + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(1); + }); + + it('records matched_indices_count for a single index pattern with wildcard', async () => { + const id = uuidv4(); + const timestamp = new Date().toISOString(); + const threatIndicatorDocument = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const document = { + id, + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getThreatMatchRuleParams({ + threat_query: '*:*', + threat_index: ['ti_1'], + query: `id : "${id}"`, + index: ['logs-test-*'], + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexThreatIndicatorDocuments([threatIndicatorDocument]); + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + + it('records matched_indices_count for multiple matching source index patterns', async () => { + const id = uuidv4(); + const timestamp = new Date().toISOString(); + const threatIndicatorDocument = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const document = { + id, + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getThreatMatchRuleParams({ + threat_query: '*:*', + threat_index: ['ti_1'], + query: `id : "${id}"`, + index: ['logs-te*,', 'logs-test-1', 'logs-test-2'], + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexThreatIndicatorDocuments([threatIndicatorDocument]); + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const id = uuidv4(); @@ -53,9 +195,9 @@ export default ({ getService }: FtrProviderContext) => { }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['logs-ti_1'], + threat_index: ['ti_1'], query: `id : "${id}"`, - index: ['logs-1'], + index: ['logs-test-1'], from: 'now-35m', interval: '30m', enabled: true, @@ -86,9 +228,9 @@ export default ({ getService }: FtrProviderContext) => { }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['logs-ti_1'], + threat_index: ['ti_1'], query: `id : "${id}"`, - index: ['logs-1'], + index: ['logs-test-1'], from: 'now-35m', interval: '30m', alert_suppression: { diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning_metrics.ts index 9b037a999a7b2..fe5c6995a30c1 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning_metrics.ts @@ -82,6 +82,30 @@ export default ({ getService }: FtrProviderContext) => { }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('does not record matched_indices_count for machine learning rules', async () => { + const createdRule = await createRule( + supertest, + log, + getMLRuleParams({ + ...sharedMlRuleRewrites, + anomaly_threshold: 30, + from: '1900-01-01T00:00:00.000Z', + enabled: true, + }) + ); + await getOpenAlerts(supertest, log, es, createdRule); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBeUndefined(); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const createdRule = await createRule( diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts index 7ce693018331d..58e595c4cee4c 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts @@ -6,6 +6,7 @@ */ import expect from 'expect'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, deleteAllAlerts } from '@kbn/detections-response-ftr-services'; import { dataGeneratorFactory, @@ -21,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments } = dataGeneratorFactory({ es, - index: 'logs-1', + index: 'logs-test-1', log, }); @@ -31,32 +32,151 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-1', + index: 'logs-test-1,logs-test-2', ignore_unavailable: true, }); - await es.indices.create({ - index: 'logs-1', - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - host: { - properties: { - name: { - type: 'keyword', - }, - ip: { - type: 'ip', - }, + + const mappings: MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + host: { + properties: { + name: { + type: 'keyword', }, }, }, }, + }; + await es.indices.create({ + index: 'logs-test-1', + mappings, + }); + await es.indices.create({ + index: 'logs-test-2', + mappings, }); }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('records matched_indices_count for one matching index pattern', async () => { + const timestamp = new Date().toISOString(); + const documents = [ + { + '@timestamp': timestamp, + host: { name: 'host-0' }, + }, + { + '@timestamp': timestamp, + host: { name: 'host-0' }, + }, + ]; + + await indexListOfDocuments(documents); + + const createdRule = await createRule( + supertest, + log, + getNewTermsRuleParams({ + index: ['logs-test-1'], + query: '*:*', + new_terms_fields: ['host.name'], + history_window_start: 'now-1h', + from: 'now-35m', + interval: '30m', + enabled: true, + }) + ); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(1); + }); + + it('records matched_indices_count for a single index pattern with wildcard', async () => { + const timestamp = new Date().toISOString(); + const documents = [ + { + '@timestamp': timestamp, + host: { name: 'host-0' }, + }, + { + '@timestamp': timestamp, + host: { name: 'host-0' }, + }, + ]; + + await indexListOfDocuments(documents); + + const createdRule = await createRule( + supertest, + log, + getNewTermsRuleParams({ + index: ['logs-test-*'], + query: '*:*', + new_terms_fields: ['host.name'], + history_window_start: 'now-1h', + from: 'now-35m', + interval: '30m', + enabled: true, + }) + ); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + + it('records matched_indices_count for multiple matching index patterns', async () => { + const timestamp = new Date().toISOString(); + const documents = [ + { + '@timestamp': timestamp, + host: { name: 'host-0' }, + }, + { + '@timestamp': timestamp, + host: { name: 'host-0' }, + }, + ]; + + await indexListOfDocuments(documents); + + const createdRule = await createRule( + supertest, + log, + getNewTermsRuleParams({ + index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + query: '*:*', + new_terms_fields: ['host.name'], + history_window_start: 'now-1h', + from: 'now-35m', + interval: '30m', + enabled: true, + }) + ); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const timestamp = new Date().toISOString(); @@ -77,7 +197,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', @@ -120,7 +240,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts index 2359744b75ba1..bd37096b76cae 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts @@ -6,6 +6,7 @@ */ import expect from 'expect'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, deleteAllAlerts } from '@kbn/detections-response-ftr-services'; import { getCustomQueryRuleParams, @@ -21,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-1', + index: 'logs-test-1', log, }); @@ -31,29 +32,127 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-1', + index: 'logs-test-1,logs-test-2', ignore_unavailable: true, }); - await es.indices.create({ - index: 'logs-1', - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - host: { - properties: { - name: { - type: 'keyword', - }, + + const mappings: MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + host: { + properties: { + name: { + type: 'keyword', }, }, }, }, + }; + await es.indices.create({ + index: 'logs-test-1', + mappings, + }); + await es.indices.create({ + index: 'logs-test-2', + mappings, }); }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('records matched_indices_count for one matching index pattern', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getCustomQueryRuleParams({ + index: ['logs-test-1'], + query: '*:*', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(1); + }); + + it('records matched_indices_count for a single index pattern with wildcard', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getCustomQueryRuleParams({ + index: ['logs-test-*'], + query: '*:*', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + + it('records matched_indices_count for multiple matching index patterns', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getCustomQueryRuleParams({ + index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + query: '*:*', + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const timestamp = new Date().toISOString(); @@ -62,7 +161,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: '*:*', from: 'now-35m', interval: '30m', @@ -89,7 +188,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: '*:*', alert_suppression: { group_by: ['host.name'], diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts index a774ec1d993d0..89f6ff005d8f6 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts @@ -6,6 +6,7 @@ */ import expect from 'expect'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, @@ -27,7 +28,7 @@ export default ({ getService }: FtrProviderContext) => { const detectionsApi = getService('detectionsApi'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-1', + index: 'logs-test-1', log, }); @@ -37,29 +38,139 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-1', + index: 'logs-test-1,logs-test-2', ignore_unavailable: true, }); - await es.indices.create({ - index: 'logs-1', - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - host: { - properties: { - name: { - type: 'keyword', - }, + + const mappings: MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + host: { + properties: { + name: { + type: 'keyword', }, }, }, }, + }; + await es.indices.create({ + index: 'logs-test-1', + mappings, + }); + await es.indices.create({ + index: 'logs-test-2', + mappings, }); }); describe('metrics collection', () => { + describe('matched_indices_count', () => { + it('records matched_indices_count for one matching index pattern', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + + await indexListOfSourceDocuments([document, document]); + + const createdRule = await createRule( + supertest, + log, + getThresholdRuleParams({ + index: ['logs-test-1'], + query: '*:*', + threshold: { + field: ['host.name'], + value: 2, + }, + from: 'now-35m', + interval: '30m', + enabled: true, + }) + ); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(1); + }); + + it('records matched_indices_count for a single index pattern with wildcard', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + + await indexListOfSourceDocuments([document, document]); + + const createdRule = await createRule( + supertest, + log, + getThresholdRuleParams({ + index: ['logs-test-*'], + query: '*:*', + threshold: { + field: ['host.name'], + value: 2, + }, + from: 'now-35m', + interval: '30m', + enabled: true, + }) + ); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + + it('records matched_indices_count for multiple matching index patterns', async () => { + const timestamp = new Date().toISOString(); + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + + await indexListOfSourceDocuments([document, document]); + + const createdRule = await createRule( + supertest, + log, + getThresholdRuleParams({ + index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + query: '*:*', + threshold: { + field: ['host.name'], + value: 2, + }, + from: 'now-35m', + interval: '30m', + enabled: true, + }) + ); + + const { matched_indices_count } = await getLatestSecurityRuleExecutionMetricsFromEventLog( + es, + log, + createdRule.id + ); + + expect(matched_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const timestamp = new Date().toISOString(); @@ -74,7 +185,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-1'], + index: ['logs-test-1'], query: '*:*', threshold: { field: ['host.name'], @@ -106,7 +217,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-1'], + index: ['logs-test-1', 'logs-test-2'], query: '*:*', threshold: { field: ['host.name'], From 622999e16af578a3db403db4c3da6f05087d04b7 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 31 Mar 2026 11:08:16 +0200 Subject: [PATCH 04/12] add matched_indicator_indices_count mapping --- .../platform/plugins/shared/event_log/generated/mappings.json | 3 +++ x-pack/platform/plugins/shared/event_log/generated/schemas.ts | 1 + x-pack/platform/plugins/shared/event_log/scripts/mappings.js | 3 +++ 3 files changed, 7 insertions(+) diff --git a/x-pack/platform/plugins/shared/event_log/generated/mappings.json b/x-pack/platform/plugins/shared/event_log/generated/mappings.json index 33a8753ca3df8..41feb581e5791 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/mappings.json +++ b/x-pack/platform/plugins/shared/event_log/generated/mappings.json @@ -480,6 +480,9 @@ "matched_indices_count": { "type": "long" }, + "matched_indicator_indices_count": { + "type": "long" + }, "frozen_indices_queried_count": { "type": "long" }, diff --git a/x-pack/platform/plugins/shared/event_log/generated/schemas.ts b/x-pack/platform/plugins/shared/event_log/generated/schemas.ts index 97987c1ebac1a..ef92d5e141e56 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/schemas.ts +++ b/x-pack/platform/plugins/shared/event_log/generated/schemas.ts @@ -210,6 +210,7 @@ export const EventSchema = schema.maybe( }) ), matched_indices_count: ecsStringOrNumber(), + matched_indicator_indices_count: ecsStringOrNumber(), frozen_indices_queried_count: ecsStringOrNumber(), rule_type_run_duration_ms: ecsStringOrNumber(), process_alerts_duration_ms: ecsStringOrNumber(), diff --git a/x-pack/platform/plugins/shared/event_log/scripts/mappings.js b/x-pack/platform/plugins/shared/event_log/scripts/mappings.js index a967afe94f310..b665d007a84e8 100644 --- a/x-pack/platform/plugins/shared/event_log/scripts/mappings.js +++ b/x-pack/platform/plugins/shared/event_log/scripts/mappings.js @@ -246,6 +246,9 @@ exports.EcsCustomPropertyMappings = { matched_indices_count: { type: 'long', }, + matched_indicator_indices_count: { + type: 'long', + }, frozen_indices_queried_count: { type: 'long', }, From 2e070af4c758dcb526a718198e9ed7dd09f0854d Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 31 Mar 2026 11:32:57 +0200 Subject: [PATCH 05/12] add matched_indicator_indices_count metric logging --- .../alerting_event_logger.ts | 1 + .../plugins/shared/alerting/server/types.ts | 1 + .../validation/run_execution_validation.ts | 12 +- .../indicator_match_metrics.ts | 154 ++++++++++++++---- 4 files changed, 135 insertions(+), 33 deletions(-) diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index 5aab5e8d7cbc8..a95fc68614f61 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -802,6 +802,7 @@ export function updateEvent(event: IEvent, opts: UpdateEventOpts) { set(event, 'kibana.alert.rule.execution.metrics', { ...event.kibana?.alert?.rule?.execution?.metrics, matched_indices_count: consumerMetrics.matched_indices_count, + matched_indicator_indices_count: consumerMetrics.matched_indicator_indices_count, alerts_candidate_count: consumerMetrics.alerts_candidate_count, alerts_suppressed_count: consumerMetrics.alerts_suppressed_count, frozen_indices_queried_count: consumerMetrics.frozen_indices_queried_count, diff --git a/x-pack/platform/plugins/shared/alerting/server/types.ts b/x-pack/platform/plugins/shared/alerting/server/types.ts index a2956725e6e86..2ebb342301663 100644 --- a/x-pack/platform/plugins/shared/alerting/server/types.ts +++ b/x-pack/platform/plugins/shared/alerting/server/types.ts @@ -458,6 +458,7 @@ export interface ConsumerExecutionMetrics { gap_duration_s: number; gap_range: { lte: string; gte: string }; matched_indices_count: number; + matched_indicator_indices_count: number; alerts_candidate_count: number; alerts_suppressed_count: number; frozen_indices_queried_count: number; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts index 33a129dd37a31..3eeea6d9eb5b7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts @@ -85,8 +85,16 @@ export const runExecutionValidation = async ( if (isThreatParams(params)) { try { - const { matchedIndexPatterns: matchedThreatIndexPatterns } = - await indexPatterns.getIndexPatternsWithMatches(params.threatIndex); + const { + matchedIndexPatterns: matchedThreatIndexPatterns, + matchedIndices: matchedThreatIndices, + } = await indexPatterns.getIndexPatternsWithMatches(params.threatIndex); + + // Collect rule execution metrics + ruleExecutionLogger.logMetric( + 'matched_indicator_indices_count', + matchedThreatIndices?.length + ); if (matchedThreatIndexPatterns.length === 0) { warnings.push( diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts index 58b48bd87a7fb..04cdaf422794c 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts @@ -6,7 +6,6 @@ */ import expect from 'expect'; -import { v4 as uuidv4 } from 'uuid'; import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { deleteAllAlerts, deleteAllRules, createRule } from '@kbn/detections-response-ftr-services'; import { @@ -23,12 +22,12 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexThreatIndicatorDocuments } = dataGeneratorFactory({ es, - index: 'ti_1', + index: 'ti_test_1', log, }); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-test-1', + index: 'test-data-1', log, }); @@ -38,7 +37,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-test-1,logs-test-2', + index: 'ti_test_1,ti_test_2,test-data-1,test-data-2', ignore_unavailable: true, }); @@ -60,11 +59,19 @@ export default ({ getService }: FtrProviderContext) => { }, }; await es.indices.create({ - index: 'logs-test-1', + index: 'ti_test_1', mappings, }); await es.indices.create({ - index: 'logs-test-2', + index: 'ti_test_2', + mappings, + }); + await es.indices.create({ + index: 'test-data-1', + mappings, + }); + await es.indices.create({ + index: 'test-data-2', mappings, }); }); @@ -72,22 +79,20 @@ export default ({ getService }: FtrProviderContext) => { describe('metrics collection', () => { describe('matched_indices_count', () => { it('records matched_indices_count for one matching source index pattern', async () => { - const id = uuidv4(); const timestamp = new Date().toISOString(); const threatIndicatorDocument = { '@timestamp': timestamp, host: { name: 'test-1' }, }; const document = { - id, '@timestamp': timestamp, host: { name: 'test-1' }, }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['ti_1'], - query: `id : "${id}"`, - index: ['logs-test-1'], + threat_index: ['ti_test_1'], + query: '*:*', + index: ['test-data-1'], from: 'now-35m', interval: '30m', enabled: true, @@ -108,22 +113,20 @@ export default ({ getService }: FtrProviderContext) => { }); it('records matched_indices_count for a single index pattern with wildcard', async () => { - const id = uuidv4(); const timestamp = new Date().toISOString(); const threatIndicatorDocument = { '@timestamp': timestamp, host: { name: 'test-1' }, }; const document = { - id, '@timestamp': timestamp, host: { name: 'test-1' }, }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['ti_1'], - query: `id : "${id}"`, - index: ['logs-test-*'], + threat_index: ['ti_test_1'], + query: '*:*', + index: ['test-data-*'], from: 'now-35m', interval: '30m', enabled: true, @@ -144,22 +147,20 @@ export default ({ getService }: FtrProviderContext) => { }); it('records matched_indices_count for multiple matching source index patterns', async () => { - const id = uuidv4(); const timestamp = new Date().toISOString(); const threatIndicatorDocument = { '@timestamp': timestamp, host: { name: 'test-1' }, }; const document = { - id, '@timestamp': timestamp, host: { name: 'test-1' }, }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['ti_1'], - query: `id : "${id}"`, - index: ['logs-te*,', 'logs-test-1', 'logs-test-2'], + threat_index: ['ti_test_1'], + query: '*:*', + index: ['test-da*,', 'test-data-1', 'test-data-2'], from: 'now-35m', interval: '30m', enabled: true, @@ -180,24 +181,117 @@ export default ({ getService }: FtrProviderContext) => { }); }); + describe('matched_indicator_indices_count', () => { + it('records matched_indicator_indices_count for one matching source index pattern', async () => { + const timestamp = new Date().toISOString(); + const threatIndicatorDocument = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getThreatMatchRuleParams({ + threat_query: '*:*', + threat_index: ['ti_test_1'], + query: '*:*', + index: ['test-data-1'], + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexThreatIndicatorDocuments([threatIndicatorDocument]); + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indicator_indices_count } = + await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id); + + expect(matched_indicator_indices_count).toBe(1); + }); + + it('records matched_indicator_indices_count for a single index pattern with wildcard', async () => { + const timestamp = new Date().toISOString(); + const threatIndicatorDocument = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getThreatMatchRuleParams({ + threat_query: '*:*', + threat_index: ['ti_test_*'], + query: '*:*', + index: ['test-data-*'], + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexThreatIndicatorDocuments([threatIndicatorDocument]); + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indicator_indices_count } = + await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id); + + expect(matched_indicator_indices_count).toBe(2); + }); + + it('records matched_indicator_indices_count for multiple matching source index patterns', async () => { + const timestamp = new Date().toISOString(); + const threatIndicatorDocument = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const document = { + '@timestamp': timestamp, + host: { name: 'test-1' }, + }; + const rule = getThreatMatchRuleParams({ + threat_query: '*:*', + threat_index: ['ti_te*', 'ti_test_1', 'ti_test_2'], + query: '*:*', + index: ['test-da*,', 'test-data-1', 'test-data-2'], + from: 'now-35m', + interval: '30m', + enabled: true, + }); + + await indexThreatIndicatorDocuments([threatIndicatorDocument]); + await indexListOfSourceDocuments([document]); + + const createdRule = await createRule(supertest, log, rule); + + const { matched_indicator_indices_count } = + await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id); + + expect(matched_indicator_indices_count).toBe(2); + }); + }); + describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { - const id = uuidv4(); const timestamp = new Date().toISOString(); const threatIndicatorDocument = { '@timestamp': timestamp, host: { name: 'test-1' }, }; const document = { - id, '@timestamp': timestamp, host: { name: 'test-1' }, }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['ti_1'], - query: `id : "${id}"`, - index: ['logs-test-1'], + threat_index: ['ti_test_1'], + query: '*:*', + index: ['test-data-1'], from: 'now-35m', interval: '30m', enabled: true, @@ -215,22 +309,20 @@ export default ({ getService }: FtrProviderContext) => { }); it('records alerts_candidate_count higher than the number of suppressed alerts', async () => { - const id = uuidv4(); const timestamp = new Date().toISOString(); const threatIndicatorDocument = { '@timestamp': timestamp, host: { name: 'test-1' }, }; const document = { - id, '@timestamp': timestamp, host: { name: 'test-1' }, }; const rule = getThreatMatchRuleParams({ threat_query: '*:*', - threat_index: ['ti_1'], - query: `id : "${id}"`, - index: ['logs-test-1'], + threat_index: ['ti_test_1'], + query: '*:*', + index: ['test-data-1'], from: 'now-35m', interval: '30m', alert_suppression: { From 6569f4c33bf0da407c8eb27f4ad9c1fa49f1441c Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 31 Mar 2026 11:33:40 +0200 Subject: [PATCH 06/12] rename indices to avoid accidental naming collision --- .../trial_license_complete_tier/eql_metrics.ts | 18 +++++++++--------- .../esql_metrics.ts | 18 +++++++++--------- .../new_terms_metrics.ts | 18 +++++++++--------- .../custom_query_metrics.ts | 18 +++++++++--------- .../threshold_metrics.ts | 18 +++++++++--------- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts index 079c744319cff..cc2ff73949bfa 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts @@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-test-1', + index: 'test-data-1', log, }); @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-test-1,logs-test-2', + index: 'test-data-1,test-data-2', ignore_unavailable: true, }); @@ -51,11 +51,11 @@ export default ({ getService }: FtrProviderContext) => { }, }; await es.indices.create({ - index: 'logs-test-1', + index: 'test-data-1', mappings, }); await es.indices.create({ - index: 'logs-test-2', + index: 'test-data-2', mappings, }); }); @@ -71,7 +71,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: 'any where true', from: 'now-35m', interval: '30m', @@ -103,7 +103,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-test-*'], + index: ['test-data-*'], query: 'any where true', from: 'now-35m', interval: '30m', @@ -135,7 +135,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + index: ['test-da*', 'test-data-1', 'test-data-2'], query: 'any where true', from: 'now-35m', interval: '30m', @@ -169,7 +169,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: 'any where true', from: 'now-35m', interval: '30m', @@ -198,7 +198,7 @@ export default ({ getService }: FtrProviderContext) => { }, }; const rule = getEqlRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: 'any where true', alert_suppression: { group_by: ['host.name'], diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts index 5a3a707302fc0..4417c27830a00 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts @@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-test-1', + index: 'test-data-1', log, }); @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-test-1,logs-test-2', + index: 'test-data-1,test-data-2', ignore_unavailable: true, }); @@ -51,11 +51,11 @@ export default ({ getService }: FtrProviderContext) => { }, }; await es.indices.create({ - index: 'logs-test-1', + index: 'test-data-1', mappings, }); await es.indices.create({ - index: 'logs-test-2', + index: 'test-data-2', mappings, }); }); @@ -69,7 +69,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-test-1 metadata _id, _index, _version', + query: 'from test-data-1 metadata _id, _index, _version', from: 'now-35m', interval: '30m', enabled: true, @@ -95,7 +95,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-test-* metadata _id, _index, _version', + query: 'from test-data-* metadata _id, _index, _version', from: 'now-35m', interval: '30m', enabled: true, @@ -121,7 +121,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-te*, logs-test-1, logs-test-2 metadata _id, _index, _version', + query: 'from test-da*, test-data-1, test-data-2 metadata _id, _index, _version', from: 'now-35m', interval: '30m', enabled: true, @@ -149,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-test-1 metadata _id, _index, _version', + query: 'from test-data-1 metadata _id, _index, _version', from: 'now-35m', interval: '30m', enabled: true, @@ -172,7 +172,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getEsqlRuleParams({ - query: 'from logs-test-1 metadata _id, _index, _version', + query: 'from test-data-1 metadata _id, _index, _version', alert_suppression: { group_by: ['host.name'], duration: { diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts index 58e595c4cee4c..c819d6abc7f17 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms_metrics.ts @@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments } = dataGeneratorFactory({ es, - index: 'logs-test-1', + index: 'test-data-1', log, }); @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-test-1,logs-test-2', + index: 'test-data-1,test-data-2', ignore_unavailable: true, }); @@ -51,11 +51,11 @@ export default ({ getService }: FtrProviderContext) => { }, }; await es.indices.create({ - index: 'logs-test-1', + index: 'test-data-1', mappings, }); await es.indices.create({ - index: 'logs-test-2', + index: 'test-data-2', mappings, }); }); @@ -81,7 +81,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', @@ -119,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-test-*'], + index: ['test-data-*'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', @@ -157,7 +157,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + index: ['test-da*', 'test-data-1', 'test-data-2'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', @@ -197,7 +197,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', @@ -240,7 +240,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getNewTermsRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', new_terms_fields: ['host.name'], history_window_start: 'now-1h', diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts index bd37096b76cae..2b8c65a6f7624 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query_metrics.ts @@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-test-1', + index: 'test-data-1', log, }); @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-test-1,logs-test-2', + index: 'test-data-1,test-data-2', ignore_unavailable: true, }); @@ -51,11 +51,11 @@ export default ({ getService }: FtrProviderContext) => { }, }; await es.indices.create({ - index: 'logs-test-1', + index: 'test-data-1', mappings, }); await es.indices.create({ - index: 'logs-test-2', + index: 'test-data-2', mappings, }); }); @@ -69,7 +69,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', from: 'now-35m', interval: '30m', @@ -99,7 +99,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-test-*'], + index: ['test-data-*'], query: '*:*', from: 'now-35m', interval: '30m', @@ -129,7 +129,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + index: ['test-da*', 'test-data-1', 'test-data-2'], query: '*:*', from: 'now-35m', interval: '30m', @@ -161,7 +161,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', from: 'now-35m', interval: '30m', @@ -188,7 +188,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: 'test-1' }, }; const rule = getCustomQueryRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', alert_suppression: { group_by: ['host.name'], diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts index 89f6ff005d8f6..864ba56d9a640 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold_metrics.ts @@ -28,7 +28,7 @@ export default ({ getService }: FtrProviderContext) => { const detectionsApi = getService('detectionsApi'); const { indexListOfDocuments: indexListOfSourceDocuments } = dataGeneratorFactory({ es, - index: 'logs-test-1', + index: 'test-data-1', log, }); @@ -38,7 +38,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllRules(supertest, log); await es.indices.delete({ - index: 'logs-test-1,logs-test-2', + index: 'test-data-1,test-data-2', ignore_unavailable: true, }); @@ -57,11 +57,11 @@ export default ({ getService }: FtrProviderContext) => { }, }; await es.indices.create({ - index: 'logs-test-1', + index: 'test-data-1', mappings, }); await es.indices.create({ - index: 'logs-test-2', + index: 'test-data-2', mappings, }); }); @@ -81,7 +81,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', threshold: { field: ['host.name'], @@ -115,7 +115,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-test-*'], + index: ['test-data-*'], query: '*:*', threshold: { field: ['host.name'], @@ -149,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-te*', 'logs-test-1', 'logs-test-2'], + index: ['test-da*', 'test-data-1', 'test-data-2'], query: '*:*', threshold: { field: ['host.name'], @@ -185,7 +185,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-test-1'], + index: ['test-data-1'], query: '*:*', threshold: { field: ['host.name'], @@ -217,7 +217,7 @@ export default ({ getService }: FtrProviderContext) => { supertest, log, getThresholdRuleParams({ - index: ['logs-test-1', 'logs-test-2'], + index: ['test-data-1', 'test-data-2'], query: '*:*', threshold: { field: ['host.name'], From 45f80f61536ce8a8f1eeb71a7238dd664653d464 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 31 Mar 2026 12:43:30 +0200 Subject: [PATCH 07/12] fix getIndexPatternMatches usage --- .../fetcher/index_patterns_fetcher.test.ts | 30 +++++++++---------- .../server/fetcher/index_patterns_fetcher.ts | 21 ++++++++----- .../internal/existing_indices.ts | 4 +-- .../knowledge_base/index.test.ts | 4 +-- .../knowledge_base/index.ts | 4 +-- .../query/create_query_alert_type.test.ts | 6 ++-- .../validation/run_execution_validation.ts | 7 +++-- 7 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts index 29511ab9b199e..26b52c84a00ff 100644 --- a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts +++ b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts @@ -101,32 +101,33 @@ describe('Index Pattern Fetcher - server', () => { expect(esClient.rollup.getRollupIndexCaps).toHaveBeenCalledTimes(0); }); - describe('getExistingIndices', () => { - test('getExistingIndices returns the valid matched indices', async () => { + describe('getIndexPatternMatches', () => { + test('returns the valid matched indices', async () => { indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); indexPatterns.getFieldsForWildcard = jest .fn() .mockResolvedValueOnce({ indices: ['length'] }) .mockResolvedValue({ indices: [] }); - const result = await indexPatterns.getIndexPatternsWithMatches([ - 'packetbeat-*', - 'filebeat-*', - ]); + const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); expect(indexPatterns.getFieldsForWildcard).toBeCalledTimes(2); - expect(result.length).toBe(1); + expect(result.matchedIndexPatterns.length).toBe(1); + expect(result.matchedIndices?.length).toBe(1); }); - test('getExistingIndices checks the positive pattern if provided with a negative pattern', async () => { + test('checks the positive pattern if provided with a negative pattern', async () => { indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); const mockFn = jest.fn().mockResolvedValue({ indices: ['length'] }); indexPatterns.getFieldsForWildcard = mockFn; - const result = await indexPatterns.getIndexPatternsWithMatches(['-filebeat-*', 'filebeat-*']); + const result = await indexPatterns.getIndexPatternMatches(['-filebeat-*', 'filebeat-*']); expect(mockFn.mock.calls[0][0].pattern).toEqual('filebeat-*'); expect(mockFn.mock.calls[1][0].pattern).toEqual('filebeat-*'); - expect(result).toEqual(['-filebeat-*', 'filebeat-*']); + expect(result).toEqual({ + matchedIndexPatterns: ['-filebeat-*', 'filebeat-*'], + matchedIndices: ['length'], + }); }); - test('getExistingIndices handles an error', async () => { + test('handles an error', async () => { indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); indexPatterns.getFieldsForWildcard = jest .fn() @@ -134,11 +135,8 @@ describe('Index Pattern Fetcher - server', () => { throw new DataViewMissingIndices('Catch me if you can!'); }) .mockImplementation(() => Promise.resolve({ indices: ['length'] })); - const result = await indexPatterns.getIndexPatternsWithMatches([ - 'packetbeat-*', - 'filebeat-*', - ]); - expect(result).toEqual(['filebeat-*']); + const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); + expect(result).toEqual({ matchedIndexPatterns: ['filebeat-*'], matchedIndices: ['length'] }); }); }); }); diff --git a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts index bfee9fc9d5ba2..b3b1f1469f329 100644 --- a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts +++ b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts @@ -48,6 +48,11 @@ interface IndexPatternsFetcherOptionalParams { rollupsEnabled?: boolean; } +interface GetIndexPatternMatchesResult { + matchedIndexPatterns: string[]; + matchedIndices?: string[]; +} + export class IndexPatternsFetcher { private readonly uiSettingsClient?: IUiSettingsClient; private readonly allowNoIndices: boolean; @@ -159,13 +164,15 @@ export class IndexPatternsFetcher { } /** - * Get existing index pattern list by providing string array index pattern list. - * @param indexPatterns - index pattern list - * @returns index pattern list of index patterns that match indices + * For each input pattern, checks whether it resolves to at least one backing index. + * + * @param indexPatterns - Index patterns to check (may include wildcards and negated entries). + * @returns Resolves to {@link GetIndexPatternMatchesResult}: + * - `matchedIndexPatterns`: input patterns that matched at least one index. + * - `matchedIndices`: deduplicated concrete index names across all matches (omitted if the + * operation fails; then `matchedIndexPatterns` is the original `indexPatterns` list). */ - async getIndexPatternsWithMatches( - indexPatterns: string[] - ): Promise<{ matchedIndexPatterns: string[]; matchedIndices?: string[] }> { + async getIndexPatternMatches(indexPatterns: string[]): Promise { const indexPatternsObs = indexPatterns.map((indexPattern) => { // when checking a negative pattern, check if the positive pattern exists const indexToQuery = indexPattern.trim().startsWith('-') @@ -185,7 +192,7 @@ export class IndexPatternsFetcher { ); }); - return new Promise<{ matchedIndexPatterns: string[]; matchedIndices?: string[] }>((resolve) => { + return new Promise((resolve) => { rateLimitingForkJoin(indexPatternsObs, 3, { fields: [], indices: [], diff --git a/src/platform/plugins/shared/data_views/server/rest_api_routes/internal/existing_indices.ts b/src/platform/plugins/shared/data_views/server/rest_api_routes/internal/existing_indices.ts index 54aa7ba824e68..fe0118930900c 100644 --- a/src/platform/plugins/shared/data_views/server/rest_api_routes/internal/existing_indices.ts +++ b/src/platform/plugins/shared/data_views/server/rest_api_routes/internal/existing_indices.ts @@ -44,8 +44,8 @@ export const handler: RequestHandler<{}, { indices: string | string[] }, string[ const elasticsearchClient = core.elasticsearch.client.asCurrentUser; const indexPatterns = new IndexPatternsFetcher(elasticsearchClient); - const response: string[] = await indexPatterns.getIndexPatternsWithMatches(indexArray); - return res.ok({ body: response }); + const { matchedIndexPatterns } = await indexPatterns.getIndexPatternMatches(indexArray); + return res.ok({ body: matchedIndexPatterns }); } catch (error) { return res.badRequest(); } diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.test.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.test.ts index 864e2115cfaba..421e783ff8ffb 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.test.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.test.ts @@ -589,9 +589,9 @@ describe('AIAssistantKnowledgeBaseDataClient', () => { describe('getAssistantTools', () => { it('should return structured tools for relevant index entries', async () => { - IndexPatternsFetcher.prototype.getIndexPatternsWithMatches = jest + IndexPatternsFetcher.prototype.getIndexPatternMatches = jest .fn() - .mockResolvedValue(['test']); + .mockResolvedValue({ matchedIndexPatterns: ['test'] }); esClientMock.search.mockReturnValue( // @ts-expect-error not full response interface getKnowledgeBaseEntrySearchEsMock('index') diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts index fa60b81f7a368..4819979d05b17 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts @@ -920,13 +920,13 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { if (results) { const entries = transformESSearchToKnowledgeBaseEntry(results.data) as IndexEntry[]; const indexPatternFetcher = new IndexPatternsFetcher(esClient); - const existingIndices = await indexPatternFetcher.getIndexPatternsWithMatches( + const { matchedIndexPatterns } = await indexPatternFetcher.getIndexPatternMatches( map(entries, 'index') ); return ( entries // Filter out any IndexEntries that don't have an existing index - .filter((entry) => existingIndices.includes(entry.index)) + .filter((entry) => matchedIndexPatterns.includes(entry.index)) .map((indexEntry) => { return getStructuredToolForIndexEntry({ indexEntry, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index de72f1086d62f..15908fe44bfef 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -24,7 +24,9 @@ import { createMockEndpointAppContextService } from '../../../../endpoint/mocks' jest.mock('@kbn/data-views-plugin/server', () => ({ ...jest.requireActual('@kbn/data-views-plugin/server'), IndexPatternsFetcher: jest.fn().mockImplementation(() => ({ - getIndexPatternsWithMatches: jest.fn().mockResolvedValue(['some-index']), + getIndexPatternsWithMatches: jest + .fn() + .mockResolvedValue({ matchedIndexPatterns: ['some-index'] }), })), })); @@ -127,7 +129,7 @@ describe('Custom Query Alerts', () => { it('short-circuits and writes a warning if no indices are found', async () => { (IndexPatternsFetcher as jest.Mock).mockImplementationOnce(() => ({ - getIndexPatternsWithMatches: jest.fn().mockResolvedValue([]), + getIndexPatternsWithMatches: jest.fn().mockResolvedValue({ matchedIndexPatterns: [] }), })); const queryAlertType = securityRuleTypeWrapper( createQueryAlertType({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts index 3eeea6d9eb5b7..bd98c9f5e3cd0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts @@ -67,8 +67,9 @@ export const runExecutionValidation = async ( const indexPatterns = new IndexPatternsFetcher(scopedClusterClient.asCurrentUser); try { - const { matchedIndexPatterns, matchedIndices } = - await indexPatterns.getIndexPatternsWithMatches(inputIndex); + const { matchedIndexPatterns, matchedIndices } = await indexPatterns.getIndexPatternMatches( + inputIndex + ); // Collect rule execution metrics ruleExecutionLogger.logMetric('matched_indices_count', matchedIndices?.length); @@ -88,7 +89,7 @@ export const runExecutionValidation = async ( const { matchedIndexPatterns: matchedThreatIndexPatterns, matchedIndices: matchedThreatIndices, - } = await indexPatterns.getIndexPatternsWithMatches(params.threatIndex); + } = await indexPatterns.getIndexPatternMatches(params.threatIndex); // Collect rule execution metrics ruleExecutionLogger.logMetric( From d39ef36ade906d477d44ee0fe04aea8d8cb4e143 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 31 Mar 2026 14:08:51 +0200 Subject: [PATCH 08/12] fix failed tests --- .../rule_types/query/create_query_alert_type.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index 15908fe44bfef..f77d15752a5bc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -24,9 +24,7 @@ import { createMockEndpointAppContextService } from '../../../../endpoint/mocks' jest.mock('@kbn/data-views-plugin/server', () => ({ ...jest.requireActual('@kbn/data-views-plugin/server'), IndexPatternsFetcher: jest.fn().mockImplementation(() => ({ - getIndexPatternsWithMatches: jest - .fn() - .mockResolvedValue({ matchedIndexPatterns: ['some-index'] }), + getIndexPatternMatches: jest.fn().mockResolvedValue({ matchedIndexPatterns: ['some-index'] }), })), })); @@ -129,7 +127,7 @@ describe('Custom Query Alerts', () => { it('short-circuits and writes a warning if no indices are found', async () => { (IndexPatternsFetcher as jest.Mock).mockImplementationOnce(() => ({ - getIndexPatternsWithMatches: jest.fn().mockResolvedValue({ matchedIndexPatterns: [] }), + getIndexPatternMatches: jest.fn().mockResolvedValue({ matchedIndexPatterns: [] }), })); const queryAlertType = securityRuleTypeWrapper( createQueryAlertType({ From 6d91d3dabebba69875ebfec74614c2a17fda2584 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 1 Apr 2026 11:51:52 +0200 Subject: [PATCH 09/12] fix index patterns in tests --- .../trial_license_complete_tier/indicator_match_metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts index 04cdaf422794c..1bc847240cd2a 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts @@ -160,7 +160,7 @@ export default ({ getService }: FtrProviderContext) => { threat_query: '*:*', threat_index: ['ti_test_1'], query: '*:*', - index: ['test-da*,', 'test-data-1', 'test-data-2'], + index: ['test-da*', 'test-data-1', 'test-data-2'], from: 'now-35m', interval: '30m', enabled: true, From c63981395255589c0bded5cec010d89d0c6d593b Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 1 Apr 2026 11:52:19 +0200 Subject: [PATCH 10/12] handle excluding index patterns --- .../fetcher/index_patterns_fetcher.test.ts | 150 +++++++++++++++--- .../server/fetcher/index_patterns_fetcher.ts | 67 +++++--- 2 files changed, 172 insertions(+), 45 deletions(-) diff --git a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts index 26b52c84a00ff..3ff6aa895da6c 100644 --- a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts +++ b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts @@ -102,28 +102,121 @@ describe('Index Pattern Fetcher - server', () => { }); describe('getIndexPatternMatches', () => { - test('returns the valid matched indices', async () => { - indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); - indexPatterns.getFieldsForWildcard = jest - .fn() - .mockResolvedValueOnce({ indices: ['length'] }) - .mockResolvedValue({ indices: [] }); - const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); - expect(indexPatterns.getFieldsForWildcard).toBeCalledTimes(2); - expect(result.matchedIndexPatterns.length).toBe(1); - expect(result.matchedIndices?.length).toBe(1); + describe('without negated index patterns', () => { + test('returns the valid matched index patterns', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + indexPatterns.getFieldsForWildcard = jest + .fn() + .mockResolvedValueOnce({ indices: ['index1'] }) + .mockResolvedValue({ indices: [] }); + + const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); + + expect(result.matchedIndexPatterns).toEqual(['packetbeat-*']); + }); + + test('returns the valid matched indices', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + indexPatterns.getFieldsForWildcard = jest + .fn() + .mockResolvedValueOnce({ indices: ['index1'] }) + .mockResolvedValue({ indices: [] }); + + const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); + + expect(result.matchedIndices).toEqual(['index1']); + }); + + test('returns the valid matched indices per index pattern', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + indexPatterns.getFieldsForWildcard = jest + .fn() + .mockResolvedValueOnce({ indices: ['index1'] }) + .mockResolvedValue({ indices: ['index2'] }); + + const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); + + expect(result.matchesByIndexPattern).toEqual({ + 'packetbeat-*': ['index1'], + 'filebeat-*': ['index2'], + }); + }); }); - test('checks the positive pattern if provided with a negative pattern', async () => { - indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); - const mockFn = jest.fn().mockResolvedValue({ indices: ['length'] }); - indexPatterns.getFieldsForWildcard = mockFn; - const result = await indexPatterns.getIndexPatternMatches(['-filebeat-*', 'filebeat-*']); - expect(mockFn.mock.calls[0][0].pattern).toEqual('filebeat-*'); - expect(mockFn.mock.calls[1][0].pattern).toEqual('filebeat-*'); - expect(result).toEqual({ - matchedIndexPatterns: ['-filebeat-*', 'filebeat-*'], - matchedIndices: ['length'], + describe('with negated index patterns', () => { + test('returns the valid matched index patterns', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + const mockFn = jest.fn().mockResolvedValue({ indices: ['index1'] }); + indexPatterns.getFieldsForWildcard = mockFn; + + const result = await indexPatterns.getIndexPatternMatches([ + '-filebeat-*', + 'filebeat-*', + 'logs-*', + '-logs-excluded-*', + ]); + + expect(result.matchedIndexPatterns).toEqual(['filebeat-*', 'logs-*']); + }); + + test('returns the valid matched indices', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + const mockFn = jest.fn().mockResolvedValue({ indices: ['index1'] }); + indexPatterns.getFieldsForWildcard = mockFn; + + const result = await indexPatterns.getIndexPatternMatches([ + '-filebeat-*', + 'filebeat-*', + 'logs-*', + '-logs-excluded-*', + ]); + + expect(result.matchedIndices).toEqual(['index1']); + }); + + test('returns the valid matched indices per index pattern', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + const mockFn = jest + .fn() + .mockResolvedValueOnce({ indices: ['index1'] }) + .mockResolvedValue({ indices: ['index2'] }); + indexPatterns.getFieldsForWildcard = mockFn; + + const result = await indexPatterns.getIndexPatternMatches([ + '-filebeat-*', + 'filebeat-*', + 'logs-*', + '-logs-excluded-*', + ]); + + expect(result.matchesByIndexPattern).toEqual({ + 'filebeat-*': ['index1'], + 'logs-*': ['index2'], + }); + }); + + test('queries each positive pattern with all negated patterns for field caps', async () => { + indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); + const mockFn = jest.fn().mockResolvedValue({ indices: ['length'] }); + indexPatterns.getFieldsForWildcard = mockFn; + + await indexPatterns.getIndexPatternMatches([ + '-filebeat-*', + 'filebeat-*', + 'logs-*', + '-logs-excluded-*', + ]); + + expect(mockFn.mock.calls[0][0].pattern).toEqual([ + 'filebeat-*', + '-filebeat-*', + '-logs-excluded-*', + ]); + expect(mockFn.mock.calls[1][0].pattern).toEqual([ + 'logs-*', + '-filebeat-*', + '-logs-excluded-*', + ]); }); }); @@ -131,12 +224,19 @@ describe('Index Pattern Fetcher - server', () => { indexPatterns = new IndexPatternsFetcher(esClient, optionalParams); indexPatterns.getFieldsForWildcard = jest .fn() - .mockImplementationOnce(async () => { - throw new DataViewMissingIndices('Catch me if you can!'); - }) - .mockImplementation(() => Promise.resolve({ indices: ['length'] })); + .mockRejectedValueOnce(new DataViewMissingIndices('Catch me if you can!')) + .mockResolvedValue({ indices: ['index1'] }); + const result = await indexPatterns.getIndexPatternMatches(['packetbeat-*', 'filebeat-*']); - expect(result).toEqual({ matchedIndexPatterns: ['filebeat-*'], matchedIndices: ['length'] }); + + expect(result).toMatchObject({ + matchedIndexPatterns: ['filebeat-*'], + matchedIndices: ['index1'], + matchesByIndexPattern: { + 'packetbeat-*': [], + 'filebeat-*': ['index1'], + }, + }); }); }); }); diff --git a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts index b3b1f1469f329..86ba62de9fd45 100644 --- a/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts +++ b/src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts @@ -48,9 +48,10 @@ interface IndexPatternsFetcherOptionalParams { rollupsEnabled?: boolean; } -interface GetIndexPatternMatchesResult { +export interface GetIndexPatternMatchesResult { matchedIndexPatterns: string[]; matchedIndices?: string[]; + matchesByIndexPattern?: Record; } export class IndexPatternsFetcher { @@ -163,59 +164,85 @@ export class IndexPatternsFetcher { return fieldCapsResponse; } + /** + * Checks whether the passed index pattern is an excluding one. + * The excluding index pattern starts with a dash, e.g. "-logs-excluded-*" + * meaning all indices matching "logs-excluded-*" will be excluded from search + * + * @param indexPattern - Index pattern to check + * @returns Whether the passed index pattern is a negated one + */ + isExcludingIndexPattern(indexPattern: string): boolean { + return indexPattern.trim().startsWith('-'); + } + /** * For each input pattern, checks whether it resolves to at least one backing index. * - * @param indexPatterns - Index patterns to check (may include wildcards and negated entries). + * Including index patterns (not starting with `-`) are checked with field caps using that pattern + * together with every excluding index pattern (starting with `-`) in the list, so resolution matches + * Elasticsearch multi-target syntax. + * + * @param indexPatterns - Index patterns to check (may include wildcards and excluded entries). * @returns Resolves to {@link GetIndexPatternMatchesResult}: * - `matchedIndexPatterns`: input patterns that matched at least one index. - * - `matchedIndices`: deduplicated concrete index names across all matches (omitted if the - * operation fails; then `matchedIndexPatterns` is the original `indexPatterns` list). + * - `matchedIndices`: deduplicated concrete index names matching index patterns (omitted on failure). + * - `matchesByIndexPattern`: per-input-pattern matched indices (omitted on failure). */ async getIndexPatternMatches(indexPatterns: string[]): Promise { - const indexPatternsObs = indexPatterns.map((indexPattern) => { - // when checking a negative pattern, check if the positive pattern exists - const indexToQuery = indexPattern.trim().startsWith('-') - ? indexPattern.trim().substring(1) - : indexPattern.trim(); + const excludingIndexPatterns = indexPatterns.filter(this.isExcludingIndexPattern); + const indexPatternsToMatch = indexPatterns + .filter((indexPattern) => !this.isExcludingIndexPattern(indexPattern)) + .map((indexPattern) => [indexPattern, ...excludingIndexPatterns]); + + const matchIndexPatterns = indexPatternsToMatch.map((pattern) => { return defer(() => from( this.getFieldsForWildcard({ - // check one field to keep request fast/small fields: ['_id'], - pattern: indexToQuery, + pattern, }) ).pipe( - map((match) => ({ ...match, indexPattern })), - catchError(() => of({ fields: [], indices: [], indexPattern })) + // expecting pattern[0] to contain an including index pattern + // and pattern[1..end] to contain excluding index patterns + map((match) => ({ ...match, indexPattern: pattern[0] })), + catchError(() => of({ fields: [], indices: [], indexPattern: pattern[0] })) ) ); }); return new Promise((resolve) => { - rateLimitingForkJoin(indexPatternsObs, 3, { + rateLimitingForkJoin(matchIndexPatterns, 3, { fields: [], indices: [], indexPattern: '', }).subscribe((indexPatternMatches) => { - const matchedIndexPatterns = new Set(); + const matchedIndexPatterns: string[] = []; const uniqueMatchedIndices = new Set(); + const matchesByIndexPattern: Record = {}; for (const indexPatternMatch of indexPatternMatches) { - if (indexPatternMatch.indices.length > 0) { - matchedIndexPatterns.add(indexPatternMatch.indexPattern); + const { indexPattern, indices } = indexPatternMatch; + + matchesByIndexPattern[indexPattern] = indices; + + if (indices.length === 0) { + continue; } - for (const index of indexPatternMatch.indices) { + matchedIndexPatterns.push(indexPattern); + + for (const index of indices) { uniqueMatchedIndices.add(index); } } resolve({ - matchedIndexPatterns: Array.from(matchedIndexPatterns), + matchedIndexPatterns, matchedIndices: Array.from(uniqueMatchedIndices), + matchesByIndexPattern, }); }); - }).catch(() => ({ matchedIndexPatterns: indexPatterns })); + }).catch(() => ({ matchedIndexPatterns: [] })); } } From c46323a0ec6651f46944aa145c5223d2383dd6c1 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 1 Apr 2026 17:02:12 +0200 Subject: [PATCH 11/12] remove matched_indicator_indices_count for now --- .../alerting_event_logger.ts | 1 - .../plugins/shared/alerting/server/types.ts | 1 - .../shared/event_log/generated/mappings.json | 1291 ++++++++--------- .../shared/event_log/generated/schemas.ts | 1 - .../shared/event_log/scripts/mappings.js | 3 - .../validation/run_execution_validation.ts | 12 +- .../indicator_match_metrics.ts | 95 -- 7 files changed, 646 insertions(+), 758 deletions(-) diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index a95fc68614f61..5aab5e8d7cbc8 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -802,7 +802,6 @@ export function updateEvent(event: IEvent, opts: UpdateEventOpts) { set(event, 'kibana.alert.rule.execution.metrics', { ...event.kibana?.alert?.rule?.execution?.metrics, matched_indices_count: consumerMetrics.matched_indices_count, - matched_indicator_indices_count: consumerMetrics.matched_indicator_indices_count, alerts_candidate_count: consumerMetrics.alerts_candidate_count, alerts_suppressed_count: consumerMetrics.alerts_suppressed_count, frozen_indices_queried_count: consumerMetrics.frozen_indices_queried_count, diff --git a/x-pack/platform/plugins/shared/alerting/server/types.ts b/x-pack/platform/plugins/shared/alerting/server/types.ts index 2ebb342301663..a2956725e6e86 100644 --- a/x-pack/platform/plugins/shared/alerting/server/types.ts +++ b/x-pack/platform/plugins/shared/alerting/server/types.ts @@ -458,7 +458,6 @@ export interface ConsumerExecutionMetrics { gap_duration_s: number; gap_range: { lte: string; gte: string }; matched_indices_count: number; - matched_indicator_indices_count: number; alerts_candidate_count: number; alerts_suppressed_count: number; frozen_indices_queried_count: number; diff --git a/x-pack/platform/plugins/shared/event_log/generated/mappings.json b/x-pack/platform/plugins/shared/event_log/generated/mappings.json index 41feb581e5791..831a129da7eea 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/mappings.json +++ b/x-pack/platform/plugins/shared/event_log/generated/mappings.json @@ -1,705 +1,702 @@ { - "dynamic": "false", - "properties": { - "@timestamp": { - "type": "date" + "dynamic": "false", + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "norms": false, + "type": "text" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" }, "message": { - "norms": false, - "type": "text" + "norms": false, + "type": "text" }, - "tags": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" } + }, + "ignore_above": 1024, + "type": "keyword" }, - "error": { - "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "stack_trace": { - "doc_values": false, - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "index": false, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kibana": { + "properties": { + "server_uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "task": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "scheduled": { + "type": "date" + }, + "schedule_delay": { + "type": "long" } + } }, - "event": { - "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" + "alerting": { + "properties": { + "instance_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "action_group_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "action_subgroup": { + "type": "keyword", + "ignore_above": 1024 + }, + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "outcome": { + "type": "keyword", + "ignore_above": 1024 + }, + "summary": { + "properties": { + "new": { + "properties": { + "count": { + "type": "long" } + } }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "long" - }, - "end": { - "type": "date" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ingested": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "doc_values": false, - "ignore_above": 1024, - "index": false, - "type": "keyword" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk_score": { - "type": "float" - }, - "risk_score_norm": { - "type": "float" - }, - "sequence": { - "type": "long" - }, - "severity": { - "type": "long" - }, - "start": { - "type": "date" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" + "ongoing": { + "properties": { + "count": { + "type": "long" } + } }, - "url": { - "ignore_above": 1024, - "type": "keyword" + "recovered": { + "properties": { + "count": { + "type": "long" + } + } } + } } + } }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "logger": { - "ignore_above": 1024, - "type": "keyword" + "alert": { + "properties": { + "flapping": { + "type": "boolean" + }, + "maintenance_window_ids": { + "type": "keyword", + "ignore_above": 1024, + "meta": { + "isArray": "true" + } + }, + "uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "deletion": { + "properties": { + "num_deleted": { + "type": "long" } - } - }, - "rule": { - "properties": { - "author": { - "ignore_above": 1024, - "type": "keyword", - "meta": { + } + }, + "rule": { + "properties": { + "consumer": { + "type": "keyword", + "ignore_above": 1024 + }, + "gap": { + "properties": { + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "range": { + "type": "date_range", + "format": "strict_date_optional_time||epoch_millis" + }, + "filled_intervals": { + "type": "date_range", + "format": "strict_date_optional_time||epoch_millis", + "meta": { "isArray": "true" + } + }, + "unfilled_intervals": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date_range", + "meta": { + "isArray": "true" + } + }, + "in_progress_intervals": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date_range", + "meta": { + "isArray": "true" + } + }, + "total_gap_duration_ms": { + "type": "long" + }, + "filled_duration_ms": { + "type": "long" + }, + "unfilled_duration_ms": { + "type": "long" + }, + "in_progress_duration_ms": { + "type": "long" + }, + "deleted": { + "type": "boolean" + }, + "updated_at": { + "type": "date" + }, + "failed_auto_fill_attempts": { + "type": "long" + }, + "reason": { + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } } + } }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "license": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - }, - "ruleset": { - "ignore_above": 1024, - "type": "keyword" - }, - "uuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user": { - "properties": { - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } + "execution": { + "properties": { + "uuid": { + "type": "keyword", + "ignore_above": 1024 }, - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "kibana": { - "properties": { - "server_uuid": { - "type": "keyword", - "ignore_above": 1024 - }, - "task": { - "properties": { + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "status_order": { + "type": "long" + }, + "backfill": { + "properties": { "id": { - "type": "keyword" + "type": "keyword", + "ignore_above": 1024 }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "scheduled": { - "type": "date" + "start": { + "type": "date" }, - "schedule_delay": { - "type": "long" + "interval": { + "type": "keyword", + "ignore_above": 1024 } - } - }, - "alerting": { - "properties": { - "instance_id": { - "type": "keyword", - "ignore_above": 1024 + } + }, + "metrics": { + "properties": { + "number_of_triggered_actions": { + "type": "long" }, - "action_group_id": { - "type": "keyword", - "ignore_above": 1024 + "number_of_generated_actions": { + "type": "long" }, - "action_subgroup": { - "type": "keyword", - "ignore_above": 1024 + "alert_counts": { + "properties": { + "active": { + "type": "long" + }, + "new": { + "type": "long" + }, + "recovered": { + "type": "long" + } + } }, - "status": { - "type": "keyword", - "ignore_above": 1024 + "number_of_delayed_alerts": { + "type": "long" }, - "outcome": { - "type": "keyword", - "ignore_above": 1024 + "number_of_searches": { + "type": "long" }, - "summary": { - "properties": { - "new": { - "properties": { - "count": { - "type": "long" - } - } - }, - "ongoing": { - "properties": { - "count": { - "type": "long" - } - } - }, - "recovered": { - "properties": { - "count": { - "type": "long" - } - } - } - } - } - } - }, - "alert": { - "properties": { - "flapping": { - "type": "boolean" + "total_indexing_duration_ms": { + "type": "long" }, - "maintenance_window_ids": { - "type": "keyword", - "ignore_above": 1024, - "meta": { - "isArray": "true" - } + "es_search_duration_ms": { + "type": "long" }, - "uuid": { - "type": "keyword", - "ignore_above": 1024 + "total_search_duration_ms": { + "type": "long" }, - "deletion": { - "properties": { - "num_deleted": { - "type": "long" - } - } + "execution_gap_duration_s": { + "type": "long" }, - "rule": { - "properties": { - "consumer": { - "type": "keyword", - "ignore_above": 1024 - }, - "gap": { - "properties": { - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "range": { - "type": "date_range", - "format": "strict_date_optional_time||epoch_millis" - }, - "filled_intervals": { - "type": "date_range", - "format": "strict_date_optional_time||epoch_millis", - "meta": { - "isArray": "true" - } - }, - "unfilled_intervals": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date_range", - "meta": { - "isArray": "true" - } - }, - "in_progress_intervals": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date_range", - "meta": { - "isArray": "true" - } - }, - "total_gap_duration_ms": { - "type": "long" - }, - "filled_duration_ms": { - "type": "long" - }, - "unfilled_duration_ms": { - "type": "long" - }, - "in_progress_duration_ms": { - "type": "long" - }, - "deleted": { - "type": "boolean" - }, - "updated_at": { - "type": "date" - }, - "failed_auto_fill_attempts": { - "type": "long" - }, - "reason": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "execution": { - "properties": { - "uuid": { - "type": "keyword", - "ignore_above": 1024 - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "status_order": { - "type": "long" - }, - "backfill": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "start": { - "type": "date" - }, - "interval": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "metrics": { - "properties": { - "number_of_triggered_actions": { - "type": "long" - }, - "number_of_generated_actions": { - "type": "long" - }, - "alert_counts": { - "properties": { - "active": { - "type": "long" - }, - "new": { - "type": "long" - }, - "recovered": { - "type": "long" - } - } - }, - "number_of_delayed_alerts": { - "type": "long" - }, - "number_of_searches": { - "type": "long" - }, - "total_indexing_duration_ms": { - "type": "long" - }, - "es_search_duration_ms": { - "type": "long" - }, - "total_search_duration_ms": { - "type": "long" - }, - "execution_gap_duration_s": { - "type": "long" - }, - "gap_range": { - "type": "date_range", - "format": "strict_date_optional_time||epoch_millis" - }, - "gap_reason": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "matched_indices_count": { - "type": "long" - }, - "matched_indicator_indices_count": { - "type": "long" - }, - "frozen_indices_queried_count": { - "type": "long" - }, - "rule_type_run_duration_ms": { - "type": "long" - }, - "process_alerts_duration_ms": { - "type": "long" - }, - "trigger_actions_duration_ms": { - "type": "long" - }, - "process_rule_duration_ms": { - "type": "long" - }, - "claim_to_start_duration_ms": { - "type": "long" - }, - "persist_alerts_duration_ms": { - "type": "long" - }, - "prepare_rule_duration_ms": { - "type": "long" - }, - "total_run_duration_ms": { - "type": "long" - }, - "total_enrichment_duration_ms": { - "type": "long" - }, - "update_alerts_duration_ms": { - "type": "long" - }, - "alerts_candidate_count": { - "type": "long" - }, - "alerts_suppressed_count": { - "type": "long" - } - } - } - } - }, - "revision": { - "type": "long" - }, - "rule_type_id": { - "type": "keyword", - "ignore_above": 1024 - } + "gap_range": { + "type": "date_range", + "format": "strict_date_optional_time||epoch_millis" + }, + "gap_reason": { + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 } - } - } - }, - "saved_objects": { - "type": "nested", - "properties": { - "rel": { - "type": "keyword", - "ignore_above": 1024 + } }, - "namespace": { - "type": "keyword", - "ignore_above": 1024 + "matched_indices_count": { + "type": "long" }, - "id": { - "type": "keyword", - "ignore_above": 1024 + "frozen_indices_queried_count": { + "type": "long" }, - "type": { - "type": "keyword", - "ignore_above": 1024 + "rule_type_run_duration_ms": { + "type": "long" + }, + "process_alerts_duration_ms": { + "type": "long" + }, + "trigger_actions_duration_ms": { + "type": "long" }, - "type_id": { - "type": "keyword", - "ignore_above": 1024 + "process_rule_duration_ms": { + "type": "long" }, - "space_agnostic": { - "type": "boolean" + "claim_to_start_duration_ms": { + "type": "long" + }, + "persist_alerts_duration_ms": { + "type": "long" + }, + "prepare_rule_duration_ms": { + "type": "long" + }, + "total_run_duration_ms": { + "type": "long" + }, + "total_enrichment_duration_ms": { + "type": "long" + }, + "update_alerts_duration_ms": { + "type": "long" + }, + "alerts_candidate_count": { + "type": "long" + }, + "alerts_suppressed_count": { + "type": "long" } + } } + } }, - "space_ids": { - "type": "keyword", - "ignore_above": 1024, - "meta": { - "isArray": "true" - } + "revision": { + "type": "long" }, - "version": { - "type": "version" + "rule_type_id": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "saved_objects": { + "type": "nested", + "properties": { + "rel": { + "type": "keyword", + "ignore_above": 1024 + }, + "namespace": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "type_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "space_agnostic": { + "type": "boolean" + } + } + }, + "space_ids": { + "type": "keyword", + "ignore_above": 1024, + "meta": { + "isArray": "true" + } + }, + "version": { + "type": "version" + }, + "action": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "type_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "execution": { + "properties": { + "source": { + "ignore_above": 1024, + "type": "keyword" }, - "action": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "type": "keyword", - "ignore_above": 1024 + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "gen_ai": { + "properties": { + "usage": { + "properties": { + "prompt_tokens": { + "type": "long" }, - "type_id": { - "type": "keyword", - "ignore_above": 1024 + "completion_tokens": { + "type": "long" }, - "execution": { - "properties": { - "source": { - "ignore_above": 1024, - "type": "keyword" - }, - "uuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "gen_ai": { - "properties": { - "usage": { - "properties": { - "prompt_tokens": { - "type": "long" - }, - "completion_tokens": { - "type": "long" - }, - "total_tokens": { - "type": "long" - } - } - } - } - }, - "usage": { - "properties": { - "request_body_bytes": { - "type": "long" - } - } - } - } + "total_tokens": { + "type": "long" } + } } + } }, - "user_api_key": { - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } + "usage": { + "properties": { + "request_body_bytes": { + "type": "long" } + } + } + } + } + } + }, + "user_api_key": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "gap_auto_fill": { + "properties": { + "execution": { + "properties": { + "status": { + "type": "keyword" }, - "gap_auto_fill": { - "properties": { - "execution": { - "properties": { - "status": { - "type": "keyword" - }, - "start": { - "type": "date" - }, - "end": { - "type": "date" - }, - "duration_ms": { - "type": "long" - }, - "rule_ids": { - "type": "keyword", - "meta": { - "isArray": "true" - } - }, - "task_params": { - "properties": { - "name": { - "type": "keyword" - }, - "num_retries": { - "type": "long" - }, - "gap_fill_range": { - "type": "keyword" - }, - "interval": { - "type": "keyword" - }, - "max_backfills": { - "type": "long" - } - } - }, - "results": { - "type": "nested", - "properties": { - "rule_id": { - "type": "keyword" - }, - "processed_gaps": { - "type": "long" - }, - "status": { - "type": "keyword" - }, - "error": { - "type": "keyword" - } - } - } - } - } + "start": { + "type": "date" + }, + "end": { + "type": "date" + }, + "duration_ms": { + "type": "long" + }, + "rule_ids": { + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "task_params": { + "properties": { + "name": { + "type": "keyword" + }, + "num_retries": { + "type": "long" + }, + "gap_fill_range": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "max_backfills": { + "type": "long" + } + } + }, + "results": { + "type": "nested", + "properties": { + "rule_id": { + "type": "keyword" + }, + "processed_gaps": { + "type": "long" + }, + "status": { + "type": "keyword" + }, + "error": { + "type": "keyword" } + } } + } } + } } + } } -} \ No newline at end of file + } +} diff --git a/x-pack/platform/plugins/shared/event_log/generated/schemas.ts b/x-pack/platform/plugins/shared/event_log/generated/schemas.ts index ef92d5e141e56..97987c1ebac1a 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/schemas.ts +++ b/x-pack/platform/plugins/shared/event_log/generated/schemas.ts @@ -210,7 +210,6 @@ export const EventSchema = schema.maybe( }) ), matched_indices_count: ecsStringOrNumber(), - matched_indicator_indices_count: ecsStringOrNumber(), frozen_indices_queried_count: ecsStringOrNumber(), rule_type_run_duration_ms: ecsStringOrNumber(), process_alerts_duration_ms: ecsStringOrNumber(), diff --git a/x-pack/platform/plugins/shared/event_log/scripts/mappings.js b/x-pack/platform/plugins/shared/event_log/scripts/mappings.js index b665d007a84e8..a967afe94f310 100644 --- a/x-pack/platform/plugins/shared/event_log/scripts/mappings.js +++ b/x-pack/platform/plugins/shared/event_log/scripts/mappings.js @@ -246,9 +246,6 @@ exports.EcsCustomPropertyMappings = { matched_indices_count: { type: 'long', }, - matched_indicator_indices_count: { - type: 'long', - }, frozen_indices_queried_count: { type: 'long', }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts index bd98c9f5e3cd0..929ac9bec1706 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/validation/run_execution_validation.ts @@ -86,16 +86,8 @@ export const runExecutionValidation = async ( if (isThreatParams(params)) { try { - const { - matchedIndexPatterns: matchedThreatIndexPatterns, - matchedIndices: matchedThreatIndices, - } = await indexPatterns.getIndexPatternMatches(params.threatIndex); - - // Collect rule execution metrics - ruleExecutionLogger.logMetric( - 'matched_indicator_indices_count', - matchedThreatIndices?.length - ); + const { matchedIndexPatterns: matchedThreatIndexPatterns } = + await indexPatterns.getIndexPatternMatches(params.threatIndex); if (matchedThreatIndexPatterns.length === 0) { warnings.push( diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts index 1bc847240cd2a..4f2f2f1762e33 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts @@ -181,101 +181,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('matched_indicator_indices_count', () => { - it('records matched_indicator_indices_count for one matching source index pattern', async () => { - const timestamp = new Date().toISOString(); - const threatIndicatorDocument = { - '@timestamp': timestamp, - host: { name: 'test-1' }, - }; - const document = { - '@timestamp': timestamp, - host: { name: 'test-1' }, - }; - const rule = getThreatMatchRuleParams({ - threat_query: '*:*', - threat_index: ['ti_test_1'], - query: '*:*', - index: ['test-data-1'], - from: 'now-35m', - interval: '30m', - enabled: true, - }); - - await indexThreatIndicatorDocuments([threatIndicatorDocument]); - await indexListOfSourceDocuments([document]); - - const createdRule = await createRule(supertest, log, rule); - - const { matched_indicator_indices_count } = - await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id); - - expect(matched_indicator_indices_count).toBe(1); - }); - - it('records matched_indicator_indices_count for a single index pattern with wildcard', async () => { - const timestamp = new Date().toISOString(); - const threatIndicatorDocument = { - '@timestamp': timestamp, - host: { name: 'test-1' }, - }; - const document = { - '@timestamp': timestamp, - host: { name: 'test-1' }, - }; - const rule = getThreatMatchRuleParams({ - threat_query: '*:*', - threat_index: ['ti_test_*'], - query: '*:*', - index: ['test-data-*'], - from: 'now-35m', - interval: '30m', - enabled: true, - }); - - await indexThreatIndicatorDocuments([threatIndicatorDocument]); - await indexListOfSourceDocuments([document]); - - const createdRule = await createRule(supertest, log, rule); - - const { matched_indicator_indices_count } = - await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id); - - expect(matched_indicator_indices_count).toBe(2); - }); - - it('records matched_indicator_indices_count for multiple matching source index patterns', async () => { - const timestamp = new Date().toISOString(); - const threatIndicatorDocument = { - '@timestamp': timestamp, - host: { name: 'test-1' }, - }; - const document = { - '@timestamp': timestamp, - host: { name: 'test-1' }, - }; - const rule = getThreatMatchRuleParams({ - threat_query: '*:*', - threat_index: ['ti_te*', 'ti_test_1', 'ti_test_2'], - query: '*:*', - index: ['test-da*,', 'test-data-1', 'test-data-2'], - from: 'now-35m', - interval: '30m', - enabled: true, - }); - - await indexThreatIndicatorDocuments([threatIndicatorDocument]); - await indexListOfSourceDocuments([document]); - - const createdRule = await createRule(supertest, log, rule); - - const { matched_indicator_indices_count } = - await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id); - - expect(matched_indicator_indices_count).toBe(2); - }); - }); - describe('alerts_candidate_count', () => { it('records alerts_candidate_count value', async () => { const timestamp = new Date().toISOString(); From 9bae245ec45b11d246e5c3a67b5091ad41868c14 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 1 Apr 2026 15:32:29 +0000 Subject: [PATCH 12/12] Changes from node scripts/telemetry_check --- .../shared/event_log/generated/mappings.json | 1288 ++++++++--------- 1 file changed, 644 insertions(+), 644 deletions(-) diff --git a/x-pack/platform/plugins/shared/event_log/generated/mappings.json b/x-pack/platform/plugins/shared/event_log/generated/mappings.json index 831a129da7eea..33a8753ca3df8 100644 --- a/x-pack/platform/plugins/shared/event_log/generated/mappings.json +++ b/x-pack/platform/plugins/shared/event_log/generated/mappings.json @@ -1,702 +1,702 @@ { - "dynamic": "false", - "properties": { - "@timestamp": { - "type": "date" - }, - "message": { - "norms": false, - "type": "text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" + "dynamic": "false", + "properties": { + "@timestamp": { + "type": "date" }, "message": { - "norms": false, - "type": "text" + "norms": false, + "type": "text" }, - "stack_trace": { - "doc_values": false, - "fields": { - "text": { - "norms": false, - "type": "text" + "tags": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" } - }, - "ignore_above": 1024, - "index": false, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" - } - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "long" - }, - "end": { - "type": "date" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ingested": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "doc_values": false, - "ignore_above": 1024, - "index": false, - "type": "keyword" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk_score": { - "type": "float" - }, - "risk_score_norm": { - "type": "float" }, - "sequence": { - "type": "long" - }, - "severity": { - "type": "long" - }, - "start": { - "type": "date" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" - } - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "logger": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "rule": { - "properties": { - "author": { - "ignore_above": 1024, - "type": "keyword", - "meta": { - "isArray": "true" - } - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "license": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - }, - "ruleset": { - "ignore_above": 1024, - "type": "keyword" - }, - "uuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user": { - "properties": { - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "kibana": { - "properties": { - "server_uuid": { - "type": "keyword", - "ignore_above": 1024 }, - "task": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "scheduled": { - "type": "date" - }, - "schedule_delay": { - "type": "long" + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } } - } }, - "alerting": { - "properties": { - "instance_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "action_group_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "action_subgroup": { - "type": "keyword", - "ignore_above": 1024 - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "outcome": { - "type": "keyword", - "ignore_above": 1024 - }, - "summary": { - "properties": { - "new": { - "properties": { - "count": { - "type": "long" - } - } + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" }, - "ongoing": { - "properties": { - "count": { - "type": "long" + "category": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" } - } }, - "recovered": { - "properties": { - "count": { - "type": "long" + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" } - } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" } - } } - } }, - "alert": { - "properties": { - "flapping": { - "type": "boolean" - }, - "maintenance_window_ids": { - "type": "keyword", - "ignore_above": 1024, - "meta": { - "isArray": "true" - } - }, - "uuid": { - "type": "keyword", - "ignore_above": 1024 - }, - "deletion": { - "properties": { - "num_deleted": { - "type": "long" + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" } - } - }, - "rule": { - "properties": { - "consumer": { - "type": "keyword", - "ignore_above": 1024 - }, - "gap": { - "properties": { - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "range": { - "type": "date_range", - "format": "strict_date_optional_time||epoch_millis" - }, - "filled_intervals": { - "type": "date_range", - "format": "strict_date_optional_time||epoch_millis", - "meta": { - "isArray": "true" - } - }, - "unfilled_intervals": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date_range", - "meta": { - "isArray": "true" - } - }, - "in_progress_intervals": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date_range", - "meta": { + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword", + "meta": { "isArray": "true" - } - }, - "total_gap_duration_ms": { - "type": "long" - }, - "filled_duration_ms": { - "type": "long" - }, - "unfilled_duration_ms": { - "type": "long" - }, - "in_progress_duration_ms": { - "type": "long" - }, - "deleted": { - "type": "boolean" - }, - "updated_at": { - "type": "date" - }, - "failed_auto_fill_attempts": { - "type": "long" - }, - "reason": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } } - } }, - "execution": { - "properties": { - "uuid": { - "type": "keyword", - "ignore_above": 1024 - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "status_order": { - "type": "long" - }, - "backfill": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "start": { - "type": "date" - }, - "interval": { - "type": "keyword", - "ignore_above": 1024 + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" } - } }, - "metrics": { - "properties": { - "number_of_triggered_actions": { - "type": "long" - }, - "number_of_generated_actions": { - "type": "long" - }, - "alert_counts": { - "properties": { - "active": { - "type": "long" - }, - "new": { - "type": "long" - }, - "recovered": { - "type": "long" - } - } + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kibana": { + "properties": { + "server_uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "task": { + "properties": { + "id": { + "type": "keyword" }, - "number_of_delayed_alerts": { - "type": "long" + "type": { + "type": "keyword", + "ignore_above": 1024 }, - "number_of_searches": { - "type": "long" + "scheduled": { + "type": "date" }, - "total_indexing_duration_ms": { - "type": "long" + "schedule_delay": { + "type": "long" + } + } + }, + "alerting": { + "properties": { + "instance_id": { + "type": "keyword", + "ignore_above": 1024 }, - "es_search_duration_ms": { - "type": "long" + "action_group_id": { + "type": "keyword", + "ignore_above": 1024 }, - "total_search_duration_ms": { - "type": "long" + "action_subgroup": { + "type": "keyword", + "ignore_above": 1024 }, - "execution_gap_duration_s": { - "type": "long" + "status": { + "type": "keyword", + "ignore_above": 1024 }, - "gap_range": { - "type": "date_range", - "format": "strict_date_optional_time||epoch_millis" + "outcome": { + "type": "keyword", + "ignore_above": 1024 }, - "gap_reason": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 + "summary": { + "properties": { + "new": { + "properties": { + "count": { + "type": "long" + } + } + }, + "ongoing": { + "properties": { + "count": { + "type": "long" + } + } + }, + "recovered": { + "properties": { + "count": { + "type": "long" + } + } + } } - } - }, - "matched_indices_count": { - "type": "long" - }, - "frozen_indices_queried_count": { - "type": "long" - }, - "rule_type_run_duration_ms": { - "type": "long" - }, - "process_alerts_duration_ms": { - "type": "long" - }, - "trigger_actions_duration_ms": { - "type": "long" + } + } + }, + "alert": { + "properties": { + "flapping": { + "type": "boolean" }, - "process_rule_duration_ms": { - "type": "long" + "maintenance_window_ids": { + "type": "keyword", + "ignore_above": 1024, + "meta": { + "isArray": "true" + } }, - "claim_to_start_duration_ms": { - "type": "long" + "uuid": { + "type": "keyword", + "ignore_above": 1024 }, - "persist_alerts_duration_ms": { - "type": "long" + "deletion": { + "properties": { + "num_deleted": { + "type": "long" + } + } }, - "prepare_rule_duration_ms": { - "type": "long" + "rule": { + "properties": { + "consumer": { + "type": "keyword", + "ignore_above": 1024 + }, + "gap": { + "properties": { + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "range": { + "type": "date_range", + "format": "strict_date_optional_time||epoch_millis" + }, + "filled_intervals": { + "type": "date_range", + "format": "strict_date_optional_time||epoch_millis", + "meta": { + "isArray": "true" + } + }, + "unfilled_intervals": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date_range", + "meta": { + "isArray": "true" + } + }, + "in_progress_intervals": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date_range", + "meta": { + "isArray": "true" + } + }, + "total_gap_duration_ms": { + "type": "long" + }, + "filled_duration_ms": { + "type": "long" + }, + "unfilled_duration_ms": { + "type": "long" + }, + "in_progress_duration_ms": { + "type": "long" + }, + "deleted": { + "type": "boolean" + }, + "updated_at": { + "type": "date" + }, + "failed_auto_fill_attempts": { + "type": "long" + }, + "reason": { + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "execution": { + "properties": { + "uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "status_order": { + "type": "long" + }, + "backfill": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "start": { + "type": "date" + }, + "interval": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "metrics": { + "properties": { + "number_of_triggered_actions": { + "type": "long" + }, + "number_of_generated_actions": { + "type": "long" + }, + "alert_counts": { + "properties": { + "active": { + "type": "long" + }, + "new": { + "type": "long" + }, + "recovered": { + "type": "long" + } + } + }, + "number_of_delayed_alerts": { + "type": "long" + }, + "number_of_searches": { + "type": "long" + }, + "total_indexing_duration_ms": { + "type": "long" + }, + "es_search_duration_ms": { + "type": "long" + }, + "total_search_duration_ms": { + "type": "long" + }, + "execution_gap_duration_s": { + "type": "long" + }, + "gap_range": { + "type": "date_range", + "format": "strict_date_optional_time||epoch_millis" + }, + "gap_reason": { + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "matched_indices_count": { + "type": "long" + }, + "frozen_indices_queried_count": { + "type": "long" + }, + "rule_type_run_duration_ms": { + "type": "long" + }, + "process_alerts_duration_ms": { + "type": "long" + }, + "trigger_actions_duration_ms": { + "type": "long" + }, + "process_rule_duration_ms": { + "type": "long" + }, + "claim_to_start_duration_ms": { + "type": "long" + }, + "persist_alerts_duration_ms": { + "type": "long" + }, + "prepare_rule_duration_ms": { + "type": "long" + }, + "total_run_duration_ms": { + "type": "long" + }, + "total_enrichment_duration_ms": { + "type": "long" + }, + "update_alerts_duration_ms": { + "type": "long" + }, + "alerts_candidate_count": { + "type": "long" + }, + "alerts_suppressed_count": { + "type": "long" + } + } + } + } + }, + "revision": { + "type": "long" + }, + "rule_type_id": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "saved_objects": { + "type": "nested", + "properties": { + "rel": { + "type": "keyword", + "ignore_above": 1024 }, - "total_run_duration_ms": { - "type": "long" + "namespace": { + "type": "keyword", + "ignore_above": 1024 }, - "total_enrichment_duration_ms": { - "type": "long" + "id": { + "type": "keyword", + "ignore_above": 1024 }, - "update_alerts_duration_ms": { - "type": "long" + "type": { + "type": "keyword", + "ignore_above": 1024 }, - "alerts_candidate_count": { - "type": "long" + "type_id": { + "type": "keyword", + "ignore_above": 1024 }, - "alerts_suppressed_count": { - "type": "long" + "space_agnostic": { + "type": "boolean" } - } } - } }, - "revision": { - "type": "long" + "space_ids": { + "type": "keyword", + "ignore_above": 1024, + "meta": { + "isArray": "true" + } }, - "rule_type_id": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "saved_objects": { - "type": "nested", - "properties": { - "rel": { - "type": "keyword", - "ignore_above": 1024 - }, - "namespace": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "type_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "space_agnostic": { - "type": "boolean" - } - } - }, - "space_ids": { - "type": "keyword", - "ignore_above": 1024, - "meta": { - "isArray": "true" - } - }, - "version": { - "type": "version" - }, - "action": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "type_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "execution": { - "properties": { - "source": { - "ignore_above": 1024, - "type": "keyword" + "version": { + "type": "version" }, - "uuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "gen_ai": { - "properties": { - "usage": { - "properties": { - "prompt_tokens": { - "type": "long" + "action": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" }, - "completion_tokens": { - "type": "long" + "id": { + "type": "keyword", + "ignore_above": 1024 }, - "total_tokens": { - "type": "long" + "type_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "execution": { + "properties": { + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "gen_ai": { + "properties": { + "usage": { + "properties": { + "prompt_tokens": { + "type": "long" + }, + "completion_tokens": { + "type": "long" + }, + "total_tokens": { + "type": "long" + } + } + } + } + }, + "usage": { + "properties": { + "request_body_bytes": { + "type": "long" + } + } + } + } } - } } - } }, - "usage": { - "properties": { - "request_body_bytes": { - "type": "long" - } - } - } - } - } - } - }, - "user_api_key": { - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - }, - "gap_auto_fill": { - "properties": { - "execution": { - "properties": { - "status": { - "type": "keyword" - }, - "start": { - "type": "date" - }, - "end": { - "type": "date" - }, - "duration_ms": { - "type": "long" - }, - "rule_ids": { - "type": "keyword", - "meta": { - "isArray": "true" - } - }, - "task_params": { - "properties": { - "name": { - "type": "keyword" - }, - "num_retries": { - "type": "long" - }, - "gap_fill_range": { - "type": "keyword" - }, - "interval": { - "type": "keyword" - }, - "max_backfills": { - "type": "long" + "user_api_key": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } } - } }, - "results": { - "type": "nested", - "properties": { - "rule_id": { - "type": "keyword" - }, - "processed_gaps": { - "type": "long" - }, - "status": { - "type": "keyword" - }, - "error": { - "type": "keyword" + "gap_auto_fill": { + "properties": { + "execution": { + "properties": { + "status": { + "type": "keyword" + }, + "start": { + "type": "date" + }, + "end": { + "type": "date" + }, + "duration_ms": { + "type": "long" + }, + "rule_ids": { + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "task_params": { + "properties": { + "name": { + "type": "keyword" + }, + "num_retries": { + "type": "long" + }, + "gap_fill_range": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "max_backfills": { + "type": "long" + } + } + }, + "results": { + "type": "nested", + "properties": { + "rule_id": { + "type": "keyword" + }, + "processed_gaps": { + "type": "long" + }, + "status": { + "type": "keyword" + }, + "error": { + "type": "keyword" + } + } + } + } + } } - } } - } } - } } - } } - } -} +} \ No newline at end of file