Skip to content

[8.19] [Security][Detection Engine] ESQL Rule Execution Logic Integration Test (#252936) #254078

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
hannahbrooks wants to merge 2 commits into from
Closed

[8.19] [Security][Detection Engine] ESQL Rule Execution Logic Integration Test (#252936) #254078

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5d7b344
[Security][Detection Engine] ESQL Rule Execution Logic Integration Te…
hannahbrooks Feb 19, 2026
2a1482b
added import i was missing
hannahbrooks Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 225 additions & 1 deletion ...s_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ import {
import { EsqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema';
import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks';
import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring';
import { ALERT_ANCESTORS } from '@kbn/security-solution-plugin/common/field_maps/field_names';
import {
ALERT_ANCESTORS,
ALERT_ORIGINAL_TIME,
} from '@kbn/security-solution-plugin/common/field_maps/field_names';

import { getMaxSignalsWarning as getMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils';
import { EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION } from '@kbn/security-solution-plugin/common/constants';
Expand Down Expand Up @@ -1602,6 +1605,227 @@ export default ({ getService }: FtrProviderContext) => {
// should return 160 alerts
expect(alertsResponse.hits.hits).toHaveLength(160);
});

describe('identical document ids across multiple indices', () => {
before(async () => {
await esArchiver.load(
'x-pack/solutions/security/test/fixtures/es_archives/security_solution/ecs_compliant_synthetic_source'
);
});

after(async () => {
await esArchiver.unload(
'x-pack/solutions/security/test/fixtures/es_archives/security_solution/ecs_compliant_synthetic_source'
);
});

it('should generate alerts from events with the same id', async () => {
const id = uuidv4();
const rule: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock(`rule-${id}`, true),
query: `from ecs_compliant, ecs_compliant_synthetic_source metadata _id, _index ${internalIdPipe(
id
)} | sort @timestamp asc`,
from: '2020-10-28T05:15:00.000Z',
to: '2020-10-28T06:00:00.000Z',
interval: '45m',
enabled: true,
};

await es.index({
index: 'ecs_compliant',
id, // id of event in index
refresh: true,
document: {
id,
'@timestamp': '2020-10-28T05:30:00.000Z',
agent: { name: 'from ecs_compliant' },
},
});
await es.index({
index: 'ecs_compliant_synthetic_source',
id, // id of event in index
refresh: true,
document: {
id,
'@timestamp': '2020-10-28T05:35:00.000Z',
agent: { name: 'from ecs_compliant_synthetic_source' },
},
});

const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:00:00.000Z'),
});

const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 200,
sort: [ALERT_ORIGINAL_TIME],
});

expect(previewAlerts).toHaveLength(2);
expect(previewAlerts[0]._source).toHaveProperty(['agent.name'], 'from ecs_compliant');
expect(previewAlerts[1]._source).toHaveProperty(
['agent.name'],
'from ecs_compliant_synthetic_source'
);
});

// since we exclude _id from the query in the next page, we should be able to generate alerts from multiple documents with the same id in different indices
it('should generate alerts over multiple pages from different indices but same event id', async () => {
const id = uuidv4();
const rule: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock(`rule-${id}`, true),
query: `from ecs_compliant, ecs_compliant_synthetic_source metadata _id, _index ${internalIdPipe(
id
)} | sort @timestamp asc`,
from: '2020-10-28T05:15:00.000Z',
to: '2020-10-28T06:00:00.000Z',
interval: '45m',
max_signals: 2, // we would paginate with page size = 2
enabled: true,
};

const document = {
id,
'@timestamp': '2020-10-28T05:30:00.000Z',
agent: { name: 'test-1' },
};

await Promise.all(
Array.from({ length: 2 }, (_, i) => i).map((i) =>
es.index({
index: 'ecs_compliant',
id: 'id-' + id + '_' + i, // id of event in index
refresh: true,
document,
})
)
);

const createdRule = await createRule(supertest, log, rule);

const alertsResponseFromFirstRuleExecution = await getAlerts(
supertest,
log,
es,
createdRule,
RuleExecutionStatusEnum.succeeded,
10
);

expect(alertsResponseFromFirstRuleExecution.hits.hits).toHaveLength(2);

await Promise.all(
Array.from({ length: 2 }, (_, i) => i).map((i) =>
es.index({
index: 'ecs_compliant_synthetic_source',
id: 'id-' + id + '_' + i, // id of event in index
refresh: true,
document: {
...document,
'@timestamp': '2020-10-28T05:53:00.000Z',
},
})
)
);

const dateRestart = new Date();

await runSoonRule(supertest, createdRule.id);

const alertsResponse = await getAlerts(
supertest,
log,
es,
createdRule,
RuleExecutionStatusEnum.succeeded,
200,
dateRestart
);

// no alert should be missed
expect(alertsResponse.hits.hits).toHaveLength(4);
});

it('should generate alerts over multiple pages from different indices but same event id for mv_expand when number alerts exceeds max signal', async () => {
const id = uuidv4();
const rule: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock(`rule-${id}`, true),
query: `from ecs_compliant, ecs_compliant_synthetic_source metadata _id, _index ${internalIdPipe(
id
)} | mv_expand agent.name | sort @timestamp asc, _index asc`, // sort by timestamp and index to ensure deterministic results, see https://github.com/elastic/kibana/issues/253849
from: '2020-10-28T05:15:00.000Z',
to: '2020-10-28T06:00:00.000Z',
interval: '45m',
enabled: true,
};

const document = {
id,
'@timestamp': '2020-10-28T05:30:00.000Z',
agent: { name: Array.from({ length: 150 }, (_, i) => `test_1_${1000 + i}`) },
};

await Promise.all(
['ecs_compliant', 'ecs_compliant_synthetic_source'].map((index, i) =>
es.index({
index,
id,
refresh: true,
document,
})
)
);

const createdRule = await createRule(supertest, log, rule);

const alertsResponseFromFirstRuleExecution = await getAlerts(
supertest,
log,
es,
createdRule,
RuleExecutionStatusEnum['partial failure'], // rule has warning, alerts were truncated, thus "partial failure" status
200
);

expect(alertsResponseFromFirstRuleExecution.hits.hits).toHaveLength(100);

const dateRestart = new Date();

await runSoonRule(supertest, createdRule.id);

const alertsResponse = await getAlerts(
supertest,
log,
es,
createdRule,
RuleExecutionStatusEnum['partial failure'], // rule has warning, alerts were truncated, thus "partial failure" status
300,
dateRestart
);

const indexCounts = alertsResponse.hits.hits.reduce<Record<string, number>>(
(acc, curr) => {
const indexName = curr._source?.[ALERT_ANCESTORS][0].index;
if (indexName) {
acc[indexName] = (acc[indexName] || 0) + 1;
}
return acc;
},
{}
);
expect(alertsResponse.hits.hits).toHaveLength(200);

expect(indexCounts).toEqual({
ecs_compliant: 100,
ecs_compliant_synthetic_source: 100,
});
});
});
});

describe('alerts enrichment', () => {
Expand Down