Skip to content

[9.1] [Detection Engine] Unskip some flaky tests, add better failure messages (#230318) #231523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rylnd merged 2 commits into from
Aug 13, 2025
Merged

[9.1] [Detection Engine] Unskip some flaky tests, add better failure messages (#230318) #231523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f66b6f
[Detection Engine] Unskip some flaky tests, add better failure messag…
rylnd Aug 11, 2025
1457524
Merge branch '9.1' into backport/9.1/pr-230318
rylnd Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions ...on_engine/exceptions/workflows/basic_license_essentials_tier/rule_exceptions_execution.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ export default ({ getService }: FtrProviderContext) => {
await waitForRuleSuccess({ supertest, log, id: createdId });
await waitForAlertsToBePresent(supertest, log, 10, [createdId]);
const alertsOpen = await getAlertsByIds(supertest, log, [createdId]);
expect(alertsOpen.hits.hits.length).toEqual(10);
expect(alertsOpen.hits.hits).toHaveLength(10);
});

it('should be able to execute against an exception list that does include valid entries and get back 0 alerts', async () => {
Expand All @@ -149,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('should be able to execute against an exception list that does include valid case sensitive entries and get back 0 alerts', async () => {
Expand Down Expand Up @@ -201,10 +201,10 @@ export default ({ getService }: FtrProviderContext) => {
const alertsOpen2 = await getOpenAlerts(supertest, log, es, createdRule2);
// Expect alerts here because all values are "Ubuntu"
// and exception is one of ["ubuntu"]
expect(alertsOpen.hits.hits.length).toEqual(10);
expect(alertsOpen.hits.hits).toHaveLength(10);
// Expect no alerts here because all values are "Ubuntu"
// and exception is one of ["ubuntu", "Ubuntu"]
expect(alertsOpen2.hits.hits.length).toEqual(0);
expect(alertsOpen2.hits.hits).toHaveLength(0);
});

it('generates no alerts when an exception is added for an EQL rule', async () => {
Expand All @@ -223,7 +223,7 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('generates no alerts when an exception is added for a threshold rule', async () => {
Expand All @@ -245,7 +245,7 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('generates no alerts when an exception is added for a threat match rule', async () => {
Expand Down Expand Up @@ -288,8 +288,9 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

describe('rules with value list exceptions', () => {
beforeEach(async () => {
await createListsIndex(supertest, log);
Expand Down Expand Up @@ -328,7 +329,7 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('generates no alerts when a value list exception is added for a threat match rule', async () => {
Expand Down Expand Up @@ -376,7 +377,7 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('generates no alerts when a value list exception is added for a threshold rule', async () => {
Expand Down Expand Up @@ -413,7 +414,7 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('generates no alerts when a value list exception is added for an EQL rule', async () => {
Expand All @@ -438,8 +439,9 @@ export default ({ getService }: FtrProviderContext) => {
],
]);
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
expect(alertsOpen.hits.hits.length).toEqual(0);
expect(alertsOpen.hits.hits).toHaveLength(0);
});

it('should Not allow deleting value list when there are references and ignoreReferences is false', async () => {
const valueListId = 'value-list-id.txt';
await importFile(supertest, log, 'keyword', ['suricata-sensor-amsterdam'], valueListId);
Expand Down
59 changes: 29 additions & 30 deletions ...ons_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,7 @@ export default ({ getService }: FtrProviderContext) => {
const auditPath = dataPathBuilder.getPath('auditbeat/hosts');
const packetBeatPath = dataPathBuilder.getPath('packetbeat/default');

// FLAKY: https://github.com/elastic/kibana/issues/220943
describe.skip('@ess @serverless @serverlessQA EQL type rules', () => {
describe('@ess @serverless @serverlessQA EQL type rules', () => {
const { indexListOfDocuments } = dataGeneratorFactory({
es,
index: 'ecs_compliant',
Expand Down Expand Up @@ -120,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(alerts.hits.hits.length).eql(1);
expect(alerts.hits.hits).toHaveLength(1);
const fullAlert = alerts.hits.hits[0]._source;
if (!fullAlert) {
return kbnExpect(fullAlert).to.be.ok();
Expand Down Expand Up @@ -289,7 +288,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxAlerts * 2 });
kbnExpect(previewAlerts.length).eql(maxAlerts);
expect(previewAlerts).toHaveLength(maxAlerts);
});

it('generates max alerts warning when circuit breaker is hit', async () => {
Expand All @@ -308,7 +307,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).eql(1);
expect(previewAlerts).toHaveLength(1);
const fullAlert = previewAlerts[0]._source;
if (!fullAlert) {
return kbnExpect(fullAlert).to.be.ok();
Expand Down Expand Up @@ -378,7 +377,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).eql(3);
expect(previewAlerts).toHaveLength(3);

const createdAtHits = previewAlerts.map((hit) => hit._source?.created_at).sort();
kbnExpect(createdAtHits).to.eql([1622676785, 1622676790, 1622676795]);
Expand All @@ -392,7 +391,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).eql(3);
expect(previewAlerts).toHaveLength(3);

const createdAtHits = previewAlerts.map((hit) => hit._source?.locale);
kbnExpect(createdAtHits).to.eql(['es', 'pt', 'ua']);
Expand Down Expand Up @@ -672,7 +671,7 @@ export default ({ getService }: FtrProviderContext) => {

const previewAlerts = await getPreviewAlerts({ es, previewId, sort: ['agent.name'] });

kbnExpect(previewAlerts).to.have.length(3);
expect(previewAlerts).toHaveLength(3);

const buildingBlockAlerts = previewAlerts.filter(
(alert) => alert._source?.['kibana.alert.building_block_type']
Expand Down Expand Up @@ -716,11 +715,11 @@ export default ({ getService }: FtrProviderContext) => {
// For EQL rules, max_alerts is the maximum number of detected sequences: each sequence has a building block
// alert for each event in the sequence, so max_alerts=200 results in 400 building blocks in addition to
// 200 regular alerts
kbnExpect(previewAlerts.length).eql(maxAlerts * 3);
expect(previewAlerts).toHaveLength(maxAlerts * 3);
const shellAlerts = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 2);
const buildingBlocks = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 1);
kbnExpect(shellAlerts.length).eql(maxAlerts);
kbnExpect(buildingBlocks.length).eql(maxAlerts * 2);
expect(shellAlerts).toHaveLength(maxAlerts);
expect(buildingBlocks).toHaveLength(maxAlerts * 2);
});

it('generates alerts when an index name contains special characters to encode', async () => {
Expand All @@ -730,7 +729,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).eql(1);
expect(previewAlerts).toHaveLength(1);
});

it('uses the provided filters', async () => {
Expand Down Expand Up @@ -776,7 +775,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).eql(2);
expect(previewAlerts).toHaveLength(2);
});

describe('with host risk index', () => {
Expand All @@ -795,7 +794,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).eql(1);
expect(previewAlerts).toHaveLength(1);
const fullAlert = previewAlerts[0]._source;
if (!fullAlert) {
return kbnExpect(fullAlert).to.be.ok();
Expand Down Expand Up @@ -850,7 +849,7 @@ export default ({ getService }: FtrProviderContext) => {
kbnExpect(_log.warnings).to.eql([expectedWarning]);

const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).to.be.greaterThan(0);
expect(previewAlerts).not.toHaveLength(0);
});

it('specifying only timestamp_override results in alert creation with an kbnExpect.expected warning', async () => {
Expand All @@ -868,7 +867,7 @@ export default ({ getService }: FtrProviderContext) => {
kbnExpect(_log.warnings).to.eql([expectedWarning]);

const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).to.be.greaterThan(0);
expect(previewAlerts).not.toHaveLength(0);
});

it('specifying both timestamp_override and timestamp_field results in alert creation with an kbnExpect.expected warning', async () => {
Expand All @@ -887,7 +886,7 @@ export default ({ getService }: FtrProviderContext) => {
kbnExpect(_log.warnings).to.eql([expectedWarning]);

const previewAlerts = await getPreviewAlerts({ es, previewId });
kbnExpect(previewAlerts.length).to.be.greaterThan(0);
expect(previewAlerts).not.toHaveLength(0);
});
});

Expand Down Expand Up @@ -959,7 +958,7 @@ export default ({ getService }: FtrProviderContext) => {
kbnExpect(_log.warnings).to.be.empty();
const previewAlerts = await getPreviewAlerts({ es, previewId });

kbnExpect(previewAlerts).to.have.length(3);
expect(previewAlerts).toHaveLength(3);
});
});

Expand Down Expand Up @@ -1034,7 +1033,7 @@ export default ({ getService }: FtrProviderContext) => {

const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(alerts.hits.hits.length).equal(3);
expect(alerts.hits.hits).toHaveLength(3);
kbnExpect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('scheduled');

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
Expand All @@ -1044,7 +1043,7 @@ export default ({ getService }: FtrProviderContext) => {

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(allNewAlerts.hits.hits.length).equal(6);
expect(allNewAlerts.hits.hits).toHaveLength(6);
kbnExpect(allNewAlerts.hits.hits[5]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('manual');

const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], {
Expand All @@ -1054,7 +1053,7 @@ export default ({ getService }: FtrProviderContext) => {

await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
const allNewAlertsAfter2ManualRuns = await getAlerts(supertest, log, es, createdRule);
kbnExpect(allNewAlertsAfter2ManualRuns.hits.hits.length).equal(6);
expect(allNewAlertsAfter2ManualRuns.hits.hits).toHaveLength(6);
});

it('does not alert if the manual run overlaps with a previous scheduled rule execution', async () => {
Expand Down Expand Up @@ -1093,7 +1092,7 @@ export default ({ getService }: FtrProviderContext) => {
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

kbnExpect(alerts.hits.hits.length).equal(3);
expect(alerts.hits.hits).toHaveLength(3);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
Expand All @@ -1102,7 +1101,7 @@ export default ({ getService }: FtrProviderContext) => {

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(allNewAlerts.hits.hits.length).equal(3);
expect(allNewAlerts.hits.hits).toHaveLength(3);
});

it('supression per rule execution should work for manual rule runs', async () => {
Expand Down Expand Up @@ -1146,7 +1145,7 @@ export default ({ getService }: FtrProviderContext) => {
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

kbnExpect(alerts.hits.hits.length).equal(0);
expect(alerts.hits.hits).toHaveLength(0);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
Expand All @@ -1155,7 +1154,7 @@ export default ({ getService }: FtrProviderContext) => {

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(allNewAlerts.hits.hits.length).equal(1);
expect(allNewAlerts.hits.hits).toHaveLength(1);

kbnExpect(allNewAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).equal(2);
});
Expand Down Expand Up @@ -1191,7 +1190,7 @@ export default ({ getService }: FtrProviderContext) => {
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

kbnExpect(alerts.hits.hits.length).equal(0);
expect(alerts.hits.hits).toHaveLength(0);

// generate alert in the past
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
Expand All @@ -1200,7 +1199,7 @@ export default ({ getService }: FtrProviderContext) => {
});
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(allNewAlerts.hits.hits.length).equal(1);
expect(allNewAlerts.hits.hits).toHaveLength(1);

// now we will ingest new event, and manual rule run should update original alert
const secondDocument = {
Expand All @@ -1220,9 +1219,9 @@ export default ({ getService }: FtrProviderContext) => {

await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
const updatedAlerts = await getAlerts(supertest, log, es, createdRule);
kbnExpect(updatedAlerts.hits.hits.length).equal(1);
expect(updatedAlerts.hits.hits).toHaveLength(1);

kbnExpect(updatedAlerts.hits.hits.length).equal(1);
expect(updatedAlerts.hits.hits).toHaveLength(1);

kbnExpect(updatedAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).equal(1);
});
Expand All @@ -1247,7 +1246,7 @@ export default ({ getService }: FtrProviderContext) => {

const requests = logs[0].requests;

kbnExpect(requests).to.have.length(1);
expect(requests).toHaveLength(1);
kbnExpect(requests![0].description).to.be('EQL request to find all matches');
kbnExpect(requests![0].request).to.contain(
'POST /auditbeat-*/_eql/search?allow_no_indices=true'
Expand Down
Loading