Skip to content

[Security Solution] Enable critical Rule Management tests in MKI periodic and 2nd quality gate pipelines #193666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
maximpn merged 5 commits into from
Sep 27, 2024
Merged

[Security Solution] Enable critical Rule Management tests in MKI periodic and 2nd quality gate pipelines #193666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e3027d7
enable specific Cypress tests in MKI periodic/2nd quality gate
maximpn Sep 22, 2024
789c777
enable specific FTR tests in MKI periodic/2nd quality gate
maximpn Sep 23, 2024
8105bec
split FTR bulk actions tests
maximpn Sep 23, 2024
795e7c9
revert bulk rule deletion
maximpn Sep 26, 2024
980b290
skip deprecated create rules bulk FTR tests in MKI
maximpn Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...tections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./perform_bulk_action_dry_run_ess'));
loadTestFile(require.resolve('./perform_bulk_action'));
loadTestFile(require.resolve('./perform_bulk_action_ess'));
loadTestFile(require.resolve('./perform_bulk_enable_disable.ts'));
});
}
36 changes: 0 additions & 36 deletions ...nse/rules_management/rule_bulk_actions/trial_license_complete_tier/perform_bulk_action.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,42 +273,6 @@ export default ({ getService }: FtrProviderContext): void => {
await fetchRule(ruleId).expect(404);
});

it('should enable rules', async () => {
const ruleId = 'ruleId';
await createRule(supertest, log, getSimpleRule(ruleId));

const { body } = await postBulkAction()
.send({ query: '', action: BulkActionTypeEnum.enable })
.expect(200);

expect(body.attributes.summary).toEqual({ failed: 0, skipped: 0, succeeded: 1, total: 1 });

// Check that the updated rule is returned with the response
expect(body.attributes.results.updated[0].enabled).toEqual(true);

// Check that the updates have been persisted
const { body: ruleBody } = await fetchRule(ruleId).expect(200);
expect(ruleBody.enabled).toEqual(true);
});

it('should disable rules', async () => {
const ruleId = 'ruleId';
await createRule(supertest, log, getSimpleRule(ruleId, true));

const { body } = await postBulkAction()
.send({ query: '', action: BulkActionTypeEnum.disable })
.expect(200);

expect(body.attributes.summary).toEqual({ failed: 0, skipped: 0, succeeded: 1, total: 1 });

// Check that the updated rule is returned with the response
expect(body.attributes.results.updated[0].enabled).toEqual(false);

// Check that the updates have been persisted
const { body: ruleBody } = await fetchRule(ruleId).expect(200);
expect(ruleBody.enabled).toEqual(false);
});

it('should duplicate rules', async () => {
const ruleId = 'ruleId';
const ruleToDuplicate = getCustomQueryRuleParams({
Expand Down
70 changes: 70 additions & 0 deletions ...s_management/rule_bulk_actions/trial_license_complete_tier/perform_bulk_enable_disable.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from 'expect';
import { BulkActionTypeEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management';
import { getCustomQueryRuleParams, fetchRule } from '../../../utils';
import { createRule, deleteAllRules } from '../../../../../../common/utils/security_solution';
import { FtrProviderContext } from '../../../../../ftr_provider_context';

export default ({ getService }: FtrProviderContext): void => {
const supertest = getService('supertest');
const securitySolutionApi = getService('securitySolutionApi');
const log = getService('log');

describe('@ess @serverless @serverlessQA Bulk enable/disable', () => {
beforeEach(async () => {
await deleteAllRules(supertest, log);
});

it('should enable rules', async () => {
const ruleId = 'ruleId';
await createRule(
supertest,
log,
getCustomQueryRuleParams({ rule_id: ruleId, enabled: false })
);

const { body } = await securitySolutionApi.performRulesBulkAction({
query: {},
body: { action: BulkActionTypeEnum.enable },
});

expect(body.attributes.summary).toEqual({ failed: 0, skipped: 0, succeeded: 1, total: 1 });

// Check that the updated rule is returned with the response
expect(body.attributes.results.updated[0].enabled).toEqual(true);

// Check that the updates have been persisted
const ruleBody = await fetchRule(supertest, { ruleId });
expect(ruleBody.enabled).toEqual(true);
});

it('should disable rules', async () => {
const ruleId = 'ruleId';
await createRule(
supertest,
log,
getCustomQueryRuleParams({ rule_id: ruleId, enabled: true })
);

const { body } = await securitySolutionApi.performRulesBulkAction({
query: {},
body: { action: BulkActionTypeEnum.disable },
});

expect(body.attributes.summary).toEqual({ failed: 0, skipped: 0, succeeded: 1, total: 1 });

// Check that the updated rule is returned with the response
expect(body.attributes.results.updated[0].enabled).toEqual(false);

// Check that the updates have been persisted
const ruleBody = await fetchRule(supertest, { ruleId });
expect(ruleBody.enabled).toEqual(false);
});
});
};
2 changes: 1 addition & 1 deletion ...ules_management/rule_creation/basic_license_essentials_tier/create_ml_rules_privileges.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ export default ({ getService }: FtrProviderContext) => {
});
});

it('@serverless should give a 200 when trying to create a single Machine Learning rule since the license is essentials', async () => {
it('@serverless @serverlessQA should give a 200 when trying to create a single Machine Learning rule since the license is essentials', async () => {
const { body } = await supertest
.post(DETECTION_ENGINE_RULES_URL)
.set('kbn-xsrf', 'true')
Expand Down
2 changes: 1 addition & 1 deletion ...ons_response/rules_management/rule_creation/basic_license_essentials_tier/create_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext) => {
const dataPathBuilder = new EsArchivePathBuilder(isServerless);
const auditbeatPath = dataPathBuilder.getPath('auditbeat/hosts');

describe('@ess @serverless create_rules', () => {
describe('@ess @serverless @serverlessQA create_rules', () => {
describe('creating rules', () => {
before(async () => {
await esArchiver.load(auditbeatPath);
Expand Down
2 changes: 1 addition & 1 deletion ...esponse/rules_management/rule_creation/basic_license_essentials_tier/create_rules_bulk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export default ({ getService }: FtrProviderContext): void => {
const auditbeatPath = dataPathBuilder.getPath('auditbeat/hosts');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless create_rules_bulk', () => {
describe('@ess @serverless @skipInServerlessMKI create_rules_bulk', () => {
describe('creating rules in bulk', () => {
before(async () => {
await esArchiver.load(auditbeatPath);
Expand Down
2 changes: 1 addition & 1 deletion ...s_response/rules_management/rule_creation/trial_license_complete_tier/create_new_terms.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export default ({ getService }: FtrProviderContext) => {
/**
* Specific api integration tests for new terms rule type
*/
describe('@serverless @ess create_new_terms', () => {
describe('@serverless @ess @serverlessQA create_new_terms', () => {
Copy link
Copy Markdown
Contributor

@jpdjere jpdjere Sep 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess we have no test coverage for creating other rule types. But just wondering why you've decided to add this one to ServerlessKibanaQAGate. Do you think that there should be similar tests to this, one for each rule type, and all should be part of ServerlessKibanaQAGate?

Copy link
Copy Markdown
Contributor Author

@maximpn maximpn Sep 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're correct we have a limited test coverage. The goal of this PR is to enable smoke tests from what's available. Since we have a test for new terms rule type it makes sense to enable it in MKI.

beforeEach(async () => {
await deleteAllRules(supertest, log);
});
Expand Down
2 changes: 1 addition & 1 deletion ...tions_response/rules_management/rule_creation/trial_license_complete_tier/create_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@serverless @ess create_rules', () => {
describe('@serverless @ess @serverlessQA create_rules', () => {
describe('rule creation', () => {
before(async () => {
await es.indices.delete({ index: 'logs-test', ignore_unavailable: true });
Expand Down
2 changes: 1 addition & 1 deletion ...ions_response/rules_management/rule_creation/trial_license_complete_tier/preview_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext) => {
const dataPathBuilder = new EsArchivePathBuilder(isServerless);
const path = dataPathBuilder.getPath('auditbeat/hosts');

describe('@serverless @ess preview_rules', () => {
describe('@serverless @ess @serverlessQA preview_rules', () => {
describe('previewing rules', () => {
before(async () => {
await esArchiver.load(path);
Expand Down
2 changes: 1 addition & 1 deletion ...tions_response/rules_management/rule_delete/basic_license_essentials_tier/delete_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless delete_rules', () => {
describe('@ess @serverless @skipInServerlessMKI delete_rules', () => {
describe('deleting rules', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ..._response/rules_management/rule_delete/basic_license_essentials_tier/delete_rules_bulk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext): void => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless delete_rules_bulk', () => {
describe('@ess @serverless @skipInServerlessMKI delete_rules_bulk', () => {
describe('deleting rules bulk using DELETE', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...ections_response/rules_management/rule_delete/trial_license_complete_tier/delete_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless delete_rules', () => {
describe('@ess @serverless @skipInServerlessMKI delete_rules', () => {
describe('deleting rules', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...esponse/rules_management/rule_import_export/basic_license_essentials_tier/export_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export default ({ getService }: FtrProviderContext): void => {
const log = getService('log');
const securitySolutionApi = getService('securitySolutionApi');

describe('@ess @serverless export_rules', () => {
describe('@ess @serverless @serverlessQA export_rules', () => {
describe('exporting rules', () => {
afterEach(async () => {
await deleteAllRules(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...esponse/rules_management/rule_import_export/basic_license_essentials_tier/import_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export default ({ getService }: FtrProviderContext): void => {
const securitySolutionApi = getService('securitySolutionApi');
const log = getService('log');

describe('@ess @serverless import_rules', () => {
describe('@ess @serverless @serverlessQA import_rules', () => {
Copy link
Copy Markdown
Contributor

@jpdjere jpdjere Sep 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reason to add import_rules to ServerlessKibanaQAGate, but not import_rules_with_overwrite?

I'm just thinking that the import function is becoming increasingly important, especially since it will be the main way to bulk create rules once the Bulk Create API is fully deprecated in 9.0

Copy link
Copy Markdown
Contributor Author

@maximpn maximpn Sep 26, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The goal is to enable smoke testing for importing functionality. There is an assumption that a scenario when import_rules_with_overwrite fails only in MKI while passes in PR and on merge pipelines but import_rules passes everywhere is quite rare. That's why the decision is to enable only basic tests.

describe('importing rules with an index', () => {
afterEach(async () => {
await deleteAllRules(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...ponse/rules_management/rule_management/basic_license_essentials_tier/coverage_overview.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ export default ({ getService }: FtrProviderContext): void => {
});

// Both serverless and ESS
describe('@serverless @ess tests', () => {
describe('@ess @serverless @skipInServerlessMKI tests', () => {
describe('base cases', () => {
it('returns an empty response if there are no rules', async () => {
const body = await getCoverageOverview(supertest);
Expand Down
2 changes: 1 addition & 1 deletion ...les_management/rule_management/trial_license_complete_tier/get_rule_management_filters.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export default ({ getService }: FtrProviderContext): void => {
const es = getService('es');
const log = getService('log');

describe('@ess @serverless get_rule_management_filters', () => {
describe('@ess @serverless @serverlessQA get_rule_management_filters', () => {
beforeEach(async () => {
await deleteAllRules(supertest, log);
});
Expand Down
2 changes: 1 addition & 1 deletion ...ections_response/rules_management/rule_patch/basic_license_essentials_tier/patch_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless patch_rules', () => {
describe('@ess @serverless @serverlessQA patch_rules', () => {
describe('patch rules', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...ns_response/rules_management/rule_patch/basic_license_essentials_tier/patch_rules_bulk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless patch_rules_bulk', () => {
describe('@ess @serverless @skipInServerlessMKI patch_rules_bulk', () => {
describe('patch rules bulk', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...etections_response/rules_management/rule_read/basic_license_essentials_tier/find_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export default ({ getService }: FtrProviderContext): void => {
const log = getService('log');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless find_rules', () => {
describe('@ess @serverless @serverlessQA find_rules', () => {
beforeEach(async () => {
await deleteAllRules(supertest, log);
});
Expand Down
2 changes: 1 addition & 1 deletion ...etections_response/rules_management/rule_read/basic_license_essentials_tier/read_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless read_rules', () => {
describe('@ess @serverless @serverlessQA read_rules', () => {
describe('reading rules', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion .../detections_response/rules_management/rule_read/trial_license_complete_tier/find_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export default ({ getService }: FtrProviderContext): void => {
const log = getService('log');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless find_rules', () => {
describe('@ess @serverless @skipInServerlessMKI find_rules', () => {
Copy link
Copy Markdown
Contributor

@jpdjere jpdjere Sep 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also here I'd like to understand the difference between Basic and Trial Licenses, and why we're adding the Basic tests to QA, but leaving Trial out

Copy link
Copy Markdown
Contributor Author

@maximpn maximpn Sep 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The goal is to have critical functionality smoke tests running in MKI. If one test fails it will block a Serverless release and require our team to do a hotfix. More tests enabled the higher such risk but of course we want to deliver stable features. The decision is to find a balance in that trade off by enabling only basic license tests. It's a critical functionality and always gives an option for a user to roll back to a basic license and wait for the next Serverless release.

Nothing blocks us from enabling more tests later on when it's necessary.

beforeEach(async () => {
await deleteAllRules(supertest, log);
});
Expand Down
2 changes: 1 addition & 1 deletion .../detections_response/rules_management/rule_read/trial_license_complete_tier/read_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless read_rules', () => {
describe('@ess @serverless @skipInServerlessMKI read_rules', () => {
describe('reading rules', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ...tions_response/rules_management/rule_update/basic_license_essentials_tier/update_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless update_rules', () => {
describe('@ess @serverless @serverlessQA update_rules', () => {
describe('update rules', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
2 changes: 1 addition & 1 deletion ..._response/rules_management/rule_update/basic_license_essentials_tier/update_rules_bulk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const utils = getService('securitySolutionUtils');

describe('@ess @serverless update_rules_bulk', () => {
describe('@ess @serverless @skipInServerlessMKI update_rules_bulk', () => {
describe('update rules bulk', () => {
beforeEach(async () => {
await createAlertsIndex(supertest, log);
Expand Down
4 changes: 2 additions & 2 deletions ...cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ const RULE_1 = createRuleAssetSavedObject({

describe(
'Detection rules, Prebuilt Rules Installation and Update Notifications',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@skipInServerlessMKI'] },
() => {
beforeEach(() => {
login();
Expand Down Expand Up @@ -72,7 +72,7 @@ describe(
});

// https://github.com/elastic/kibana/issues/179968
describe('Notifications', { tags: ['@skipInServerlessMKI'] }, () => {
describe('Notifications', () => {
beforeEach(() => {
installPrebuiltRuleAssets([RULE_1]);
});
Expand Down
2 changes: 1 addition & 1 deletion ...ypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ const PREVIEW_TABS = {

describe(
'Detection rules, Prebuilt Rules Installation and Update workflow',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@skipInServerlessMKI'] },
() => {
const commonProperties: Partial<PrebuiltRuleAsset> = {
author: ['Elastic', 'Another author'],
Expand Down
2 changes: 1 addition & 1 deletion ...press/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.cy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import { visitRulesManagementTable } from '../../../../tasks/rules_management';

describe(
'Detection rules, Prebuilt Rules Installation and Update workflow',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@skipInServerlessMKI'] },
() => {
describe('Upgrade of prebuilt rules', () => {
const RULE_1_ID = 'rule_1';
Expand Down
Loading