diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index 52e92332cd506..e1271601de41f 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -17,6 +17,9 @@ disabled: # Playwright - x-pack/solutions/security/test/security_solution_playwright/serverless_config.ts + # QA suites that are run out-of-band + - x-pack/solutions/security/test/serverless/functional/configs/config.cloud_security_posture.cloud.ts + # MKI only configs files - x-pack/solutions/security/test/serverless/functional/configs/config.mki_only.ts diff --git a/x-pack/solutions/security/test/serverless/functional/configs/config.cloud_security_posture.cloud.ts b/x-pack/solutions/security/test/serverless/functional/configs/config.cloud_security_posture.cloud.ts new file mode 100644 index 0000000000000..ea9d1942c028e --- /dev/null +++ b/x-pack/solutions/security/test/serverless/functional/configs/config.cloud_security_posture.cloud.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '@kbn/test-suites-xpack-platform/serverless/functional/config.base'; +import { services } from '../services'; +import { pageObjects } from '../page_objects'; + +export default createTestConfig({ + serverlessProject: 'security', + pageObjects, + services, + junit: { + reportName: 'Serverless Security Cloud Security Functional Tests', + }, + + // load tests in the index file + testFiles: [require.resolve('../test_suites/ftr/cloud_security_posture/cloud_tests')], +}); diff --git a/x-pack/solutions/security/test/serverless/functional/page_objects/index.ts b/x-pack/solutions/security/test/serverless/functional/page_objects/index.ts index 73a9331e7c37f..7acf3b7a19fd1 100644 --- a/x-pack/solutions/security/test/serverless/functional/page_objects/index.ts +++ b/x-pack/solutions/security/test/serverless/functional/page_objects/index.ts @@ -10,6 +10,8 @@ import { SvlSecLandingPageProvider } from './svl_sec_landing_page'; import { CspSecurityCommonProvider } from './security_common'; import { CspDashboardPageProvider } from '../../../cloud_security_posture_functional/page_objects/csp_dashboard_page'; import { AddCisIntegrationFormPageProvider } from '../../../cloud_security_posture_functional/page_objects/add_cis_integration_form_page'; +import { BenchmarkPagePageProvider } from '../../../cloud_security_posture_functional/page_objects/benchmark_page'; +import { FindingsPageProvider } from '../../../cloud_security_posture_functional/page_objects/findings_page'; export const pageObjects = { ...svlPlatformPageObjects, @@ -18,4 +20,6 @@ export const pageObjects = { cloudPostureDashboard: CspDashboardPageProvider, cisAddIntegration: AddCisIntegrationFormPageProvider, cspSecurity: CspSecurityCommonProvider, + cspBenchmarkPage: BenchmarkPagePageProvider, + cspFindingsPage: FindingsPageProvider, }; diff --git a/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/benchmark_sanity.ts b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/benchmark_sanity.ts new file mode 100644 index 0000000000000..22ea8284f1ddb --- /dev/null +++ b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/benchmark_sanity.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import type { FtrProviderContext } from '../../../../ftr_provider_context'; +export default function ({ getPageObjects }: FtrProviderContext) { + const pageObjects = getPageObjects([ + 'settings', + 'common', + 'svlCommonPage', + 'header', + 'cspBenchmarkPage', + ]); + + describe('Benchmark Page - Sanity Tests', function () { + this.tags(['cloud_security_posture_ui_sanity']); + let benchmark: typeof pageObjects.cspBenchmarkPage; + + before(async () => { + benchmark = pageObjects.cspBenchmarkPage; + await pageObjects.svlCommonPage.loginWithRole('admin'); + await benchmark.navigateToBenchnmarkPage(); + await benchmark.waitForPluginInitialized(); + }); + + it('Benchmark table exists', async () => { + expect(await benchmark.benchmarkPage.doesBenchmarkTableExists()); + }); + + it('Benchmarks count is more than 0', async () => { + const benchmarksRows = await benchmark.benchmarkPage.getBenchmarkTableRows(); + expect(benchmarksRows.length).to.be.greaterThan(0); + }); + + it('For each benchmark, evaluation and complience are not empty', async () => { + const benchmarksRows = await benchmark.benchmarkPage.getBenchmarkTableRows(); + for (const row of benchmarksRows) { + const benchmarkName = await benchmark.benchmarkPage.getCisNameCellData(row); + const isEvaluationEmpty = await benchmark.benchmarkPage.isEvaluationEmpty(row); + const isComplianceEmpty = await benchmark.benchmarkPage.isComplianceEmpty(row); + + expect(isEvaluationEmpty).to.eql( + false, + `The ${benchmarkName} does not have evaluated data` + ); + + expect(isComplianceEmpty).to.eql( + false, + `The ${benchmarkName} does not have compliance data` + ); + } + }); + }); +} diff --git a/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/dashboard_sanity.ts b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/dashboard_sanity.ts new file mode 100644 index 0000000000000..b2fa3986c1599 --- /dev/null +++ b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/dashboard_sanity.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import type { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + const retry = getService('retry'); + const pageObjects = getPageObjects([ + 'common', + 'cloudPostureDashboard', + 'header', + 'svlCommonPage', + 'cspFindingsPage', + ]); + + describe('Cloud Posture Dashboard Page - Sanity Tests', function () { + this.tags(['cloud_security_posture_ui_sanity']); + let cspDashboard: typeof pageObjects.cloudPostureDashboard; + let dashboard: typeof pageObjects.cloudPostureDashboard.dashboard; + let findings: typeof pageObjects.cspFindingsPage; + let TAB_TYPES: typeof pageObjects.cloudPostureDashboard.TAB_TYPES; + + before(async () => { + cspDashboard = pageObjects.cloudPostureDashboard; + dashboard = pageObjects.cloudPostureDashboard.dashboard; + findings = pageObjects.cspFindingsPage; + TAB_TYPES = pageObjects.cloudPostureDashboard.TAB_TYPES; + await pageObjects.svlCommonPage.loginWithRole('admin'); + await cspDashboard.waitForPluginInitialized(); + await cspDashboard.navigateToComplianceDashboardPage(); + await retry.waitFor( + 'Cloud posture integration dashboard to be displayed', + async () => !!dashboard.getIntegrationDashboardContainer() + ); + }); + + describe('Cloud Dashboard', () => { + it('displays compliance score greater than 40', async () => { + await pageObjects.header.waitUntilLoadingHasFinished(); + const scoreElement = await dashboard.getCloudComplianceScore(); + const score = parseInt((await scoreElement.getVisibleText()).replace('%', ''), 10); + expect(score).to.be.greaterThan(40); + }); + + it('displays all compliance scores', async () => { + const scoresElements = await dashboard.getAllCloudComplianceScores(); + const scores: string[] = []; + for (const scoreElement of scoresElements) { + scores.push(await scoreElement.getVisibleText()); + } + // 3 scores for each cloud provider + 1 summary score + expect(scores.length).to.be(4); + }); + + it('displays a number of resources evaluated greater than 1500', async () => { + const resourcesEvaluated = await dashboard.getCloudResourcesEvaluated(); + const visibleText = await resourcesEvaluated.getVisibleText(); + const resourcesEvaluatedCount = parseInt(visibleText.replace(/,/g, ''), 10); + expect(resourcesEvaluatedCount).greaterThan(1500); + }); + + it('Compliance By CIS sections have non empty values', async () => { + const complianceScoresChartPanel = await dashboard.getAllComplianceScoresByCisSection( + TAB_TYPES.CLOUD + ); + expect(complianceScoresChartPanel.length).to.be.greaterThan(0); + for (const score of complianceScoresChartPanel) { + const scoreValue = await score.getVisibleText(); + // Check if the score is a percentage + expect(scoreValue).to.match(/^\d+%$/); + } + }); + + it('Navigation to Findings page', async () => { + const findingsLinkCount = await dashboard.getFindingsLinksCount(TAB_TYPES.CLOUD); + for (let i = 0; i < findingsLinkCount; i++) { + const link = await dashboard.getFindingsLinkAtIndex(TAB_TYPES.CLOUD, i); + await link.click(); + await pageObjects.header.waitUntilLoadingHasFinished(); + const groupSelector = await findings.groupSelector(); + await groupSelector.openDropDown(); + await groupSelector.setValue('None'); + expect( + await findings.createDataTableObject('latest_findings_table').getRowsCount() + ).to.be.greaterThan(0); + await cspDashboard.navigateToComplianceDashboardPage(); + await pageObjects.header.waitUntilLoadingHasFinished(); + } + }); + }); + + describe('Kubernetes Dashboard', () => { + it('displays compliance score greater than 80', async () => { + await pageObjects.header.waitUntilLoadingHasFinished(); + const scoreElement = await dashboard.getKubernetesComplianceScore(); + const score = parseInt((await scoreElement.getVisibleText()).replace('%', ''), 10); + expect(score).to.be.greaterThan(80); + }); + + it('displays a number of resources evaluated greater than 150', async () => { + const resourcesEvaluated = await dashboard.getKubernetesResourcesEvaluated(); + const resourcesEvaluatedCount = parseInt( + (await resourcesEvaluated.getVisibleText()).replace(/,/g, ''), + 10 + ); + expect(resourcesEvaluatedCount).greaterThan(150); + }); + + it('Compliance By CIS sections have non empty values', async () => { + const complianceScoresChartPanel = await dashboard.getAllComplianceScoresByCisSection( + 'Kubernetes' + ); + expect(complianceScoresChartPanel.length).to.be.greaterThan(0); + for (const score of complianceScoresChartPanel) { + const scoreValue = await score.getVisibleText(); + // Check if the score is a percentage + expect(scoreValue).to.match(/^\d+%$/); + } + }); + + it('Navigation to Findings page', async () => { + const findingsLinkCount = await dashboard.getFindingsLinksCount(TAB_TYPES.KUBERNETES); + for (let i = 0; i < findingsLinkCount; i++) { + const link = await dashboard.getFindingsLinkAtIndex(TAB_TYPES.KUBERNETES, i); + await link.click(); + await pageObjects.header.waitUntilLoadingHasFinished(); + const groupSelector = await findings.groupSelector(); + await groupSelector.openDropDown(); + await groupSelector.setValue('None'); + expect( + await findings.createDataTableObject('latest_findings_table').getRowsCount() + ).to.be.greaterThan(0); + await cspDashboard.navigateToComplianceDashboardPage(); + await pageObjects.header.waitUntilLoadingHasFinished(); + await dashboard.getKubernetesDashboard(); + } + }); + }); + }); +}; diff --git a/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/findings_sanity.ts b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/findings_sanity.ts new file mode 100644 index 0000000000000..0d1e11b2f825d --- /dev/null +++ b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/findings_sanity.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import type { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + const pageObjects = getPageObjects(['common', 'svlCommonPage', 'cspFindingsPage', 'header']); + const queryBar = getService('queryBar'); + const testSubjects = getService('testSubjects'); + + describe('Findings Page - Sanity Tests', function () { + this.tags(['cloud_security_posture_ui_sanity']); + let findings: typeof pageObjects.cspFindingsPage; + let latestFindingsTable: typeof findings.latestFindingsTable; + + before(async () => { + findings = pageObjects.cspFindingsPage; + latestFindingsTable = pageObjects.cspFindingsPage.latestFindingsTable; + + await pageObjects.svlCommonPage.loginWithRole('admin'); + await findings.navigateToLatestFindingsPage(); + await pageObjects.header.waitUntilLoadingHasFinished(); + }); + + describe('Findings - Querying data', () => { + afterEach(async () => { + // Reset the group selector to None + const groupSelector = await findings.groupSelector(); + await groupSelector.openDropDown(); + await groupSelector.setValue('None'); + // Reset search query + await queryBar.clearQuery(); + await queryBar.submitQuery(); + }); + + const testCases = [ + { + searchQuery: + 'cloud.provider : "aws" and cloud.region : "eu-west-3" and result.evaluation : "failed" and rule.tags : "CIS 5.4"', + provider: 'aws', + expectedRowsCount: 3, + expectedGroupCount: '1 cloud account', + expectedUnitCount: '3 findings', + }, + { + searchQuery: + 'cloud.provider : "gcp" and rule.benchmark.rule_number : "3.1" and result.evaluation : "failed"', + provider: 'gcp', + expectedRowsCount: 1, + expectedGroupCount: '1 cloud account', + expectedUnitCount: '1 finding', + }, + { + searchQuery: + 'cloud.provider : "azure" and rule.benchmark.rule_number : "9.1" and result.evaluation : "failed"', + provider: 'azure', + expectedRowsCount: 1, + expectedGroupCount: '1 cloud account', + expectedUnitCount: '1 finding', + }, + { + searchQuery: + 'rule.benchmark.id : "cis_k8s" and rule.benchmark.rule_number : "4.2.4" and result.evaluation : "failed"', + provider: 'k8s', + expectedRowsCount: 2, + expectedGroupCount: '0 cloud accounts', + expectedUnitCount: '2 findings', + }, + { + searchQuery: 'rule.benchmark.id : "cis_eks" and rule.benchmark.rule_number : "3.1.1"', + provider: 'eks', + expectedRowsCount: 2, + expectedGroupCount: '0 cloud accounts', + expectedUnitCount: '2 findings', + }, + ]; + + testCases.forEach( + ({ searchQuery, provider, expectedRowsCount, expectedGroupCount, expectedUnitCount }) => { + it(`Querying ${provider} provider data`, async () => { + // Execute the query + await queryBar.setQuery(searchQuery); + await queryBar.submitQuery(); + // Get the number of rows in the data table + const rowsCount = await findings + .createDataTableObject('latest_findings_table') + .getRowsCount(); + + // Check that the number of rows matches the expected count + expect(rowsCount).to.be(expectedRowsCount); + const groupSelector = await findings.groupSelector(); + await groupSelector.openDropDown(); + await groupSelector.setValue('Cloud account ID'); + const grouping = await findings.findingsGrouping(); + // Check that the group count and unit count matches the expected values + const groupCount = await grouping.getGroupCount(); + expect(groupCount).to.be(expectedGroupCount); + + const unitCount = await grouping.getUnitCount(); + expect(unitCount).to.be(expectedUnitCount); + }); + } + ); + }); + + describe('Findings - Sorting data', () => { + afterEach(async () => { + const paginationBtn = await testSubjects.find('tablePaginationPopoverButton'); + await paginationBtn.click(); + const pageSizeOption = await testSubjects.find('tablePagination-50-rows'); + await pageSizeOption.click(); + }); + + type SortDirection = 'asc' | 'desc'; + const paginationAndsortingTestCases: Array<{ + searchQuery: string; + paginationRowsCount: string; + columnName: string; + sortType: SortDirection; + expectedResult: string; + }> = [ + { + searchQuery: + 'cloud.provider : "aws" and resource.sub_type : "aws-iam-user" and result.evaluation : "passed"', + paginationRowsCount: '250', + columnName: 'rule.benchmark.rule_number', + sortType: 'desc', + expectedResult: '1.7', + }, + { + searchQuery: 'cloud.provider : "azure" and result.evaluation : "failed"', + paginationRowsCount: '500', + columnName: 'rule.benchmark.rule_number', + sortType: 'asc', + expectedResult: '1.23', + }, + { + searchQuery: 'cloud.provider : "gcp" and result.evaluation : "passed"', + paginationRowsCount: '500', + columnName: 'resource.sub_type', + sortType: 'desc', + expectedResult: 'gcp-storage-bucket', + }, + ]; + + paginationAndsortingTestCases.forEach( + ({ searchQuery, paginationRowsCount, columnName, sortType, expectedResult }) => { + it(`Paginating and sorting data`, async () => { + // Run query + await queryBar.clearQuery(); + await queryBar.setQuery(searchQuery); + await queryBar.submitQuery(); + // Update latest findings table pagination + const paginationBtn = await testSubjects.find('tablePaginationPopoverButton'); + await paginationBtn.click(); + const pageSizeOption = await testSubjects.find( + `tablePagination-${paginationRowsCount}-rows` + ); + await pageSizeOption.click(); + // Sort by column + await latestFindingsTable.toggleColumnSort(columnName, sortType); + await pageObjects.header.waitUntilLoadingHasFinished(); + const values = (await latestFindingsTable.getColumnValues(columnName)).filter(Boolean); + // Check that the first value matches the expected result + // Whole sorting logic functionality is checked in the findings.ts + expect(values[0]).to.equal(expectedResult); + }); + } + ); + }); + }); +}; diff --git a/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/index.ts b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/index.ts new file mode 100644 index 0000000000000..13f2b44eef4cb --- /dev/null +++ b/x-pack/solutions/security/test/serverless/functional/test_suites/ftr/cloud_security_posture/cloud_tests/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('cloud_security_posture', function () { + this.tags(['cloud_security_posture_sanity_tests']); + loadTestFile(require.resolve('./benchmark_sanity')); + loadTestFile(require.resolve('./findings_sanity')); + loadTestFile(require.resolve('./dashboard_sanity')); + }); +}