diff --git a/x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts b/x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts index 19297cb91f977..453391d6612c6 100644 --- a/x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts +++ b/x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts @@ -41,8 +41,21 @@ export const STATUS_API_CURRENT_VERSION = '1'; /** The base path for all cloud security posture pages. */ export const CLOUD_SECURITY_POSTURE_BASE_PATH = '/cloud_security_posture'; +// Array of legacy data view IDs for migration purposes +export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS = [ + 'cloud_security_posture-303eea10-c475-11ec-af18-c5b9b437dbbe', // legacy version 8.x version (logs-cloud_security_posture.findings_latest-*) + 'cloud_security_posture-9129a080-7f48-11ec-8249-431333f83c5f', // legacy version 8.x version (logs-cloud_security_posture.findings-*) +]; +// Array of old data view IDs for migration purposes +// Add new deprecated versions here when updating to a new version +export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS = [ + 'security_solution_cdr_latest_misconfigurations', // v1 +]; + +// Current data view ID - increment version when making breaking changes export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX = - 'security_solution_cdr_latest_misconfigurations'; + 'security_solution_cdr_latest_misconfigurations_v2'; + export const SECURITY_DEFAULT_DATA_VIEW_ID = 'security-solution-default'; export const CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN = @@ -52,6 +65,23 @@ export const CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN = export const CDR_VULNERABILITIES_INDEX_PATTERN = `${CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN},${CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN}`; export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d'; +export const CDR_VULNERABILITIES_DATA_VIEW_NAME = 'Latest Cloud Security Vulnerabilities'; + +// Array of legacy vulnerabilities data view IDs for migration purposes +export const CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS = [ + 'cloud_security_posture-c406d945-a359-4c04-9a6a-65d66de8706b', // legacy 8.x version (logs-cloud_security_posture.vulnerabilities-*) + 'cloud_security_posture-07a5e6d6-982d-4c7c-a845-5f2be43279c9', // legacy 8.x version (logs-cloud_security_posture.vulnerabilities_latest-*) +]; +// Array of old vulnerabilities data view IDs for migration purposes +// Add new deprecated versions here when updating to a new version +export const CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS = [ + 'security_solution_cdr_latest_vulnerabilities', // v1 +]; + +// Current vulnerabilities data view ID - increment version when making breaking changes +export const CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX = + 'security_solution_cdr_latest_vulnerabilities_v2'; + // meant as a temp workaround to get good enough posture view for 3rd party integrations, see https://github.com/elastic/security-team/issues/10683 and https://github.com/elastic/security-team/issues/10801 export const CDR_EXTENDED_VULN_RETENTION_POLICY = '90d'; diff --git a/x-pack/solutions/security/plugins/cloud_security_posture/README.md b/x-pack/solutions/security/plugins/cloud_security_posture/README.md index 90dfb4556f64f..3e90a5790a44f 100755 --- a/x-pack/solutions/security/plugins/cloud_security_posture/README.md +++ b/x-pack/solutions/security/plugins/cloud_security_posture/README.md @@ -8,6 +8,67 @@ Cloud Posture automates the identification and remediation of risks across cloud Read [Kibana Contributing Guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for more details +### DataView Migration Logic + +The data view migration is split into two parts: + +1. Deletion of old and legacy data views during the plugin initialization (only runs once when the CSP package is installed or when Kibana is started) +2. Creation of new data views when the user navigates to the CSP page (the check runs every time the user navigates to the CSP page to see if the data views need to be created) + +When making changes to CSP data views, follow these guidelines: + +#### When to Update Data View Version + +Create a new data view version when: + +1. **Index Pattern Changes**: Updating the underlying index pattern (e.g., from `logs-*` to `security_solution-*`) +2. **Field Mapping Updates**: Making significant changes to field mappings that could affect existing queries +3. **Breaking Changes**: Any change that would break existing saved searches, visualizations, or dashboards +4. **Data Source Migration**: Moving from one data source to another (e.g., from native to CDR indices) + +#### How to Update Data View Version + +1. **Update Constants** in `packages/kbn-cloud-security-posture/common/constants.ts`: + + - Add the current version to the OLD_VERSIONS array + - Update the main constant to the new version `_v{n+1}` + + ```typescript + // Array of old data view IDs for migration purposes + export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS = [ + 'security_solution_cdr_latest_misconfigurations', // v1 + 'security_solution_cdr_latest_misconfigurations_v2', // v2 - Add current version here when moving to v3 + // Future deprecated versions will be added here + ]; + + // Current data view ID - increment version when making breaking changes + export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX = + 'security_solution_cdr_latest_misconfigurations_v3'; // Updated to v3 + ``` + +2. **Update Tests** in `test/cloud_security_posture_functional/data_views/data_views.ts`: + - Test deletion from v1 to current version (with space suffix) + - Test deletion from legacy to current version (global to space-specific) + - Test deletion of old and legacy data views during plugin initialization + - Test creation of new data views when the user navigates to the CSP page + +#### Example: Moving from v2 to v3 + +```typescript +// Step 1: Update the OLD_VERSIONS array in packages/kbn-cloud-security-posture/common/constants.ts +export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS = [ + 'security_solution_cdr_latest_misconfigurations', // v1 + 'security_solution_cdr_latest_misconfigurations_v2', // v2 - Added current version +]; + +// Step 2: Update the current version +export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX = + 'security_solution_cdr_latest_misconfigurations_v3'; // Now v3 + +// Note: Legacy versions (global data views) are tracked separately and rarely change +export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS = []; +``` + ## Testing For general guidelines, read [Kibana Testing Guide](https://www.elastic.co/guide/en/kibana/current/development-tests.html) for more details @@ -117,6 +178,13 @@ yarn test:ftr:server --config x-pack/solutions/security/test/cloud_security_post yarn test:ftr:runner --config x-pack/solutions/security/test/cloud_security_posture_functional/config.ts ``` +run data view migration tests: + +```bash +yarn test:ftr:server --config x-pack/solutions/security/test/cloud_security_posture_functional/data_views/config.ts +yarn test:ftr:runner --config x-pack/solutions/security/test/cloud_security_posture_functional/data_views/config.ts +``` + run serverless api integration tests: ```bash diff --git a/x-pack/solutions/security/plugins/cloud_security_posture/common/constants.ts b/x-pack/solutions/security/plugins/cloud_security_posture/common/constants.ts index 585a0160e90ec..cccab923fc846 100644 --- a/x-pack/solutions/security/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/solutions/security/plugins/cloud_security_posture/common/constants.ts @@ -52,10 +52,6 @@ export const BENCHMARK_SCORE_INDEX_TEMPLATE_NAME = 'logs-cloud_security_posture. export const BENCHMARK_SCORE_INDEX_PATTERN = 'logs-cloud_security_posture.scores-*'; export const BENCHMARK_SCORE_INDEX_DEFAULT_NS = 'logs-cloud_security_posture.scores-default'; -export const CDR_VULNERABILITIES_DATA_VIEW_NAME = 'Latest Cloud Security Vulnerabilities'; -export const CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX = - 'security_solution_cdr_latest_vulnerabilities'; - export const VULNERABILITIES_INDEX_NAME = 'logs-cloud_security_posture.vulnerabilities'; export const VULNERABILITIES_INDEX_PATTERN = 'logs-cloud_security_posture.vulnerabilities-default*'; export const VULNERABILITIES_INDEX_DEFAULT_NS = diff --git a/x-pack/solutions/security/plugins/cloud_security_posture/public/pages/vulnerabilities/vulnerabilities.tsx b/x-pack/solutions/security/plugins/cloud_security_posture/public/pages/vulnerabilities/vulnerabilities.tsx index f1e9e26f7f3b8..c0878eaad366e 100644 --- a/x-pack/solutions/security/plugins/cloud_security_posture/public/pages/vulnerabilities/vulnerabilities.tsx +++ b/x-pack/solutions/security/plugins/cloud_security_posture/public/pages/vulnerabilities/vulnerabilities.tsx @@ -10,8 +10,8 @@ import { findingsNavigation } from '@kbn/cloud-security-posture'; import { useCspSetupStatusApi } from '@kbn/cloud-security-posture/src/hooks/use_csp_setup_status_api'; import { useDataView } from '@kbn/cloud-security-posture/src/hooks/use_data_view'; import { EuiSpacer } from '@elastic/eui'; +import { CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX } from '@kbn/cloud-security-posture-common'; import { VULNERABILITIES_PAGE } from './test_subjects'; -import { CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX } from '../../../common/constants'; import { NoVulnerabilitiesStates } from '../../components/no_vulnerabilities_states'; import { CloudPosturePage } from '../../components/cloud_posture_page'; import { LatestVulnerabilitiesContainer } from './latest_vulnerabilities_container'; diff --git a/x-pack/solutions/security/plugins/cloud_security_posture/server/plugin.ts b/x-pack/solutions/security/plugins/cloud_security_posture/server/plugin.ts index 6d0bb54825186..905a15377c868 100755 --- a/x-pack/solutions/security/plugins/cloud_security_posture/server/plugin.ts +++ b/x-pack/solutions/security/plugins/cloud_security_posture/server/plugin.ts @@ -49,6 +49,7 @@ import type { } from './types'; import { setupRoutes } from './routes/setup_routes'; import { cspBenchmarkRule, cspSettings } from './saved_objects'; +import { deleteOldAndLegacyCdrDataViewsForAllSpaces } from './saved_objects/data_views'; import { initializeCspIndices } from './create_indices/create_indices'; import { deletePreviousTransformsVersions, @@ -239,8 +240,10 @@ export class CspPlugin ): Promise { this.logger.debug('initialize'); const esClient = core.elasticsearch.client.asInternalUser; + const soClient = core.savedObjects.createInternalRepository(); const isIntegrationVersionIncludesTransformAsset = isTransformAssetIncluded(packagePolicyVersion); + await initializeCspIndices( esClient, this.config, @@ -252,8 +255,13 @@ export class CspPlugin isIntegrationVersionIncludesTransformAsset, this.logger ); + await scheduleFindingsStatsTask(taskManager, this.logger); await this.initializeIndexAlias(esClient, this.logger); + + // Delete old and legacy CDR data views for all spaces + await deleteOldAndLegacyCdrDataViewsForAllSpaces(soClient, this.logger); + this.#isInitialized = true; } diff --git a/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.test.ts b/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.test.ts new file mode 100644 index 0000000000000..102456311e69e --- /dev/null +++ b/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.test.ts @@ -0,0 +1,673 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, ISavedObjectsRepository } from '@kbn/core/server'; +import type { KibanaRequest, Logger } from '@kbn/core/server'; +import type { SpacesServiceStart } from '@kbn/spaces-plugin/server'; +import type { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; +import { + CDR_MISCONFIGURATIONS_INDEX_PATTERN, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS, + CDR_MISCONFIGURATIONS_DATA_VIEW_NAME, + CDR_VULNERABILITIES_INDEX_PATTERN, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS, + CDR_VULNERABILITIES_DATA_VIEW_NAME, +} from '@kbn/cloud-security-posture-common'; +import { + installDataView, + deleteOldAndLegacyCdrDataViewsForAllSpaces, + setupCdrDataViews, +} from './data_views'; + +describe('data_views', () => { + let mockSoClient: jest.Mocked; + let mockEsClient: jest.Mocked; + let mockLogger: jest.Mocked; + let mockSpacesService: jest.Mocked; + let mockDataViewsService: jest.Mocked; + let mockRequest: jest.Mocked; + + beforeEach(() => { + mockSoClient = { + get: jest.fn(), + find: jest.fn(), + delete: jest.fn(), + create: jest.fn(), + bulkGet: jest.fn(), + } as any; + + mockEsClient = {} as any; + + mockLogger = { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + } as any; + + mockSpacesService = { + getSpaceId: jest.fn().mockReturnValue(DEFAULT_SPACE_ID), + } as any; + + const mockDataViewsClient = { + createAndSave: jest.fn(), + }; + + mockDataViewsService = { + dataViewsServiceFactory: jest.fn().mockResolvedValue(mockDataViewsClient), + } as any; + + mockRequest = {} as any; + }); + + describe('installDataView', () => { + it('should create a new data view when it does not exist', async () => { + mockSoClient.get.mockRejectedValue(new Error('Not found')); + + const mockDataViewsClient = { + createAndSave: jest.fn(), + }; + mockDataViewsService.dataViewsServiceFactory.mockResolvedValue(mockDataViewsClient as any); + + await installDataView( + mockEsClient, + mockSoClient, + mockSpacesService, + mockDataViewsService, + mockRequest, + CDR_MISCONFIGURATIONS_DATA_VIEW_NAME, + CDR_MISCONFIGURATIONS_INDEX_PATTERN, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + mockLogger + ); + + expect(mockDataViewsClient.createAndSave).toHaveBeenCalledWith( + { + id: `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`, + title: CDR_MISCONFIGURATIONS_INDEX_PATTERN, + name: `${CDR_MISCONFIGURATIONS_DATA_VIEW_NAME} - ${DEFAULT_SPACE_ID} `, + namespaces: [DEFAULT_SPACE_ID], + allowNoIndex: true, + timeFieldName: '@timestamp', + }, + true + ); + }); + + it('should not create a data view when it already exists', async () => { + mockSoClient.get.mockResolvedValue({ + id: `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`, + type: 'index-pattern', + attributes: {}, + references: [], + }); + + const mockDataViewsClient = { + createAndSave: jest.fn(), + }; + mockDataViewsService.dataViewsServiceFactory.mockResolvedValue(mockDataViewsClient as any); + + await installDataView( + mockEsClient, + mockSoClient, + mockSpacesService, + mockDataViewsService, + mockRequest, + CDR_MISCONFIGURATIONS_DATA_VIEW_NAME, + CDR_MISCONFIGURATIONS_INDEX_PATTERN, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + mockLogger + ); + + expect(mockDataViewsClient.createAndSave).not.toHaveBeenCalled(); + }); + + it('should handle errors gracefully', async () => { + mockSoClient.get.mockRejectedValue(new Error('Not found')); + mockDataViewsService.dataViewsServiceFactory.mockRejectedValue( + new Error('Service unavailable') + ); + + await installDataView( + mockEsClient, + mockSoClient, + mockSpacesService, + mockDataViewsService, + mockRequest, + CDR_MISCONFIGURATIONS_DATA_VIEW_NAME, + CDR_MISCONFIGURATIONS_INDEX_PATTERN, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + mockLogger + ); + + expect(mockLogger.error).toHaveBeenCalledWith('Failed to setup data view', expect.any(Error)); + }); + + it('should use custom space ID when spaces service is available', async () => { + const customSpaceId = 'custom-space'; + mockSpacesService.getSpaceId.mockReturnValue(customSpaceId); + mockSoClient.get.mockRejectedValue(new Error('Not found')); + + const mockDataViewsClient = { + createAndSave: jest.fn(), + }; + mockDataViewsService.dataViewsServiceFactory.mockResolvedValue(mockDataViewsClient as any); + + await installDataView( + mockEsClient, + mockSoClient, + mockSpacesService, + mockDataViewsService, + mockRequest, + CDR_MISCONFIGURATIONS_DATA_VIEW_NAME, + CDR_MISCONFIGURATIONS_INDEX_PATTERN, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + mockLogger + ); + + expect(mockDataViewsClient.createAndSave).toHaveBeenCalledWith( + expect.objectContaining({ + id: `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${customSpaceId}`, + namespaces: [customSpaceId], + }), + true + ); + }); + }); + + describe('deleteOldAndLegacyCdrDataViewsForAllSpaces', () => { + it('should find and delete old misconfigurations data views', async () => { + const oldDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + const newDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: oldDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + { + id: newDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + ], + total: 2, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.find).toHaveBeenCalledWith({ + type: 'index-pattern', + namespaces: ['*'], + perPage: 1000, + }); + + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldDataViewId, { + namespace: DEFAULT_SPACE_ID, + }); + + expect(mockSoClient.delete).not.toHaveBeenCalledWith('index-pattern', newDataViewId, { + namespace: DEFAULT_SPACE_ID, + }); + }); + + it('should find and delete old vulnerabilities data views', async () => { + const oldDataViewId = `${CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: oldDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + ], + total: 1, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldDataViewId, { + namespace: DEFAULT_SPACE_ID, + }); + }); + + it('should find and delete legacy misconfigurations data views (wildcard, not space-specific)', async () => { + const legacyDataViewId = CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: legacyDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['*'], // Legacy data views used wildcards + }, + ], + total: 1, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + // For wildcard namespaces, delete uses force: true instead of namespace: '*' + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', legacyDataViewId, { + force: true, + }); + }); + + it('should find and delete legacy vulnerabilities data views (wildcard, not space-specific)', async () => { + const legacyDataViewId = CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: legacyDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['*'], // Legacy data views used wildcards + }, + ], + total: 1, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + // For wildcard namespaces, delete uses force: true instead of namespace: '*' + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', legacyDataViewId, { + force: true, + }); + }); + + it('should handle multiple old data views across different spaces', async () => { + const oldMisconfigId1 = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + const oldMisconfigId2 = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-custom-space`; + const oldVulnId = `${CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: oldMisconfigId1, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + { + id: oldMisconfigId2, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['custom-space'], + }, + { + id: oldVulnId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + ], + total: 3, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.delete).toHaveBeenCalledTimes(3); + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldMisconfigId1, { + namespace: DEFAULT_SPACE_ID, + }); + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldMisconfigId2, { + namespace: 'custom-space', + }); + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldVulnId, { + namespace: DEFAULT_SPACE_ID, + }); + }); + + it('should handle both legacy and old data views together', async () => { + // Legacy IDs don't have space suffix (wildcard) + const legacyMisconfigId = CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + const legacyVulnId = CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + + // Old (v1) IDs have space suffix + const oldMisconfigId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + const oldVulnId = `${CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + + // Current (v2) IDs have space suffix + const currentMisconfigId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`; + const currentVulnId = `${CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: legacyMisconfigId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['*'], // Legacy used wildcards + }, + { + id: oldMisconfigId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + { + id: legacyVulnId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['*'], // Legacy used wildcards + }, + { + id: oldVulnId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + { + id: currentMisconfigId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + { + id: currentVulnId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + ], + total: 6, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + // Should delete all 4 old/legacy data views but not the 2 current ones + expect(mockSoClient.delete).toHaveBeenCalledTimes(4); + // Legacy data views use force: true (because they have wildcard namespaces) + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', legacyMisconfigId, { + force: true, + }); + // Old (v1) data views use namespace: space-id + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldMisconfigId, { + namespace: DEFAULT_SPACE_ID, + }); + // Legacy data views use force: true (because they have wildcard namespaces) + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', legacyVulnId, { + force: true, + }); + // Old (v1) data views use namespace: space-id + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldVulnId, { + namespace: DEFAULT_SPACE_ID, + }); + + // Should not delete current data views + expect(mockSoClient.delete).not.toHaveBeenCalledWith('index-pattern', currentMisconfigId, { + namespace: DEFAULT_SPACE_ID, + }); + expect(mockSoClient.delete).not.toHaveBeenCalledWith('index-pattern', currentVulnId, { + namespace: DEFAULT_SPACE_ID, + }); + }); + + it('should handle all legacy versions when multiple exist (wildcard namespaces)', async () => { + // Legacy IDs don't have space suffix - they used wildcards + const legacyIds = CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS; + + mockSoClient.find.mockResolvedValue({ + saved_objects: legacyIds.map((id) => ({ + id, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['*'], // Legacy data views used wildcards + })), + total: legacyIds.length, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.delete).toHaveBeenCalledTimes(legacyIds.length); + // Legacy data views with wildcard namespaces use force: true + legacyIds.forEach((legacyId) => { + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', legacyId, { + force: true, + }); + }); + }); + + it('should handle legacy data views in specific namespace (edge case)', async () => { + const legacyDataViewId = CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: legacyDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: ['custom-space'], // Edge case: legacy in specific space + }, + ], + total: 1, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', legacyDataViewId, { + namespace: 'custom-space', + }); + }); + + it('should not delete the Current data view', async () => { + const oldDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + const currentDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: oldDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + { + id: currentDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + ], + total: 2, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + // Should only delete the old data view, not the new one + expect(mockSoClient.delete).toHaveBeenCalledTimes(1); + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldDataViewId, { + namespace: DEFAULT_SPACE_ID, + }); + }); + + it('should warn when total data views exceeds page limit', async () => { + mockSoClient.find.mockResolvedValue({ + saved_objects: [], + total: 1500, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Total data views (1500) exceeds page limit (1000). Some old data views may not be deleted.' + ); + }); + + it('should handle no old data views gracefully', async () => { + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + namespaces: [DEFAULT_SPACE_ID], + }, + ], + total: 1, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.delete).not.toHaveBeenCalled(); + }); + + it('should handle errors gracefully and not throw', async () => { + mockSoClient.find.mockRejectedValue(new Error('Service unavailable')); + + await expect( + deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger) + ).resolves.not.toThrow(); + + expect(mockLogger.error).toHaveBeenCalledWith( + 'Failed to delete old and legacy CDR data views', + expect.any(Error) + ); + }); + + it('should use default space ID when namespace is not provided', async () => { + const oldDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${DEFAULT_SPACE_ID}`; + + mockSoClient.find.mockResolvedValue({ + saved_objects: [ + { + id: oldDataViewId, + type: 'index-pattern', + attributes: {}, + references: [], + score: 1, + // No namespaces property + }, + ], + total: 1, + per_page: 1000, + page: 1, + }); + + await deleteOldAndLegacyCdrDataViewsForAllSpaces(mockSoClient, mockLogger); + + expect(mockSoClient.delete).toHaveBeenCalledWith('index-pattern', oldDataViewId, { + namespace: DEFAULT_SPACE_ID, + }); + }); + }); + + describe('setupCdrDataViews', () => { + it('should install both misconfigurations and vulnerabilities data views', async () => { + mockSoClient.get.mockRejectedValue(new Error('Not found')); + + const mockDataViewsClient = { + createAndSave: jest.fn(), + }; + mockDataViewsService.dataViewsServiceFactory.mockResolvedValue(mockDataViewsClient as any); + + await setupCdrDataViews( + mockEsClient, + mockSoClient, + mockSpacesService, + mockDataViewsService, + mockRequest, + mockLogger + ); + + expect(mockDataViewsClient.createAndSave).toHaveBeenCalledTimes(2); + + // Check misconfigurations data view + expect(mockDataViewsClient.createAndSave).toHaveBeenCalledWith( + expect.objectContaining({ + id: `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`, + title: CDR_MISCONFIGURATIONS_INDEX_PATTERN, + name: `${CDR_MISCONFIGURATIONS_DATA_VIEW_NAME} - ${DEFAULT_SPACE_ID} `, + }), + true + ); + + // Check vulnerabilities data view + expect(mockDataViewsClient.createAndSave).toHaveBeenCalledWith( + expect.objectContaining({ + id: `${CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX}-${DEFAULT_SPACE_ID}`, + title: CDR_VULNERABILITIES_INDEX_PATTERN, + name: `${CDR_VULNERABILITIES_DATA_VIEW_NAME} - ${DEFAULT_SPACE_ID} `, + }), + true + ); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.ts b/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.ts index f8cc2790d61f0..b72564b8caf49 100644 --- a/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.ts +++ b/x-pack/solutions/security/plugins/cloud_security_posture/server/saved_objects/data_views.ts @@ -13,13 +13,15 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; import { CDR_MISCONFIGURATIONS_INDEX_PATTERN, CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS, CDR_MISCONFIGURATIONS_DATA_VIEW_NAME, CDR_VULNERABILITIES_INDEX_PATTERN, -} from '@kbn/cloud-security-posture-common'; -import { CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS, CDR_VULNERABILITIES_DATA_VIEW_NAME, -} from '../../common/constants'; + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS, +} from '@kbn/cloud-security-posture-common'; const DATA_VIEW_TIME_FIELD = '@timestamp'; @@ -50,6 +52,25 @@ const getCurrentSpaceId = ( return spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID; }; +const deleteDataViewSafe = async ( + soClient: ISavedObjectsRepository, + dataViewId: string, + namespace: string, + logger: Logger +): Promise => { + try { + if (namespace === '*') { + await soClient.delete('index-pattern', dataViewId, { force: true }); + } else { + await soClient.delete('index-pattern', dataViewId, { namespace }); + } + logger.info(`Deleted old data view: ${dataViewId}`); + } catch (e) { + // Ignore if doesn't exist - expected behavior for new installations + return; + } +}; + export const installDataView = async ( esClient: ElasticsearchClient, soClient: ISavedObjectsRepository, @@ -97,6 +118,90 @@ export const installDataView = async ( } }; +export const deleteOldAndLegacyCdrDataViewsForAllSpaces = async ( + soClient: ISavedObjectsRepository, + logger: Logger +) => { + try { + logger.info('Starting deletion of old and legacy CDR data views across all spaces'); + + // Get all data views matching old prefixes + const oldMisconfigurationsPrefixes = CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS; + const oldVulnerabilitiesPrefixes = CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS; + const legacyMisconfigurationsPrefixes = + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS; + const legacyVulnerabilitiesPrefixes = CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS; + // Search for all data views across all namespaces and filter by old prefixes + // We can't use wildcard on _id field, so we fetch all index-patterns and filter in memory + const allDataViewsResult = await soClient.find({ + type: 'index-pattern', + namespaces: ['*'], // Search across all spaces + perPage: 1000, + }); + + logger.info(`Found ${allDataViewsResult.saved_objects.length} total data views to check`); + + if (allDataViewsResult.total > 1000) { + logger.warn( + `Total data views (${allDataViewsResult.total}) exceeds page limit (1000). Some old data views may not be deleted.` + ); + } + + // Filter data views that match old prefixes and legacy ids + // Include the dash (-) in the check to avoid matching current data views + const oldMisconfigurationsDataViews = allDataViewsResult.saved_objects.filter((obj) => + oldMisconfigurationsPrefixes.some((prefix) => obj.id.startsWith(`${prefix}-`)) + ); + + const legacyMisconfigurationsDataViews = allDataViewsResult.saved_objects.filter((obj) => + legacyMisconfigurationsPrefixes.some((prefix) => obj.id === prefix) + ); + + const oldVulnerabilitiesDataViews = allDataViewsResult.saved_objects.filter((obj) => + oldVulnerabilitiesPrefixes.some((prefix) => obj.id.startsWith(`${prefix}-`)) + ); + + const legacyVulnerabilitiesDataViews = allDataViewsResult.saved_objects.filter((obj) => + legacyVulnerabilitiesPrefixes.some((prefix) => obj.id === prefix) + ); + + // Delete legacy misconfigurations data views + for (const dataView of legacyMisconfigurationsDataViews) { + const namespace = dataView.namespaces?.[0] || DEFAULT_SPACE_ID; + logger.info( + `Found legacy misconfigurations data view: ${dataView.id} in namespace: ${dataView.namespaces}, deleting...` + ); + await deleteDataViewSafe(soClient, dataView.id, namespace, logger); + } + + // Delete legacy vulnerabilities data views + for (const dataView of legacyVulnerabilitiesDataViews) { + logger.info(`Found legacy vulnerabilities data view: ${dataView.id}, deleting...`); + const namespace = dataView.namespaces?.[0] || DEFAULT_SPACE_ID; + await deleteDataViewSafe(soClient, dataView.id, namespace, logger); + } + + // Delete old misconfigurations data views + for (const dataView of oldMisconfigurationsDataViews) { + logger.info(`Found old misconfigurations data view: ${dataView.id}, deleting...`); + const namespace = dataView.namespaces?.[0] || DEFAULT_SPACE_ID; + await deleteDataViewSafe(soClient, dataView.id, namespace, logger); + } + + // Delete old vulnerabilities data views + for (const dataView of oldVulnerabilitiesDataViews) { + logger.info(`Found old vulnerabilities data view: ${dataView.id}, deleting...`); + const namespace = dataView.namespaces?.[0] || DEFAULT_SPACE_ID; + await deleteDataViewSafe(soClient, dataView.id, namespace, logger); + } + + logger.info('Deletion of old and legacy CDR data views completed successfully'); + } catch (error) { + logger.error('Failed to delete old and legacy CDR data views', error); + // Don't throw - deletion failure shouldn't block plugin initialization + } +}; + export const setupCdrDataViews = async ( esClient: ElasticsearchClient, soClient: ISavedObjectsRepository, diff --git a/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/config.ts b/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/config.ts index cfb5a88d0f891..ccf2b94ce7c2d 100644 --- a/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/config.ts +++ b/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/config.ts @@ -7,6 +7,7 @@ import { resolve } from 'path'; import type { FtrConfigProviderContext } from '@kbn/test'; +import { CLOUD_SECURITY_PLUGIN_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants'; import { pageObjects } from '../page_objects'; import { services } from '../services'; @@ -39,6 +40,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { * 1. release a new package to EPR * 2. merge the updated version number change to kibana */ + `--xpack.fleet.packages.0.name=cloud_security_posture`, + `--xpack.fleet.packages.0.version=${CLOUD_SECURITY_PLUGIN_VERSION}`, `--xpack.fleet.agents.fleet_server.hosts=["https://ftr.kibana:8220"]`, `--xpack.fleet.internal.fleetServerStandalone=true`, ], diff --git a/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/data_views.ts b/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/data_views.ts index 7fe3876b1566d..dc54090c207f0 100644 --- a/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/data_views.ts +++ b/x-pack/solutions/security/test/cloud_security_posture_functional/data_views/data_views.ts @@ -6,10 +6,18 @@ */ import expect from '@kbn/expect'; +import type SuperTest from 'supertest'; import type { DataViewAttributes } from '@kbn/data-views-plugin/common'; -import { CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX } from '@kbn/cloud-security-posture-common'; -import { CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX } from '@kbn/cloud-security-posture-plugin/common/constants'; +import { + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS, + CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS, + CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS, +} from '@kbn/cloud-security-posture-common'; import type { KbnClientSavedObjects } from '@kbn/test/src/kbn_client/kbn_client_saved_objects'; +import { CLOUD_SECURITY_PLUGIN_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants'; import type { FtrProviderContext } from '../ftr_provider_context'; const TEST_SPACE = 'space-1'; @@ -35,13 +43,101 @@ const getDataViewSafe = async ( return false; } }; + +const createDataView = async ( + supertest: SuperTest.Agent, + id: string, + name: string, + title: string, + space?: string +): Promise<{ data_view: { id: string } }> => { + const basePath = space ? `/s/${space}` : ''; + const { body } = await supertest + .post(`${basePath}/api/data_views/data_view`) + .set('kbn-xsrf', 'foo') + .send({ data_view: { id, title, name, timeFieldName: '@timestamp', allowNoIndex: true } }) + .expect(200); + return body; +}; + // eslint-disable-next-line import/no-default-export export default ({ getService, getPageObjects }: FtrProviderContext) => { const kibanaServer = getService('kibanaServer'); const spacesService = getService('spaces'); const retry = getService('retry'); + const supertest = getService('supertest'); const fetchingOfDataViewsTimeout = 1000 * 30; // 30 seconds + /** + * Installs the Cloud Security Posture package, which triggers plugin initialization and migration + */ + const installCspPackageAndPackagePolicy = async () => { + // Create agent policy with unique name + const policyName = `Test CSP Policy ${Date.now()}`; + const agentPolicyResponse = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xx') + .send({ + name: policyName, + namespace: 'default', + description: 'Test policy for CSP data views', + monitoring_enabled: ['logs', 'metrics'], + }) + .expect(200); + + const agentPolicyId = agentPolicyResponse.body.item.id; + + // Create a package policy for the CSP package + const { body: packagePolicyResponse } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + force: true, + name: `cloud_security_posture-${agentPolicyId}`, + description: '', + namespace: 'default', + policy_id: agentPolicyId, + enabled: true, + inputs: [ + { + type: 'cloudbeat/cis_aws', + policy_template: 'cspm', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'cloud_security_posture.findings', + }, + }, + ], + }, + ], + package: { + name: 'cloud_security_posture', + title: 'Security Posture Management', + version: CLOUD_SECURITY_PLUGIN_VERSION, + }, + vars: { + deployment: { + value: 'aws', + type: 'text', + }, + posture: { + value: 'cspm', + type: 'text', + }, + }, + }) + .expect(200); + + return { + agentPolicyId, + packagePolicyId: packagePolicyResponse.item.id, + }; + }; + const pageObjects = getPageObjects([ 'common', 'findings', @@ -87,7 +183,14 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { }); await spacesService.delete(TEST_SPACE); - await pageObjects.security.forceLogout(); + + // Wrap logout in try-catch as it can be flaky and shouldn't fail the entire test + try { + await pageObjects.security.forceLogout(); + } catch (e) { + // eslint-disable-next-line no-console + console.log('Warning: forceLogout failed during cleanup, but test cleanup will continue'); + } }); DATA_VIEW_PREFIXES.forEach((dataViewPrefix) => { @@ -165,11 +268,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { DATA_VIEW_PREFIXES.forEach((dataViewPrefix) => { it('Verify data view is created once user reach the findings page - non default space', async () => { - await pageObjects.common.navigateToApp('home'); await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); - await pageObjects.spaceSelector.openSpacesNav(); - await pageObjects.spaceSelector.clickSpaceAvatar(TEST_SPACE); - await pageObjects.spaceSelector.expectHomePage(TEST_SPACE); const expectedDataViewId = `${dataViewPrefix}-${TEST_SPACE}`; if (await getDataViewSafe(kibanaServer.savedObjects, expectedDataViewId, TEST_SPACE)) { @@ -204,11 +303,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { DATA_VIEW_PREFIXES.forEach((dataViewPrefix) => { it('Verify data view is created once user reach the dashboard page - non default space', async () => { - await pageObjects.common.navigateToApp('home'); await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); - await pageObjects.spaceSelector.openSpacesNav(); - await pageObjects.spaceSelector.clickSpaceAvatar(TEST_SPACE); - await pageObjects.spaceSelector.expectHomePage(TEST_SPACE); const expectedDataViewId = `${dataViewPrefix}-${TEST_SPACE}`; if (await getDataViewSafe(kibanaServer.savedObjects, expectedDataViewId, TEST_SPACE)) { @@ -225,6 +320,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { }); } + // Navigate directly to dashboard page in the test space const cspDashboard = pageObjects.cloudPostureDashboard; await cspDashboard.navigateToComplianceDashboardPage(TEST_SPACE); await waitForDataViews({ @@ -244,8 +340,13 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { DATA_VIEW_PREFIXES.forEach((dataViewPrefix) => { it('Verify data view is created once user with read permissions reach the dashboard page', async () => { - await pageObjects.common.navigateToApp('home'); - await cspSecurity.logout(); + // Ensure we're logged out first before attempting to login with read user + try { + await pageObjects.security.forceLogout(); + } catch (e) { + // If logout fails, continue - we might already be logged out + } + await cspSecurity.login('csp_read_user'); const expectedDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX}-default`; @@ -279,5 +380,181 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { }); }); }); + + describe('Misconfigurations old Data View removal', () => { + it('Should delete old data view when installing CSP package', async () => { + await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); + + const oldDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${TEST_SPACE}`; + + // Create old v1 data view in test space + const body = await createDataView( + supertest, + oldDataViewId, + 'Old Misconfiguration Data View v1', + 'security_solution-*.misconfiguration_latest', + TEST_SPACE + ); + expect(body.data_view.id).to.eql(oldDataViewId); + + // Install CSP package - this triggers plugin initialization which runs the migration + await installCspPackageAndPackagePolicy(); + + // Verify old v1 data view is deleted + await retry.tryForTime(60000, async () => { + const oldDataViewExistsAfterMigration = await getDataViewSafe( + kibanaServer.savedObjects, + oldDataViewId, + TEST_SPACE + ); + expect(oldDataViewExistsAfterMigration).to.be(false); + }); + }); + + it('Should delete legacy data view when installing CSP package', async () => { + await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); + + // Legacy data views don't have space suffix - they were global with wildcard namespaces + const legacyDataViewId = CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + + // Create legacy data view (no space suffix, uses wildcard namespace) + const body = await createDataView( + supertest, + legacyDataViewId, + 'Old Legacy Misconfiguration Data View', + 'logs-cloud_security_posture.findings_latest-*' + ); + expect(body.data_view.id).to.eql(legacyDataViewId); + + // Install CSP package - this triggers plugin initialization which runs the migration + await installCspPackageAndPackagePolicy(); + + // Verify legacy data view is deleted (check in default space as it was global) + await retry.tryForTime(60000, async () => { + const legacyDataViewExistsAfterMigration = await getDataViewSafe( + kibanaServer.savedObjects, + legacyDataViewId, + 'default' + ); + expect(legacyDataViewExistsAfterMigration).to.be(false); + }); + }); + + it('Should not delete unrelated dataviews when installing CSP package', async () => { + await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); + + // Create a random unrelated data view that should not be deleted + const unrelatedDataViewId = 'test-unrelated-dataview-id'; + + const unrelatedBody = await createDataView( + supertest, + unrelatedDataViewId, + 'Test Unrelated Data View', + 'test-unrelated-dataview-id', + TEST_SPACE + ); + expect(unrelatedBody.data_view.id).to.eql(unrelatedDataViewId); + + // Create old CSP data view to trigger migration + const oldDataViewId = `${CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${TEST_SPACE}`; + + const oldBody = await createDataView( + supertest, + oldDataViewId, + 'Old Misconfiguration Data View v1', + 'security_solution-*.misconfiguration_latest', + TEST_SPACE + ); + expect(oldBody.data_view.id).to.eql(oldDataViewId); + + // Install CSP package - this triggers plugin initialization which runs the migration + await installCspPackageAndPackagePolicy(); + + // Verify old CSP data view is deleted as expected + await retry.tryForTime(60000, async () => { + const oldDataViewExists = await getDataViewSafe( + kibanaServer.savedObjects, + oldDataViewId, + TEST_SPACE + ); + expect(oldDataViewExists).to.be(false); + }); + + // Verify the unrelated data view still exists after migration + const unrelatedDataViewExistsAfterMigration = await getDataViewSafe( + kibanaServer.savedObjects, + unrelatedDataViewId, + TEST_SPACE + ); + expect(unrelatedDataViewExistsAfterMigration).to.be(true); + + // Clean up the unrelated data view + await kibanaServer.savedObjects.delete({ + type: 'index-pattern', + id: unrelatedDataViewId, + space: TEST_SPACE, + }); + }); + }); + + describe('Vulnerabilities old Data View removal', () => { + it('Should delete old data view when installing CSP package', async () => { + await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); + + const oldDataViewId = `${CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_OLD_VERSIONS[0]}-${TEST_SPACE}`; + + // Create old v1 vulnerabilities data view in test space + const body = await createDataView( + supertest, + oldDataViewId, + 'Old Vulnerabilities Data View v1', + 'security_solution-*.vulnerability_latest,logs-cloud_security_posture.vulnerabilities_latest-default', + TEST_SPACE + ); + expect(body.data_view.id).to.eql(oldDataViewId); + + // Install CSP package - this triggers plugin initialization which runs the migration + await installCspPackageAndPackagePolicy(); + + // Verify old v1 vulnerabilities data view is deleted + await retry.tryForTime(60000, async () => { + const oldDataViewExistsAfterMigration = await getDataViewSafe( + kibanaServer.savedObjects, + oldDataViewId, + TEST_SPACE + ); + expect(oldDataViewExistsAfterMigration).to.be(false); + }); + }); + + it('Should delete legacy data view when installing CSP package', async () => { + await spacesService.create({ id: TEST_SPACE, name: 'space_one', disabledFeatures: [] }); + + // Legacy data views don't have space suffix - they were global with wildcard namespaces + const legacyDataViewId = CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX_LEGACY_VERSIONS[0]; + + // Create legacy vulnerabilities data view (no space suffix, uses wildcard namespace) + const body = await createDataView( + supertest, + legacyDataViewId, + 'Old Legacy Vulnerabilities Data View', + 'logs-cloud_security_posture.vulnerabilities-*' + ); + expect(body.data_view.id).to.eql(legacyDataViewId); + + // Install CSP package - this triggers plugin initialization which runs the migration + await installCspPackageAndPackagePolicy(); + + // Verify legacy vulnerabilities data view is deleted (check in default space as it was global) + await retry.tryForTime(60000, async () => { + const legacyDataViewExistsAfterMigration = await getDataViewSafe( + kibanaServer.savedObjects, + legacyDataViewId, + 'default' + ); + expect(legacyDataViewExistsAfterMigration).to.be(false); + }); + }); + }); }); };