diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml index 269be53f635bd..9dcc4cba31b4f 100644 --- a/.buildkite/ftr_security_stateful_configs.yml +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -106,7 +106,6 @@ enabled: - x-pack/test/cloud_security_posture_functional/data_views/config.ts - x-pack/test/automatic_import_api_integration/apis/config_basic.ts - x-pack/test/automatic_import_api_integration/apis/config_graphs.ts - - x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/spaces_api_integration/deployment_agnostic/spaces_only/config.ts - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/stateful.config_basic.ts - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/stateful.config_trial.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 78db89dce3024..85cebb36db5a6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2533,7 +2533,6 @@ x-pack/solutions/security/plugins/security_solution/public/common/components/ses x-pack/solutions/security/plugins/security_solution/public/cloud_defend @elastic/kibana-cloud-security-posture x-pack/solutions/security/plugins/security_solution/public/cloud_security_posture @elastic/kibana-cloud-security-posture x-pack/solutions/security/plugins/security_solution/public/kubernetes @elastic/kibana-cloud-security-posture -x-pack/test/security_solution_api_integration/test_suites/asset_inventory @elastic/kibana-cloud-security-posture x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory @elastic/kibana-cloud-security-posture ## Fleet plugin (co-owned with Fleet team) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 5ce3bec194f0e..4a7d19dc608b3 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -51821,7 +51821,6 @@ components: - user - host - service - - universal type: string Security_Entity_Analytics_API_HostEntity: type: object @@ -51897,7 +51896,6 @@ components: - host.name - user.name - service.name - - related.entity type: string Security_Entity_Analytics_API_IndexPattern: type: string diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index af37bdad32dec..c82df68a096de 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -58588,7 +58588,6 @@ components: - user - host - service - - universal type: string Security_Entity_Analytics_API_HostEntity: type: object @@ -58664,7 +58663,6 @@ components: - host.name - user.name - service.name - - related.entity type: string Security_Entity_Analytics_API_IndexPattern: type: string diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts index 2c9cc0760f210..acae48846c82c 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts @@ -17,7 +17,7 @@ import { z } from '@kbn/zod'; export type IdField = z.infer; -export const IdField = z.enum(['host.name', 'user.name', 'service.name', 'related.entity']); +export const IdField = z.enum(['host.name', 'user.name', 'service.name']); export type IdFieldEnum = typeof IdField.enum; export const IdFieldEnum = IdField.enum; diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml index 97485bb2c6605..32bbfc5122efb 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml @@ -29,7 +29,6 @@ components: - 'host.name' - 'user.name' - 'service.name' - - 'related.entity' AssetCriticalityRecordIdParts: type: object properties: diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts index 628d0fb295a8e..f1cbc070792f8 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts @@ -17,7 +17,7 @@ import { z } from '@kbn/zod'; export type EntityType = z.infer; -export const EntityType = z.enum(['user', 'host', 'service', 'universal']); +export const EntityType = z.enum(['user', 'host', 'service']); export type EntityTypeEnum = typeof EntityType.enum; export const EntityTypeEnum = EntityType.enum; diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml index 4bcfba9fec891..82fc5b379a70c 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml @@ -12,7 +12,6 @@ components: - user - host - service - - universal EngineDescriptor: type: object diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/utils.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/utils.ts index 697ca1858ef23..ae3b7fe8fc075 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/utils.ts @@ -7,17 +7,11 @@ import type { ExperimentalFeatures } from '../../experimental_features'; import { getAllEntityTypes, getDisabledEntityTypes } from '../utils'; -import { EntityType } from '../types'; - -const ASSET_CRITICALITY_UNAVAILABLE_TYPES = [EntityType.universal]; // TODO delete this function when the universal entity support is added export const getAssetCriticalityEntityTypes = (experimentalFeatures: ExperimentalFeatures) => { const allEntityTypes = getAllEntityTypes(); const disabledEntityTypes = getDisabledEntityTypes(experimentalFeatures); - return allEntityTypes.filter( - (value) => - !disabledEntityTypes.includes(value) && !ASSET_CRITICALITY_UNAVAILABLE_TYPES.includes(value) - ); + return allEntityTypes.filter((value) => !disabledEntityTypes.includes(value)); }; diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/risk_engine/utils.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/risk_engine/utils.ts index 005bfca6644cf..af38542da14f1 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/risk_engine/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/risk_engine/utils.ts @@ -6,7 +6,6 @@ */ import * as t from 'io-ts'; -import { EntityType } from '../types'; import { getAllEntityTypes, getDisabledEntityTypes } from '../utils'; import type { ExperimentalFeatures } from '../../experimental_features'; @@ -28,15 +27,10 @@ export function fromEnum( ); } -const RISK_ENGINE_UNAVAILABLE_TYPES = [EntityType.universal]; - // TODO delete this function when the universal entity support is added export const getRiskEngineEntityTypes = (experimentalFeatures: ExperimentalFeatures) => { const allEntityTypes = getAllEntityTypes(); const disabledEntityTypes = getDisabledEntityTypes(experimentalFeatures); - return allEntityTypes.filter( - (value) => - !disabledEntityTypes.includes(value) && !RISK_ENGINE_UNAVAILABLE_TYPES.includes(value) - ); + return allEntityTypes.filter((value) => !disabledEntityTypes.includes(value)); }; diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/types.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/types.ts index ff8047da9877a..a47dd4dd18903 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/types.ts @@ -15,19 +15,16 @@ export enum EntityType { user = 'user', host = 'host', service = 'service', - universal = 'universal', } export enum EntityIdentifierFields { hostName = 'host.name', userName = 'user.name', serviceName = 'service.name', - universal = 'related.entity', } export const EntityTypeToIdentifierField: Record = { [EntityType.host]: EntityIdentifierFields.hostName, [EntityType.user]: EntityIdentifierFields.userName, [EntityType.service]: EntityIdentifierFields.serviceName, - [EntityType.universal]: EntityIdentifierFields.universal, }; diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.test.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.test.ts index 4a878669c6f61..205f43ca0dfda 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.test.ts @@ -4,13 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { getAllEntityTypes, getDisabledEntityTypes } from './utils'; +import { getAllEntityTypes } from './utils'; import { EntityType } from './types'; -import type { ExperimentalFeatures } from '../experimental_features'; -import { mockGlobalState } from '../../public/common/mock'; - -const mockedExperimentalFeatures = mockGlobalState.app.enableExperimental; describe('utils', () => { describe('getAllEntityTypes', () => { @@ -19,24 +14,4 @@ describe('utils', () => { expect(entityTypes).toEqual(Object.values(EntityType)); }); }); - - describe('getDisabledEntityTypes', () => { - it('should return disabled entity types when assetInventoryStoreEnabled is false', () => { - const experimentalFeatures: ExperimentalFeatures = { - ...mockedExperimentalFeatures, - assetInventoryStoreEnabled: false, - }; - const disabledEntityTypes = getDisabledEntityTypes(experimentalFeatures); - expect(disabledEntityTypes).toEqual([EntityType.universal]); - }); - - it('should return no disabled entity types when both features are true', () => { - const experimentalFeatures: ExperimentalFeatures = { - ...mockedExperimentalFeatures, - assetInventoryStoreEnabled: true, - }; - const disabledEntityTypes = getDisabledEntityTypes(experimentalFeatures); - expect(disabledEntityTypes).toEqual([]); - }); - }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.ts index 062f0909b0231..26e116ec28bd8 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/utils.ts @@ -13,11 +13,6 @@ export const getDisabledEntityTypes = ( experimentalFeatures: ExperimentalFeatures ): EntityType[] => { const disabledEntityTypes: EntityType[] = []; - const isUniversalEntityStoreEnabled = experimentalFeatures.assetInventoryStoreEnabled; - - if (!isUniversalEntityStoreEnabled) { - disabledEntityTypes.push(EntityType.universal); - } return disabledEntityTypes; }; diff --git a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts index 569e2acdc1c3f..25eedeb55178a 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts @@ -262,12 +262,6 @@ export const allowedExperimentalValues = Object.freeze({ */ crowdstrikeRunScriptEnabled: true, - /** - * Enables the Asset Inventory Entity Store feature. - * Allows initializing the Universal Entity Store via the API. - */ - assetInventoryStoreEnabled: false, - /** * Enables the Asset Inventory feature */ diff --git a/x-pack/solutions/security/plugins/security_solution/common/search_strategy/security_solution/risk_score/all/index.ts b/x-pack/solutions/security/plugins/security_solution/common/search_strategy/security_solution/risk_score/all/index.ts index 9f31484a430ab..0062ed6c061df 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/search_strategy/security_solution/risk_score/all/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/search_strategy/security_solution/risk_score/all/index.ts @@ -93,12 +93,10 @@ export const EntityTypeToLevelField: Record = { [EntityType.host]: RiskScoreFields.hostRisk, [EntityType.user]: RiskScoreFields.userRisk, [EntityType.service]: RiskScoreFields.serviceRisk, - [EntityType.universal]: RiskScoreFields.unsupported, // We don't calculate risk for the universal entity }; export const EntityTypeToScoreField: Record = { [EntityType.host]: RiskScoreFields.hostRiskScore, [EntityType.user]: RiskScoreFields.userRiskScore, [EntityType.service]: RiskScoreFields.serviceRiskScore, - [EntityType.universal]: RiskScoreFields.unsupported, // We don't calculate risk for the universal entity }; diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index eef0bb58a723a..cf69df24bbc85 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -1204,7 +1204,6 @@ components: - user - host - service - - universal type: string HostEntity: type: object @@ -1280,7 +1279,6 @@ components: - host.name - user.name - service.name - - related.entity type: string IndexPattern: type: string diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index 4a923d1771e61..14ddf9436b4aa 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -1204,7 +1204,6 @@ components: - user - host - service - - universal type: string HostEntity: type: object @@ -1280,7 +1279,6 @@ components: - host.name - user.name - service.name - - related.entity type: string IndexPattern: type: string diff --git a/x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/helpers.tsx b/x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/helpers.tsx index 3ae178b22ac5a..a8096fda7b09a 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/helpers.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/helpers.tsx @@ -35,7 +35,6 @@ export const EntityIconByType: Record = { [EntityType.user]: 'user', [EntityType.host]: 'storage', [EntityType.service]: 'node', - [EntityType.universal]: 'globe', // random value since we don't support universal entity type }; export const sourceFieldToText = (source: string) => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/flyout/entity_details/shared/constants.ts b/x-pack/solutions/security/plugins/security_solution/public/flyout/entity_details/shared/constants.ts index ae123243aa04e..3002d187aae41 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/flyout/entity_details/shared/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/flyout/entity_details/shared/constants.ts @@ -32,7 +32,6 @@ export const EntityPanelKeyByType: Record = { [EntityType.host]: HostPanelKey, [EntityType.user]: UserPanelKey, [EntityType.service]: ServicePanelKey, - [EntityType.universal]: undefined, // TODO create universal flyout? }; // TODO rename all params and merged them as 'entityName' @@ -40,5 +39,4 @@ export const EntityPanelParamByType: Record = { [EntityType.host]: 'hostName', [EntityType.user]: 'userName', [EntityType.service]: 'serviceName', - [EntityType.universal]: undefined, // TODO create universal flyout? }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.ts index 96d397ed252b5..b3f20b5d545f0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.ts @@ -34,11 +34,7 @@ export class AssetInventoryDataClient { // Initializes the asset inventory by validating experimental feature flags and triggering asynchronous setup. public async init() { - const { experimentalFeatures, logger } = this.options; - - if (!experimentalFeatures.assetInventoryStoreEnabled) { - throw new Error('Universal entity store is not enabled'); - } + const { logger } = this.options; logger.debug(`Initializing asset inventory`); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/index.ts deleted file mode 100644 index 41c9c45587e1c..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './keyword_builder_ingest_pipeline'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/keyword_builder_ingest_pipeline.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/keyword_builder_ingest_pipeline.ts deleted file mode 100644 index 5b2ca91be98bb..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/keyword_builder_ingest_pipeline.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types'; - -const PIPELINE_ID = 'entity-keyword-builder@platform'; - -export const buildIngestPipeline = (): IngestProcessorContainer[] => { - return [ - { - script: { - lang: 'painless', - on_failure: [ - { - set: { - field: 'error.message', - value: - 'Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}', - }, - }, - ], - - // There are two layers of language to string scape on this script. - // - One is in javascript - // - Another one is in painless. - // - // .e.g, in painless we want the following line: - // entry.getKey().replace("\"", "\\\""); - // - // To do so we must scape each backslash in javascript, otherwise the backslashes will only scape the next character - // and the backslashes won't end up in the painless layer - // - // The code then becomes: - // entry.getKey().replace("\\"", "\\\\\\""); - // That is one extra backslash per backslash (there is no need to scape quotes in the javascript layer) - source: ` - String jsonFromMap(Map map) { - StringBuilder json = new StringBuilder("{"); - boolean first = true; - for (entry in map.entrySet()) { - if (!first) { - json.append(","); - } - first = false; - String key = entry.getKey().replace("\\"", "\\\\\\""); - Object value = entry.getValue(); - json.append("\\"").append(key).append("\\":"); - - if (value instanceof String) { - String escapedValue = ((String) value).replace("\\"", "\\\\\\"").replace("=", ":"); - json.append("\\"").append(escapedValue).append("\\""); - } else if (value instanceof Map) { - json.append(jsonFromMap((Map) value)); - } else if (value instanceof List) { - json.append(jsonFromList((List) value)); - } else if (value instanceof Boolean || value instanceof Number) { - json.append(value.toString()); - } else { - // For other types, treat as string - String escapedValue = value.toString().replace("\\"", "\\\\\\"").replace("=", ":"); - json.append("\\"").append(escapedValue).append("\\""); - } - } - json.append("}"); - return json.toString(); - } - - String jsonFromList(List list) { - StringBuilder json = new StringBuilder("["); - boolean first = true; - for (item in list) { - if (!first) { - json.append(","); - } - first = false; - if (item instanceof String) { - String escapedItem = ((String) item).replace("\\"", "\\\\\\"").replace("=", ":"); - json.append("\\"").append(escapedItem).append("\\""); - } else if (item instanceof Map) { - json.append(jsonFromMap((Map) item)); - } else if (item instanceof List) { - json.append(jsonFromList((List) item)); - } else if (item instanceof Boolean || item instanceof Number) { - json.append(item.toString()); - } else { - // For other types, treat as string - String escapedItem = item.toString().replace("\\"", "\\\\\\"").replace("=", ":"); - json.append("\\"").append(escapedItem).append("\\""); - } - } - json.append("]"); - return json.toString(); - } - - if (ctx.entities?.metadata == null) { - return; - } - - def keywords = []; - for (key in ctx.entities.metadata.keySet()) { - def value = ctx.entities.metadata[key]; - def metadata = jsonFromMap([key: value]); - keywords.add(metadata); - } - - ctx['entities']['keyword'] = keywords; - `, - }, - }, - { - set: { - field: 'event.ingested', - value: '{{{_ingest.timestamp}}}', - }, - }, - ]; -}; - -// developing the pipeline is a bit tricky, so we have a debug mode -// set xpack.securitySolution.entityAnalytics.entityStore.developer.pipelineDebugMode -// to true to keep the enrich field and the context field in the document to help with debugging. -export const createKeywordBuilderPipeline = async ({ - logger, - esClient, -}: { - logger: Logger; - esClient: ElasticsearchClient; -}) => { - const pipeline = { - id: PIPELINE_ID, - body: { - _meta: { - managed_by: 'entity_store', - managed: true, - }, - description: `Serialize entities.metadata into a keyword field`, - processors: buildIngestPipeline(), - }, - }; - - logger.debug(`Attempting to create pipeline: ${JSON.stringify(pipeline)}`); - - await esClient.ingest.putPipeline(pipeline); -}; - -export const deleteKeywordBuilderPipeline = ({ - logger, - esClient, -}: { - logger: Logger; - esClient: ElasticsearchClient; -}) => { - logger.debug(`Attempting to delete pipeline: ${PIPELINE_ID}`); - return esClient.ingest.deletePipeline( - { - id: PIPELINE_ID, - }, - { - ignore: [404], - } - ); -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts index f3aa9fa5b0d09..5ae034fe5d86d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts @@ -82,7 +82,6 @@ const entityTypeByIdField = { 'host.name': 'host', 'user.name': 'user', 'service.name': 'service', - 'related.entity': 'universal', } as const; export const getImplicitEntityFields = (record: AssetCriticalityUpsertWithDeleted) => { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts index c3665155c5ac7..a934b186f875e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts @@ -8,7 +8,6 @@ import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store'; import { HOST_DEFINITION_VERSION, - UNIVERSAL_DEFINITION_VERSION, USER_DEFINITION_VERSION, SERVICE_DEFINITION_VERSION, } from './entity_descriptions'; @@ -16,7 +15,6 @@ import { export const VERSIONS_BY_ENTITY_TYPE: Record = { host: HOST_DEFINITION_VERSION, user: USER_DEFINITION_VERSION, - universal: UNIVERSAL_DEFINITION_VERSION, service: SERVICE_DEFINITION_VERSION, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts index 8984c4aac0eca..58c9ce601094f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts @@ -8,5 +8,4 @@ export * from './host'; export * from './user'; export * from './service'; -export * from './universal'; export { getCommonFieldDescriptions } from './common'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts deleted file mode 100644 index e0dd82f0b330d..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types'; -import type { EntityDescription } from '../types'; -import { collectValues } from './field_utils'; - -export const UNIVERSAL_DEFINITION_VERSION = '1.0.0'; -export const UNIVERSAL_IDENTITY_FIELD = 'related.entity'; - -export const entityMetadataExtractorProcessor = { - script: { - tag: 'entity_metadata_extractor', - on_failure: [ - { - set: { - field: 'error.message', - value: - 'Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}', - }, - }, - ], - lang: 'painless', - source: /* java */ ` - // Array, boolean, integer, ip, bytes, anything that is not a map, is a leaf field - void overwriteLeafFields(Object toBeOverwritten, Object toOverwrite) { - if (!(toBeOverwritten instanceof Map)) { - // We can't override anything that isn't a map - return; - } - if (toOverwrite instanceof Map) { - Map mapToBeOverwritten = (Map) toBeOverwritten; - for (entryToOverwrite in ((Map) toOverwrite).entrySet()) { - String keyToOverwrite = entryToOverwrite.getKey(); - Object valueToOverwrite = entryToOverwrite.getValue(); - - if (valueToOverwrite instanceof Map) { - // If no initial value, we just put everything we have to overwrite - if (mapToBeOverwritten.get(keyToOverwrite) == null) { - mapToBeOverwritten.put(keyToOverwrite, valueToOverwrite) - } else { - overwriteLeafFields(mapToBeOverwritten.get(keyToOverwrite), valueToOverwrite); - } - } else { - mapToBeOverwritten.put(keyToOverwrite, valueToOverwrite) - } - } - } - } - - def id = ctx.entity.id; - Map merged = ctx; - for (meta in ctx.collected.metadata) { - Object json = Processors.json(meta); - if (((Map)json)[id] == null) { - continue; - } - - if (((Map)json)[id] != null) { - overwriteLeafFields(merged, ((Map)json)[id]); - } - } - - merged.entity.id = id; - ctx = merged; - `, - }, -}; - -export const universalEntityEngineDescription: EntityDescription = { - version: UNIVERSAL_DEFINITION_VERSION, - entityType: 'universal', - identityField: UNIVERSAL_IDENTITY_FIELD, - fields: [collectValues({ source: 'entities.keyword', destination: 'collected.metadata' })], - settings: { - timestampField: 'event.ingested', - }, - pipeline: (processors: IngestProcessorContainer[]) => { - const index = processors.findIndex((p) => Boolean(p.enrich)); - - if (index === -1) { - throw new Error('Enrich processor not found'); - } - - const init = processors.slice(0, index); - const tail = processors.slice(index); - const pipe = [...init, entityMetadataExtractorProcessor, ...tail]; - - return pipe; - }, - dynamic: true, -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts index 9f6783ccc1df2..c085d857ba8cb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts @@ -363,14 +363,5 @@ describe('EntityStoreDataClient', () => { expect(spyInit).toHaveBeenCalledWith(EntityType.host, expect.anything(), expect.anything()); }); - - it('does not enable engine when the given entity type is disabled', async () => { - await dataClient.enable({ - ...defaultOptions, - entityTypes: [EntityType.universal], - }); - - expect(spyInit).not.toHaveBeenCalled(); - }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 5c863d356a3a1..1747c5a53401a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -96,10 +96,6 @@ import { import { CRITICALITY_VALUES } from '../asset_criticality/constants'; import { createEngineDescription } from './installation/engine_description'; import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion'; -import { - createKeywordBuilderPipeline, - deleteKeywordBuilderPipeline, -} from '../../asset_inventory/ingest_pipelines'; import type { ApiKeyManager } from './auth/api_key'; @@ -310,12 +306,6 @@ export class EntityStoreDataClient { requestBody: InitEntityEngineRequestBody, { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {} ): Promise { - const { experimentalFeatures } = this.options; - - if (entityType === EntityType.universal && !experimentalFeatures.assetInventoryStoreEnabled) { - throw new Error('Universal entity store is not enabled'); - } - if (!this.options.taskManager) { throw new Error('Task Manager is not available'); } @@ -406,14 +396,6 @@ export class EntityStoreDataClient { }); this.log(`debug`, entityType, `Created entity definition`); - if (entityType === EntityType.universal) { - logger.debug('creating keyword builder pipeline'); - await createKeywordBuilderPipeline({ - logger, - esClient: this.esClient, - }); - } - // the index must be in place with the correct mapping before the enrich policy is created // this is because the enrich policy will fail if the index does not exist with the correct fields await createEntityIndexComponentTemplate(description, this.esClient); @@ -698,15 +680,6 @@ export class EntityStoreDataClient { }); this.log('debug', entityType, `Deleted field retention enrich policy`); - if (entityType === EntityType.universal) { - logger.debug(`Deleting asset inventory keyword builder pipeline`); - - await deleteKeywordBuilderPipeline({ - logger, - esClient: this.esClient, - }); - } - if (deleteData) { await deleteEntityIndex({ entityType, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts index 2b09bd41e5497..7face22fbf959 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts @@ -14,7 +14,6 @@ import { generateIndexMappings } from '../elasticsearch_assets'; import { hostEntityEngineDescription, userEntityEngineDescription, - universalEntityEngineDescription, serviceEntityEngineDescription, } from '../entity_definitions/entity_descriptions'; import type { EntityStoreConfig } from '../types'; @@ -27,7 +26,6 @@ import { defaultOptions } from '../constants'; const engineDescriptionRegistry: Record = { host: hostEntityEngineDescription, user: userEntityEngineDescription, - universal: universalEntityEngineDescription, service: serviceEntityEngineDescription, }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts deleted file mode 100644 index 03814bdd3daba..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrConfigProviderContext } from '@kbn/test'; -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const functionalConfig = await readConfigFile( - require.resolve('../../../../../config/ess/config.base.trial') - ); - - return { - ...functionalConfig.getAll(), - kbnTestServer: { - ...functionalConfig.get('kbnTestServer'), - serverArgs: [ - ...functionalConfig.get('kbnTestServer.serverArgs'), - `--xpack.securitySolution.enableExperimental=${JSON.stringify([])}`, - ], - }, - testFiles: [require.resolve('..')], - junit: { - reportName: 'Asset Inventory Integration Tests - ESS Env - Basic License', - }, - }; -} diff --git a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/index.ts deleted file mode 100644 index 715fa42083f35..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../../ftr_provider_context'; - -export default function ({ loadTestFile }: FtrProviderContext) { - describe('Asset Inventory - Entity Store', function () { - loadTestFile(require.resolve('./keyword_builder_ingest_pipeline')); - }); -} diff --git a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/keyword_builder_ingest_pipeline.ts b/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/keyword_builder_ingest_pipeline.ts deleted file mode 100644 index 67b2238176a3a..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/keyword_builder_ingest_pipeline.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; - -import { buildIngestPipeline } from '@kbn/security-solution-plugin/server/lib/asset_inventory/ingest_pipelines'; -import { applyIngestProcessorToDoc } from '../utils/ingest'; -import { FtrProviderContext } from '../../../../ftr_provider_context'; - -export default ({ getService }: FtrProviderContext) => { - const es = getService('es'); - const log = getService('log'); - - const applyOperatorToDoc = async (docSource: any): Promise => { - const steps = buildIngestPipeline(); - - return applyIngestProcessorToDoc(steps, docSource, es, log); - }; - - describe('@ess @skipInServerlessMKI Asset Inventory - Entity store - Keyword builder pipeline', () => { - it('should build entities.keyword when entities.metadata is provided ', async () => { - const doc = { - related: { - entity: ['entity-id-1', 'entity-id-2', 'entity-id-3'], - }, - entities: { - metadata: { - 'entity-id-1': { - entity: { - type: 'SomeType', - }, - cloud: { - region: 'us-east-1', - }, - someECSfield: 'someECSfieldValue', - SomeNonECSField: 'SomeNonECSValue', - }, - 'entity-id-2': { - entity: { - type: 'SomeOtherType', - }, - SomeNonECSField: 'SomeNonECSValue', - }, - 'entity-id-3': { - someECSfield: 'someECSfieldValue', - }, - }, - }, - }; - - const resultDoc = await applyOperatorToDoc(doc); - - expect(resultDoc.entities.keyword).to.eql([ - '{"entity-id-3":{"someECSfield":"someECSfieldValue"}}', - '{"entity-id-2":{"SomeNonECSField":"SomeNonECSValue","entity":{"type":"SomeOtherType"}}}', - '{"entity-id-1":{"cloud":{"region":"us-east-1"},"SomeNonECSField":"SomeNonECSValue","someECSfield":"someECSfieldValue","entity":{"type":"SomeType"}}}', - ]); - }); - }); -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/utils/ingest.ts b/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/utils/ingest.ts deleted file mode 100644 index aec5812332ab2..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/utils/ingest.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Client } from '@elastic/elasticsearch'; -import { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { ToolingLog } from '@kbn/tooling-log'; - -export const applyIngestProcessorToDoc = async ( - steps: IngestProcessorContainer[], - docSource: any, - es: Client, - log: ToolingLog -): Promise => { - const doc = { - _index: 'index', - _id: 'id', - _source: docSource, - }; - - const res = await es.ingest.simulate({ - pipeline: { - description: 'test', - processors: steps, - }, - docs: [doc], - }); - - const firstDoc = res.docs?.[0]; - - const error = firstDoc?.error; - if (error) { - log.error('Full painless error below: '); - log.error(JSON.stringify(error, null, 2)); - throw new Error('Painless error running pipeline see logs for full detail : ' + error?.type); - } - - return firstDoc?.doc?._source; -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/asset_inventory_pipeline.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/asset_inventory_pipeline.ts deleted file mode 100644 index ca8d7b15cb81e..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/asset_inventory_pipeline.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { entityMetadataExtractorProcessor } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal'; -import { dynamicNewestRetentionSteps } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention/dynamic_retention'; -import { FtrProviderContext } from '../../../../ftr_provider_context'; -import { applyIngestProcessorToDoc } from '../utils/ingest'; - -export default ({ getService }: FtrProviderContext) => { - const es = getService('es'); - const log = getService('log'); - describe('@ess @serverless @skipInServerlessMKI Asset Inventory - universal entity engine pipeline ', () => { - describe('Entity metadata extractor processor step', () => { - it('should extract metadata from "collected.metadata" and add it to the document', async () => { - const metadata = { - test: { - cloud: { super: 123 }, - okta: { foo: { baz: { qux: 1 } } }, - }, - }; - const doc = { - collected: { - metadata: [JSON.stringify(metadata)], - }, - entity: { - id: 'test', - }, - }; - - const result = await applyIngestProcessorToDoc( - [entityMetadataExtractorProcessor], - doc, - es, - log - ); - - const processed = { - ...doc, - ...metadata.test, - }; - - return expect(result).to.eql(processed); - }); - }); - - describe('prefer newest value for dynamic entities', () => { - it('should return latest value if no history value', async () => { - const metadata = { - cloud: { super: 123 }, - }; - - const doc = metadata; - - const processor = dynamicNewestRetentionSteps([]); - const result = await applyIngestProcessorToDoc([processor], doc, es, log); - - return expect(result).to.eql(doc); - }); - - it('should return history value if no latest value is found', async () => { - const metadata = { - cloud: { super: 123 }, - }; - - const doc = { - historical: metadata, - }; - - const processor = dynamicNewestRetentionSteps([]); - const result = await applyIngestProcessorToDoc([processor], doc, es, log); - - return expect(result).to.eql({ - ...doc, - ...metadata, - }); - }); - - it('should return latest value if both historical and latest values exist', async () => { - const metadata = { - cloud: { super: 123 }, - }; - - const historical = { - cloud: { super: 456 }, - }; - - const doc = { - historical, - ...metadata, - }; - - const processor = dynamicNewestRetentionSteps([]); - const result = await applyIngestProcessorToDoc([processor], doc, es, log); - - return expect(result).to.eql(doc); - }); - - it('should merge nested object preserving historical values not found in latest', async () => { - const metadata = { - cloud: { host: 'test' }, - okta: { foo: { bar: { baz: 1 } } }, - }; - - const historical = { - cloud: { user: 'agent' }, - okta: { foo: { bar: { qux: 11 } } }, - }; - - const doc = { - historical, - ...metadata, - }; - - const processor = dynamicNewestRetentionSteps([]); - const result = await applyIngestProcessorToDoc([processor], doc, es, log); - - return expect(result).to.eql({ - historical, - cloud: { host: 'test', user: 'agent' }, - okta: { foo: { bar: { baz: 1, qux: 11 } } }, - }); - }); - - it('should ignore historical static fields', async () => { - const metadata = { - cloud: { host: 'test' }, - }; - - const historical = { - static: 'static', - cloud: { user: 'agent' }, - okta: { foo: { bar: { qux: 1 } } }, - }; - - const doc = { - historical, - ...metadata, - }; - - const staticFields = ['static']; - const processor = dynamicNewestRetentionSteps(staticFields); - const result = await applyIngestProcessorToDoc([processor], doc, es, log); - - return expect(result).to.eql({ - historical, - cloud: { host: 'test', user: 'agent' }, - okta: { foo: { bar: { qux: 1 } } }, - }); - }); - }); - }); -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/index.ts index a6fb4cf805f2d..899dbc68102f3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/index.ts @@ -13,6 +13,5 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./entity_store')); loadTestFile(require.resolve('./field_retention_operators')); loadTestFile(require.resolve('./entity_store_nondefault_spaces')); - loadTestFile(require.resolve('./asset_inventory_pipeline')); }); }