From de04f1bd4a2204a1d7a415f847ada65f49d148f8 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Thu, 5 Feb 2026 11:26:01 +0100 Subject: [PATCH 01/69] add boilerplate --- .../server/domain/asset_manager.ts | 5 + .../server/domain/entity_manager.ts | 100 ++++++++++++++++++ .../errors/document_version_conflict.ts | 14 +++ .../server/domain/errors/entity_not_found.ts | 12 +++ .../server/domain/errors/index.ts | 9 ++ .../server/routes/apis/crud_upsert.ts | 62 +++++++++++ .../entity_store/server/routes/constants.ts | 7 ++ 7 files changed, 209 insertions(+) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/errors/document_version_conflict.ts create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_not_found.ts create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts create mode 100644 x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts index 1ed077872508b..348f75fe12944 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts @@ -123,4 +123,9 @@ export class AssetManager { throw error; } } + + public async isInstalled(type: EntityType): Promise { + const response = await this.engineDescriptorClient.find(type); + return response.total > 0; + } } diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts new file mode 100644 index 0000000000000..7601729328654 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { Result } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { getLatestEntitiesIndexName } from './assets/latest_index'; +import type { EntityType } from './definitions/entity_schema'; +import { DocumentVersionConflictError, EntityNotFoundError } from './errors'; + +interface EntityManagerDependencies { + logger: Logger; + esClient: ElasticsearchClient; + namespace: string; +} + +// TODO: Temporary interface to avoid all the error logs +interface Entity { + entity: { + type: EntityType; + }; +} + +export class EntityManager { + private readonly logger: Logger; + private readonly esClient: ElasticsearchClient; + private readonly namespace: string; + + constructor(deps: EntityManagerDependencies) { + this.logger = deps.logger; + this.esClient = deps.esClient; + this.namespace = deps.namespace; + } + + private getEntityId(document: Entity): string { + // TODO: NOT IMPLEMENTED + throw new Error(`getEntityId() not implemented`); + return 'TODO'; + } + + // TODO: Bulk upsert + + public async upsertEntity(document: Entity) { + // From: x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_crud_client.ts + // TODO: normalizeToECS() + // TODO: getFlattenedObject() + + const id = this.getEntityId(document); + this.logger.info(`Upserting entity ID ${id}`); + const { result } = await this.esClient.update({ + // TODO: remove entity type after single index merge + index: getLatestEntitiesIndexName(document.entity.type, this.namespace), + id, + doc: document, + doc_as_upsert: true, + }); + + switch (result as Result) { + case 'deleted': + case 'not_found': + throw new Error(`Could not upsert entity ID ${id}`); + case 'created': + this.logger.info(`Entity ID ${id} created`); + break; + case 'updated': + this.logger.info(`Entity ID ${id} updated`); + break; + case 'noop': + this.logger.info(`Entity ID ${id} updated (no change)`); + break; + } + } + + public async deleteEntity(id: string) { + const resp = await this.esClient.deleteByQuery({ + // TODO: remove entity type after single index merge + index: getLatestEntitiesIndexName('generic', this.namespace), + query: { + term: { + 'entity.id': id, + }, + }, + conflicts: 'proceed', + }); + + if (resp.failures !== undefined && resp.failures.length > 0) { + throw new Error(`Failed to delete entity ID ${id}`); + } + if (resp.version_conflicts) { + throw new DocumentVersionConflictError(); + } + if (!resp.deleted) { + throw new EntityNotFoundError(id); + } + } +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/document_version_conflict.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/document_version_conflict.ts new file mode 100644 index 0000000000000..76fa10410abc9 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/document_version_conflict.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DocumentVersionConflictError extends Error { + constructor() { + super( + `Attempted to update document while another document update was happening. Try again in a few seconds.` + ); + } +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_not_found.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_not_found.ts new file mode 100644 index 0000000000000..9170e5fa585f6 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_not_found.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class EntityNotFoundError extends Error { + constructor(id: string) { + super(`Entity ID '${id}' not found`); + } +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts new file mode 100644 index 0000000000000..d4c2f56d39418 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { EntityNotFoundError } from './entity_not_found'; +export { DocumentVersionConflictError } from './document_version_conflict'; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts new file mode 100644 index 0000000000000..d184f3dc78d53 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { z } from '@kbn/zod'; +import type { IKibanaResponse } from '@kbn/core-http-server'; +import { + API_VERSIONS, + DEFAULT_ENTITY_STORE_PERMISSIONS, + EngineNotRunningError, +} from '../constants'; +import type { EntityStorePluginRouter } from '../../types'; +import { wrapMiddlewares } from '../middleware'; +import { EntityType } from '../../domain/definitions/entity_schema'; + +// TODO: openapi schema for body +const bodySchema = z.object({}); + +export function registerCRUDUpsert(router: EntityStorePluginRouter) { + router.versioned + .put({ + path: '/api/entity-store/entities', + access: 'public', + security: { + authz: DEFAULT_ENTITY_STORE_PERMISSIONS, + }, + enableQueryVersion: true, + }) + .addVersion( + { + version: API_VERSIONS.internal.v2, + validate: { + request: { + body: buildRouteValidationWithZod(bodySchema), + }, + }, + }, + wrapMiddlewares(async (ctx, req, res): Promise => { + const entityStoreCtx = await ctx.entityStore; + const { logger, assetManager } = entityStoreCtx; + + logger.debug('CRUD Upsert api called'); + const entityType = req.body.entity.type; + if (!(await assetManager.isInstalled(entityType))) { + return res.customError({ statusCode: 503, body: new EngineNotRunningError(entityType) }); + } + // TODO: Check if skipping ?force flag does anything + // TODO: use req.body as entity to create + // TODO: ask guys do we REALLY need generated schemas + + return res.ok({ + body: { + ok: true, + }, + }); + }) + ); +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts index 0c60c2026d340..0eac5655b7b35 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts @@ -8,6 +8,7 @@ import type { AuthzEnabled } from '@kbn/core/server'; import type { z } from '@kbn/zod'; import { LogExtractionState } from '../domain/definitions/saved_objects'; +import type { EntityType } from '../domain/definitions/entity_schema'; export const DEFAULT_ENTITY_STORE_PERMISSIONS: AuthzEnabled = { requiredPrivileges: ['securitySolution'], @@ -34,3 +35,9 @@ export const LogExtractionBodyParams = LogExtractionState.pick({ delay: true, docsLimit: true, }).partial(); + +export class EngineNotRunningError extends Error { + constructor(engine: EntityType) { + super(`Entity Engine '${engine}' is not running`); + } +} From 5b2b0c6c68e7fd55c84c88a52490e8e8e61dd7db Mon Sep 17 00:00:00 2001 From: kubasobon Date: Thu, 5 Feb 2026 12:10:50 +0100 Subject: [PATCH 02/69] add crud delete --- .../server/domain/asset_manager.ts | 17 ++++- .../errors/entity_store_not_installed.ts | 12 +++ .../server/domain/errors/index.ts | 1 + .../server/request_context_factory.ts | 8 ++ .../server/routes/apis/crud_delete.ts | 74 +++++++++++++++++++ .../server/routes/apis/crud_upsert.ts | 21 +++--- .../entity_store/server/routes/constants.ts | 7 -- .../plugins/entity_store/server/types.ts | 2 + 8 files changed, 121 insertions(+), 21 deletions(-) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts create mode 100644 x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts index 348f75fe12944..7a497d7a714dd 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts @@ -9,7 +9,11 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { getEntityDefinition } from './definitions/registry'; -import type { EntityType, ManagedEntityDefinition } from './definitions/entity_schema'; +import { + ALL_ENTITY_TYPES, + type EntityType, + type ManagedEntityDefinition, +} from './definitions/entity_schema'; import { scheduleExtractEntityTask, stopExtractEntityTask } from '../tasks/extract_entity_task'; import { installElasticsearchAssets, uninstallElasticsearchAssets } from './assets/install_assets'; import type { EngineDescriptorClient, LogExtractionState } from './definitions/saved_objects'; @@ -124,8 +128,13 @@ export class AssetManager { } } - public async isInstalled(type: EntityType): Promise { - const response = await this.engineDescriptorClient.find(type); - return response.total > 0; + public async isInstalled(): Promise { + for (const type of ALL_ENTITY_TYPES) { + const response = await this.engineDescriptorClient.find(type); + if (response.total > 0) { + return true; + } + } + return false; } } diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts new file mode 100644 index 0000000000000..fa38ba4e24b74 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class EntityStoreNotInstalledError extends Error { + constructor() { + super(`No EntityStore engine is running`); + } +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts index d4c2f56d39418..1956988d00ed8 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts @@ -7,3 +7,4 @@ export { EntityNotFoundError } from './entity_not_found'; export { DocumentVersionConflictError } from './document_version_conflict'; +export { EntityStoreNotInstalledError } from './entity_store_not_installed'; diff --git a/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts b/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts index bd2ba055b098e..80e04bed89bcb 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts @@ -17,6 +17,7 @@ import { AssetManager } from './domain/asset_manager'; import { FeatureFlags } from './infra/feature_flags'; import { EngineDescriptorClient } from './domain/definitions/saved_objects'; import { LogsExtractionClient } from './domain/logs_extraction_client'; +import { EntityManager } from './domain/entity_manager'; interface EntityStoreApiRequestHandlerContextDeps { coreSetup: CoreSetup; @@ -49,6 +50,12 @@ export async function createRequestHandlerContext({ logger ); + const entityManager = new EntityManager({ + logger, + esClient: core.elasticsearch.client.asCurrentUser, + namespace, + }); + return { core, logger, @@ -59,6 +66,7 @@ export async function createRequestHandlerContext({ engineDescriptorClient, namespace, }), + entityManager, featureFlags: new FeatureFlags(core.uiSettings.client), logsExtractionClient: new LogsExtractionClient( logger, diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts new file mode 100644 index 0000000000000..cf53163a28085 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { z } from '@kbn/zod'; +import type { IKibanaResponse } from '@kbn/core-http-server'; +import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; +import type { EntityStorePluginRouter } from '../../types'; +import { wrapMiddlewares } from '../middleware'; +import { + DocumentVersionConflictError, + EntityNotFoundError, + EntityStoreNotInstalledError, +} from '../../domain/errors'; + +const paramsSchema = z.object({ + id: z.string(), +}); + +export function registerCRUDDelete(router: EntityStorePluginRouter) { + router.versioned + .delete({ + path: '/api/entity-store/entities/{id}', + access: 'public', + security: { + authz: DEFAULT_ENTITY_STORE_PERMISSIONS, + }, + enableQueryVersion: true, + }) + .addVersion( + { + version: API_VERSIONS.internal.v2, + validate: { + request: { + params: buildRouteValidationWithZod(paramsSchema), + }, + }, + }, + wrapMiddlewares(async (ctx, req, res): Promise => { + const entityStoreCtx = await ctx.entityStore; + const { logger, assetManager, entityManager } = entityStoreCtx; + + logger.debug('CRUD Delete api called'); + if (!(await assetManager.isInstalled())) { + return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); + } + + try { + await entityManager.deleteEntity(req.params.id); + } catch (error) { + if (error instanceof DocumentVersionConflictError) { + return res.customError({ + statusCode: 409, + body: error as DocumentVersionConflictError, + }); + } + if (error instanceof EntityNotFoundError) { + return res.customError({ + statusCode: 404, + body: error as EntityNotFoundError, + }); + } + logger.error(error); + throw error; + } + + return res.ok(); + }) + ); +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index d184f3dc78d53..155c219dbaf10 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -8,14 +8,10 @@ import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { z } from '@kbn/zod'; import type { IKibanaResponse } from '@kbn/core-http-server'; -import { - API_VERSIONS, - DEFAULT_ENTITY_STORE_PERMISSIONS, - EngineNotRunningError, -} from '../constants'; +import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; -import { EntityType } from '../../domain/definitions/entity_schema'; +import { EntityStoreNotInstalledError } from '../../domain/errors'; // TODO: openapi schema for body const bodySchema = z.object({}); @@ -41,16 +37,21 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { }, wrapMiddlewares(async (ctx, req, res): Promise => { const entityStoreCtx = await ctx.entityStore; - const { logger, assetManager } = entityStoreCtx; + const { logger, assetManager, entityManager } = entityStoreCtx; logger.debug('CRUD Upsert api called'); - const entityType = req.body.entity.type; - if (!(await assetManager.isInstalled(entityType))) { - return res.customError({ statusCode: 503, body: new EngineNotRunningError(entityType) }); + if (!(await assetManager.isInstalled())) { + return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); } // TODO: Check if skipping ?force flag does anything // TODO: use req.body as entity to create // TODO: ask guys do we REALLY need generated schemas + try { + await entityManager.upsertEntity(req.body); + } catch (error) { + logger.error(error); + throw error; + } return res.ok({ body: { diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts index 0eac5655b7b35..0c60c2026d340 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/constants.ts @@ -8,7 +8,6 @@ import type { AuthzEnabled } from '@kbn/core/server'; import type { z } from '@kbn/zod'; import { LogExtractionState } from '../domain/definitions/saved_objects'; -import type { EntityType } from '../domain/definitions/entity_schema'; export const DEFAULT_ENTITY_STORE_PERMISSIONS: AuthzEnabled = { requiredPrivileges: ['securitySolution'], @@ -35,9 +34,3 @@ export const LogExtractionBodyParams = LogExtractionState.pick({ delay: true, docsLimit: true, }).partial(); - -export class EngineNotRunningError extends Error { - constructor(engine: EntityType) { - super(`Entity Engine '${engine}' is not running`); - } -} diff --git a/x-pack/solutions/security/plugins/entity_store/server/types.ts b/x-pack/solutions/security/plugins/entity_store/server/types.ts index 07b4146380bd6..f1c2fa238301e 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/types.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/types.ts @@ -26,6 +26,7 @@ import type { CoreSetup } from '@kbn/core/server'; import type { AssetManager } from './domain/asset_manager'; import type { FeatureFlags } from './infra/feature_flags'; import type { LogsExtractionClient } from './domain/logs_extraction_client'; +import type { EntityManager } from './domain/entity_manager'; export interface EntityStoreSetupPlugins { taskManager: TaskManagerSetupContract; @@ -45,6 +46,7 @@ export interface EntityStoreApiRequestHandlerContext { core: CoreRequestHandlerContext; logger: Logger; assetManager: AssetManager; + entityManager: EntityManager; featureFlags: FeatureFlags; logsExtractionClient: LogsExtractionClient; } From 37fb4d96f0503413c2fc1c63883ba0c81a20e1f5 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 10:41:15 +0100 Subject: [PATCH 03/69] add upsert bulk --- .../server/domain/entity_manager.ts | 4 ++ .../server/routes/apis/crud_upsert_bulk.ts | 63 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts index 7601729328654..5d7baae94878c 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -75,6 +75,10 @@ export class EntityManager { } } + public async upsertEntitiesBulk(documents: Entity[]) { + await Promise.all(documents.map((document) => this.upsertEntity(document))); + } + public async deleteEntity(id: string) { const resp = await this.esClient.deleteByQuery({ // TODO: remove entity type after single index merge diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts new file mode 100644 index 0000000000000..91f32718eea08 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { z } from '@kbn/zod'; +import type { IKibanaResponse } from '@kbn/core-http-server'; +import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; +import type { EntityStorePluginRouter } from '../../types'; +import { wrapMiddlewares } from '../middleware'; +import { EntityStoreNotInstalledError } from '../../domain/errors'; + +// TODO: openapi schema for body +const bodySchema = z.array(z.object({})); + +export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { + router.versioned + .put({ + path: '/api/entity-store/entities/bulk', + access: 'public', + security: { + authz: DEFAULT_ENTITY_STORE_PERMISSIONS, + }, + enableQueryVersion: true, + }) + .addVersion( + { + version: API_VERSIONS.internal.v2, + validate: { + request: { + body: buildRouteValidationWithZod(bodySchema), + }, + }, + }, + wrapMiddlewares(async (ctx, req, res): Promise => { + const entityStoreCtx = await ctx.entityStore; + const { logger, assetManager, entityManager } = entityStoreCtx; + + logger.debug('CRUD Upsert Bulk api called'); + if (!(await assetManager.isInstalled())) { + return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); + } + // TODO: Check if skipping ?force flag does anything + // TODO: use req.body as entity to create + // TODO: ask guys do we REALLY need generated schemas + try { + await entityManager.upsertEntitiesBulk(req.body); + } catch (error) { + logger.error(error); + throw error; + } + + return res.ok({ + body: { + ok: true, + }, + }); + }) + ); +} From 653726f63b5b2121d43518d1f051ede17eff5f62 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 10:44:45 +0100 Subject: [PATCH 04/69] add entity schema --- .../server/domain/schemas/entity.schema.yaml | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml new file mode 100644 index 0000000000000..32af53a9113c6 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml @@ -0,0 +1,335 @@ +openapi: 3.0.0 +info: + title: Common Entities Schemas + description: Common Entities schemas for the Entity Store + version: '1' +paths: {} +components: + schemas: + EngineMetadata: + type: object + additionalProperties: false + required: + - Type + properties: + Type: + type: string + + EntityField: + type: object + additionalProperties: false + required: + - id + properties: + id: + type: string + name: + type: string + type: + type: string + sub_type: + type: string + source: + type: string + EngineMetadata: + $ref: '#/components/schemas/EngineMetadata' + attributes: + type: object + additionalProperties: false + properties: + privileged: + type: boolean + asset: + type: boolean + managed: + type: boolean + mfa_enabled: + type: boolean + behaviors: + type: object + additionalProperties: false + properties: + brute_force_victim: + type: boolean + new_country_login: + type: boolean + used_usb_device: + type: boolean + lifecycle: + type: object + additionalProperties: false + properties: + first_seen: + type: string + format: date-time + last_activity: + type: string + format: date-time + relationships: + type: object + additionalProperties: false + properties: + communicates_with: + type: array + items: + type: string + depends_on: + type: array + items: + type: string + dependent_of: + type: array + items: + type: string + owns: + type: array + items: + type: string + owned_by: + type: array + items: + type: string + accesses_frequently: + type: array + items: + type: string + accessed_frequently_by: + type: array + items: + type: string + supervises: + type: array + items: + type: string + supervised_by: + type: array + items: + type: string + risk: + type: object + additionalProperties: false + properties: + calculated_level: + $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskLevels' + example: 'Critical' + description: Lexical description of the entity's risk. + calculated_score: + type: number + format: double + description: The raw numeric value of the given entity's risk score. + calculated_score_norm: + type: number + format: double + minimum: 0 + maximum: 100 + description: The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. + + Asset: + type: object + additionalProperties: false + properties: + id: + type: string + name: + type: string + owner: + type: string + serial_number: + type: string + model: + type: string + vendor: + type: string + environment: + type: string + criticality: + $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel' + business_unit: + type: string + + UserEntity: + type: object + additionalProperties: false + required: + - entity + properties: + "@timestamp": + type: string + format: date-time + entity: + $ref: '#/components/schemas/EntityField' + user: + type: object + additionalProperties: false + properties: + full_name: + type: array + items: + type: string + domain: + type: array + items: + type: string + roles: + type: array + items: + type: string + name: + type: string + id: + type: array + items: + type: string + email: + type: array + items: + type: string + hash: + type: array + items: + type: string + risk: + $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + additionalProperties: false + required: + - name + asset: + $ref: '#/components/schemas/Asset' + additionalProperties: false + event: + type: object + additionalProperties: false + properties: + ingested: + type: string + format: date-time + + HostEntity: + type: object + additionalProperties: false + required: + - entity + properties: + "@timestamp": + type: string + format: date-time + entity: + $ref: '#/components/schemas/EntityField' + host: + type: object + additionalProperties: false + properties: + hostname: + type: array + items: + type: string + domain: + type: array + items: + type: string + ip: + type: array + items: + type: string + name: + type: string + id: + type: array + items: + type: string + type: + type: array + items: + type: string + mac: + type: array + items: + type: string + architecture: + type: array + items: + type: string + risk: + $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + entity: + $ref: '#/components/schemas/EntityField' + required: + - name + asset: + $ref: '#/components/schemas/Asset' + additionalProperties: false + event: + type: object + additionalProperties: false + properties: + ingested: + type: string + format: date-time + + ServiceEntity: + type: object + additionalProperties: false + required: + - entity + properties: + "@timestamp": + type: string + format: date-time + entity: + $ref: '#/components/schemas/EntityField' + service: + type: object + additionalProperties: false + properties: + name: + type: string + risk: + $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + entity: + $ref: '#/components/schemas/EntityField' + required: + - name + asset: + $ref: '#/components/schemas/Asset' + additionalProperties: false + event: + type: object + additionalProperties: false + properties: + ingested: + type: string + format: date-time + + # The Generic Entity definition maps more than just entity. + # however I don't see a reason to duplicate the definition + # of all the fields just for the sake of doing. + # Thus the current mapping maps entity and asset only + # (used in code). If you end up needing the fields mapped + # in the schema just add it. + GenericEntity: + type: object + additionalProperties: false + required: + - entity + properties: + "@timestamp": + type: string + format: date-time + entity: + $ref: '#/components/schemas/EntityField' + asset: + $ref: '#/components/schemas/Asset' + additionalProperties: false + + # These entities represent the API of Entity Store + # where entity is a root field, as it's in the final + # index for entities. Note that this is different from the + # source logs mapping where `entity` will be nested under + # `host`, `user` and `service` + Entity: + oneOf: + - $ref: '#/components/schemas/UserEntity' + - $ref: '#/components/schemas/HostEntity' + - $ref: '#/components/schemas/ServiceEntity' + - $ref: '#/components/schemas/GenericEntity' + From 05f2d209cb6af58093d6dcfb46a0391750b42037 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 10:57:58 +0100 Subject: [PATCH 05/69] change schema to omit AC/Risk for now --- .../server/domain/schemas/entity.schema.yaml | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml index 32af53a9113c6..73ef7d7ab6515 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml @@ -109,10 +109,11 @@ components: type: object additionalProperties: false properties: - calculated_level: - $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskLevels' - example: 'Critical' - description: Lexical description of the entity's risk. + ## TODO: KUBA I'll skip risk and asset criticality for now + # calculated_level: + # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskLevels' + # example: 'Critical' + # description: Lexical description of the entity's risk. calculated_score: type: number format: double @@ -142,8 +143,9 @@ components: type: string environment: type: string - criticality: - $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel' + ## TODO: KUBA I'll skip risk and asset criticality for now + # criticality: + # $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel' business_unit: type: string @@ -188,9 +190,10 @@ components: type: array items: type: string - risk: - $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' - additionalProperties: false + ## TODO: KUBA I'll skip risk and asset criticality for now + # risk: + # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + # additionalProperties: false required: - name asset: @@ -249,8 +252,9 @@ components: type: array items: type: string - risk: - $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + ## TODO: KUBA I'll skip risk and asset criticality for now + # risk: + # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' entity: $ref: '#/components/schemas/EntityField' required: @@ -283,8 +287,9 @@ components: properties: name: type: string - risk: - $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + ## TODO: KUBA I'll skip risk and asset criticality for now# + # risk: + # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' entity: $ref: '#/components/schemas/EntityField' required: From 232573fb702126d71f08437a93c32ca29b8c12bf Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 11:03:00 +0100 Subject: [PATCH 06/69] generate temp schema and fix types --- .../server/domain/entity_manager.ts | 13 +- .../server/domain/schemas/entity.gen.ts | 193 ++++++++++++++++++ .../server/routes/apis/crud_upsert.ts | 7 +- .../server/routes/apis/crud_upsert_bulk.ts | 4 +- 4 files changed, 202 insertions(+), 15 deletions(-) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.gen.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts index 5d7baae94878c..19f340cfd8ec8 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -11,6 +11,7 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { getLatestEntitiesIndexName } from './assets/latest_index'; import type { EntityType } from './definitions/entity_schema'; import { DocumentVersionConflictError, EntityNotFoundError } from './errors'; +import type { Entity } from './schemas/entity.gen'; interface EntityManagerDependencies { logger: Logger; @@ -18,13 +19,6 @@ interface EntityManagerDependencies { namespace: string; } -// TODO: Temporary interface to avoid all the error logs -interface Entity { - entity: { - type: EntityType; - }; -} - export class EntityManager { private readonly logger: Logger; private readonly esClient: ElasticsearchClient; @@ -50,10 +44,13 @@ export class EntityManager { // TODO: getFlattenedObject() const id = this.getEntityId(document); + if (document.entity.type === undefined) { + throw new Error(`Entity ID ${id} type undefined`); + } this.logger.info(`Upserting entity ID ${id}`); const { result } = await this.esClient.update({ // TODO: remove entity type after single index merge - index: getLatestEntitiesIndexName(document.entity.type, this.namespace), + index: getLatestEntitiesIndexName(document.entity.type as EntityType, this.namespace), id, doc: document, doc_as_upsert: true, diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.gen.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.gen.ts new file mode 100644 index 0000000000000..4687b2f1b1909 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.gen.ts @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Common Entities Schemas + * version: 1 + */ + +import { z } from '@kbn/zod'; + +export type EngineMetadata = z.infer; +export const EngineMetadata = z + .object({ + Type: z.string(), + }) + .strict(); + +export type EntityField = z.infer; +export const EntityField = z + .object({ + id: z.string(), + name: z.string().optional(), + type: z.string().optional(), + sub_type: z.string().optional(), + source: z.string().optional(), + EngineMetadata: EngineMetadata.optional(), + attributes: z + .object({ + privileged: z.boolean().optional(), + asset: z.boolean().optional(), + managed: z.boolean().optional(), + mfa_enabled: z.boolean().optional(), + }) + .strict() + .optional(), + behaviors: z + .object({ + brute_force_victim: z.boolean().optional(), + new_country_login: z.boolean().optional(), + used_usb_device: z.boolean().optional(), + }) + .strict() + .optional(), + lifecycle: z + .object({ + first_seen: z.string().datetime().optional(), + last_activity: z.string().datetime().optional(), + }) + .strict() + .optional(), + relationships: z + .object({ + communicates_with: z.array(z.string()).optional(), + depends_on: z.array(z.string()).optional(), + dependent_of: z.array(z.string()).optional(), + owns: z.array(z.string()).optional(), + owned_by: z.array(z.string()).optional(), + accesses_frequently: z.array(z.string()).optional(), + accessed_frequently_by: z.array(z.string()).optional(), + supervises: z.array(z.string()).optional(), + supervised_by: z.array(z.string()).optional(), + }) + .strict() + .optional(), + risk: z + .object({ + /** + * The raw numeric value of the given entity's risk score. + */ + calculated_score: z.number().optional(), + /** + * The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. + */ + calculated_score_norm: z.number().min(0).max(100).optional(), + }) + .strict() + .optional(), + }) + .strict(); + +export type Asset = z.infer; +export const Asset = z + .object({ + id: z.string().optional(), + name: z.string().optional(), + owner: z.string().optional(), + serial_number: z.string().optional(), + model: z.string().optional(), + vendor: z.string().optional(), + environment: z.string().optional(), + business_unit: z.string().optional(), + }) + .strict(); + +export type UserEntity = z.infer; +export const UserEntity = z + .object({ + '@timestamp': z.string().datetime().optional(), + entity: EntityField, + user: z + .object({ + full_name: z.array(z.string()).optional(), + domain: z.array(z.string()).optional(), + roles: z.array(z.string()).optional(), + name: z.string(), + id: z.array(z.string()).optional(), + email: z.array(z.string()).optional(), + hash: z.array(z.string()).optional(), + }) + .strict() + .optional(), + asset: Asset.optional(), + event: z + .object({ + ingested: z.string().datetime().optional(), + }) + .strict() + .optional(), + }) + .strict(); + +export type HostEntity = z.infer; +export const HostEntity = z + .object({ + '@timestamp': z.string().datetime().optional(), + entity: EntityField, + host: z + .object({ + hostname: z.array(z.string()).optional(), + domain: z.array(z.string()).optional(), + ip: z.array(z.string()).optional(), + name: z.string(), + id: z.array(z.string()).optional(), + type: z.array(z.string()).optional(), + mac: z.array(z.string()).optional(), + architecture: z.array(z.string()).optional(), + entity: EntityField.optional(), + }) + .strict() + .optional(), + asset: Asset.optional(), + event: z + .object({ + ingested: z.string().datetime().optional(), + }) + .strict() + .optional(), + }) + .strict(); + +export type ServiceEntity = z.infer; +export const ServiceEntity = z + .object({ + '@timestamp': z.string().datetime().optional(), + entity: EntityField, + service: z + .object({ + name: z.string(), + entity: EntityField.optional(), + }) + .strict() + .optional(), + asset: Asset.optional(), + event: z + .object({ + ingested: z.string().datetime().optional(), + }) + .strict() + .optional(), + }) + .strict(); + +export type GenericEntity = z.infer; +export const GenericEntity = z + .object({ + '@timestamp': z.string().datetime().optional(), + entity: EntityField, + asset: Asset.optional(), + }) + .strict(); + +export const EntityInternal = z.union([UserEntity, HostEntity, ServiceEntity, GenericEntity]); + +export type Entity = z.infer; +export const Entity = EntityInternal as z.ZodType; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index 155c219dbaf10..541a3bb3790b7 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -6,15 +6,12 @@ */ import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import { z } from '@kbn/zod'; import type { IKibanaResponse } from '@kbn/core-http-server'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; - -// TODO: openapi schema for body -const bodySchema = z.object({}); +import { Entity } from '../../domain/schemas/entity.gen'; export function registerCRUDUpsert(router: EntityStorePluginRouter) { router.versioned @@ -31,7 +28,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { version: API_VERSIONS.internal.v2, validate: { request: { - body: buildRouteValidationWithZod(bodySchema), + body: buildRouteValidationWithZod(Entity), }, }, }, diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index 91f32718eea08..f5762a2661df6 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -12,9 +12,9 @@ import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; +import { Entity } from '../../domain/schemas/entity.gen'; -// TODO: openapi schema for body -const bodySchema = z.array(z.object({})); +const bodySchema = z.array(Entity); export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { router.versioned From 597fe1d849b874d8211f37f456f9fef36aa39deb Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 11:36:25 +0100 Subject: [PATCH 07/69] register routes --- .../entity_store/server/routes/apis/crud_delete.ts | 2 +- .../entity_store/server/routes/apis/crud_upsert.ts | 2 +- .../server/routes/apis/crud_upsert_bulk.ts | 2 +- .../entity_store/server/routes/apis/index.ts | 4 ++++ .../entity_store/server/routes/register_routes.ts | 14 ++++++++++++-- x-pack/yarn.lock | 4 ++++ 6 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 x-pack/yarn.lock diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts index cf53163a28085..4ea83593f9493 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts @@ -33,7 +33,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.internal.v2, + version: API_VERSIONS.public.v1, validate: { request: { params: buildRouteValidationWithZod(paramsSchema), diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index 541a3bb3790b7..1eeb32e448b99 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -25,7 +25,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.internal.v2, + version: API_VERSIONS.public.v1, // TODO: KUBA: SHould it be public? Really? validate: { request: { body: buildRouteValidationWithZod(Entity), diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index f5762a2661df6..c9c1a849f9785 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -28,7 +28,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.internal.v2, + version: API_VERSIONS.public.v1, validate: { request: { body: buildRouteValidationWithZod(bodySchema), diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts index 8d93f1119b245..311980ac6faa7 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts @@ -8,3 +8,7 @@ export { registerInstall } from './install'; export { registerStop } from './stop'; export { registerForceLogExtraction } from './force_log_extraction'; +export { registerUninstall } from './uninstall'; +export { registerCRUDUpsert } from './crud_upsert'; +export { registerCRUDUpsertBulk } from './crud_upsert_bulk'; +export { registerCRUDDelete } from './crud_delete'; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/register_routes.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/register_routes.ts index 406df53a52857..0f9a9fac49d45 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/register_routes.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/register_routes.ts @@ -5,13 +5,23 @@ * 2.0. */ -import { registerInstall, registerStop, registerForceLogExtraction } from './apis'; +import { + registerInstall, + registerStop, + registerForceLogExtraction, + registerUninstall, + registerCRUDUpsert, + registerCRUDUpsertBulk, + registerCRUDDelete, +} from './apis'; import type { EntityStorePluginRouter } from '../types'; -import { registerUninstall } from './apis/uninstall'; export function registerRoutes(router: EntityStorePluginRouter) { registerInstall(router); registerStop(router); registerUninstall(router); registerForceLogExtraction(router); + registerCRUDUpsert(router); + registerCRUDUpsertBulk(router); + registerCRUDDelete(router); } diff --git a/x-pack/yarn.lock b/x-pack/yarn.lock new file mode 100644 index 0000000000000..fb57ccd13afbd --- /dev/null +++ b/x-pack/yarn.lock @@ -0,0 +1,4 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + From a95d6e721fb6239b679fd23e1a45654c431ee744 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 11:43:07 +0100 Subject: [PATCH 08/69] remove dependency on entityType --- .../entity_store/server/domain/entity_manager.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts index 19f340cfd8ec8..e4b0e7b3b4b66 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -9,7 +9,6 @@ import type { Logger } from '@kbn/logging'; import type { Result } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { getLatestEntitiesIndexName } from './assets/latest_index'; -import type { EntityType } from './definitions/entity_schema'; import { DocumentVersionConflictError, EntityNotFoundError } from './errors'; import type { Entity } from './schemas/entity.gen'; @@ -44,13 +43,9 @@ export class EntityManager { // TODO: getFlattenedObject() const id = this.getEntityId(document); - if (document.entity.type === undefined) { - throw new Error(`Entity ID ${id} type undefined`); - } this.logger.info(`Upserting entity ID ${id}`); const { result } = await this.esClient.update({ - // TODO: remove entity type after single index merge - index: getLatestEntitiesIndexName(document.entity.type as EntityType, this.namespace), + index: getLatestEntitiesIndexName(this.namespace), id, doc: document, doc_as_upsert: true, @@ -78,8 +73,7 @@ export class EntityManager { public async deleteEntity(id: string) { const resp = await this.esClient.deleteByQuery({ - // TODO: remove entity type after single index merge - index: getLatestEntitiesIndexName('generic', this.namespace), + index: getLatestEntitiesIndexName(this.namespace), query: { term: { 'entity.id': id, From b0487514141affb8986fac0d83f239646c834bf6 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 6 Feb 2026 11:56:42 +0100 Subject: [PATCH 09/69] delete files added by mistake --- x-pack/yarn.lock | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 x-pack/yarn.lock diff --git a/x-pack/yarn.lock b/x-pack/yarn.lock deleted file mode 100644 index fb57ccd13afbd..0000000000000 --- a/x-pack/yarn.lock +++ /dev/null @@ -1,4 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - From b7ffde1756cc6785decf1124cb25495bd7fab008 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 11 Feb 2026 11:31:48 +0100 Subject: [PATCH 10/69] fix imports --- .../entity_store/server/domain/asset_manager.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts index 14cdcd2afdcbb..d70b1e0bd7b65 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts @@ -9,12 +9,6 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; -import { getEntityDefinition } from './definitions/registry'; -import { - ALL_ENTITY_TYPES, - type EntityType, - type ManagedEntityDefinition, -} from './definitions/entity_schema'; import { scheduleExtractEntityTask, stopExtractEntityTask } from '../tasks/extract_entity_task'; import { installElasticsearchAssets, uninstallElasticsearchAssets } from './assets/install_assets'; import type { @@ -40,6 +34,9 @@ import { } from './assets/component_templates'; import { getUpdatesEntitiesDataStreamName } from './assets/updates_data_stream'; import type { LogsExtractionClient } from './logs_extraction_client'; +import { EntityType } from '@kbn/entity-store/common'; +import { ALL_ENTITY_TYPES, ManagedEntityDefinition } from '@kbn/entity-store/common/domain/definitions/entity_schema'; +import { getEntityDefinition } from '@kbn/entity-store/common/domain/definitions/registry'; interface AssetManagerDependencies { logger: Logger; @@ -177,7 +174,7 @@ export class AssetManager { } return true; } - + public async getStatus(withComponents: boolean = false): Promise { try { const engines = await this.engineDescriptorClient.getAll(); From 24dd5f19ef9e37a968e4ec5a51d0fc153dff6555 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 11 Feb 2026 12:51:11 +0100 Subject: [PATCH 11/69] update with extra validation --- .../domain/definitions}/entity.gen.ts | 0 .../domain/definitions}/entity.schema.yaml | 0 .../server/domain/entity_manager.ts | 139 ++++++++++++++++-- .../domain/errors/bad_crud_request_error.ts | 5 + .../server/domain/errors/index.ts | 1 + .../server/routes/apis/crud_upsert.ts | 16 +- .../server/routes/apis/crud_upsert_bulk.ts | 14 +- 7 files changed, 147 insertions(+), 28 deletions(-) rename x-pack/solutions/security/plugins/entity_store/{server/domain/schemas => common/domain/definitions}/entity.gen.ts (100%) rename x-pack/solutions/security/plugins/entity_store/{server/domain/schemas => common/domain/definitions}/entity.schema.yaml (100%) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.gen.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts similarity index 100% rename from x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.gen.ts rename to x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml similarity index 100% rename from x-pack/solutions/security/plugins/entity_store/server/domain/schemas/entity.schema.yaml rename to x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts index e4b0e7b3b4b66..3cbd962f72838 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -9,8 +9,21 @@ import type { Logger } from '@kbn/logging'; import type { Result } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { getLatestEntitiesIndexName } from './assets/latest_index'; -import { DocumentVersionConflictError, EntityNotFoundError } from './errors'; -import type { Entity } from './schemas/entity.gen'; +import { BadCRUDRequestError, DocumentVersionConflictError, EntityNotFoundError } from './errors'; +import type { Entity } from '../../common/domain/definitions/entity.gen'; +import { getFlattenedObject } from '@kbn/std'; +import { getEntityDefinition } from '@kbn/entity-store/common/domain/definitions/registry'; +import { EntityType } from '@kbn/entity-store/common'; +import { EntityField, ManagedEntityDefinition } from '@kbn/entity-store/common/domain/definitions/entity_schema'; +import { getEuidFromObject } from '@kbn/entity-store/common/domain/euid'; + +const PROTECTED_FIELDS = [ +'entity.hashedId', +'entity.id', +'entity.EngineMetadata.UntypedId', +'entity.EngineMetadata.Type', +] + interface EntityManagerDependencies { logger: Logger; @@ -29,24 +42,40 @@ export class EntityManager { this.namespace = deps.namespace; } - private getEntityId(document: Entity): string { - // TODO: NOT IMPLEMENTED - throw new Error(`getEntityId() not implemented`); - return 'TODO'; + private getEntityId(entityType: EntityType, document: Entity): string { + const id = getEuidFromObject(entityType, document); + if (id === undefined) { + throw new Error(`Could not get EUID for document`); + } + return id; } - // TODO: Bulk upsert + public async upsertEntity(document: Entity, force: boolean) { + const entityType = document.entity?.type; + if (entityType == null) { + throw new BadCRUDRequestError('', 'entity.type is required'); + } + const id = this.getEntityId(entityType as EntityType, document); + this.logger.info(`Upserting entity ID ${id}`); + + // check protected fields + const flat = getFlattenedObject(document); + const definition = getEntityDefinition(entityType as EntityType, this.namespace); + const fieldDescriptions = getFieldDescriptions(id, flat, definition); + assertNoEUIDFields(id, fieldDescriptions); + if (!force) { + assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); + } - public async upsertEntity(document: Entity) { - // From: x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_crud_client.ts - // TODO: normalizeToECS() - // TODO: getFlattenedObject() + // TODO(kuba): + // - hash ID (MD5) + const hashedId: string = 'TODO TODO TODO TODO'; + // - validate entity.id + // - validate entity.hashedId - const id = this.getEntityId(document); - this.logger.info(`Upserting entity ID ${id}`); const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), - id, + id: hashedId, doc: document, doc_as_upsert: true, }); @@ -67,8 +96,8 @@ export class EntityManager { } } - public async upsertEntitiesBulk(documents: Entity[]) { - await Promise.all(documents.map((document) => this.upsertEntity(document))); + public async upsertEntitiesBulk(documents: Entity[], force: boolean) { + await Promise.all(documents.map((document) => this.upsertEntity(document, force))); } public async deleteEntity(id: string) { @@ -93,3 +122,81 @@ export class EntityManager { } } } + +function getFieldDescriptions( + id: string, + flatProps: Record, + description: ManagedEntityDefinition +): Record { + const allFieldDescriptions = description.fields.reduce((obj, field) => { + obj[field.destination || field.source] = field; + return obj; + }, {} as Record); + + const invalid: string[] = []; + const descriptions: Record = {}; + + for (const [key, value] of Object.entries(flatProps)) { + if (description.identityField.requiresOneOfFields.includes(key)) { + // eslint-disable-next-line no-continue + continue; + } + + if (!allFieldDescriptions[key]) { + invalid.push(key); + } else { + descriptions[key] = { + ...allFieldDescriptions[key], + value, + }; + } + } + + // This will catch differences between + // API and entity store definition + if (invalid.length > 0) { + const invalidString = invalid.join(', '); + throw new BadCRUDRequestError( + id, + `The following attributes are not allowed to be updated: ${invalidString}` + ); + } + + return descriptions; +} + +function assertNoEUIDFields(id: string, fields: Record) { + const notAllowedProps = []; + for (const [name, _description] of Object.entries(fields)) { + if (PROTECTED_FIELDS.includes(name)) { + notAllowedProps.push(name); + } + } + + if (notAllowedProps.length > 0) { + const notAllowedPropsString = notAllowedProps.join(', '); + throw new BadCRUDRequestError( + id, + `The fields are not allowed to be updated: ${notAllowedPropsString}` + ); + } +} + +function assertOnlyNonForcedAttributesInReq(id: string, fields: Record) { + const notAllowedProps = []; + + for (const [name, description] of Object.entries(fields)) { + if (!description.allowAPIUpdate && !PROTECTED_FIELDS.includes(name)) { + notAllowedProps.push(name); + } + } + + if (notAllowedProps.length > 0) { + const notAllowedPropsString = notAllowedProps.join(', '); + throw new BadCRUDRequestError( + id, + `The following attributes are not allowed to be ` + + `updated without forcing it (?force=true): ${notAllowedPropsString}` + ); + } +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts new file mode 100644 index 0000000000000..081b891870016 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts @@ -0,0 +1,5 @@ +export class BadCRUDRequestError extends Error { + constructor(id: string, reason: string) { + super(`Error for Entity ID '${id}': ${reason}`); + } +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts index 1956988d00ed8..485936c56d5dc 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts @@ -8,3 +8,4 @@ export { EntityNotFoundError } from './entity_not_found'; export { DocumentVersionConflictError } from './document_version_conflict'; export { EntityStoreNotInstalledError } from './entity_store_not_installed'; +export { BadCRUDRequestError } from './bad_crud_request_error'; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index 1eeb32e448b99..112ca9df68fba 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -5,13 +5,18 @@ * 2.0. */ -import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { BooleanFromString, buildRouteValidationWithZod } from '@kbn/zod-helpers'; import type { IKibanaResponse } from '@kbn/core-http-server'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; -import { Entity } from '../../domain/schemas/entity.gen'; +import { Entity } from '../../../common/domain/definitions/entity.gen'; +import { z } from '@kbn/zod'; + +const querySchema = z.object({ + force: BooleanFromString.optional().default(false), +}); export function registerCRUDUpsert(router: EntityStorePluginRouter) { router.versioned @@ -29,6 +34,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { validate: { request: { body: buildRouteValidationWithZod(Entity), + query: buildRouteValidationWithZod(querySchema), }, }, }, @@ -40,11 +46,9 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { if (!(await assetManager.isInstalled())) { return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); } - // TODO: Check if skipping ?force flag does anything - // TODO: use req.body as entity to create - // TODO: ask guys do we REALLY need generated schemas + try { - await entityManager.upsertEntity(req.body); + await entityManager.upsertEntity(req.body, req.query.force); } catch (error) { logger.error(error); throw error; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index c9c1a849f9785..26e9439e8dac5 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -5,16 +5,19 @@ * 2.0. */ -import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { BooleanFromString, buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { z } from '@kbn/zod'; import type { IKibanaResponse } from '@kbn/core-http-server'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; -import { Entity } from '../../domain/schemas/entity.gen'; +import { Entity } from '../../../common/domain/definitions/entity.gen'; const bodySchema = z.array(Entity); +const querySchema = z.object({ + force: BooleanFromString.optional().default(false), +}); export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { router.versioned @@ -32,6 +35,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { validate: { request: { body: buildRouteValidationWithZod(bodySchema), + query: buildRouteValidationWithZod(querySchema), }, }, }, @@ -43,11 +47,9 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { if (!(await assetManager.isInstalled())) { return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); } - // TODO: Check if skipping ?force flag does anything - // TODO: use req.body as entity to create - // TODO: ask guys do we REALLY need generated schemas + try { - await entityManager.upsertEntitiesBulk(req.body); + await entityManager.upsertEntitiesBulk(req.body, req.query.force); } catch (error) { logger.error(error); throw error; From d05f012b6daf3d7087ba034e05a6d6ae84c61cd5 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 11 Feb 2026 13:06:49 +0100 Subject: [PATCH 12/69] drop EUID method --- .../server/domain/entity_manager.ts | 38 +++---------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts index 3cbd962f72838..de689c54f5619 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -17,14 +17,6 @@ import { EntityType } from '@kbn/entity-store/common'; import { EntityField, ManagedEntityDefinition } from '@kbn/entity-store/common/domain/definitions/entity_schema'; import { getEuidFromObject } from '@kbn/entity-store/common/domain/euid'; -const PROTECTED_FIELDS = [ -'entity.hashedId', -'entity.id', -'entity.EngineMetadata.UntypedId', -'entity.EngineMetadata.Type', -] - - interface EntityManagerDependencies { logger: Logger; esClient: ElasticsearchClient; @@ -45,12 +37,12 @@ export class EntityManager { private getEntityId(entityType: EntityType, document: Entity): string { const id = getEuidFromObject(entityType, document); if (id === undefined) { - throw new Error(`Could not get EUID for document`); + throw new BadCRUDRequestError('', `Could not derive entity EUID from document`); } return id; } - public async upsertEntity(document: Entity, force: boolean) { + public async upsertEntity(document: Entity, force: boolean): Promise { const entityType = document.entity?.type; if (entityType == null) { throw new BadCRUDRequestError('', 'entity.type is required'); @@ -59,11 +51,10 @@ export class EntityManager { this.logger.info(`Upserting entity ID ${id}`); // check protected fields - const flat = getFlattenedObject(document); - const definition = getEntityDefinition(entityType as EntityType, this.namespace); - const fieldDescriptions = getFieldDescriptions(id, flat, definition); - assertNoEUIDFields(id, fieldDescriptions); if (!force) { + const flat = getFlattenedObject(document); + const definition = getEntityDefinition(entityType as EntityType, this.namespace); + const fieldDescriptions = getFieldDescriptions(id, flat, definition); assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); } @@ -165,28 +156,11 @@ function getFieldDescriptions( return descriptions; } -function assertNoEUIDFields(id: string, fields: Record) { - const notAllowedProps = []; - for (const [name, _description] of Object.entries(fields)) { - if (PROTECTED_FIELDS.includes(name)) { - notAllowedProps.push(name); - } - } - - if (notAllowedProps.length > 0) { - const notAllowedPropsString = notAllowedProps.join(', '); - throw new BadCRUDRequestError( - id, - `The fields are not allowed to be updated: ${notAllowedPropsString}` - ); - } -} - function assertOnlyNonForcedAttributesInReq(id: string, fields: Record) { const notAllowedProps = []; for (const [name, description] of Object.entries(fields)) { - if (!description.allowAPIUpdate && !PROTECTED_FIELDS.includes(name)) { + if (!description.allowAPIUpdate) { notAllowedProps.push(name); } } From ed80d21dd55ce186c72ac8e1152b06ad54d3fdf3 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 11 Feb 2026 13:22:09 +0100 Subject: [PATCH 13/69] add hashed ids --- .../server/domain/entity_manager.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts index de689c54f5619..e276a6ab1d0ec 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts @@ -8,6 +8,7 @@ import type { Logger } from '@kbn/logging'; import type { Result } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; +import { createHash } from 'crypto'; import { getLatestEntitiesIndexName } from './assets/latest_index'; import { BadCRUDRequestError, DocumentVersionConflictError, EntityNotFoundError } from './errors'; import type { Entity } from '../../common/domain/definitions/entity.gen'; @@ -47,10 +48,17 @@ export class EntityManager { if (entityType == null) { throw new BadCRUDRequestError('', 'entity.type is required'); } - const id = this.getEntityId(entityType as EntityType, document); + const rawId = this.getEntityId(entityType as EntityType, document); + const id: string = createHash('md5').update(rawId).digest('hex'); this.logger.info(`Upserting entity ID ${id}`); - // check protected fields + if (!document.entity.id) { + document.entity.id = id; + } + if (document.entity.id !== id) { + throw new BadCRUDRequestError(id, `Entity ID ${document.entity.id} does not match calculated ID ${id}`); + } + if (!force) { const flat = getFlattenedObject(document); const definition = getEntityDefinition(entityType as EntityType, this.namespace); @@ -58,15 +66,9 @@ export class EntityManager { assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); } - // TODO(kuba): - // - hash ID (MD5) - const hashedId: string = 'TODO TODO TODO TODO'; - // - validate entity.id - // - validate entity.hashedId - const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), - id: hashedId, + id, doc: document, doc_as_upsert: true, }); From 32184f51e5e428038dc365a61b165bfd9125b3bd Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 11 Feb 2026 13:32:08 +0100 Subject: [PATCH 14/69] update schema --- .../common/domain/definitions/entity.gen.ts | 117 ++++++++++++ .../domain/definitions/entity.schema.yaml | 179 ++++++++++++++++-- 2 files changed, 278 insertions(+), 18 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts index 4687b2f1b1909..84c5edd605307 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts @@ -23,6 +23,11 @@ export const EngineMetadata = z }) .strict(); +export type EntityRiskLevels = z.infer; +export const EntityRiskLevels = z.enum(['Unknown', 'Low', 'Moderate', 'High', 'Critical']); +export type EntityRiskLevelsEnum = typeof EntityRiskLevels.enum; +export const EntityRiskLevelsEnum = EntityRiskLevels.enum; + export type EntityField = z.infer; export const EntityField = z .object({ @@ -72,6 +77,10 @@ export const EntityField = z .optional(), risk: z .object({ + /** + * Lexical description of the entity's risk. + */ + calculated_level: EntityRiskLevels.optional(), /** * The raw numeric value of the given entity's risk score. */ @@ -86,6 +95,19 @@ export const EntityField = z }) .strict(); +/** + * The criticality level of the asset. + */ +export type AssetCriticalityLevel = z.infer; +export const AssetCriticalityLevel = z.enum([ + 'low_impact', + 'medium_impact', + 'high_impact', + 'extreme_impact', +]); +export type AssetCriticalityLevelEnum = typeof AssetCriticalityLevel.enum; +export const AssetCriticalityLevelEnum = AssetCriticalityLevel.enum; + export type Asset = z.infer; export const Asset = z .object({ @@ -96,10 +118,102 @@ export const Asset = z model: z.string().optional(), vendor: z.string().optional(), environment: z.string().optional(), + criticality: AssetCriticalityLevel.optional(), business_unit: z.string().optional(), }) .strict(); +/** + * A generic representation of a document contributing to a Risk Score. + */ +export type RiskScoreInput = z.infer; +export const RiskScoreInput = z.object({ + /** + * The unique identifier (`_id`) of the original source document + */ + id: z.string(), + /** + * The unique index (`_index`) of the original source document + */ + index: z.string(), + /** + * The risk category of the risk input document. + */ + category: z.string(), + /** + * A human-readable description of the risk input document. + */ + description: z.string(), + /** + * The weighted risk score of the risk input document. + */ + risk_score: z.number().min(0).max(100).optional(), + /** + * The @timestamp of the risk input document. + */ + timestamp: z.string().optional(), + contribution_score: z.number().optional(), +}); + +export type EntityRiskScoreRecord = z.infer; +export const EntityRiskScoreRecord = z.object({ + /** + * The time at which the risk score was calculated. + */ + '@timestamp': z.string().datetime(), + /** + * The identifier field defining this risk score. Coupled with `id_value`, uniquely identifies the entity being scored. + */ + id_field: z.string(), + /** + * The identifier value defining this risk score. Coupled with `id_field`, uniquely identifies the entity being scored. + */ + id_value: z.string(), + /** + * Lexical description of the entity's risk. + */ + calculated_level: EntityRiskLevels, + /** + * The raw numeric value of the given entity's risk score. + */ + calculated_score: z.number(), + /** + * The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. + */ + calculated_score_norm: z.number().min(0).max(100), + /** + * The contribution of Category 1 to the overall risk score (`calculated_score`). Category 1 contains Detection Engine Alerts. + */ + category_1_score: z.number(), + /** + * The number of risk input documents that contributed to the Category 1 score (`category_1_score`). + */ + category_1_count: z.number().int(), + /** + * A list of the highest-risk documents contributing to this risk score. Useful for investigative purposes. + */ + inputs: z.array(RiskScoreInput), + category_2_score: z.number().optional(), + category_2_count: z.number().int().optional(), + notes: z.array(z.string()), + criticality_modifier: z.number().optional(), + criticality_level: AssetCriticalityLevel.optional(), + /** + * A list of modifiers that were applied to the risk score calculation. + */ + modifiers: z + .array( + z.object({ + type: z.string(), + subtype: z.string().optional(), + modifier_value: z.number().optional(), + contribution: z.number(), + metadata: z.object({}).catchall(z.unknown()).optional(), + }) + ) + .optional(), +}); + export type UserEntity = z.infer; export const UserEntity = z .object({ @@ -114,6 +228,7 @@ export const UserEntity = z id: z.array(z.string()).optional(), email: z.array(z.string()).optional(), hash: z.array(z.string()).optional(), + risk: EntityRiskScoreRecord.optional(), }) .strict() .optional(), @@ -142,6 +257,7 @@ export const HostEntity = z type: z.array(z.string()).optional(), mac: z.array(z.string()).optional(), architecture: z.array(z.string()).optional(), + risk: EntityRiskScoreRecord.optional(), entity: EntityField.optional(), }) .strict() @@ -164,6 +280,7 @@ export const ServiceEntity = z service: z .object({ name: z.string(), + risk: EntityRiskScoreRecord.optional(), entity: EntityField.optional(), }) .strict() diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml index 73ef7d7ab6515..bd6af8587f646 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml @@ -109,11 +109,10 @@ components: type: object additionalProperties: false properties: - ## TODO: KUBA I'll skip risk and asset criticality for now - # calculated_level: - # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskLevels' - # example: 'Critical' - # description: Lexical description of the entity's risk. + calculated_level: + $ref: '#/components/schemas/EntityRiskLevels' + example: 'Critical' + description: Lexical description of the entity's risk. calculated_score: type: number format: double @@ -143,9 +142,8 @@ components: type: string environment: type: string - ## TODO: KUBA I'll skip risk and asset criticality for now - # criticality: - # $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel' + criticality: + $ref: '#/components/schemas/AssetCriticalityLevel' business_unit: type: string @@ -190,10 +188,9 @@ components: type: array items: type: string - ## TODO: KUBA I'll skip risk and asset criticality for now - # risk: - # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' - # additionalProperties: false + risk: + $ref: '#/components/schemas/EntityRiskScoreRecord' + additionalProperties: false required: - name asset: @@ -252,9 +249,8 @@ components: type: array items: type: string - ## TODO: KUBA I'll skip risk and asset criticality for now - # risk: - # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + risk: + $ref: '#/components/schemas/EntityRiskScoreRecord' entity: $ref: '#/components/schemas/EntityField' required: @@ -287,9 +283,8 @@ components: properties: name: type: string - ## TODO: KUBA I'll skip risk and asset criticality for now# - # risk: - # $ref: '../../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord' + risk: + $ref: '#/components/schemas/EntityRiskScoreRecord' entity: $ref: '#/components/schemas/EntityField' required: @@ -338,3 +333,151 @@ components: - $ref: '#/components/schemas/ServiceEntity' - $ref: '#/components/schemas/GenericEntity' + # Temporary addition of risk score and asset criticality assets + AssetCriticalityLevel: + type: string + enum: + - low_impact + - medium_impact + - high_impact + - extreme_impact + description: The criticality level of the asset. + + EntityRiskLevels: + type: string + enum: + - "Unknown" + - "Low" + - "Moderate" + - "High" + - "Critical" + + RiskScoreInput: + description: A generic representation of a document contributing to a Risk Score. + type: object + required: + - id + - index + - description + - category + properties: + id: + type: string + example: 91a93376a507e86cfbf282166275b89f9dbdb1f0be6c8103c6ff2909ca8e1a1c + description: The unique identifier (`_id`) of the original source document + index: + type: string + example: .internal.alerts-security.alerts-default-000001 + description: The unique index (`_index`) of the original source document + category: + type: string + example: category_1 + description: The risk category of the risk input document. + description: + type: string + example: "Generated from Detection Engine Rule: Malware Prevention Alert" + description: A human-readable description of the risk input document. + risk_score: + type: number + format: double + minimum: 0 + maximum: 100 + description: The weighted risk score of the risk input document. + timestamp: + type: string + example: "2017-07-21T17:32:28Z" + description: The @timestamp of the risk input document. + contribution_score: + type: number + format: double + + EntityRiskScoreRecord: + type: object + required: + - "@timestamp" + - id_field + - id_value + - calculated_level + - calculated_score + - calculated_score_norm + - category_1_score + - category_1_count + - inputs + - notes + properties: + "@timestamp": + type: string + format: "date-time" + example: "2017-07-21T17:32:28Z" + description: The time at which the risk score was calculated. + id_field: + type: string + example: "host.name" + description: The identifier field defining this risk score. Coupled with `id_value`, uniquely identifies the entity being scored. + id_value: + type: string + example: "example.host" + description: The identifier value defining this risk score. Coupled with `id_field`, uniquely identifies the entity being scored. + calculated_level: + $ref: "#/components/schemas/EntityRiskLevels" + example: "Critical" + description: Lexical description of the entity's risk. + calculated_score: + type: number + format: double + description: The raw numeric value of the given entity's risk score. + calculated_score_norm: + type: number + format: double + minimum: 0 + maximum: 100 + description: The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. + category_1_score: + type: number + format: double + description: The contribution of Category 1 to the overall risk score (`calculated_score`). Category 1 contains Detection Engine Alerts. + category_1_count: + type: integer + description: The number of risk input documents that contributed to the Category 1 score (`category_1_score`). + inputs: + type: array + description: A list of the highest-risk documents contributing to this risk score. Useful for investigative purposes. + items: + $ref: "#/components/schemas/RiskScoreInput" + category_2_score: + type: number + format: double + category_2_count: + type: integer + notes: + type: array + items: + type: string + criticality_modifier: + type: number + format: double + criticality_level: + $ref: "#/components/schemas/AssetCriticalityLevel" + + modifiers: + type: array + description: A list of modifiers that were applied to the risk score calculation. + items: + type: object + properties: + type: + type: string + subtype: + type: string + modifier_value: + type: number + format: double + contribution: + type: number + format: double + metadata: + type: object + additionalProperties: true + required: + - type + - contribution \ No newline at end of file From cf8738293d609c684fe1a7eb70235c1523e15c5d Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 11 Feb 2026 16:34:12 +0100 Subject: [PATCH 15/69] rename to crud_client and reintroduce entityType --- .../{entity_manager.ts => crud_client.ts} | 21 ++++++++++--------- .../server/request_context_factory.ts | 4 ++-- .../server/routes/apis/crud_upsert.ts | 12 ++++++++--- .../server/routes/apis/crud_upsert_bulk.ts | 7 ++++++- .../plugins/entity_store/server/types.ts | 4 ++-- 5 files changed, 30 insertions(+), 18 deletions(-) rename x-pack/solutions/security/plugins/entity_store/server/domain/{entity_manager.ts => crud_client.ts} (90%) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts similarity index 90% rename from x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts rename to x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index e276a6ab1d0ec..7f8fe30f7a80a 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/entity_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -24,7 +24,12 @@ interface EntityManagerDependencies { namespace: string; } -export class EntityManager { +interface BulkObject { + type: EntityType, + document: Entity, +} + +export class CRUDClient { private readonly logger: Logger; private readonly esClient: ElasticsearchClient; private readonly namespace: string; @@ -43,12 +48,8 @@ export class EntityManager { return id; } - public async upsertEntity(document: Entity, force: boolean): Promise { - const entityType = document.entity?.type; - if (entityType == null) { - throw new BadCRUDRequestError('', 'entity.type is required'); - } - const rawId = this.getEntityId(entityType as EntityType, document); + public async upsertEntity(entityType: EntityType, document: Entity, force: boolean): Promise { + const rawId = this.getEntityId(entityType, document); const id: string = createHash('md5').update(rawId).digest('hex'); this.logger.info(`Upserting entity ID ${id}`); @@ -61,7 +62,7 @@ export class EntityManager { if (!force) { const flat = getFlattenedObject(document); - const definition = getEntityDefinition(entityType as EntityType, this.namespace); + const definition = getEntityDefinition(entityType, this.namespace); const fieldDescriptions = getFieldDescriptions(id, flat, definition); assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); } @@ -89,8 +90,8 @@ export class EntityManager { } } - public async upsertEntitiesBulk(documents: Entity[], force: boolean) { - await Promise.all(documents.map((document) => this.upsertEntity(document, force))); + public async upsertEntitiesBulk(objects: BulkObject[], force: boolean) { + await Promise.all(objects.map((obj) => this.upsertEntity(obj.type, obj.document, force))); } public async deleteEntity(id: string) { diff --git a/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts b/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts index 5cd1b1bc889c2..6688a06b66052 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts @@ -17,7 +17,7 @@ import { AssetManager } from './domain/asset_manager'; import { FeatureFlags } from './infra/feature_flags'; import { EngineDescriptorClient } from './domain/definitions/saved_objects'; import { LogsExtractionClient } from './domain/logs_extraction_client'; -import { EntityManager } from './domain/entity_manager'; +import { CRUDClient } from './domain/crud_client'; interface EntityStoreApiRequestHandlerContextDeps { coreSetup: CoreSetup; @@ -52,7 +52,7 @@ export async function createRequestHandlerContext({ logger ); - const entityManager = new EntityManager({ + const entityManager = new CRUDClient({ logger, esClient: core.elasticsearch.client.asCurrentUser, namespace, diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index 112ca9df68fba..f7f2caf7208c9 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -13,6 +13,11 @@ import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; import { Entity } from '../../../common/domain/definitions/entity.gen'; import { z } from '@kbn/zod'; +import { EntityType } from '@kbn/entity-store/common/domain/definitions/entity_schema'; + +const paramsSchema = z.object({ + entityType: EntityType, +}).required(); const querySchema = z.object({ force: BooleanFromString.optional().default(false), @@ -21,7 +26,7 @@ const querySchema = z.object({ export function registerCRUDUpsert(router: EntityStorePluginRouter) { router.versioned .put({ - path: '/api/entity-store/entities', + path: '/api/entity-store/entities/{entityType}', access: 'public', security: { authz: DEFAULT_ENTITY_STORE_PERMISSIONS, @@ -30,10 +35,11 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.public.v1, // TODO: KUBA: SHould it be public? Really? + version: API_VERSIONS.public.v1, // TODO(kuba): public or internal? validate: { request: { body: buildRouteValidationWithZod(Entity), + params: buildRouteValidationWithZod(paramsSchema), query: buildRouteValidationWithZod(querySchema), }, }, @@ -48,7 +54,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { } try { - await entityManager.upsertEntity(req.body, req.query.force); + await entityManager.upsertEntity(req.params.entityType, req.body, req.query.force); } catch (error) { logger.error(error); throw error; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index 26e9439e8dac5..11eb394b42190 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -13,8 +13,13 @@ import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; import { Entity } from '../../../common/domain/definitions/entity.gen'; +import { EntityType } from '@kbn/entity-store/common/domain/definitions/entity_schema'; + +const bodySchema = z.array(z.object({ + type: EntityType, + document: Entity +})); -const bodySchema = z.array(Entity); const querySchema = z.object({ force: BooleanFromString.optional().default(false), }); diff --git a/x-pack/solutions/security/plugins/entity_store/server/types.ts b/x-pack/solutions/security/plugins/entity_store/server/types.ts index f1c2fa238301e..38c30c4be34cc 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/types.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/types.ts @@ -26,7 +26,7 @@ import type { CoreSetup } from '@kbn/core/server'; import type { AssetManager } from './domain/asset_manager'; import type { FeatureFlags } from './infra/feature_flags'; import type { LogsExtractionClient } from './domain/logs_extraction_client'; -import type { EntityManager } from './domain/entity_manager'; +import type { CRUDClient } from './domain/crud_client'; export interface EntityStoreSetupPlugins { taskManager: TaskManagerSetupContract; @@ -46,7 +46,7 @@ export interface EntityStoreApiRequestHandlerContext { core: CoreRequestHandlerContext; logger: Logger; assetManager: AssetManager; - entityManager: EntityManager; + entityManager: CRUDClient; featureFlags: FeatureFlags; logsExtractionClient: LogsExtractionClient; } From ed7a859ae21449d410a83b6bb9821e00f5647bfd Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 13 Feb 2026 14:40:47 +0100 Subject: [PATCH 16/69] update upsert & delete behavior --- .../entity_store/server/domain/crud_client.ts | 145 ++++++++++++------ .../server/request_context_factory.ts | 6 +- .../server/routes/apis/crud_delete.ts | 18 +-- .../server/routes/apis/crud_upsert.ts | 22 ++- .../server/routes/apis/crud_upsert_bulk.ts | 16 +- .../plugins/entity_store/server/types.ts | 2 +- 6 files changed, 129 insertions(+), 80 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 7f8fe30f7a80a..33f8bd068f8ed 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -9,14 +9,18 @@ import type { Logger } from '@kbn/logging'; import type { Result } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { createHash } from 'crypto'; +import { unset } from 'lodash'; +import { getFlattenedObject } from '@kbn/std'; import { getLatestEntitiesIndexName } from './assets/latest_index'; -import { BadCRUDRequestError, DocumentVersionConflictError, EntityNotFoundError } from './errors'; +import { BadCRUDRequestError, EntityNotFoundError } from './errors'; import type { Entity } from '../../common/domain/definitions/entity.gen'; -import { getFlattenedObject } from '@kbn/std'; -import { getEntityDefinition } from '@kbn/entity-store/common/domain/definitions/registry'; -import { EntityType } from '@kbn/entity-store/common'; -import { EntityField, ManagedEntityDefinition } from '@kbn/entity-store/common/domain/definitions/entity_schema'; -import { getEuidFromObject } from '@kbn/entity-store/common/domain/euid'; +import { getEntityDefinition } from '../../common/domain/definitions/registry'; +import type { EntityType } from '../../common'; +import type { + EntityField, + ManagedEntityDefinition, +} from '../../common/domain/definitions/entity_schema'; +import { getEuidFromObject } from '../../common/domain/euid'; interface EntityManagerDependencies { logger: Logger; @@ -25,8 +29,8 @@ interface EntityManagerDependencies { } interface BulkObject { - type: EntityType, - document: Entity, + type: EntityType; + document: Entity; } export class CRUDClient { @@ -40,47 +44,63 @@ export class CRUDClient { this.namespace = deps.namespace; } - private getEntityId(entityType: EntityType, document: Entity): string { - const id = getEuidFromObject(entityType, document); - if (id === undefined) { + public async upsertEntity( + entityType: EntityType, + document: Entity, + force: boolean + ): Promise { + const rawId = getEuidFromObject(entityType, document); + if (rawId === undefined) { throw new BadCRUDRequestError('', `Could not derive entity EUID from document`); } - return id; - } - - public async upsertEntity(entityType: EntityType, document: Entity, force: boolean): Promise { - const rawId = this.getEntityId(entityType, document); + // EUID generation uses MD5. It is not a security-related feature. + // eslint-disable-next-line @kbn/eslint/no_unsafe_hash const id: string = createHash('md5').update(rawId).digest('hex'); this.logger.info(`Upserting entity ID ${id}`); - if (!document.entity.id) { document.entity.id = id; } - if (document.entity.id !== id) { - throw new BadCRUDRequestError(id, `Entity ID ${document.entity.id} does not match calculated ID ${id}`); - } + // TODO: + // - reimplement ID logic to follow Romulo discussion + // - extract checks and ID fixes to a single function (for bulk as well) + // - update bulk to actually use esClient.bulk() and create all docs in updates index + // - add buildDocumentToUpdate() to format the docs + + const definition = getEntityDefinition(entityType, this.namespace); if (!force) { const flat = getFlattenedObject(document); - const definition = getEntityDefinition(entityType, this.namespace); const fieldDescriptions = getFieldDescriptions(id, flat, definition); assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); } + prepareDocumentForUpsert(entityType, document); + + try { + await this.esClient.create({ + index: getLatestEntitiesIndexName(this.namespace), + id, + document, + refresh: 'wait_for', + }); + this.logger.info(`Created entity ID ${id}`); + return; + } catch (error) { + if (error.statusCode !== 409) { + throw error; + } + } + removeEUIDFields(definition, document); const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), id, doc: document, - doc_as_upsert: true, }); switch (result as Result) { case 'deleted': case 'not_found': throw new Error(`Could not upsert entity ID ${id}`); - case 'created': - this.logger.info(`Entity ID ${id} created`); - break; case 'updated': this.logger.info(`Entity ID ${id} updated`); break; @@ -90,30 +110,23 @@ export class CRUDClient { } } - public async upsertEntitiesBulk(objects: BulkObject[], force: boolean) { + public async upsertEntitiesBulk(objects: BulkObject[], force: boolean) { await Promise.all(objects.map((obj) => this.upsertEntity(obj.type, obj.document, force))); } public async deleteEntity(id: string) { - const resp = await this.esClient.deleteByQuery({ - index: getLatestEntitiesIndexName(this.namespace), - query: { - term: { - 'entity.id': id, - }, - }, - conflicts: 'proceed', - }); - - if (resp.failures !== undefined && resp.failures.length > 0) { - throw new Error(`Failed to delete entity ID ${id}`); - } - if (resp.version_conflicts) { - throw new DocumentVersionConflictError(); - } - if (!resp.deleted) { - throw new EntityNotFoundError(id); + try { + await this.esClient.delete({ + index: getLatestEntitiesIndexName(this.namespace), + id, + }); + } catch (error) { + if (error.statusCode === 404) { + throw new EntityNotFoundError(id); + } + throw error; } + return { deleted: true }; } } @@ -132,7 +145,6 @@ function getFieldDescriptions( for (const [key, value] of Object.entries(flatProps)) { if (description.identityField.requiresOneOfFields.includes(key)) { - // eslint-disable-next-line no-continue continue; } @@ -171,9 +183,48 @@ function assertOnlyNonForcedAttributesInReq(id: string, fields: Record 0) { const notAllowedPropsString = notAllowedProps.join(', '); throw new BadCRUDRequestError( - id, + id, `The following attributes are not allowed to be ` + `updated without forcing it (?force=true): ${notAllowedPropsString}` ); } -} \ No newline at end of file +} + +function prepareDocumentForUpsert(type: EntityType, data: Partial) { + const now = new Date().toISOString(); + + if (type === 'generic') { + return { + '@timestamp': now, + ...data, + }; + } + + // Get host, user, service field + const typeData = (data[type as keyof typeof data] || {}) as Record; + + // Force name to be picked by the store + typeData.name = data.entity?.id; + // Nest entity under type data + typeData.entity = data.entity; + + const doc: Record = { + '@timestamp': now, + ...data, + }; + + // Remove entity from root + delete doc.entity; + + // override the host, user service + // field with the built value + doc[type as keyof typeof doc] = typeData; + + return doc; +} + +function removeEUIDFields(definition: ManagedEntityDefinition, document: Entity) { + for (const euidField of definition.identityField.requiresOneOfFields) { + unset(document, euidField); + } +} diff --git a/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts b/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts index 6688a06b66052..f5c2c2aaebcc0 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/request_context_factory.ts @@ -52,12 +52,12 @@ export async function createRequestHandlerContext({ logger ); - const entityManager = new CRUDClient({ + const crudClient = new CRUDClient({ logger, esClient: core.elasticsearch.client.asCurrentUser, namespace, }); - + const logsExtractionClient = new LogsExtractionClient( logger, namespace, @@ -78,7 +78,7 @@ export async function createRequestHandlerContext({ isServerless, logsExtractionClient, }), - entityManager, + crudClient, featureFlags: new FeatureFlags(core.uiSettings.client), logsExtractionClient, }; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts index 4ea83593f9493..3908032cedd3b 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts @@ -11,11 +11,7 @@ import type { IKibanaResponse } from '@kbn/core-http-server'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; -import { - DocumentVersionConflictError, - EntityNotFoundError, - EntityStoreNotInstalledError, -} from '../../domain/errors'; +import { EntityNotFoundError, EntityStoreNotInstalledError } from '../../domain/errors'; const paramsSchema = z.object({ id: z.string(), @@ -42,7 +38,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { }, wrapMiddlewares(async (ctx, req, res): Promise => { const entityStoreCtx = await ctx.entityStore; - const { logger, assetManager, entityManager } = entityStoreCtx; + const { logger, assetManager, crudClient } = entityStoreCtx; logger.debug('CRUD Delete api called'); if (!(await assetManager.isInstalled())) { @@ -50,14 +46,8 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { } try { - await entityManager.deleteEntity(req.params.id); + await crudClient.deleteEntity(req.params.id); } catch (error) { - if (error instanceof DocumentVersionConflictError) { - return res.customError({ - statusCode: 409, - body: error as DocumentVersionConflictError, - }); - } if (error instanceof EntityNotFoundError) { return res.customError({ statusCode: 404, @@ -68,7 +58,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { throw error; } - return res.ok(); + return res.ok({ body: { deleted: true } }); }) ); } diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index f7f2caf7208c9..3760a614f8151 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -7,17 +7,19 @@ import { BooleanFromString, buildRouteValidationWithZod } from '@kbn/zod-helpers'; import type { IKibanaResponse } from '@kbn/core-http-server'; +import { z } from '@kbn/zod'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; -import { EntityStoreNotInstalledError } from '../../domain/errors'; +import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../domain/errors'; import { Entity } from '../../../common/domain/definitions/entity.gen'; -import { z } from '@kbn/zod'; -import { EntityType } from '@kbn/entity-store/common/domain/definitions/entity_schema'; +import { EntityType } from '../../../common/domain/definitions/entity_schema'; -const paramsSchema = z.object({ - entityType: EntityType, -}).required(); +const paramsSchema = z + .object({ + entityType: EntityType, + }) + .required(); const querySchema = z.object({ force: BooleanFromString.optional().default(false), @@ -46,7 +48,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { }, wrapMiddlewares(async (ctx, req, res): Promise => { const entityStoreCtx = await ctx.entityStore; - const { logger, assetManager, entityManager } = entityStoreCtx; + const { logger, assetManager, crudClient } = entityStoreCtx; logger.debug('CRUD Upsert api called'); if (!(await assetManager.isInstalled())) { @@ -54,8 +56,12 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { } try { - await entityManager.upsertEntity(req.params.entityType, req.body, req.query.force); + await crudClient.upsertEntity(req.params.entityType, req.body, req.query.force); } catch (error) { + if (error instanceof BadCRUDRequestError) { + return res.badRequest({ body: error as BadCRUDRequestError }); + } + logger.error(error); throw error; } diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index 11eb394b42190..614cc921d682a 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -8,17 +8,19 @@ import { BooleanFromString, buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { z } from '@kbn/zod'; import type { IKibanaResponse } from '@kbn/core-http-server'; +import { EntityType } from '../../../common/domain/definitions/entity_schema'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; import { EntityStoreNotInstalledError } from '../../domain/errors'; import { Entity } from '../../../common/domain/definitions/entity.gen'; -import { EntityType } from '@kbn/entity-store/common/domain/definitions/entity_schema'; -const bodySchema = z.array(z.object({ - type: EntityType, - document: Entity -})); +const bodySchema = z.array( + z.object({ + type: EntityType, + document: Entity, + }) +); const querySchema = z.object({ force: BooleanFromString.optional().default(false), @@ -46,7 +48,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { }, wrapMiddlewares(async (ctx, req, res): Promise => { const entityStoreCtx = await ctx.entityStore; - const { logger, assetManager, entityManager } = entityStoreCtx; + const { logger, assetManager, crudClient } = entityStoreCtx; logger.debug('CRUD Upsert Bulk api called'); if (!(await assetManager.isInstalled())) { @@ -54,7 +56,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { } try { - await entityManager.upsertEntitiesBulk(req.body, req.query.force); + await crudClient.upsertEntitiesBulk(req.body, req.query.force); } catch (error) { logger.error(error); throw error; diff --git a/x-pack/solutions/security/plugins/entity_store/server/types.ts b/x-pack/solutions/security/plugins/entity_store/server/types.ts index 38c30c4be34cc..ce688d526064c 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/types.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/types.ts @@ -46,7 +46,7 @@ export interface EntityStoreApiRequestHandlerContext { core: CoreRequestHandlerContext; logger: Logger; assetManager: AssetManager; - entityManager: CRUDClient; + crudClient: CRUDClient; featureFlags: FeatureFlags; logsExtractionClient: LogsExtractionClient; } From e1c83e4646b238dc6804fd9b5f2b1d7184afbc8b Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 13 Feb 2026 15:04:23 +0100 Subject: [PATCH 17/69] handle bulkUpsert entities --- .../entity_store/server/domain/crud_client.ts | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 33f8bd068f8ed..3a48294296e8b 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -6,7 +6,11 @@ */ import type { Logger } from '@kbn/logging'; -import type { Result } from '@elastic/elasticsearch/lib/api/types'; +import type { + BulkOperationContainer, + BulkUpdateAction, + Result, +} from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { createHash } from 'crypto'; import { unset } from 'lodash'; @@ -21,6 +25,7 @@ import type { ManagedEntityDefinition, } from '../../common/domain/definitions/entity_schema'; import { getEuidFromObject } from '../../common/domain/euid'; +import { getUpdatesEntitiesDataStreamName } from './assets/updates_data_stream'; interface EntityManagerDependencies { logger: Logger; @@ -61,25 +66,19 @@ export class CRUDClient { document.entity.id = id; } - // TODO: - // - reimplement ID logic to follow Romulo discussion - // - extract checks and ID fixes to a single function (for bulk as well) - // - update bulk to actually use esClient.bulk() and create all docs in updates index - // - add buildDocumentToUpdate() to format the docs - const definition = getEntityDefinition(entityType, this.namespace); if (!force) { const flat = getFlattenedObject(document); const fieldDescriptions = getFieldDescriptions(id, flat, definition); assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); } - prepareDocumentForUpsert(entityType, document); + const preparedDoc = prepareDocumentForUpsert(entityType, document); try { await this.esClient.create({ index: getLatestEntitiesIndexName(this.namespace), id, - document, + document: preparedDoc, refresh: 'wait_for', }); this.logger.info(`Created entity ID ${id}`); @@ -90,11 +89,11 @@ export class CRUDClient { } } - removeEUIDFields(definition, document); + removeEUIDFields(definition, preparedDoc); const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), id, - doc: document, + doc: preparedDoc, }); switch (result as Result) { @@ -111,7 +110,25 @@ export class CRUDClient { } public async upsertEntitiesBulk(objects: BulkObject[], force: boolean) { - await Promise.all(objects.map((obj) => this.upsertEntity(obj.type, obj.document, force))); + const operations: (BulkOperationContainer | BulkUpdateAction)[] = []; + + for (const { type: entityType, document } of objects) { + const definition = getEntityDefinition(entityType, this.namespace); + if (!force) { + const flat = getFlattenedObject(document); + // TODO: make these two throw regular Errors and try-catch this block as BadCRUDRequest. + const fieldDescriptions = getFieldDescriptions('', flat, definition); + assertOnlyNonForcedAttributesInReq('', fieldDescriptions); + } + const preparedDoc = prepareDocumentForUpsert(entityType, document); + + operations.push({ create: {} }, preparedDoc); + } + + await this.esClient.bulk({ + index: getUpdatesEntitiesDataStreamName(this.namespace), + operations, + }); } public async deleteEntity(id: string) { @@ -164,7 +181,7 @@ function getFieldDescriptions( const invalidString = invalid.join(', '); throw new BadCRUDRequestError( id, - `The following attributes are not allowed to be updated: ${invalidString}` + `The following attributes are not allowed to be updated: [${invalidString}]` ); } @@ -190,7 +207,10 @@ function assertOnlyNonForcedAttributesInReq(id: string, fields: Record) { +function prepareDocumentForUpsert( + type: EntityType, + data: Partial +): Record { const now = new Date().toISOString(); if (type === 'generic') { @@ -223,7 +243,7 @@ function prepareDocumentForUpsert(type: EntityType, data: Partial) { return doc; } -function removeEUIDFields(definition: ManagedEntityDefinition, document: Entity) { +function removeEUIDFields(definition: ManagedEntityDefinition, document: Record) { for (const euidField of definition.identityField.requiresOneOfFields) { unset(document, euidField); } From da763476b93cbe65e30db55ea41772152be15ca8 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 13 Feb 2026 15:13:12 +0100 Subject: [PATCH 18/69] handle request errors better --- .../entity_store/server/domain/crud_client.ts | 16 ++++++---------- .../domain/errors/bad_crud_request_error.ts | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 3a48294296e8b..7be8f8c2ae9a3 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -56,7 +56,7 @@ export class CRUDClient { ): Promise { const rawId = getEuidFromObject(entityType, document); if (rawId === undefined) { - throw new BadCRUDRequestError('', `Could not derive entity EUID from document`); + throw new BadCRUDRequestError(`Could not derive entity EUID from document`); } // EUID generation uses MD5. It is not a security-related feature. // eslint-disable-next-line @kbn/eslint/no_unsafe_hash @@ -69,8 +69,8 @@ export class CRUDClient { const definition = getEntityDefinition(entityType, this.namespace); if (!force) { const flat = getFlattenedObject(document); - const fieldDescriptions = getFieldDescriptions(id, flat, definition); - assertOnlyNonForcedAttributesInReq(id, fieldDescriptions); + const fieldDescriptions = getFieldDescriptions(flat, definition); + assertOnlyNonForcedAttributesInReq(fieldDescriptions); } const preparedDoc = prepareDocumentForUpsert(entityType, document); @@ -116,9 +116,8 @@ export class CRUDClient { const definition = getEntityDefinition(entityType, this.namespace); if (!force) { const flat = getFlattenedObject(document); - // TODO: make these two throw regular Errors and try-catch this block as BadCRUDRequest. - const fieldDescriptions = getFieldDescriptions('', flat, definition); - assertOnlyNonForcedAttributesInReq('', fieldDescriptions); + const fieldDescriptions = getFieldDescriptions(flat, definition); + assertOnlyNonForcedAttributesInReq(fieldDescriptions); } const preparedDoc = prepareDocumentForUpsert(entityType, document); @@ -148,7 +147,6 @@ export class CRUDClient { } function getFieldDescriptions( - id: string, flatProps: Record, description: ManagedEntityDefinition ): Record { @@ -180,7 +178,6 @@ function getFieldDescriptions( if (invalid.length > 0) { const invalidString = invalid.join(', '); throw new BadCRUDRequestError( - id, `The following attributes are not allowed to be updated: [${invalidString}]` ); } @@ -188,7 +185,7 @@ function getFieldDescriptions( return descriptions; } -function assertOnlyNonForcedAttributesInReq(id: string, fields: Record) { +function assertOnlyNonForcedAttributesInReq(fields: Record) { const notAllowedProps = []; for (const [name, description] of Object.entries(fields)) { @@ -200,7 +197,6 @@ function assertOnlyNonForcedAttributesInReq(id: string, fields: Record 0) { const notAllowedPropsString = notAllowedProps.join(', '); throw new BadCRUDRequestError( - id, `The following attributes are not allowed to be ` + `updated without forcing it (?force=true): ${notAllowedPropsString}` ); diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts index 081b891870016..0219fe87c0e01 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/bad_crud_request_error.ts @@ -1,5 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + export class BadCRUDRequestError extends Error { - constructor(id: string, reason: string) { - super(`Error for Entity ID '${id}': ${reason}`); + constructor(reason: string, id?: string) { + if (id) { + super(`Bad request for Entity ID ${id}: ${reason}`); + } else { + super(`Bad request: ${reason}`); + } } -} \ No newline at end of file +} From d34f03ee6972931ab852b0702bada92ea5120d80 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 13 Feb 2026 15:35:16 +0100 Subject: [PATCH 19/69] add missing entity id field clause --- .../domain/definitions/common_fields.ts | 2 ++ .../entity_store/server/domain/crud_client.ts | 3 ++- .../server/routes/apis/crud_delete.ts | 1 + .../server/routes/apis/crud_upsert_bulk.ts | 22 ++++++++++++------- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts index e02361c558445..07456ca68c264 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts @@ -8,6 +8,8 @@ import type { EntityType, EntityField } from './entity_schema'; import { oldestValue, newestValue } from './field_retention_operations'; +export const ENTITY_ID_FIELD = 'entity.id'; + // Copied from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts export const getCommonFieldDescriptions = ( diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 7be8f8c2ae9a3..22250f30bd7eb 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -15,6 +15,7 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { createHash } from 'crypto'; import { unset } from 'lodash'; import { getFlattenedObject } from '@kbn/std'; +import { ENTITY_ID_FIELD } from '../../common/domain/definitions/common_fields'; import { getLatestEntitiesIndexName } from './assets/latest_index'; import { BadCRUDRequestError, EntityNotFoundError } from './errors'; import type { Entity } from '../../common/domain/definitions/entity.gen'; @@ -159,7 +160,7 @@ function getFieldDescriptions( const descriptions: Record = {}; for (const [key, value] of Object.entries(flatProps)) { - if (description.identityField.requiresOneOfFields.includes(key)) { + if (key === ENTITY_ID_FIELD || description.identityField.requiresOneOfFields.includes(key)) { continue; } diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts index 3908032cedd3b..0a8cd3c6509cf 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts @@ -54,6 +54,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { body: error as EntityNotFoundError, }); } + logger.error(error); throw error; } diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index 614cc921d682a..2c9c453e11746 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -12,15 +12,17 @@ import { EntityType } from '../../../common/domain/definitions/entity_schema'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; import type { EntityStorePluginRouter } from '../../types'; import { wrapMiddlewares } from '../middleware'; -import { EntityStoreNotInstalledError } from '../../domain/errors'; +import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../domain/errors'; import { Entity } from '../../../common/domain/definitions/entity.gen'; -const bodySchema = z.array( - z.object({ - type: EntityType, - document: Entity, - }) -); +const bodySchema = z.object({ + entities: z.array( + z.object({ + type: EntityType, + document: Entity, + }) + ), +}); const querySchema = z.object({ force: BooleanFromString.optional().default(false), @@ -56,8 +58,12 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { } try { - await crudClient.upsertEntitiesBulk(req.body, req.query.force); + await crudClient.upsertEntitiesBulk(req.body.entities, req.query.force); } catch (error) { + if (error instanceof BadCRUDRequestError) { + return res.badRequest({ body: error as BadCRUDRequestError }); + } + logger.error(error); throw error; } From 82bb6af78626e22e741e54b7e5d058c4d4c1dc7f Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 10:01:35 +0100 Subject: [PATCH 20/69] start working on Scout tests --- .../test/scout/api/fixtures/constants.ts | 3 + .../test/scout/api/tests/crud_api.spec.ts | 88 +++++++++++++++++++ .../test/scout/api/tests/install.spec.ts | 2 +- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts index 558f6ac81d7da..f95528da869fa 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts @@ -24,8 +24,11 @@ export const ENTITY_STORE_ROUTES = { UNINSTALL: 'internal/security/entity-store/uninstall', FORCE_LOG_EXTRACTION: (entityType: string) => `internal/security/entity-store/${entityType}/force-log-extraction`, + CRUD_UPSERT: (entityType: string) => + `api/entity-store/entities/${entityType}`, } as const; export const ENTITY_STORE_TAGS = [...tags.stateful.classic, ...tags.serverless.security.complete]; +export const LATEST_INDEX = '.entities.v2.latest.security_default'; export const UPDATES_INDEX = '.entities.v2.updates.security_default'; diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts new file mode 100644 index 0000000000000..8cc2b1b4d97c5 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { apiTest } from '@kbn/scout-security'; +import { expect } from '@kbn/scout-security/api'; +import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS, LATEST_INDEX } from '../fixtures/constants'; +import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; +import { Entity } from '@kbn/entity-store/common/domain/definitions/entity.gen'; +import {API_VERSIONS} from '@kbn/entity-store/server/routes/constants'; + +const genericEntity: Entity = { + entity: { + id: 'required-id', + // type: 'generic' + } +} + + +apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { + let defaultHeaders: Record; + let crudHeaders: Record; + + apiTest.beforeAll(async ({ apiClient, kbnClient, samlAuth }) => { + const credentials = await samlAuth.asInteractiveUser('admin'); + defaultHeaders = { + ...credentials.cookieHeader, + ...COMMON_HEADERS, + }; + crudHeaders = { + ...defaultHeaders, + 'elastic-api-version': API_VERSIONS.public.v1, + }; + + // enable feature flag + await kbnClient.uiSettings.update({ + [FF_ENABLE_ENTITY_STORE_V2]: true, + }); + + // Install the entity store + const response = await apiClient.post(ENTITY_STORE_ROUTES.INSTALL, { + headers: defaultHeaders, + responseType: 'json', + body: {}, + }); + expect(response.statusCode).toBe(200); + }); + + apiTest.afterAll(async ({ apiClient }) => { + const response = await apiClient.post(ENTITY_STORE_ROUTES.UNINSTALL, { + headers: defaultHeaders, + responseType: 'json', + body: {}, + }); + expect(response.statusCode).toBe(200); + }); + + apiTest( + 'KUBA Should create an entity', + async ({ apiClient }) => { + + const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { + headers: crudHeaders, + responseType: 'json', + body: genericEntity, + }); + expect(create.body).toStrictEqual({ ok: true }); + expect(create.statusCode).toBe(200); + + const search = await apiClient.post(LATEST_INDEX + '/_search', { + headers: crudHeaders, + responseType: 'json', + body: { + query: { + term: { + 'entity.id': genericEntity.entity.id, + } + } + }, + }) + expect(search.statusCode).toBe(200); + expect(search.body).toBe({}); + } + ); +}); diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts index 3515a7511e332..f2020b1c15796 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts @@ -14,7 +14,7 @@ apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { let defaultHeaders: Record; apiTest.beforeAll(async ({ samlAuth }) => { - const credentials = await samlAuth.asInteractiveUser('admin'); + const credentials = await samlAuth.asInteract.iveUser('admin'); const cookieHeader = credentials.cookieHeader; defaultHeaders = { ...cookieHeader, From 2609597b1ac8c684a765ffbeab681c8b8d3552af Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 10:45:09 +0100 Subject: [PATCH 21/69] add first scout test - working --- .../test/scout/api/tests/crud_api.spec.ts | 67 +++++++++---------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 8cc2b1b4d97c5..044f82dd7178d 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -7,18 +7,21 @@ import { apiTest } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/api'; -import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS, LATEST_INDEX } from '../fixtures/constants'; +import type { Entity } from '../../../../common/domain/definitions/entity.gen'; +import { + COMMON_HEADERS, + ENTITY_STORE_ROUTES, + ENTITY_STORE_TAGS, + LATEST_INDEX, +} from '../fixtures/constants'; import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; -import { Entity } from '@kbn/entity-store/common/domain/definitions/entity.gen'; -import {API_VERSIONS} from '@kbn/entity-store/server/routes/constants'; +import { API_VERSIONS } from '../../../../server/routes/constants'; const genericEntity: Entity = { - entity: { - id: 'required-id', - // type: 'generic' - } -} - + entity: { + id: 'required-id', + }, +}; apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { let defaultHeaders: Record; @@ -31,8 +34,8 @@ apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { ...COMMON_HEADERS, }; crudHeaders = { - ...defaultHeaders, - 'elastic-api-version': API_VERSIONS.public.v1, + ...defaultHeaders, + 'elastic-api-version': API_VERSIONS.public.v1, }; // enable feature flag @@ -58,31 +61,23 @@ apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { expect(response.statusCode).toBe(200); }); - apiTest( - 'KUBA Should create an entity', - async ({ apiClient }) => { - - const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { - headers: crudHeaders, - responseType: 'json', - body: genericEntity, - }); - expect(create.body).toStrictEqual({ ok: true }); - expect(create.statusCode).toBe(200); + apiTest('Should create an entity', async ({ apiClient, esClient }) => { + const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { + headers: crudHeaders, + responseType: 'json', + body: genericEntity, + }); + expect(create.statusCode).toBe(200); + expect(create.body).toStrictEqual({ ok: true }); - const search = await apiClient.post(LATEST_INDEX + '/_search', { - headers: crudHeaders, - responseType: 'json', - body: { - query: { - term: { - 'entity.id': genericEntity.entity.id, - } - } + const entities = await esClient.search({ + index: LATEST_INDEX, + query: { + term: { + 'entity.id': genericEntity.entity.id, }, - }) - expect(search.statusCode).toBe(200); - expect(search.body).toBe({}); - } - ); + }, + }); + expect(entities.hits.hits).toHaveLength(1); + }); }); From 0728382da7c834f8cfcb26f3638c4453ded1cb8c Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 11:34:22 +0100 Subject: [PATCH 22/69] update tests to reflect new routes --- .../entity_store/test/scout/api/fixtures/constants.ts | 2 +- .../entity_store/test/scout/api/tests/crud_api.spec.ts | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts index f95528da869fa..814bde5d0b012 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts @@ -25,7 +25,7 @@ export const ENTITY_STORE_ROUTES = { FORCE_LOG_EXTRACTION: (entityType: string) => `internal/security/entity-store/${entityType}/force-log-extraction`, CRUD_UPSERT: (entityType: string) => - `api/entity-store/entities/${entityType}`, + `internal/security/entity-store/entities/${entityType}`, } as const; export const ENTITY_STORE_TAGS = [...tags.stateful.classic, ...tags.serverless.security.complete]; diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 044f82dd7178d..bc86ced08504d 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -23,9 +23,8 @@ const genericEntity: Entity = { }, }; -apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { +apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () => { let defaultHeaders: Record; - let crudHeaders: Record; apiTest.beforeAll(async ({ apiClient, kbnClient, samlAuth }) => { const credentials = await samlAuth.asInteractiveUser('admin'); @@ -33,10 +32,6 @@ apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { ...credentials.cookieHeader, ...COMMON_HEADERS, }; - crudHeaders = { - ...defaultHeaders, - 'elastic-api-version': API_VERSIONS.public.v1, - }; // enable feature flag await kbnClient.uiSettings.update({ @@ -63,7 +58,7 @@ apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { apiTest('Should create an entity', async ({ apiClient, esClient }) => { const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { - headers: crudHeaders, + headers: defaultHeaders, responseType: 'json', body: genericEntity, }); From a0675ed38210800b9a288ef6371058821ff24401 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 11:34:32 +0100 Subject: [PATCH 23/69] change v2 api to internal --- .../plugins/entity_store/server/routes/apis/crud_delete.ts | 6 +++--- .../plugins/entity_store/server/routes/apis/crud_upsert.ts | 6 +++--- .../entity_store/server/routes/apis/crud_upsert_bulk.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts index 0a8cd3c6509cf..02814d9e1824a 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts @@ -20,8 +20,8 @@ const paramsSchema = z.object({ export function registerCRUDDelete(router: EntityStorePluginRouter) { router.versioned .delete({ - path: '/api/entity-store/entities/{id}', - access: 'public', + path: '/internal/security/entity-store/entities/{id}', + access: 'internal', security: { authz: DEFAULT_ENTITY_STORE_PERMISSIONS, }, @@ -29,7 +29,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.public.v1, + version: API_VERSIONS.internal.v2, validate: { request: { params: buildRouteValidationWithZod(paramsSchema), diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index 3760a614f8151..3f3fc270b1bab 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -28,8 +28,8 @@ const querySchema = z.object({ export function registerCRUDUpsert(router: EntityStorePluginRouter) { router.versioned .put({ - path: '/api/entity-store/entities/{entityType}', - access: 'public', + path: '/internal/security/entity-store/entities/{entityType}', + access: 'internal', security: { authz: DEFAULT_ENTITY_STORE_PERMISSIONS, }, @@ -37,7 +37,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.public.v1, // TODO(kuba): public or internal? + version: API_VERSIONS.internal.v2, validate: { request: { body: buildRouteValidationWithZod(Entity), diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index 2c9c453e11746..7fc2aed5ea59c 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -31,8 +31,8 @@ const querySchema = z.object({ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { router.versioned .put({ - path: '/api/entity-store/entities/bulk', - access: 'public', + path: '/internal/security/entity-store/entities/bulk', + access: 'internal', security: { authz: DEFAULT_ENTITY_STORE_PERMISSIONS, }, @@ -40,7 +40,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { }) .addVersion( { - version: API_VERSIONS.public.v1, + version: API_VERSIONS.internal.v2, validate: { request: { body: buildRouteValidationWithZod(bodySchema), From 303d3a8e724fad9e05eb9ecdf8477d1feac845d1 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 12:31:02 +0100 Subject: [PATCH 24/69] test force flag --- .../test/scout/api/tests/crud_api.spec.ts | 64 ++++++++++++++++--- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index bc86ced08504d..88ceee02c29ab 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -7,6 +7,7 @@ import { apiTest } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/api'; +import type { Client } from '@elastic/elasticsearch'; import type { Entity } from '../../../../common/domain/definitions/entity.gen'; import { COMMON_HEADERS, @@ -15,13 +16,6 @@ import { LATEST_INDEX, } from '../fixtures/constants'; import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; -import { API_VERSIONS } from '../../../../server/routes/constants'; - -const genericEntity: Entity = { - entity: { - id: 'required-id', - }, -}; apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () => { let defaultHeaders: Record; @@ -57,22 +51,72 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = }); apiTest('Should create an entity', async ({ apiClient, esClient }) => { + const entityObj: Entity = { + entity: { + id: 'required-id-create', + }, + }; const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { headers: defaultHeaders, responseType: 'json', - body: genericEntity, + body: entityObj, }); expect(create.statusCode).toBe(200); expect(create.body).toStrictEqual({ ok: true }); + expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(1); + }); + + apiTest('Should require a force flag', async ({ apiClient, esClient }) => { + const entityObj: Entity = { + entity: { + id: 'required-id-force', + }, + host: { + name: 'needs-force-flag', + }, + }; + + const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { + headers: defaultHeaders, + responseType: 'json', + body: entityObj, + }); + expect(create.statusCode).toBe(400); + expect(create.body.message).toContain('not allowed to be updated without forcing it'); + + expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(0); + + const createWithForce = await apiClient.put( + ENTITY_STORE_ROUTES.CRUD_UPSERT('generic') + '?force=true', + { + headers: defaultHeaders, + responseType: 'json', + body: entityObj, + } + ); + expect(createWithForce.statusCode).toBe(200); + const entities = await esClient.search({ index: LATEST_INDEX, query: { term: { - 'entity.id': genericEntity.entity.id, + 'entity.id': entityObj.entity.id, }, }, }); - expect(entities.hits.hits).toHaveLength(1); + expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(1); }); }); + +async function countEntitiesByID(esClient: Client, index: string, id: string): Promise { + const resp = await esClient.search({ + index, + query: { + term: { + 'entity.id': id, + }, + }, + }); + return resp.hits.hits.length; +} From 39b22916fe54db65d461061af2005290dd9448cf Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 12:55:25 +0100 Subject: [PATCH 25/69] add upsert test --- .../test/scout/api/tests/crud_api.spec.ts | 57 +++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 88ceee02c29ab..2620bce7b8782 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -8,7 +8,7 @@ import { apiTest } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/api'; import type { Client } from '@elastic/elasticsearch'; -import type { Entity } from '../../../../common/domain/definitions/entity.gen'; +import type { HostEntity } from '../../../../common/domain/definitions/entity.gen'; import { COMMON_HEADERS, ENTITY_STORE_ROUTES, @@ -96,16 +96,62 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = } ); expect(createWithForce.statusCode).toBe(200); + expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(1); + }); + + apiTest('Should perform an upsert', async ({ apiClient, esClient }) => { + const entityObj: Entity = { + entity: { + id: 'this-is-upsert', + }, + host: { + name: 'this-is-upsert', + entity: { + id: 'this-is-upsert', + }, + }, + }; + + const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('host'), { + headers: defaultHeaders, + responseType: 'json', + body: entityObj, + }); + expect(create.statusCode).toBe(200); + expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(1); + + const upsert = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('host') + '?force=true', { + headers: defaultHeaders, + responseType: 'json', + body: { + entity: { + id: entityObj.host.entity.id, + name: 'this-is-upsert', + }, + host: { + name: 'this-is-upsert', + }, + }, + }); + expect(upsert.statusCode).toBe(200); const entities = await esClient.search({ index: LATEST_INDEX, query: { term: { - 'entity.id': entityObj.entity.id, + 'host.entity.id': entityObj.entity.id, }, }, }); - expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(1); + expect(entities.hits.hits).toHaveLength(1); + const received = entities.hits.hits[0]._source as HostEntity; + expect(received.entity).toBeUndefined(); + expect(received.host).toMatchObject({ + entity: { + id: entityObj.host.entity.id, + }, + name: 'this-is-upsert', + }); }); }); @@ -113,8 +159,9 @@ async function countEntitiesByID(esClient: Client, index: string, id: string): P const resp = await esClient.search({ index, query: { - term: { - 'entity.id': id, + bool: { + should: [{ term: { 'entity.id': id } }, { term: { 'host.entity.id': id } }], + minimum_should_match: 1, }, }, }); From 9c4011b4d8a8f4141ceabab596e3b75f03d9cb1f Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 14:42:38 +0100 Subject: [PATCH 26/69] updates --- .../entity_store/server/domain/crud_client.ts | 1 + .../test/scout/api/fixtures/constants.ts | 6 +-- .../test/scout/api/tests/crud_api.spec.ts | 48 +++++++++++++++++-- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 22250f30bd7eb..ed329d952ff2d 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -125,6 +125,7 @@ export class CRUDClient { operations.push({ create: {} }, preparedDoc); } + this.logger.info(`Upserting ${operations.length / 2} entities`); await this.esClient.bulk({ index: getUpdatesEntitiesDataStreamName(this.namespace), operations, diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts index 814bde5d0b012..9b838ff8e2957 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts @@ -24,11 +24,11 @@ export const ENTITY_STORE_ROUTES = { UNINSTALL: 'internal/security/entity-store/uninstall', FORCE_LOG_EXTRACTION: (entityType: string) => `internal/security/entity-store/${entityType}/force-log-extraction`, - CRUD_UPSERT: (entityType: string) => - `internal/security/entity-store/entities/${entityType}`, + CRUD_UPSERT: (entityType: string) => `internal/security/entity-store/entities/${entityType}`, + CRUD_UPSERT_BULK: 'internal/security/entity-store/entities/bulk', } as const; export const ENTITY_STORE_TAGS = [...tags.stateful.classic, ...tags.serverless.security.complete]; export const LATEST_INDEX = '.entities.v2.latest.security_default'; -export const UPDATES_INDEX = '.entities.v2.updates.security_default'; +export const UPDATES_DATASTREAM = '.entities.v2.updates.security_default'; diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 2620bce7b8782..0fe4b9f781401 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -8,12 +8,13 @@ import { apiTest } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/api'; import type { Client } from '@elastic/elasticsearch'; -import type { HostEntity } from '../../../../common/domain/definitions/entity.gen'; +import type { Entity, HostEntity } from '../../../../common/domain/definitions/entity.gen'; import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS, LATEST_INDEX, + UPDATES_DATASTREAM, } from '../fixtures/constants'; import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; @@ -125,7 +126,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = responseType: 'json', body: { entity: { - id: entityObj.host.entity.id, + id: entityObj.host?.entity?.id, name: 'this-is-upsert', }, host: { @@ -148,11 +149,50 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = expect(received.entity).toBeUndefined(); expect(received.host).toMatchObject({ entity: { - id: entityObj.host.entity.id, + id: entityObj.host?.entity?.id, }, name: 'this-is-upsert', }); }); + + apiTest('Should perform a bulk upsert', async ({ apiClient, esClient }) => { + const bulkBody = { + entities: [ + { + type: 'generic', + document: { + entity: { + id: 'required-id-1-bulk', + }, + }, + }, + { + type: 'generic', + document: { + entity: { + id: 'required-id-2-bulk', + }, + }, + }, + ], + }; + + const bulkUpsert = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT_BULK, { + headers: defaultHeaders, + responseType: 'json', + body: bulkBody, + }); + // expect(bulkUpsert.body).toBe('ok'); + expect(bulkUpsert.statusCode).toBe(200); + + const resp = await esClient.search({ + index: UPDATES_DATASTREAM, + query: { + match_all: {}, + }, + }); + expect(resp.hits.hits).toHaveLength(2); + }); }); async function countEntitiesByID(esClient: Client, index: string, id: string): Promise { @@ -160,7 +200,7 @@ async function countEntitiesByID(esClient: Client, index: string, id: string): P index, query: { bool: { - should: [{ term: { 'entity.id': id } }, { term: { 'host.entity.id': id } }], + should: [{ wildcard: { 'entity.id': id } }, { wildcard: { 'host.entity.id': id } }], minimum_should_match: 1, }, }, From b923c6596542482bcf1929f1f79ce7db345614b6 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 14:52:14 +0100 Subject: [PATCH 27/69] fix bulk update test --- .../plugins/entity_store/server/domain/crud_client.ts | 1 + .../entity_store/test/scout/api/tests/crud_api.spec.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index ed329d952ff2d..a93e9ef0d16e4 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -129,6 +129,7 @@ export class CRUDClient { await this.esClient.bulk({ index: getUpdatesEntitiesDataStreamName(this.namespace), operations, + refresh: 'wait_for', }); } diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 0fe4b9f781401..02d7b8ede2d78 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -188,7 +188,9 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = const resp = await esClient.search({ index: UPDATES_DATASTREAM, query: { - match_all: {}, + wildcard: { + 'entity.id': 'required-id-*-bulk', + }, }, }); expect(resp.hits.hits).toHaveLength(2); From 67e65aaef794a3509117b903425c87936441edb0 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 15:31:07 +0100 Subject: [PATCH 28/69] add working scout test suite --- .../test/scout/api/fixtures/constants.ts | 1 + .../test/scout/api/tests/crud_api.spec.ts | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts index 9b838ff8e2957..8a80e41035eef 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts @@ -26,6 +26,7 @@ export const ENTITY_STORE_ROUTES = { `internal/security/entity-store/${entityType}/force-log-extraction`, CRUD_UPSERT: (entityType: string) => `internal/security/entity-store/entities/${entityType}`, CRUD_UPSERT_BULK: 'internal/security/entity-store/entities/bulk', + CRUD_DELETE: (id: string) => `internal/security/entity-store/entities/${id}`, } as const; export const ENTITY_STORE_TAGS = [...tags.stateful.classic, ...tags.serverless.security.complete]; diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 02d7b8ede2d78..0c61dbada4d49 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -182,7 +182,6 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = responseType: 'json', body: bulkBody, }); - // expect(bulkUpsert.body).toBe('ok'); expect(bulkUpsert.statusCode).toBe(200); const resp = await esClient.search({ @@ -195,6 +194,51 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = }); expect(resp.hits.hits).toHaveLength(2); }); + + apiTest('Should delete an entity', async ({ apiClient, esClient }) => { + const entityObj: Entity = { + entity: { + id: 'required-id-delete', + }, + }; + const create = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT('generic'), { + headers: defaultHeaders, + responseType: 'json', + body: entityObj, + }); + expect(create.statusCode).toBe(200); + const resp = await esClient.search({ + index: LATEST_INDEX, + query: { + match: { + 'entity.id': entityObj.entity.id, + }, + }, + }); + expect(resp.hits.hits).toHaveLength(1); + expect(resp.hits.hits[0]._id).toBeDefined(); + const entityId = resp.hits.hits[0]._id as string; + + const del = await apiClient.delete(ENTITY_STORE_ROUTES.CRUD_DELETE(entityId), { + headers: defaultHeaders, + responseType: 'json', + }); + expect(del.body).toStrictEqual({ deleted: true }); + expect(del.statusCode).toBe(200); + + expect( + esClient.get({ + index: LATEST_INDEX, + id: entityId, + }) + ).rejects.toThrow(`"found":false`); + + const apiNotFound = await apiClient.delete(ENTITY_STORE_ROUTES.CRUD_DELETE(entityId), { + headers: defaultHeaders, + responseType: 'json', + }); + expect(apiNotFound.body.statusCode).toBe(404); + }); }); async function countEntitiesByID(esClient: Client, index: string, id: string): Promise { From b0d445f47e6e45662968f57b4a023280c54a0727 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 15:35:10 +0100 Subject: [PATCH 29/69] fix naming issue --- .../plugins/entity_store/server/domain/crud_client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index a93e9ef0d16e4..7943bf42e9c7e 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -28,7 +28,7 @@ import type { import { getEuidFromObject } from '../../common/domain/euid'; import { getUpdatesEntitiesDataStreamName } from './assets/updates_data_stream'; -interface EntityManagerDependencies { +interface CRUDClientDependencies { logger: Logger; esClient: ElasticsearchClient; namespace: string; @@ -44,7 +44,7 @@ export class CRUDClient { private readonly esClient: ElasticsearchClient; private readonly namespace: string; - constructor(deps: EntityManagerDependencies) { + constructor(deps: CRUDClientDependencies) { this.logger = deps.logger; this.esClient = deps.esClient; this.namespace = deps.namespace; From 567f0ddc91d3ad00e99dbfb8941a50cddb6b4f49 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 15:36:06 +0100 Subject: [PATCH 30/69] delete entity id --- .../security/plugins/entity_store/server/domain/crud_client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 7943bf42e9c7e..96d48f6ab0836 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -135,6 +135,7 @@ export class CRUDClient { public async deleteEntity(id: string) { try { + this.logger.info(`Deleting Entity ID ${id}g`); await this.esClient.delete({ index: getLatestEntitiesIndexName(this.namespace), id, From 6cd8bf30019062c4ac3c46822b9a4de34d8011da Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 16 Feb 2026 15:03:12 +0000 Subject: [PATCH 31/69] Changes from node scripts/lint_ts_projects --fix --- x-pack/solutions/security/plugins/entity_store/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/tsconfig.json b/x-pack/solutions/security/plugins/entity_store/tsconfig.json index 5e41931f3ec50..5d753b6bdcc00 100644 --- a/x-pack/solutions/security/plugins/entity_store/tsconfig.json +++ b/x-pack/solutions/security/plugins/entity_store/tsconfig.json @@ -34,5 +34,6 @@ "@kbn/encrypted-saved-objects-plugin", "@kbn/scout-security", "@kbn/core-saved-objects-server", + "@kbn/std", ] } From 334d814bc6d7f37baae29d7e7af6a5e4675129ea Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 16 Feb 2026 15:09:27 +0000 Subject: [PATCH 32/69] Changes from node scripts/regenerate_moon_projects.js --update --- x-pack/solutions/security/plugins/entity_store/moon.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/moon.yml b/x-pack/solutions/security/plugins/entity_store/moon.yml index 7851ea648ffc9..1a145c8eb49f6 100644 --- a/x-pack/solutions/security/plugins/entity_store/moon.yml +++ b/x-pack/solutions/security/plugins/entity_store/moon.yml @@ -38,6 +38,7 @@ dependsOn: - '@kbn/encrypted-saved-objects-plugin' - '@kbn/scout-security' - '@kbn/core-saved-objects-server' + - '@kbn/std' tags: - plugin - prod From 25dec945ee7503d3fd049da4f523c1187cae4fa3 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 16:22:26 +0100 Subject: [PATCH 33/69] fix type errors --- .../test/scout/api/tests/dsl_translation.spec.ts | 16 ++++++++-------- .../scout/api/tests/esql_translation.spec.ts | 16 ++++++++-------- .../test/scout/api/tests/install.spec.ts | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/dsl_translation.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/dsl_translation.spec.ts index 938fca2b14c92..2ff69ecbfca7c 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/dsl_translation.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/dsl_translation.spec.ts @@ -11,7 +11,7 @@ import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS, - UPDATES_INDEX, + UPDATES_DATASTREAM, } from '../fixtures/constants'; import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; import { getEuidDslFilterBasedOnDocument } from '../../../../common/domain/euid/dsl'; @@ -65,7 +65,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); @@ -85,7 +85,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); @@ -108,7 +108,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); @@ -129,7 +129,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); @@ -153,7 +153,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); @@ -175,7 +175,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); @@ -196,7 +196,7 @@ apiTest.describe('DSL query translation', { tag: ENTITY_STORE_TAGS }, () => { expect(dsl).toBeDefined(); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, query: { ...dsl }, size: 10, }); diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/esql_translation.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/esql_translation.spec.ts index e3f3bf58f9435..48a19aed09eee 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/esql_translation.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/esql_translation.spec.ts @@ -11,7 +11,7 @@ import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS, - UPDATES_INDEX, + UPDATES_DATASTREAM, } from '../fixtures/constants'; import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; import { getEuidEsqlFilterBasedOnDocument } from '../../../../common/domain/euid/esql'; @@ -58,7 +58,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('generic', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, @@ -79,7 +79,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('host', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, @@ -103,7 +103,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('host', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, @@ -124,7 +124,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('user', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, @@ -148,7 +148,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('user', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, @@ -172,7 +172,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('service', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, @@ -193,7 +193,7 @@ apiTest.describe('ESQL query translation', { tag: ENTITY_STORE_TAGS }, () => { const filter = getEuidEsqlFilterBasedOnDocument('service', docSource); expect(filter).toBeDefined(); - const query = `FROM ${UPDATES_INDEX} | WHERE ${filter} | LIMIT 10`; + const query = `FROM ${UPDATES_DATASTREAM} | WHERE ${filter} | LIMIT 10`; const result = await esClient.esql.query({ query, drop_null_columns: true, diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts index f2020b1c15796..3515a7511e332 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/install.spec.ts @@ -14,7 +14,7 @@ apiTest.describe('Entity Store API tests', { tag: ENTITY_STORE_TAGS }, () => { let defaultHeaders: Record; apiTest.beforeAll(async ({ samlAuth }) => { - const credentials = await samlAuth.asInteract.iveUser('admin'); + const credentials = await samlAuth.asInteractiveUser('admin'); const cookieHeader = credentials.cookieHeader; defaultHeaders = { ...cookieHeader, From b7b92acfe0b263ee60d3a170ec2599970de87341 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 16:35:09 +0100 Subject: [PATCH 34/69] update last test --- .../test/scout/api/tests/painless_translation.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/painless_translation.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/painless_translation.spec.ts index 4137a10aa060b..e5252c0eb4026 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/painless_translation.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/painless_translation.spec.ts @@ -11,7 +11,7 @@ import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS, - UPDATES_INDEX, + UPDATES_DATASTREAM, } from '../fixtures/constants'; import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; import { getEuidPainlessEvaluation } from '../../../../common/domain/euid/painless'; @@ -69,7 +69,7 @@ apiTest.describe('Painless runtime field translation', { tag: ENTITY_STORE_TAGS const emitScript = toRuntimeFieldEmitScript(returnScript); const result = await esClient.search({ - index: UPDATES_INDEX, + index: UPDATES_DATASTREAM, body: { query: { match_all: {} }, runtime_mappings: { From d3459f32f2bf618f409987ea2e27890b48c1b402 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 16 Feb 2026 15:57:02 +0000 Subject: [PATCH 35/69] Changes from node scripts/eslint_all_files --no-cache --fix --- .../entity_store/server/domain/asset_manager.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts index d70b1e0bd7b65..91da4cbd4c7b9 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts @@ -9,6 +9,7 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import type { EntityType } from '../../common'; import { scheduleExtractEntityTask, stopExtractEntityTask } from '../tasks/extract_entity_task'; import { installElasticsearchAssets, uninstallElasticsearchAssets } from './assets/install_assets'; import type { @@ -34,9 +35,9 @@ import { } from './assets/component_templates'; import { getUpdatesEntitiesDataStreamName } from './assets/updates_data_stream'; import type { LogsExtractionClient } from './logs_extraction_client'; -import { EntityType } from '@kbn/entity-store/common'; -import { ALL_ENTITY_TYPES, ManagedEntityDefinition } from '@kbn/entity-store/common/domain/definitions/entity_schema'; -import { getEntityDefinition } from '@kbn/entity-store/common/domain/definitions/registry'; +import type { ManagedEntityDefinition } from '../../common/domain/definitions/entity_schema'; +import { ALL_ENTITY_TYPES } from '../../common/domain/definitions/entity_schema'; +import { getEntityDefinition } from '../../common/domain/definitions/registry'; interface AssetManagerDependencies { logger: Logger; @@ -168,8 +169,8 @@ export class AssetManager { for (const type of ALL_ENTITY_TYPES) { try { await this.engineDescriptorClient.findOrThrow(type); - } catch { - return false + } catch { + return false; } } return true; From 1f84f43bc8f08fdcdd5af477c0add1ace4bbca7a Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 16 Feb 2026 17:13:58 +0100 Subject: [PATCH 36/69] add eslint marker --- .../plugins/entity_store/test/scout/api/tests/crud_api.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 0c61dbada4d49..30649cb4642e0 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -227,6 +227,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = expect(del.statusCode).toBe(200); expect( + // @typescript-eslint/no-floating-promises esClient.get({ index: LATEST_INDEX, id: entityId, From 70660e902cf3fb9918f8c0ac5ab63609286992fc Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 12:10:42 +0100 Subject: [PATCH 37/69] lowercase the attributes/relationships/behaviors etc. --- .../domain/definitions/common_fields.ts | 36 +++++++++---------- .../common/domain/definitions/host.ts | 20 +++++------ .../common/domain/definitions/service.ts | 12 +++---- .../common/domain/definitions/user.ts | 16 ++++----- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts index 07456ca68c264..867e63bdacf59 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts @@ -55,57 +55,57 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => { newestValue({ source: `${prefix}.url`, destination: 'entity.url' }), newestValue({ - source: `${prefix}.attributes.Privileged`, - destination: 'entity.attributes.Privileged', + source: `${prefix}.attributes.privileged`, + destination: 'entity.attributes.privileged', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), newestValue({ - source: `${prefix}.attributes.Asset`, - destination: 'entity.attributes.Asset', + source: `${prefix}.attributes.asset`, + destination: 'entity.attributes.asset', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), newestValue({ - source: `${prefix}.attributes.Managed`, - destination: 'entity.attributes.Managed', + source: `${prefix}.attributes.managed`, + destination: 'entity.attributes.managed', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), newestValue({ - source: `${prefix}.attributes.Mfa_enabled`, - destination: 'entity.attributes.Mfa_enabled', + source: `${prefix}.attributes.mfa_enabled`, + destination: 'entity.attributes.mfa_enabled', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), /* Lifecycle fields should not allow update via the API */ newestValue({ - source: `${prefix}.lifecycle.First_seen`, - destination: 'entity.lifecycle.First_seen', + source: `${prefix}.lifecycle.first_seen`, + destination: 'entity.lifecycle.first_seen', mapping: { type: 'date' }, }), newestValue({ - source: `${prefix}.lifecycle.Last_activity`, - destination: 'entity.lifecycle.Last_activity', + source: `${prefix}.lifecycle.last_activity`, + destination: 'entity.lifecycle.last_activity', mapping: { type: 'date' }, }), newestValue({ - source: `${prefix}.behaviors.Brute_force_victim`, - destination: 'entity.behaviors.Brute_force_victim', + source: `${prefix}.behaviors.brute_force_victim`, + destination: 'entity.behaviors.brute_force_victim', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), newestValue({ - source: `${prefix}.behaviors.New_country_login`, - destination: 'entity.behaviors.New_country_login', + source: `${prefix}.behaviors.new_country_login`, + destination: 'entity.behaviors.new_country_login', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), newestValue({ - source: `${prefix}.behaviors.Used_usb_device`, - destination: 'entity.behaviors.Used_usb_device', + source: `${prefix}.behaviors.used_usb_device`, + destination: 'entity.behaviors.used_usb_device', mapping: { type: 'boolean' }, allowAPIUpdate: true, }), diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/host.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/host.ts index 1c0f9039fbf66..ed80f73cd8e84 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/host.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/host.ts @@ -60,33 +60,33 @@ export const hostEntityDefinition: EntityDefinitionWithoutId = { ...getEntityFieldsDescriptions('host'), collect({ - source: `host.entity.relationships.Communicates_with`, - destination: 'entity.relationships.Communicates_with', + source: `host.entity.relationships.communicates_with`, + destination: 'entity.relationships.communicates_with', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `host.entity.relationships.Depends_on`, - destination: 'entity.relationships.Depends_on', + source: `host.entity.relationships.depends_on`, + destination: 'entity.relationships.depends_on', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `host.entity.relationships.Dependent_of`, - destination: 'entity.relationships.Dependent_of', + source: `host.entity.relationships.dependent_of`, + destination: 'entity.relationships.dependent_of', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `host.entity.relationships.Owned_by`, - destination: 'entity.relationships.Owned_by', + source: `host.entity.relationships.owned_by`, + destination: 'entity.relationships.owned_by', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `host.entity.relationships.Accessed_frequently_by`, - destination: 'entity.relationships.Accessed_frequently_by', + source: `host.entity.relationships.accessed_frequently_by`, + destination: 'entity.relationships.accessed_frequently_by', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/service.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/service.ts index e8243fed9b16c..e26d6e3f6ec6e 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/service.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/service.ts @@ -37,20 +37,20 @@ export const serviceEntityDefinition: EntityDefinitionWithoutId = { ...getEntityFieldsDescriptions('service'), collect({ - source: `service.entity.relationships.Communicates_with`, - destination: 'entity.relationships.Communicates_with', + source: `service.entity.relationships.communicates_with`, + destination: 'entity.relationships.communicates_with', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `service.entity.relationships.Depends_on`, - destination: 'entity.relationships.Depends_on', + source: `service.entity.relationships.depends_on`, + destination: 'entity.relationships.depends_on', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `service.entity.relationships.Dependent_of`, - destination: 'entity.relationships.Dependent_of', + source: `service.entity.relationships.dependent_of`, + destination: 'entity.relationships.dependent_of', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/user.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/user.ts index bc5d63a0959ce..1d11a559e29dc 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/user.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/user.ts @@ -57,27 +57,27 @@ export const userEntityDefinition: EntityDefinitionWithoutId = { collect({ source: 'host.name' }), collect({ - source: `user.entity.relationships.Accesses_frequently`, - destination: 'entity.relationships.Accesses_frequently', + source: `user.entity.relationships.accesses_frequently`, + destination: 'entity.relationships.accesses_frequently', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `user.entity.relationships.Owns`, - destination: 'entity.relationships.Owns', + source: `user.entity.relationships.owns`, + destination: 'entity.relationships.owns', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `user.entity.relationships.Supervises`, - destination: 'entity.relationships.Supervises', + source: `user.entity.relationships.supervises`, + destination: 'entity.relationships.supervises', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), collect({ - source: `user.entity.relationships.Supervised_by`, - destination: 'entity.relationships.Supervised_by', + source: `user.entity.relationships.supervised_by`, + destination: 'entity.relationships.supervised_by', mapping: { type: 'keyword' }, allowAPIUpdate: true, }), From f9cdbdcf63c93b1fe49925d22700ed3558c2e2d5 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 12:44:00 +0100 Subject: [PATCH 38/69] fix hanging promise issue --- .../entity_store/test/scout/api/tests/crud_api.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index c53fc18f94126..9cf0214637c6c 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -39,7 +39,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = responseType: 'json', body: {}, }); - expect(response.statusCode).toBe(200); + expect(response.statusCode).toBe(201); }); apiTest.afterAll(async ({ apiClient }) => { @@ -226,8 +226,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = expect(del.body).toStrictEqual({ deleted: true }); expect(del.statusCode).toBe(200); - expect( - // @typescript-eslint/no-floating-promises + await expect( esClient.get({ index: LATEST_INDEX, id: entityId, From 184b01c0cab6ebbb799d5270d3a8e0243a97fe84 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 12:50:42 +0100 Subject: [PATCH 39/69] fix ID/Hashed ID issues --- .../plugins/entity_store/server/domain/crud_client.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 96d48f6ab0836..932b5a1947d83 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -55,13 +55,13 @@ export class CRUDClient { document: Entity, force: boolean ): Promise { - const rawId = getEuidFromObject(entityType, document); - if (rawId === undefined) { + const id = getEuidFromObject(entityType, document); + if (id === undefined) { throw new BadCRUDRequestError(`Could not derive entity EUID from document`); } // EUID generation uses MD5. It is not a security-related feature. // eslint-disable-next-line @kbn/eslint/no_unsafe_hash - const id: string = createHash('md5').update(rawId).digest('hex'); + const hashedId: string = createHash('md5').update(id).digest('hex'); this.logger.info(`Upserting entity ID ${id}`); if (!document.entity.id) { document.entity.id = id; @@ -78,7 +78,7 @@ export class CRUDClient { try { await this.esClient.create({ index: getLatestEntitiesIndexName(this.namespace), - id, + id: hashedId, document: preparedDoc, refresh: 'wait_for', }); @@ -93,7 +93,7 @@ export class CRUDClient { removeEUIDFields(definition, preparedDoc); const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), - id, + id: hashedId, doc: preparedDoc, }); From b93afbc5bbf2417173f1bf327d414e4916d72f11 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 12:54:41 +0100 Subject: [PATCH 40/69] crud_client: use logger.debug --- .../entity_store/server/domain/crud_client.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 932b5a1947d83..b4a5d4d13467e 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -62,7 +62,7 @@ export class CRUDClient { // EUID generation uses MD5. It is not a security-related feature. // eslint-disable-next-line @kbn/eslint/no_unsafe_hash const hashedId: string = createHash('md5').update(id).digest('hex'); - this.logger.info(`Upserting entity ID ${id}`); + this.logger.debug(`Upserting entity ID ${id}`); if (!document.entity.id) { document.entity.id = id; } @@ -82,7 +82,7 @@ export class CRUDClient { document: preparedDoc, refresh: 'wait_for', }); - this.logger.info(`Created entity ID ${id}`); + this.logger.debug(`Created entity ID ${id}`); return; } catch (error) { if (error.statusCode !== 409) { @@ -102,10 +102,10 @@ export class CRUDClient { case 'not_found': throw new Error(`Could not upsert entity ID ${id}`); case 'updated': - this.logger.info(`Entity ID ${id} updated`); + this.logger.debug(`Entity ID ${id} updated`); break; case 'noop': - this.logger.info(`Entity ID ${id} updated (no change)`); + this.logger.debug(`Entity ID ${id} updated (no change)`); break; } } @@ -125,7 +125,7 @@ export class CRUDClient { operations.push({ create: {} }, preparedDoc); } - this.logger.info(`Upserting ${operations.length / 2} entities`); + this.logger.debug(`Upserting ${operations.length / 2} entities`); await this.esClient.bulk({ index: getUpdatesEntitiesDataStreamName(this.namespace), operations, @@ -135,7 +135,7 @@ export class CRUDClient { public async deleteEntity(id: string) { try { - this.logger.info(`Deleting Entity ID ${id}g`); + this.logger.debug(`Deleting Entity ID ${id}`); await this.esClient.delete({ index: getLatestEntitiesIndexName(this.namespace), id, From a8fc7a477ac4b5ae5ea9ef3f36655430330126dd Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 13:28:44 +0100 Subject: [PATCH 41/69] crud_client: extract create/update methods, add explicit return values --- .../entity_store/server/domain/crud_client.ts | 134 ++++++++++-------- .../server/routes/apis/crud_upsert_bulk.ts | 2 +- .../test/scout/api/tests/crud_api.spec.ts | 4 +- 3 files changed, 81 insertions(+), 59 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index b4a5d4d13467e..3c3f0e65f3cfb 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -36,7 +36,22 @@ interface CRUDClientDependencies { interface BulkObject { type: EntityType; - document: Entity; + doc: Entity; +} + +function validateAndTransformDoc( + entityType: EntityType, + namespace: string, + doc: Entity, + force: boolean +): Record { + const definition = getEntityDefinition(entityType, namespace); + if (!force) { + const flat = getFlattenedObject(doc); + const fieldDescriptions = getFieldDescriptions(flat, definition); + assertOnlyNonForcedAttributesInReq(fieldDescriptions); + } + return transformDocForUpsert(entityType, doc); } export class CRUDClient { @@ -50,12 +65,53 @@ export class CRUDClient { this.namespace = deps.namespace; } + private async createEntity( + hashedId: string, + doc: Record + ): Promise { + this.logger.debug(`Creating entity ID: ${hashedId}`); + await this.esClient.create({ + index: getLatestEntitiesIndexName(this.namespace), + id: hashedId, + document: doc, + refresh: 'wait_for', + }); + this.logger.debug(`Created entity ID ${hashedId}`); + } + + private async updateEntity( + hashedId: string, + entityType: EntityType, + doc: Record, + ): Promise { + this.logger.debug(`Updating entity ID: ${hashedId}`); + const definition = getEntityDefinition(entityType, this.namespace); + removeEUIDFields(definition, doc); + const { result } = await this.esClient.update({ + index: getLatestEntitiesIndexName(this.namespace), + id: hashedId, + doc, + }); + + switch (result as Result) { + case 'deleted': + case 'not_found': + throw new Error(`Could not update entity ID ${hashedId}`); + case 'updated': + this.logger.debug(`Updated entity ID ${hashedId}`); + break; + case 'noop': + this.logger.debug(`Updated entity ID ${hashedId} (no change)`); + break; + } + } + public async upsertEntity( entityType: EntityType, - document: Entity, + doc: Entity, force: boolean ): Promise { - const id = getEuidFromObject(entityType, document); + const id = getEuidFromObject(entityType, doc); if (id === undefined) { throw new BadCRUDRequestError(`Could not derive entity EUID from document`); } @@ -63,77 +119,45 @@ export class CRUDClient { // eslint-disable-next-line @kbn/eslint/no_unsafe_hash const hashedId: string = createHash('md5').update(id).digest('hex'); this.logger.debug(`Upserting entity ID ${id}`); - if (!document.entity.id) { - document.entity.id = id; - } - const definition = getEntityDefinition(entityType, this.namespace); - if (!force) { - const flat = getFlattenedObject(document); - const fieldDescriptions = getFieldDescriptions(flat, definition); - assertOnlyNonForcedAttributesInReq(fieldDescriptions); + + if (!doc.entity?.id) { + doc.entity.id = id; } - const preparedDoc = prepareDocumentForUpsert(entityType, document); + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force) try { - await this.esClient.create({ - index: getLatestEntitiesIndexName(this.namespace), - id: hashedId, - document: preparedDoc, - refresh: 'wait_for', - }); - this.logger.debug(`Created entity ID ${id}`); - return; + await this.createEntity(hashedId, readyDoc) } catch (error) { if (error.statusCode !== 409) { throw error; } + this.logger.debug(`Conflict while creating entity ID ${id}, updating instead`); } - removeEUIDFields(definition, preparedDoc); - const { result } = await this.esClient.update({ - index: getLatestEntitiesIndexName(this.namespace), - id: hashedId, - doc: preparedDoc, - }); - - switch (result as Result) { - case 'deleted': - case 'not_found': - throw new Error(`Could not upsert entity ID ${id}`); - case 'updated': - this.logger.debug(`Entity ID ${id} updated`); - break; - case 'noop': - this.logger.debug(`Entity ID ${id} updated (no change)`); - break; - } + await this.updateEntity(hashedId, entityType, readyDoc) + return; } - public async upsertEntitiesBulk(objects: BulkObject[], force: boolean) { + public async upsertEntitiesBulk(objects: BulkObject[], force: boolean): Promise { const operations: (BulkOperationContainer | BulkUpdateAction)[] = []; - for (const { type: entityType, document } of objects) { - const definition = getEntityDefinition(entityType, this.namespace); - if (!force) { - const flat = getFlattenedObject(document); - const fieldDescriptions = getFieldDescriptions(flat, definition); - assertOnlyNonForcedAttributesInReq(fieldDescriptions); - } - const preparedDoc = prepareDocumentForUpsert(entityType, document); - - operations.push({ create: {} }, preparedDoc); + this.logger.debug(`Preparing ${objects.length} entities for bulk upsert`); + for (const { type: entityType, doc } of objects) { + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force) + operations.push({ create: {} }, readyDoc); } - this.logger.debug(`Upserting ${operations.length / 2} entities`); + this.logger.debug(`Bulk upserting ${objects.length} entities`); await this.esClient.bulk({ index: getUpdatesEntitiesDataStreamName(this.namespace), operations, refresh: 'wait_for', }); + return; } - public async deleteEntity(id: string) { + public async deleteEntity(id: string): Promise { try { this.logger.debug(`Deleting Entity ID ${id}`); await this.esClient.delete({ @@ -146,7 +170,6 @@ export class CRUDClient { } throw error; } - return { deleted: true }; } } @@ -207,12 +230,11 @@ function assertOnlyNonForcedAttributesInReq(fields: Record) } } -function prepareDocumentForUpsert( +function transformDocForUpsert( type: EntityType, data: Partial ): Record { const now = new Date().toISOString(); - if (type === 'generic') { return { '@timestamp': now, @@ -243,8 +265,8 @@ function prepareDocumentForUpsert( return doc; } -function removeEUIDFields(definition: ManagedEntityDefinition, document: Record) { +function removeEUIDFields(definition: ManagedEntityDefinition, doc: Record) { for (const euidField of definition.identityField.requiresOneOfFields) { - unset(document, euidField); + unset(doc, euidField); } } diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index 7fc2aed5ea59c..e151c15982a0a 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -19,7 +19,7 @@ const bodySchema = z.object({ entities: z.array( z.object({ type: EntityType, - document: Entity, + doc: Entity, }) ), }); diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 9cf0214637c6c..21d7f9af782e6 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -160,7 +160,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = entities: [ { type: 'generic', - document: { + doc: { entity: { id: 'required-id-1-bulk', }, @@ -168,7 +168,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = }, { type: 'generic', - document: { + doc: { entity: { id: 'required-id-2-bulk', }, From b505a38026157cd6bc970251a8758e8bc5fda922 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 13:31:33 +0100 Subject: [PATCH 42/69] crud_client: retry 3x on update conflict --- .../security/plugins/entity_store/server/domain/crud_client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 3c3f0e65f3cfb..61f6ef24e4c5e 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -91,6 +91,7 @@ export class CRUDClient { index: getLatestEntitiesIndexName(this.namespace), id: hashedId, doc, + retry_on_conflict: 3, }); switch (result as Result) { From e3c38915fdaf4a2d3fa5816a7d169438cc216156 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 13:39:09 +0100 Subject: [PATCH 43/69] crud_api: remove redundant type casts --- .../plugins/entity_store/server/routes/apis/crud_delete.ts | 2 +- .../plugins/entity_store/server/routes/apis/crud_upsert.ts | 2 +- .../plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts index 02814d9e1824a..0a48ec134ceb6 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts @@ -51,7 +51,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { if (error instanceof EntityNotFoundError) { return res.customError({ statusCode: 404, - body: error as EntityNotFoundError, + body: error, }); } diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts index 3f3fc270b1bab..3c07e35706470 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts @@ -59,7 +59,7 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { await crudClient.upsertEntity(req.params.entityType, req.body, req.query.force); } catch (error) { if (error instanceof BadCRUDRequestError) { - return res.badRequest({ body: error as BadCRUDRequestError }); + return res.badRequest({ body: error }); } logger.error(error); diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts index e151c15982a0a..38f3a235edd87 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts @@ -61,7 +61,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { await crudClient.upsertEntitiesBulk(req.body.entities, req.query.force); } catch (error) { if (error instanceof BadCRUDRequestError) { - return res.badRequest({ body: error as BadCRUDRequestError }); + return res.badRequest({ body: error }); } logger.error(error); From 640870ed098b7cd1586b03ad83cb09061c0d03c9 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 13:41:49 +0100 Subject: [PATCH 44/69] crud_api: move endpoints to a common directory --- .../routes/apis/{crud_delete.ts => crud/delete.ts} | 8 ++++---- .../routes/apis/{crud_upsert.ts => crud/upsert.ts} | 12 ++++++------ .../{crud_upsert_bulk.ts => crud/upsert_bulk.ts} | 12 ++++++------ .../plugins/entity_store/server/routes/apis/index.ts | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) rename x-pack/solutions/security/plugins/entity_store/server/routes/apis/{crud_delete.ts => crud/delete.ts} (92%) rename x-pack/solutions/security/plugins/entity_store/server/routes/apis/{crud_upsert.ts => crud/upsert.ts} (87%) rename x-pack/solutions/security/plugins/entity_store/server/routes/apis/{crud_upsert_bulk.ts => crud/upsert_bulk.ts} (87%) diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts similarity index 92% rename from x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts rename to x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts index 0a48ec134ceb6..bc7b3bef041f1 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts @@ -8,10 +8,10 @@ import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { z } from '@kbn/zod'; import type { IKibanaResponse } from '@kbn/core-http-server'; -import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; -import type { EntityStorePluginRouter } from '../../types'; -import { wrapMiddlewares } from '../middleware'; -import { EntityNotFoundError, EntityStoreNotInstalledError } from '../../domain/errors'; +import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../../constants'; +import type { EntityStorePluginRouter } from '../../../types'; +import { wrapMiddlewares } from '../../middleware'; +import { EntityNotFoundError, EntityStoreNotInstalledError } from '../../../domain/errors'; const paramsSchema = z.object({ id: z.string(), diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts similarity index 87% rename from x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts rename to x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts index 3c07e35706470..a97cf60c88132 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts @@ -8,12 +8,12 @@ import { BooleanFromString, buildRouteValidationWithZod } from '@kbn/zod-helpers'; import type { IKibanaResponse } from '@kbn/core-http-server'; import { z } from '@kbn/zod'; -import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; -import type { EntityStorePluginRouter } from '../../types'; -import { wrapMiddlewares } from '../middleware'; -import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../domain/errors'; -import { Entity } from '../../../common/domain/definitions/entity.gen'; -import { EntityType } from '../../../common/domain/definitions/entity_schema'; +import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../../constants'; +import type { EntityStorePluginRouter } from '../../../types'; +import { wrapMiddlewares } from '../../middleware'; +import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../../domain/errors'; +import { Entity } from '../../../../common/domain/definitions/entity.gen'; +import { EntityType } from '../../../../common/domain/definitions/entity_schema'; const paramsSchema = z .object({ diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts similarity index 87% rename from x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts rename to x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts index 38f3a235edd87..2d30ee2561cfc 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud_upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts @@ -8,12 +8,12 @@ import { BooleanFromString, buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { z } from '@kbn/zod'; import type { IKibanaResponse } from '@kbn/core-http-server'; -import { EntityType } from '../../../common/domain/definitions/entity_schema'; -import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../constants'; -import type { EntityStorePluginRouter } from '../../types'; -import { wrapMiddlewares } from '../middleware'; -import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../domain/errors'; -import { Entity } from '../../../common/domain/definitions/entity.gen'; +import { EntityType } from '../../../../common/domain/definitions/entity_schema'; +import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../../constants'; +import type { EntityStorePluginRouter } from '../../../types'; +import { wrapMiddlewares } from '../../middleware'; +import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../../domain/errors'; +import { Entity } from '../../../../common/domain/definitions/entity.gen'; const bodySchema = z.object({ entities: z.array( diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts index 0704911f29580..792bde4084fe7 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/index.ts @@ -10,7 +10,7 @@ export { registerStop } from './stop'; export { registerStatus } from './status'; export { registerForceLogExtraction } from './force_log_extraction'; export { registerUninstall } from './uninstall'; -export { registerCRUDUpsert } from './crud_upsert'; -export { registerCRUDUpsertBulk } from './crud_upsert_bulk'; -export { registerCRUDDelete } from './crud_delete'; +export { registerCRUDUpsert } from './crud/upsert'; +export { registerCRUDUpsertBulk } from './crud/upsert_bulk'; +export { registerCRUDDelete } from './crud/delete'; export { registerStart } from './start'; From 339a3d6d5c787c808829ce01810a79252e318ea1 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 13:49:16 +0100 Subject: [PATCH 45/69] crud_api: extend async/bulk upsert test with forced extraction --- .../test/scout/api/tests/crud_api.spec.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index 21d7f9af782e6..c561466346400 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -193,6 +193,22 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = }, }); expect(resp.hits.hits).toHaveLength(2); + + const extractionResponse = await apiClient.post( + ENTITY_STORE_ROUTES.FORCE_LOG_EXTRACTION('generic'), + { + headers: defaultHeaders, + responseType: 'json', + body: { + fromDateISO: new Date(Date.now() - 5 * 60 * 1000).toISOString(), // 5 minutes ago + toDateISO: new Date().toISOString(), + }, + } + ); + expect(extractionResponse.statusCode).toBe(200); + expect(extractionResponse.body.success).toBe(true); + expect(extractionResponse.body.pages).toBe(1); + expect(extractionResponse.body.count).toBe(2); }); apiTest('Should delete an entity', async ({ apiClient, esClient }) => { From 1d521565f4791c4470f48cdd4b6c854411def622 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 14:03:28 +0100 Subject: [PATCH 46/69] crud_api: remove two update results that never occur --- .../security/plugins/entity_store/server/domain/crud_client.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 61f6ef24e4c5e..33190c111a452 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -95,9 +95,6 @@ export class CRUDClient { }); switch (result as Result) { - case 'deleted': - case 'not_found': - throw new Error(`Could not update entity ID ${hashedId}`); case 'updated': this.logger.debug(`Updated entity ID ${hashedId}`); break; From 9df544cf71d9cb7c6f522e91611b6f40a88cfa63 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:25:47 +0000 Subject: [PATCH 47/69] Changes from node scripts/eslint_all_files --no-cache --fix --- .../entity_store/server/domain/crud_client.ts | 41 +++++++------------ 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 33190c111a452..a3a5073cba919 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -45,13 +45,13 @@ function validateAndTransformDoc( doc: Entity, force: boolean ): Record { - const definition = getEntityDefinition(entityType, namespace); - if (!force) { - const flat = getFlattenedObject(doc); - const fieldDescriptions = getFieldDescriptions(flat, definition); - assertOnlyNonForcedAttributesInReq(fieldDescriptions); - } - return transformDocForUpsert(entityType, doc); + const definition = getEntityDefinition(entityType, namespace); + if (!force) { + const flat = getFlattenedObject(doc); + const fieldDescriptions = getFieldDescriptions(flat, definition); + assertOnlyNonForcedAttributesInReq(fieldDescriptions); + } + return transformDocForUpsert(entityType, doc); } export class CRUDClient { @@ -65,10 +65,7 @@ export class CRUDClient { this.namespace = deps.namespace; } - private async createEntity( - hashedId: string, - doc: Record - ): Promise { + private async createEntity(hashedId: string, doc: Record): Promise { this.logger.debug(`Creating entity ID: ${hashedId}`); await this.esClient.create({ index: getLatestEntitiesIndexName(this.namespace), @@ -82,7 +79,7 @@ export class CRUDClient { private async updateEntity( hashedId: string, entityType: EntityType, - doc: Record, + doc: Record ): Promise { this.logger.debug(`Updating entity ID: ${hashedId}`); const definition = getEntityDefinition(entityType, this.namespace); @@ -104,11 +101,7 @@ export class CRUDClient { } } - public async upsertEntity( - entityType: EntityType, - doc: Entity, - force: boolean - ): Promise { + public async upsertEntity(entityType: EntityType, doc: Entity, force: boolean): Promise { const id = getEuidFromObject(entityType, doc); if (id === undefined) { throw new BadCRUDRequestError(`Could not derive entity EUID from document`); @@ -118,14 +111,13 @@ export class CRUDClient { const hashedId: string = createHash('md5').update(id).digest('hex'); this.logger.debug(`Upserting entity ID ${id}`); - if (!doc.entity?.id) { doc.entity.id = id; } - const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force) + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); try { - await this.createEntity(hashedId, readyDoc) + await this.createEntity(hashedId, readyDoc); } catch (error) { if (error.statusCode !== 409) { throw error; @@ -133,7 +125,7 @@ export class CRUDClient { this.logger.debug(`Conflict while creating entity ID ${id}, updating instead`); } - await this.updateEntity(hashedId, entityType, readyDoc) + await this.updateEntity(hashedId, entityType, readyDoc); return; } @@ -142,7 +134,7 @@ export class CRUDClient { this.logger.debug(`Preparing ${objects.length} entities for bulk upsert`); for (const { type: entityType, doc } of objects) { - const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force) + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); operations.push({ create: {} }, readyDoc); } @@ -228,10 +220,7 @@ function assertOnlyNonForcedAttributesInReq(fields: Record) } } -function transformDocForUpsert( - type: EntityType, - data: Partial -): Record { +function transformDocForUpsert(type: EntityType, data: Partial): Record { const now = new Date().toISOString(); if (type === 'generic') { return { From d5f4add323ec41246ca897e57f0fd719fbb67eba Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 15:39:12 +0100 Subject: [PATCH 48/69] crud_api: let upsert errors bubble up --- .../entity_store/server/domain/crud_client.ts | 71 +++++++++++-------- .../server/routes/apis/crud/upsert_bulk.ts | 16 ++--- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts index 33190c111a452..1cef82b35d7d5 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts @@ -39,19 +39,26 @@ interface BulkObject { doc: Entity; } +interface BulkObjectResponse { + _id: string; + status: number; + type: string; + reason: string; +} + function validateAndTransformDoc( entityType: EntityType, namespace: string, doc: Entity, force: boolean ): Record { - const definition = getEntityDefinition(entityType, namespace); - if (!force) { - const flat = getFlattenedObject(doc); - const fieldDescriptions = getFieldDescriptions(flat, definition); - assertOnlyNonForcedAttributesInReq(fieldDescriptions); - } - return transformDocForUpsert(entityType, doc); + const definition = getEntityDefinition(entityType, namespace); + if (!force) { + const flat = getFlattenedObject(doc); + const fieldDescriptions = getFieldDescriptions(flat, definition); + assertOnlyNonForcedAttributesInReq(fieldDescriptions); + } + return transformDocForUpsert(entityType, doc); } export class CRUDClient { @@ -65,10 +72,7 @@ export class CRUDClient { this.namespace = deps.namespace; } - private async createEntity( - hashedId: string, - doc: Record - ): Promise { + private async createEntity(hashedId: string, doc: Record): Promise { this.logger.debug(`Creating entity ID: ${hashedId}`); await this.esClient.create({ index: getLatestEntitiesIndexName(this.namespace), @@ -82,7 +86,7 @@ export class CRUDClient { private async updateEntity( hashedId: string, entityType: EntityType, - doc: Record, + doc: Record ): Promise { this.logger.debug(`Updating entity ID: ${hashedId}`); const definition = getEntityDefinition(entityType, this.namespace); @@ -104,11 +108,7 @@ export class CRUDClient { } } - public async upsertEntity( - entityType: EntityType, - doc: Entity, - force: boolean - ): Promise { + public async upsertEntity(entityType: EntityType, doc: Entity, force: boolean): Promise { const id = getEuidFromObject(entityType, doc); if (id === undefined) { throw new BadCRUDRequestError(`Could not derive entity EUID from document`); @@ -118,14 +118,13 @@ export class CRUDClient { const hashedId: string = createHash('md5').update(id).digest('hex'); this.logger.debug(`Upserting entity ID ${id}`); - if (!doc.entity?.id) { doc.entity.id = id; } - const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force) + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); try { - await this.createEntity(hashedId, readyDoc) + await this.createEntity(hashedId, readyDoc); } catch (error) { if (error.statusCode !== 409) { throw error; @@ -133,26 +132,43 @@ export class CRUDClient { this.logger.debug(`Conflict while creating entity ID ${id}, updating instead`); } - await this.updateEntity(hashedId, entityType, readyDoc) + await this.updateEntity(hashedId, entityType, readyDoc); return; } - public async upsertEntitiesBulk(objects: BulkObject[], force: boolean): Promise { + public async upsertEntitiesBulk( + objects: BulkObject[], + force: boolean + ): Promise { const operations: (BulkOperationContainer | BulkUpdateAction)[] = []; this.logger.debug(`Preparing ${objects.length} entities for bulk upsert`); for (const { type: entityType, doc } of objects) { - const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force) + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); operations.push({ create: {} }, readyDoc); } this.logger.debug(`Bulk upserting ${objects.length} entities`); - await this.esClient.bulk({ + const resp = await this.esClient.bulk({ index: getUpdatesEntitiesDataStreamName(this.namespace), operations, refresh: 'wait_for', }); - return; + + if (!resp.errors) { + this.logger.debug(`Successfully bulk upserted ${objects.length} entities`); + return []; + } + this.logger.debug(`Bulk upserted ${objects.length} entities with errors`); + return resp.items.map((item) => { + const [, value] = Object.entries(item)[0]; + return { + _id: value._id, + status: value.status, + type: value.error?.type, + reason: value.error?.reason, + } as BulkObjectResponse; + }); } public async deleteEntity(id: string): Promise { @@ -228,10 +244,7 @@ function assertOnlyNonForcedAttributesInReq(fields: Record) } } -function transformDocForUpsert( - type: EntityType, - data: Partial -): Record { +function transformDocForUpsert(type: EntityType, data: Partial): Record { const now = new Date().toISOString(); if (type === 'generic') { return { diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts index 2d30ee2561cfc..3718d378f6d9c 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts @@ -58,21 +58,21 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { } try { - await crudClient.upsertEntitiesBulk(req.body.entities, req.query.force); + const errors = await crudClient.upsertEntitiesBulk(req.body.entities, req.query.force); + return res.ok({ + body: { + ok: true, + errors, + }, + }); } catch (error) { if (error instanceof BadCRUDRequestError) { - return res.badRequest({ body: error }); + return res.badRequest({ body: error }); } logger.error(error); throw error; } - - return res.ok({ - body: { - ok: true, - }, - }); }) ); } From fdd4fbc96dbb3441680fa5ca0d3bc2026f021f49 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 15:51:02 +0100 Subject: [PATCH 49/69] crud_api: update tests --- .../plugins/entity_store/test/scout/api/tests/crud_api.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index c561466346400..e71d9fab58d22 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -183,6 +183,7 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = body: bulkBody, }); expect(bulkUpsert.statusCode).toBe(200); + expect(bulkUpsert.body.errors).toHaveLength(0); const resp = await esClient.search({ index: UPDATES_INDEX, From 71f5b722f82eb9c31e31050cae42cc3aeae6ef43 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 20 Feb 2026 16:23:15 +0100 Subject: [PATCH 50/69] update snapshots --- ...logs_extraction_query_builder.test.ts.snap | 456 +++++++++--------- 1 file changed, 228 insertions(+), 228 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap index 0bbbae8ea5cc5..77b72efe1ad00 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap @@ -15,15 +15,15 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e recent.entity.type = LAST(TO_STRING(entity.type), @timestamp) WHERE entity.type IS NOT NULL, recent.entity.sub_type = LAST(TO_STRING(entity.sub_type), @timestamp) WHERE entity.sub_type IS NOT NULL, recent.entity.url = LAST(TO_STRING(entity.url), @timestamp) WHERE entity.url IS NOT NULL, - recent.entity.attributes.Privileged = LAST(TO_BOOLEAN(entity.attributes.Privileged), @timestamp) WHERE entity.attributes.Privileged IS NOT NULL, - recent.entity.attributes.Asset = LAST(TO_BOOLEAN(entity.attributes.Asset), @timestamp) WHERE entity.attributes.Asset IS NOT NULL, - recent.entity.attributes.Managed = LAST(TO_BOOLEAN(entity.attributes.Managed), @timestamp) WHERE entity.attributes.Managed IS NOT NULL, - recent.entity.attributes.Mfa_enabled = LAST(TO_BOOLEAN(entity.attributes.Mfa_enabled), @timestamp) WHERE entity.attributes.Mfa_enabled IS NOT NULL, - recent.entity.lifecycle.First_seen = LAST(TO_DATETIME(entity.lifecycle.First_seen), @timestamp) WHERE entity.lifecycle.First_seen IS NOT NULL, - recent.entity.lifecycle.Last_activity = LAST(TO_DATETIME(entity.lifecycle.Last_activity), @timestamp) WHERE entity.lifecycle.Last_activity IS NOT NULL, - recent.entity.behaviors.Brute_force_victim = LAST(TO_BOOLEAN(entity.behaviors.Brute_force_victim), @timestamp) WHERE entity.behaviors.Brute_force_victim IS NOT NULL, - recent.entity.behaviors.New_country_login = LAST(TO_BOOLEAN(entity.behaviors.New_country_login), @timestamp) WHERE entity.behaviors.New_country_login IS NOT NULL, - recent.entity.behaviors.Used_usb_device = LAST(TO_BOOLEAN(entity.behaviors.Used_usb_device), @timestamp) WHERE entity.behaviors.Used_usb_device IS NOT NULL, + recent.entity.attributes.privileged = LAST(TO_BOOLEAN(entity.attributes.privileged), @timestamp) WHERE entity.attributes.privileged IS NOT NULL, + recent.entity.attributes.asset = LAST(TO_BOOLEAN(entity.attributes.asset), @timestamp) WHERE entity.attributes.asset IS NOT NULL, + recent.entity.attributes.managed = LAST(TO_BOOLEAN(entity.attributes.managed), @timestamp) WHERE entity.attributes.managed IS NOT NULL, + recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(entity.attributes.mfa_enabled), @timestamp) WHERE entity.attributes.mfa_enabled IS NOT NULL, + recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(entity.lifecycle.first_seen), @timestamp) WHERE entity.lifecycle.first_seen IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(entity.lifecycle.last_activity), @timestamp) WHERE entity.lifecycle.last_activity IS NOT NULL, + recent.entity.behaviors.brute_force_victim = LAST(TO_BOOLEAN(entity.behaviors.brute_force_victim), @timestamp) WHERE entity.behaviors.brute_force_victim IS NOT NULL, + recent.entity.behaviors.new_country_login = LAST(TO_BOOLEAN(entity.behaviors.new_country_login), @timestamp) WHERE entity.behaviors.new_country_login IS NOT NULL, + recent.entity.behaviors.used_usb_device = LAST(TO_BOOLEAN(entity.behaviors.used_usb_device), @timestamp) WHERE entity.behaviors.used_usb_device IS NOT NULL, recent.entity.risk.calculated_level = LAST(TO_STRING(entity.risk.calculated_level), @timestamp) WHERE entity.risk.calculated_level IS NOT NULL, recent.entity.risk.calculated_score = LAST(entity.risk.calculated_score, @timestamp) WHERE entity.risk.calculated_score IS NOT NULL, recent.entity.risk.calculated_score_norm = LAST(entity.risk.calculated_score_norm, @timestamp) WHERE entity.risk.calculated_score_norm IS NOT NULL, @@ -103,15 +103,15 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e entity.type = COALESCE(recent.entity.type, entity.type), entity.sub_type = COALESCE(recent.entity.sub_type, entity.sub_type), entity.url = COALESCE(recent.entity.url, entity.url), - entity.attributes.Privileged = COALESCE(recent.entity.attributes.Privileged, entity.attributes.Privileged), - entity.attributes.Asset = COALESCE(recent.entity.attributes.Asset, entity.attributes.Asset), - entity.attributes.Managed = COALESCE(recent.entity.attributes.Managed, entity.attributes.Managed), - entity.attributes.Mfa_enabled = COALESCE(recent.entity.attributes.Mfa_enabled, entity.attributes.Mfa_enabled), - entity.lifecycle.First_seen = COALESCE(recent.entity.lifecycle.First_seen, entity.lifecycle.First_seen), - entity.lifecycle.Last_activity = COALESCE(recent.entity.lifecycle.Last_activity, entity.lifecycle.Last_activity), - entity.behaviors.Brute_force_victim = COALESCE(recent.entity.behaviors.Brute_force_victim, entity.behaviors.Brute_force_victim), - entity.behaviors.New_country_login = COALESCE(recent.entity.behaviors.New_country_login, entity.behaviors.New_country_login), - entity.behaviors.Used_usb_device = COALESCE(recent.entity.behaviors.Used_usb_device, entity.behaviors.Used_usb_device), + entity.attributes.privileged = COALESCE(recent.entity.attributes.privileged, entity.attributes.privileged), + entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), + entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), + entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), + entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), + entity.behaviors.brute_force_victim = COALESCE(recent.entity.behaviors.brute_force_victim, entity.behaviors.brute_force_victim), + entity.behaviors.new_country_login = COALESCE(recent.entity.behaviors.new_country_login, entity.behaviors.new_country_login), + entity.behaviors.used_usb_device = COALESCE(recent.entity.behaviors.used_usb_device, entity.behaviors.used_usb_device), entity.risk.calculated_level = COALESCE(recent.entity.risk.calculated_level, entity.risk.calculated_level), entity.risk.calculated_score = COALESCE(recent.entity.risk.calculated_score, entity.risk.calculated_score), entity.risk.calculated_score_norm = COALESCE(recent.entity.risk.calculated_score_norm, entity.risk.calculated_score_norm), @@ -191,15 +191,15 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e entity.type, entity.sub_type, entity.url, - entity.attributes.Privileged, - entity.attributes.Asset, - entity.attributes.Managed, - entity.attributes.Mfa_enabled, - entity.lifecycle.First_seen, - entity.lifecycle.Last_activity, - entity.behaviors.Brute_force_victim, - entity.behaviors.New_country_login, - entity.behaviors.Used_usb_device, + entity.attributes.privileged, + entity.attributes.asset, + entity.attributes.managed, + entity.attributes.mfa_enabled, + entity.lifecycle.first_seen, + entity.lifecycle.last_activity, + entity.behaviors.brute_force_victim, + entity.behaviors.new_country_login, + entity.behaviors.used_usb_device, entity.risk.calculated_level, entity.risk.calculated_score, entity.risk.calculated_score_norm, @@ -320,23 +320,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti recent.entity.type = LAST(TO_STRING(host.entity.type), @timestamp) WHERE host.entity.type IS NOT NULL, recent.entity.sub_type = LAST(TO_STRING(host.entity.sub_type), @timestamp) WHERE host.entity.sub_type IS NOT NULL, recent.entity.url = LAST(TO_STRING(host.entity.url), @timestamp) WHERE host.entity.url IS NOT NULL, - recent.entity.attributes.Privileged = LAST(TO_BOOLEAN(host.entity.attributes.Privileged), @timestamp) WHERE host.entity.attributes.Privileged IS NOT NULL, - recent.entity.attributes.Asset = LAST(TO_BOOLEAN(host.entity.attributes.Asset), @timestamp) WHERE host.entity.attributes.Asset IS NOT NULL, - recent.entity.attributes.Managed = LAST(TO_BOOLEAN(host.entity.attributes.Managed), @timestamp) WHERE host.entity.attributes.Managed IS NOT NULL, - recent.entity.attributes.Mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.Mfa_enabled), @timestamp) WHERE host.entity.attributes.Mfa_enabled IS NOT NULL, - recent.entity.lifecycle.First_seen = LAST(TO_DATETIME(host.entity.lifecycle.First_seen), @timestamp) WHERE host.entity.lifecycle.First_seen IS NOT NULL, - recent.entity.lifecycle.Last_activity = LAST(TO_DATETIME(host.entity.lifecycle.Last_activity), @timestamp) WHERE host.entity.lifecycle.Last_activity IS NOT NULL, - recent.entity.behaviors.Brute_force_victim = LAST(TO_BOOLEAN(host.entity.behaviors.Brute_force_victim), @timestamp) WHERE host.entity.behaviors.Brute_force_victim IS NOT NULL, - recent.entity.behaviors.New_country_login = LAST(TO_BOOLEAN(host.entity.behaviors.New_country_login), @timestamp) WHERE host.entity.behaviors.New_country_login IS NOT NULL, - recent.entity.behaviors.Used_usb_device = LAST(TO_BOOLEAN(host.entity.behaviors.Used_usb_device), @timestamp) WHERE host.entity.behaviors.Used_usb_device IS NOT NULL, + recent.entity.attributes.privileged = LAST(TO_BOOLEAN(host.entity.attributes.privileged), @timestamp) WHERE host.entity.attributes.privileged IS NOT NULL, + recent.entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, + recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, + recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, + recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.behaviors.brute_force_victim = LAST(TO_BOOLEAN(host.entity.behaviors.brute_force_victim), @timestamp) WHERE host.entity.behaviors.brute_force_victim IS NOT NULL, + recent.entity.behaviors.new_country_login = LAST(TO_BOOLEAN(host.entity.behaviors.new_country_login), @timestamp) WHERE host.entity.behaviors.new_country_login IS NOT NULL, + recent.entity.behaviors.used_usb_device = LAST(TO_BOOLEAN(host.entity.behaviors.used_usb_device), @timestamp) WHERE host.entity.behaviors.used_usb_device IS NOT NULL, recent.entity.risk.calculated_level = LAST(TO_STRING(host.entity.risk.calculated_level), @timestamp) WHERE host.entity.risk.calculated_level IS NOT NULL, recent.entity.risk.calculated_score = LAST(host.entity.risk.calculated_score, @timestamp) WHERE host.entity.risk.calculated_score IS NOT NULL, recent.entity.risk.calculated_score_norm = LAST(host.entity.risk.calculated_score_norm, @timestamp) WHERE host.entity.risk.calculated_score_norm IS NOT NULL, - recent.entity.relationships.Communicates_with = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Communicates_with)), 10) WHERE host.entity.relationships.Communicates_with IS NOT NULL, - recent.entity.relationships.Depends_on = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Depends_on)), 10) WHERE host.entity.relationships.Depends_on IS NOT NULL, - recent.entity.relationships.Dependent_of = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Dependent_of)), 10) WHERE host.entity.relationships.Dependent_of IS NOT NULL, - recent.entity.relationships.Owned_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Owned_by)), 10) WHERE host.entity.relationships.Owned_by IS NOT NULL, - recent.entity.relationships.Accessed_frequently_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Accessed_frequently_by)), 10) WHERE host.entity.relationships.Accessed_frequently_by IS NOT NULL + recent.entity.relationships.communicates_with = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.communicates_with)), 10) WHERE host.entity.relationships.communicates_with IS NOT NULL, + recent.entity.relationships.depends_on = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.depends_on)), 10) WHERE host.entity.relationships.depends_on IS NOT NULL, + recent.entity.relationships.dependent_of = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.dependent_of)), 10) WHERE host.entity.relationships.dependent_of IS NOT NULL, + recent.entity.relationships.owned_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.owned_by)), 10) WHERE host.entity.relationships.owned_by IS NOT NULL, + recent.entity.relationships.accessed_frequently_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.accessed_frequently_by)), 10) WHERE host.entity.relationships.accessed_frequently_by IS NOT NULL BY recent.entity.EngineMetadata.UntypedId | SORT entity.EngineMetadata.FirstSeenLogInPage ASC, recent.entity.EngineMetadata.UntypedId ASC @@ -373,23 +373,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti entity.type = COALESCE(recent.entity.type, entity.type), entity.sub_type = COALESCE(recent.entity.sub_type, entity.sub_type), entity.url = COALESCE(recent.entity.url, entity.url), - entity.attributes.Privileged = COALESCE(recent.entity.attributes.Privileged, entity.attributes.Privileged), - entity.attributes.Asset = COALESCE(recent.entity.attributes.Asset, entity.attributes.Asset), - entity.attributes.Managed = COALESCE(recent.entity.attributes.Managed, entity.attributes.Managed), - entity.attributes.Mfa_enabled = COALESCE(recent.entity.attributes.Mfa_enabled, entity.attributes.Mfa_enabled), - entity.lifecycle.First_seen = COALESCE(recent.entity.lifecycle.First_seen, entity.lifecycle.First_seen), - entity.lifecycle.Last_activity = COALESCE(recent.entity.lifecycle.Last_activity, entity.lifecycle.Last_activity), - entity.behaviors.Brute_force_victim = COALESCE(recent.entity.behaviors.Brute_force_victim, entity.behaviors.Brute_force_victim), - entity.behaviors.New_country_login = COALESCE(recent.entity.behaviors.New_country_login, entity.behaviors.New_country_login), - entity.behaviors.Used_usb_device = COALESCE(recent.entity.behaviors.Used_usb_device, entity.behaviors.Used_usb_device), + entity.attributes.privileged = COALESCE(recent.entity.attributes.privileged, entity.attributes.privileged), + entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), + entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), + entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), + entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), + entity.behaviors.brute_force_victim = COALESCE(recent.entity.behaviors.brute_force_victim, entity.behaviors.brute_force_victim), + entity.behaviors.new_country_login = COALESCE(recent.entity.behaviors.new_country_login, entity.behaviors.new_country_login), + entity.behaviors.used_usb_device = COALESCE(recent.entity.behaviors.used_usb_device, entity.behaviors.used_usb_device), entity.risk.calculated_level = COALESCE(recent.entity.risk.calculated_level, entity.risk.calculated_level), entity.risk.calculated_score = COALESCE(recent.entity.risk.calculated_score, entity.risk.calculated_score), entity.risk.calculated_score_norm = COALESCE(recent.entity.risk.calculated_score_norm, entity.risk.calculated_score_norm), - entity.relationships.Communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.Communicates_with, entity.relationships.Communicates_with), 0, 9), - entity.relationships.Depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.Depends_on, entity.relationships.Depends_on), 0, 9), - entity.relationships.Dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.Dependent_of, entity.relationships.Dependent_of), 0, 9), - entity.relationships.Owned_by = MV_SLICE(MV_UNION(recent.entity.relationships.Owned_by, entity.relationships.Owned_by), 0, 9), - entity.relationships.Accessed_frequently_by = MV_SLICE(MV_UNION(recent.entity.relationships.Accessed_frequently_by, entity.relationships.Accessed_frequently_by), 0, 9) + entity.relationships.communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.communicates_with, entity.relationships.communicates_with), 0, 9), + entity.relationships.depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.depends_on, entity.relationships.depends_on), 0, 9), + entity.relationships.dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.dependent_of, entity.relationships.dependent_of), 0, 9), + entity.relationships.owned_by = MV_SLICE(MV_UNION(recent.entity.relationships.owned_by, entity.relationships.owned_by), 0, 9), + entity.relationships.accessed_frequently_by = MV_SLICE(MV_UNION(recent.entity.relationships.accessed_frequently_by, entity.relationships.accessed_frequently_by), 0, 9) | EVAL @timestamp = recent.timestamp, entity.name = CASE(entity.name IS NOT NULL AND entity.name != \\"\\", entity.name, recent.entity.EngineMetadata.UntypedId), entity.EngineMetadata.Type = \\"host\\", @@ -427,23 +427,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti entity.type, entity.sub_type, entity.url, - entity.attributes.Privileged, - entity.attributes.Asset, - entity.attributes.Managed, - entity.attributes.Mfa_enabled, - entity.lifecycle.First_seen, - entity.lifecycle.Last_activity, - entity.behaviors.Brute_force_victim, - entity.behaviors.New_country_login, - entity.behaviors.Used_usb_device, + entity.attributes.privileged, + entity.attributes.asset, + entity.attributes.managed, + entity.attributes.mfa_enabled, + entity.lifecycle.first_seen, + entity.lifecycle.last_activity, + entity.behaviors.brute_force_victim, + entity.behaviors.new_country_login, + entity.behaviors.used_usb_device, entity.risk.calculated_level, entity.risk.calculated_score, entity.risk.calculated_score_norm, - entity.relationships.Communicates_with, - entity.relationships.Depends_on, - entity.relationships.Dependent_of, - entity.relationships.Owned_by, - entity.relationships.Accessed_frequently_by, + entity.relationships.communicates_with, + entity.relationships.depends_on, + entity.relationships.dependent_of, + entity.relationships.owned_by, + entity.relationships.accessed_frequently_by, @timestamp, entity.id, entity.name, @@ -497,23 +497,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with recent.entity.type = LAST(TO_STRING(host.entity.type), @timestamp) WHERE host.entity.type IS NOT NULL, recent.entity.sub_type = LAST(TO_STRING(host.entity.sub_type), @timestamp) WHERE host.entity.sub_type IS NOT NULL, recent.entity.url = LAST(TO_STRING(host.entity.url), @timestamp) WHERE host.entity.url IS NOT NULL, - recent.entity.attributes.Privileged = LAST(TO_BOOLEAN(host.entity.attributes.Privileged), @timestamp) WHERE host.entity.attributes.Privileged IS NOT NULL, - recent.entity.attributes.Asset = LAST(TO_BOOLEAN(host.entity.attributes.Asset), @timestamp) WHERE host.entity.attributes.Asset IS NOT NULL, - recent.entity.attributes.Managed = LAST(TO_BOOLEAN(host.entity.attributes.Managed), @timestamp) WHERE host.entity.attributes.Managed IS NOT NULL, - recent.entity.attributes.Mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.Mfa_enabled), @timestamp) WHERE host.entity.attributes.Mfa_enabled IS NOT NULL, - recent.entity.lifecycle.First_seen = LAST(TO_DATETIME(host.entity.lifecycle.First_seen), @timestamp) WHERE host.entity.lifecycle.First_seen IS NOT NULL, - recent.entity.lifecycle.Last_activity = LAST(TO_DATETIME(host.entity.lifecycle.Last_activity), @timestamp) WHERE host.entity.lifecycle.Last_activity IS NOT NULL, - recent.entity.behaviors.Brute_force_victim = LAST(TO_BOOLEAN(host.entity.behaviors.Brute_force_victim), @timestamp) WHERE host.entity.behaviors.Brute_force_victim IS NOT NULL, - recent.entity.behaviors.New_country_login = LAST(TO_BOOLEAN(host.entity.behaviors.New_country_login), @timestamp) WHERE host.entity.behaviors.New_country_login IS NOT NULL, - recent.entity.behaviors.Used_usb_device = LAST(TO_BOOLEAN(host.entity.behaviors.Used_usb_device), @timestamp) WHERE host.entity.behaviors.Used_usb_device IS NOT NULL, + recent.entity.attributes.privileged = LAST(TO_BOOLEAN(host.entity.attributes.privileged), @timestamp) WHERE host.entity.attributes.privileged IS NOT NULL, + recent.entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, + recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, + recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, + recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.behaviors.brute_force_victim = LAST(TO_BOOLEAN(host.entity.behaviors.brute_force_victim), @timestamp) WHERE host.entity.behaviors.brute_force_victim IS NOT NULL, + recent.entity.behaviors.new_country_login = LAST(TO_BOOLEAN(host.entity.behaviors.new_country_login), @timestamp) WHERE host.entity.behaviors.new_country_login IS NOT NULL, + recent.entity.behaviors.used_usb_device = LAST(TO_BOOLEAN(host.entity.behaviors.used_usb_device), @timestamp) WHERE host.entity.behaviors.used_usb_device IS NOT NULL, recent.entity.risk.calculated_level = LAST(TO_STRING(host.entity.risk.calculated_level), @timestamp) WHERE host.entity.risk.calculated_level IS NOT NULL, recent.entity.risk.calculated_score = LAST(host.entity.risk.calculated_score, @timestamp) WHERE host.entity.risk.calculated_score IS NOT NULL, recent.entity.risk.calculated_score_norm = LAST(host.entity.risk.calculated_score_norm, @timestamp) WHERE host.entity.risk.calculated_score_norm IS NOT NULL, - recent.entity.relationships.Communicates_with = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Communicates_with)), 10) WHERE host.entity.relationships.Communicates_with IS NOT NULL, - recent.entity.relationships.Depends_on = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Depends_on)), 10) WHERE host.entity.relationships.Depends_on IS NOT NULL, - recent.entity.relationships.Dependent_of = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Dependent_of)), 10) WHERE host.entity.relationships.Dependent_of IS NOT NULL, - recent.entity.relationships.Owned_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Owned_by)), 10) WHERE host.entity.relationships.Owned_by IS NOT NULL, - recent.entity.relationships.Accessed_frequently_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Accessed_frequently_by)), 10) WHERE host.entity.relationships.Accessed_frequently_by IS NOT NULL + recent.entity.relationships.communicates_with = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.communicates_with)), 10) WHERE host.entity.relationships.communicates_with IS NOT NULL, + recent.entity.relationships.depends_on = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.depends_on)), 10) WHERE host.entity.relationships.depends_on IS NOT NULL, + recent.entity.relationships.dependent_of = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.dependent_of)), 10) WHERE host.entity.relationships.dependent_of IS NOT NULL, + recent.entity.relationships.owned_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.owned_by)), 10) WHERE host.entity.relationships.owned_by IS NOT NULL, + recent.entity.relationships.accessed_frequently_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.accessed_frequently_by)), 10) WHERE host.entity.relationships.accessed_frequently_by IS NOT NULL BY recent.entity.EngineMetadata.UntypedId | SORT entity.EngineMetadata.FirstSeenLogInPage ASC, recent.entity.EngineMetadata.UntypedId ASC | WHERE entity.EngineMetadata.FirstSeenLogInPage > TO_DATETIME(\\"2022-01-01T00:00:00.000Z\\") @@ -552,23 +552,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.type = COALESCE(recent.entity.type, entity.type), entity.sub_type = COALESCE(recent.entity.sub_type, entity.sub_type), entity.url = COALESCE(recent.entity.url, entity.url), - entity.attributes.Privileged = COALESCE(recent.entity.attributes.Privileged, entity.attributes.Privileged), - entity.attributes.Asset = COALESCE(recent.entity.attributes.Asset, entity.attributes.Asset), - entity.attributes.Managed = COALESCE(recent.entity.attributes.Managed, entity.attributes.Managed), - entity.attributes.Mfa_enabled = COALESCE(recent.entity.attributes.Mfa_enabled, entity.attributes.Mfa_enabled), - entity.lifecycle.First_seen = COALESCE(recent.entity.lifecycle.First_seen, entity.lifecycle.First_seen), - entity.lifecycle.Last_activity = COALESCE(recent.entity.lifecycle.Last_activity, entity.lifecycle.Last_activity), - entity.behaviors.Brute_force_victim = COALESCE(recent.entity.behaviors.Brute_force_victim, entity.behaviors.Brute_force_victim), - entity.behaviors.New_country_login = COALESCE(recent.entity.behaviors.New_country_login, entity.behaviors.New_country_login), - entity.behaviors.Used_usb_device = COALESCE(recent.entity.behaviors.Used_usb_device, entity.behaviors.Used_usb_device), + entity.attributes.privileged = COALESCE(recent.entity.attributes.privileged, entity.attributes.privileged), + entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), + entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), + entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), + entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), + entity.behaviors.brute_force_victim = COALESCE(recent.entity.behaviors.brute_force_victim, entity.behaviors.brute_force_victim), + entity.behaviors.new_country_login = COALESCE(recent.entity.behaviors.new_country_login, entity.behaviors.new_country_login), + entity.behaviors.used_usb_device = COALESCE(recent.entity.behaviors.used_usb_device, entity.behaviors.used_usb_device), entity.risk.calculated_level = COALESCE(recent.entity.risk.calculated_level, entity.risk.calculated_level), entity.risk.calculated_score = COALESCE(recent.entity.risk.calculated_score, entity.risk.calculated_score), entity.risk.calculated_score_norm = COALESCE(recent.entity.risk.calculated_score_norm, entity.risk.calculated_score_norm), - entity.relationships.Communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.Communicates_with, entity.relationships.Communicates_with), 0, 9), - entity.relationships.Depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.Depends_on, entity.relationships.Depends_on), 0, 9), - entity.relationships.Dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.Dependent_of, entity.relationships.Dependent_of), 0, 9), - entity.relationships.Owned_by = MV_SLICE(MV_UNION(recent.entity.relationships.Owned_by, entity.relationships.Owned_by), 0, 9), - entity.relationships.Accessed_frequently_by = MV_SLICE(MV_UNION(recent.entity.relationships.Accessed_frequently_by, entity.relationships.Accessed_frequently_by), 0, 9) + entity.relationships.communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.communicates_with, entity.relationships.communicates_with), 0, 9), + entity.relationships.depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.depends_on, entity.relationships.depends_on), 0, 9), + entity.relationships.dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.dependent_of, entity.relationships.dependent_of), 0, 9), + entity.relationships.owned_by = MV_SLICE(MV_UNION(recent.entity.relationships.owned_by, entity.relationships.owned_by), 0, 9), + entity.relationships.accessed_frequently_by = MV_SLICE(MV_UNION(recent.entity.relationships.accessed_frequently_by, entity.relationships.accessed_frequently_by), 0, 9) | EVAL @timestamp = recent.timestamp, entity.name = CASE(entity.name IS NOT NULL AND entity.name != \\"\\", entity.name, recent.entity.EngineMetadata.UntypedId), entity.EngineMetadata.Type = \\"host\\", @@ -606,23 +606,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.type, entity.sub_type, entity.url, - entity.attributes.Privileged, - entity.attributes.Asset, - entity.attributes.Managed, - entity.attributes.Mfa_enabled, - entity.lifecycle.First_seen, - entity.lifecycle.Last_activity, - entity.behaviors.Brute_force_victim, - entity.behaviors.New_country_login, - entity.behaviors.Used_usb_device, + entity.attributes.privileged, + entity.attributes.asset, + entity.attributes.managed, + entity.attributes.mfa_enabled, + entity.lifecycle.first_seen, + entity.lifecycle.last_activity, + entity.behaviors.brute_force_victim, + entity.behaviors.new_country_login, + entity.behaviors.used_usb_device, entity.risk.calculated_level, entity.risk.calculated_score, entity.risk.calculated_score_norm, - entity.relationships.Communicates_with, - entity.relationships.Depends_on, - entity.relationships.Dependent_of, - entity.relationships.Owned_by, - entity.relationships.Accessed_frequently_by, + entity.relationships.communicates_with, + entity.relationships.depends_on, + entity.relationships.dependent_of, + entity.relationships.owned_by, + entity.relationships.accessed_frequently_by, @timestamp, entity.id, entity.name, @@ -676,23 +676,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with recent.entity.type = LAST(TO_STRING(host.entity.type), @timestamp) WHERE host.entity.type IS NOT NULL, recent.entity.sub_type = LAST(TO_STRING(host.entity.sub_type), @timestamp) WHERE host.entity.sub_type IS NOT NULL, recent.entity.url = LAST(TO_STRING(host.entity.url), @timestamp) WHERE host.entity.url IS NOT NULL, - recent.entity.attributes.Privileged = LAST(TO_BOOLEAN(host.entity.attributes.Privileged), @timestamp) WHERE host.entity.attributes.Privileged IS NOT NULL, - recent.entity.attributes.Asset = LAST(TO_BOOLEAN(host.entity.attributes.Asset), @timestamp) WHERE host.entity.attributes.Asset IS NOT NULL, - recent.entity.attributes.Managed = LAST(TO_BOOLEAN(host.entity.attributes.Managed), @timestamp) WHERE host.entity.attributes.Managed IS NOT NULL, - recent.entity.attributes.Mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.Mfa_enabled), @timestamp) WHERE host.entity.attributes.Mfa_enabled IS NOT NULL, - recent.entity.lifecycle.First_seen = LAST(TO_DATETIME(host.entity.lifecycle.First_seen), @timestamp) WHERE host.entity.lifecycle.First_seen IS NOT NULL, - recent.entity.lifecycle.Last_activity = LAST(TO_DATETIME(host.entity.lifecycle.Last_activity), @timestamp) WHERE host.entity.lifecycle.Last_activity IS NOT NULL, - recent.entity.behaviors.Brute_force_victim = LAST(TO_BOOLEAN(host.entity.behaviors.Brute_force_victim), @timestamp) WHERE host.entity.behaviors.Brute_force_victim IS NOT NULL, - recent.entity.behaviors.New_country_login = LAST(TO_BOOLEAN(host.entity.behaviors.New_country_login), @timestamp) WHERE host.entity.behaviors.New_country_login IS NOT NULL, - recent.entity.behaviors.Used_usb_device = LAST(TO_BOOLEAN(host.entity.behaviors.Used_usb_device), @timestamp) WHERE host.entity.behaviors.Used_usb_device IS NOT NULL, + recent.entity.attributes.privileged = LAST(TO_BOOLEAN(host.entity.attributes.privileged), @timestamp) WHERE host.entity.attributes.privileged IS NOT NULL, + recent.entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, + recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, + recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, + recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.behaviors.brute_force_victim = LAST(TO_BOOLEAN(host.entity.behaviors.brute_force_victim), @timestamp) WHERE host.entity.behaviors.brute_force_victim IS NOT NULL, + recent.entity.behaviors.new_country_login = LAST(TO_BOOLEAN(host.entity.behaviors.new_country_login), @timestamp) WHERE host.entity.behaviors.new_country_login IS NOT NULL, + recent.entity.behaviors.used_usb_device = LAST(TO_BOOLEAN(host.entity.behaviors.used_usb_device), @timestamp) WHERE host.entity.behaviors.used_usb_device IS NOT NULL, recent.entity.risk.calculated_level = LAST(TO_STRING(host.entity.risk.calculated_level), @timestamp) WHERE host.entity.risk.calculated_level IS NOT NULL, recent.entity.risk.calculated_score = LAST(host.entity.risk.calculated_score, @timestamp) WHERE host.entity.risk.calculated_score IS NOT NULL, recent.entity.risk.calculated_score_norm = LAST(host.entity.risk.calculated_score_norm, @timestamp) WHERE host.entity.risk.calculated_score_norm IS NOT NULL, - recent.entity.relationships.Communicates_with = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Communicates_with)), 10) WHERE host.entity.relationships.Communicates_with IS NOT NULL, - recent.entity.relationships.Depends_on = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Depends_on)), 10) WHERE host.entity.relationships.Depends_on IS NOT NULL, - recent.entity.relationships.Dependent_of = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Dependent_of)), 10) WHERE host.entity.relationships.Dependent_of IS NOT NULL, - recent.entity.relationships.Owned_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Owned_by)), 10) WHERE host.entity.relationships.Owned_by IS NOT NULL, - recent.entity.relationships.Accessed_frequently_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.Accessed_frequently_by)), 10) WHERE host.entity.relationships.Accessed_frequently_by IS NOT NULL + recent.entity.relationships.communicates_with = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.communicates_with)), 10) WHERE host.entity.relationships.communicates_with IS NOT NULL, + recent.entity.relationships.depends_on = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.depends_on)), 10) WHERE host.entity.relationships.depends_on IS NOT NULL, + recent.entity.relationships.dependent_of = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.dependent_of)), 10) WHERE host.entity.relationships.dependent_of IS NOT NULL, + recent.entity.relationships.owned_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.owned_by)), 10) WHERE host.entity.relationships.owned_by IS NOT NULL, + recent.entity.relationships.accessed_frequently_by = TOP(MV_DEDUPE(TO_STRING(host.entity.relationships.accessed_frequently_by)), 10) WHERE host.entity.relationships.accessed_frequently_by IS NOT NULL BY recent.entity.EngineMetadata.UntypedId | SORT entity.EngineMetadata.FirstSeenLogInPage ASC, recent.entity.EngineMetadata.UntypedId ASC | WHERE entity.EngineMetadata.FirstSeenLogInPage > TO_DATETIME(\\"2022-01-01T00:00:00.000Z\\") @@ -731,23 +731,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.type = COALESCE(recent.entity.type, entity.type), entity.sub_type = COALESCE(recent.entity.sub_type, entity.sub_type), entity.url = COALESCE(recent.entity.url, entity.url), - entity.attributes.Privileged = COALESCE(recent.entity.attributes.Privileged, entity.attributes.Privileged), - entity.attributes.Asset = COALESCE(recent.entity.attributes.Asset, entity.attributes.Asset), - entity.attributes.Managed = COALESCE(recent.entity.attributes.Managed, entity.attributes.Managed), - entity.attributes.Mfa_enabled = COALESCE(recent.entity.attributes.Mfa_enabled, entity.attributes.Mfa_enabled), - entity.lifecycle.First_seen = COALESCE(recent.entity.lifecycle.First_seen, entity.lifecycle.First_seen), - entity.lifecycle.Last_activity = COALESCE(recent.entity.lifecycle.Last_activity, entity.lifecycle.Last_activity), - entity.behaviors.Brute_force_victim = COALESCE(recent.entity.behaviors.Brute_force_victim, entity.behaviors.Brute_force_victim), - entity.behaviors.New_country_login = COALESCE(recent.entity.behaviors.New_country_login, entity.behaviors.New_country_login), - entity.behaviors.Used_usb_device = COALESCE(recent.entity.behaviors.Used_usb_device, entity.behaviors.Used_usb_device), + entity.attributes.privileged = COALESCE(recent.entity.attributes.privileged, entity.attributes.privileged), + entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), + entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), + entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), + entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), + entity.behaviors.brute_force_victim = COALESCE(recent.entity.behaviors.brute_force_victim, entity.behaviors.brute_force_victim), + entity.behaviors.new_country_login = COALESCE(recent.entity.behaviors.new_country_login, entity.behaviors.new_country_login), + entity.behaviors.used_usb_device = COALESCE(recent.entity.behaviors.used_usb_device, entity.behaviors.used_usb_device), entity.risk.calculated_level = COALESCE(recent.entity.risk.calculated_level, entity.risk.calculated_level), entity.risk.calculated_score = COALESCE(recent.entity.risk.calculated_score, entity.risk.calculated_score), entity.risk.calculated_score_norm = COALESCE(recent.entity.risk.calculated_score_norm, entity.risk.calculated_score_norm), - entity.relationships.Communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.Communicates_with, entity.relationships.Communicates_with), 0, 9), - entity.relationships.Depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.Depends_on, entity.relationships.Depends_on), 0, 9), - entity.relationships.Dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.Dependent_of, entity.relationships.Dependent_of), 0, 9), - entity.relationships.Owned_by = MV_SLICE(MV_UNION(recent.entity.relationships.Owned_by, entity.relationships.Owned_by), 0, 9), - entity.relationships.Accessed_frequently_by = MV_SLICE(MV_UNION(recent.entity.relationships.Accessed_frequently_by, entity.relationships.Accessed_frequently_by), 0, 9) + entity.relationships.communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.communicates_with, entity.relationships.communicates_with), 0, 9), + entity.relationships.depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.depends_on, entity.relationships.depends_on), 0, 9), + entity.relationships.dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.dependent_of, entity.relationships.dependent_of), 0, 9), + entity.relationships.owned_by = MV_SLICE(MV_UNION(recent.entity.relationships.owned_by, entity.relationships.owned_by), 0, 9), + entity.relationships.accessed_frequently_by = MV_SLICE(MV_UNION(recent.entity.relationships.accessed_frequently_by, entity.relationships.accessed_frequently_by), 0, 9) | EVAL @timestamp = recent.timestamp, entity.name = CASE(entity.name IS NOT NULL AND entity.name != \\"\\", entity.name, recent.entity.EngineMetadata.UntypedId), entity.EngineMetadata.Type = \\"host\\", @@ -785,23 +785,23 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.type, entity.sub_type, entity.url, - entity.attributes.Privileged, - entity.attributes.Asset, - entity.attributes.Managed, - entity.attributes.Mfa_enabled, - entity.lifecycle.First_seen, - entity.lifecycle.Last_activity, - entity.behaviors.Brute_force_victim, - entity.behaviors.New_country_login, - entity.behaviors.Used_usb_device, + entity.attributes.privileged, + entity.attributes.asset, + entity.attributes.managed, + entity.attributes.mfa_enabled, + entity.lifecycle.first_seen, + entity.lifecycle.last_activity, + entity.behaviors.brute_force_victim, + entity.behaviors.new_country_login, + entity.behaviors.used_usb_device, entity.risk.calculated_level, entity.risk.calculated_score, entity.risk.calculated_score_norm, - entity.relationships.Communicates_with, - entity.relationships.Depends_on, - entity.relationships.Dependent_of, - entity.relationships.Owned_by, - entity.relationships.Accessed_frequently_by, + entity.relationships.communicates_with, + entity.relationships.depends_on, + entity.relationships.dependent_of, + entity.relationships.owned_by, + entity.relationships.accessed_frequently_by, @timestamp, entity.id, entity.name, @@ -852,21 +852,21 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e recent.entity.type = LAST(TO_STRING(service.entity.type), @timestamp) WHERE service.entity.type IS NOT NULL, recent.entity.sub_type = LAST(TO_STRING(service.entity.sub_type), @timestamp) WHERE service.entity.sub_type IS NOT NULL, recent.entity.url = LAST(TO_STRING(service.entity.url), @timestamp) WHERE service.entity.url IS NOT NULL, - recent.entity.attributes.Privileged = LAST(TO_BOOLEAN(service.entity.attributes.Privileged), @timestamp) WHERE service.entity.attributes.Privileged IS NOT NULL, - recent.entity.attributes.Asset = LAST(TO_BOOLEAN(service.entity.attributes.Asset), @timestamp) WHERE service.entity.attributes.Asset IS NOT NULL, - recent.entity.attributes.Managed = LAST(TO_BOOLEAN(service.entity.attributes.Managed), @timestamp) WHERE service.entity.attributes.Managed IS NOT NULL, - recent.entity.attributes.Mfa_enabled = LAST(TO_BOOLEAN(service.entity.attributes.Mfa_enabled), @timestamp) WHERE service.entity.attributes.Mfa_enabled IS NOT NULL, - recent.entity.lifecycle.First_seen = LAST(TO_DATETIME(service.entity.lifecycle.First_seen), @timestamp) WHERE service.entity.lifecycle.First_seen IS NOT NULL, - recent.entity.lifecycle.Last_activity = LAST(TO_DATETIME(service.entity.lifecycle.Last_activity), @timestamp) WHERE service.entity.lifecycle.Last_activity IS NOT NULL, - recent.entity.behaviors.Brute_force_victim = LAST(TO_BOOLEAN(service.entity.behaviors.Brute_force_victim), @timestamp) WHERE service.entity.behaviors.Brute_force_victim IS NOT NULL, - recent.entity.behaviors.New_country_login = LAST(TO_BOOLEAN(service.entity.behaviors.New_country_login), @timestamp) WHERE service.entity.behaviors.New_country_login IS NOT NULL, - recent.entity.behaviors.Used_usb_device = LAST(TO_BOOLEAN(service.entity.behaviors.Used_usb_device), @timestamp) WHERE service.entity.behaviors.Used_usb_device IS NOT NULL, + recent.entity.attributes.privileged = LAST(TO_BOOLEAN(service.entity.attributes.privileged), @timestamp) WHERE service.entity.attributes.privileged IS NOT NULL, + recent.entity.attributes.asset = LAST(TO_BOOLEAN(service.entity.attributes.asset), @timestamp) WHERE service.entity.attributes.asset IS NOT NULL, + recent.entity.attributes.managed = LAST(TO_BOOLEAN(service.entity.attributes.managed), @timestamp) WHERE service.entity.attributes.managed IS NOT NULL, + recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(service.entity.attributes.mfa_enabled), @timestamp) WHERE service.entity.attributes.mfa_enabled IS NOT NULL, + recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(service.entity.lifecycle.first_seen), @timestamp) WHERE service.entity.lifecycle.first_seen IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(service.entity.lifecycle.last_activity), @timestamp) WHERE service.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.behaviors.brute_force_victim = LAST(TO_BOOLEAN(service.entity.behaviors.brute_force_victim), @timestamp) WHERE service.entity.behaviors.brute_force_victim IS NOT NULL, + recent.entity.behaviors.new_country_login = LAST(TO_BOOLEAN(service.entity.behaviors.new_country_login), @timestamp) WHERE service.entity.behaviors.new_country_login IS NOT NULL, + recent.entity.behaviors.used_usb_device = LAST(TO_BOOLEAN(service.entity.behaviors.used_usb_device), @timestamp) WHERE service.entity.behaviors.used_usb_device IS NOT NULL, recent.entity.risk.calculated_level = LAST(TO_STRING(service.entity.risk.calculated_level), @timestamp) WHERE service.entity.risk.calculated_level IS NOT NULL, recent.entity.risk.calculated_score = LAST(service.entity.risk.calculated_score, @timestamp) WHERE service.entity.risk.calculated_score IS NOT NULL, recent.entity.risk.calculated_score_norm = LAST(service.entity.risk.calculated_score_norm, @timestamp) WHERE service.entity.risk.calculated_score_norm IS NOT NULL, - recent.entity.relationships.Communicates_with = TOP(MV_DEDUPE(TO_STRING(service.entity.relationships.Communicates_with)), 10) WHERE service.entity.relationships.Communicates_with IS NOT NULL, - recent.entity.relationships.Depends_on = TOP(MV_DEDUPE(TO_STRING(service.entity.relationships.Depends_on)), 10) WHERE service.entity.relationships.Depends_on IS NOT NULL, - recent.entity.relationships.Dependent_of = TOP(MV_DEDUPE(TO_STRING(service.entity.relationships.Dependent_of)), 10) WHERE service.entity.relationships.Dependent_of IS NOT NULL + recent.entity.relationships.communicates_with = TOP(MV_DEDUPE(TO_STRING(service.entity.relationships.communicates_with)), 10) WHERE service.entity.relationships.communicates_with IS NOT NULL, + recent.entity.relationships.depends_on = TOP(MV_DEDUPE(TO_STRING(service.entity.relationships.depends_on)), 10) WHERE service.entity.relationships.depends_on IS NOT NULL, + recent.entity.relationships.dependent_of = TOP(MV_DEDUPE(TO_STRING(service.entity.relationships.dependent_of)), 10) WHERE service.entity.relationships.dependent_of IS NOT NULL BY recent.entity.EngineMetadata.UntypedId | SORT entity.EngineMetadata.FirstSeenLogInPage ASC, recent.entity.EngineMetadata.UntypedId ASC @@ -904,21 +904,21 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e entity.type = COALESCE(recent.entity.type, entity.type), entity.sub_type = COALESCE(recent.entity.sub_type, entity.sub_type), entity.url = COALESCE(recent.entity.url, entity.url), - entity.attributes.Privileged = COALESCE(recent.entity.attributes.Privileged, entity.attributes.Privileged), - entity.attributes.Asset = COALESCE(recent.entity.attributes.Asset, entity.attributes.Asset), - entity.attributes.Managed = COALESCE(recent.entity.attributes.Managed, entity.attributes.Managed), - entity.attributes.Mfa_enabled = COALESCE(recent.entity.attributes.Mfa_enabled, entity.attributes.Mfa_enabled), - entity.lifecycle.First_seen = COALESCE(recent.entity.lifecycle.First_seen, entity.lifecycle.First_seen), - entity.lifecycle.Last_activity = COALESCE(recent.entity.lifecycle.Last_activity, entity.lifecycle.Last_activity), - entity.behaviors.Brute_force_victim = COALESCE(recent.entity.behaviors.Brute_force_victim, entity.behaviors.Brute_force_victim), - entity.behaviors.New_country_login = COALESCE(recent.entity.behaviors.New_country_login, entity.behaviors.New_country_login), - entity.behaviors.Used_usb_device = COALESCE(recent.entity.behaviors.Used_usb_device, entity.behaviors.Used_usb_device), + entity.attributes.privileged = COALESCE(recent.entity.attributes.privileged, entity.attributes.privileged), + entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), + entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), + entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), + entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), + entity.behaviors.brute_force_victim = COALESCE(recent.entity.behaviors.brute_force_victim, entity.behaviors.brute_force_victim), + entity.behaviors.new_country_login = COALESCE(recent.entity.behaviors.new_country_login, entity.behaviors.new_country_login), + entity.behaviors.used_usb_device = COALESCE(recent.entity.behaviors.used_usb_device, entity.behaviors.used_usb_device), entity.risk.calculated_level = COALESCE(recent.entity.risk.calculated_level, entity.risk.calculated_level), entity.risk.calculated_score = COALESCE(recent.entity.risk.calculated_score, entity.risk.calculated_score), entity.risk.calculated_score_norm = COALESCE(recent.entity.risk.calculated_score_norm, entity.risk.calculated_score_norm), - entity.relationships.Communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.Communicates_with, entity.relationships.Communicates_with), 0, 9), - entity.relationships.Depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.Depends_on, entity.relationships.Depends_on), 0, 9), - entity.relationships.Dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.Dependent_of, entity.relationships.Dependent_of), 0, 9) + entity.relationships.communicates_with = MV_SLICE(MV_UNION(recent.entity.relationships.communicates_with, entity.relationships.communicates_with), 0, 9), + entity.relationships.depends_on = MV_SLICE(MV_UNION(recent.entity.relationships.depends_on, entity.relationships.depends_on), 0, 9), + entity.relationships.dependent_of = MV_SLICE(MV_UNION(recent.entity.relationships.dependent_of, entity.relationships.dependent_of), 0, 9) | EVAL @timestamp = recent.timestamp, entity.name = CASE(entity.name IS NOT NULL AND entity.name != \\"\\", entity.name, recent.entity.EngineMetadata.UntypedId), entity.EngineMetadata.Type = \\"service\\", @@ -957,21 +957,21 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e entity.type, entity.sub_type, entity.url, - entity.attributes.Privileged, - entity.attributes.Asset, - entity.attributes.Managed, - entity.attributes.Mfa_enabled, - entity.lifecycle.First_seen, - entity.lifecycle.Last_activity, - entity.behaviors.Brute_force_victim, - entity.behaviors.New_country_login, - entity.behaviors.Used_usb_device, + entity.attributes.privileged, + entity.attributes.asset, + entity.attributes.managed, + entity.attributes.mfa_enabled, + entity.lifecycle.first_seen, + entity.lifecycle.last_activity, + entity.behaviors.brute_force_victim, + entity.behaviors.new_country_login, + entity.behaviors.used_usb_device, entity.risk.calculated_level, entity.risk.calculated_score, entity.risk.calculated_score_norm, - entity.relationships.Communicates_with, - entity.relationships.Depends_on, - entity.relationships.Dependent_of, + entity.relationships.communicates_with, + entity.relationships.depends_on, + entity.relationships.dependent_of, @timestamp, entity.id, entity.name, @@ -1024,25 +1024,25 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti recent.entity.type = LAST(TO_STRING(user.entity.type), @timestamp) WHERE user.entity.type IS NOT NULL, recent.entity.sub_type = LAST(TO_STRING(user.entity.sub_type), @timestamp) WHERE user.entity.sub_type IS NOT NULL, recent.entity.url = LAST(TO_STRING(user.entity.url), @timestamp) WHERE user.entity.url IS NOT NULL, - recent.entity.attributes.Privileged = LAST(TO_BOOLEAN(user.entity.attributes.Privileged), @timestamp) WHERE user.entity.attributes.Privileged IS NOT NULL, - recent.entity.attributes.Asset = LAST(TO_BOOLEAN(user.entity.attributes.Asset), @timestamp) WHERE user.entity.attributes.Asset IS NOT NULL, - recent.entity.attributes.Managed = LAST(TO_BOOLEAN(user.entity.attributes.Managed), @timestamp) WHERE user.entity.attributes.Managed IS NOT NULL, - recent.entity.attributes.Mfa_enabled = LAST(TO_BOOLEAN(user.entity.attributes.Mfa_enabled), @timestamp) WHERE user.entity.attributes.Mfa_enabled IS NOT NULL, - recent.entity.lifecycle.First_seen = LAST(TO_DATETIME(user.entity.lifecycle.First_seen), @timestamp) WHERE user.entity.lifecycle.First_seen IS NOT NULL, - recent.entity.lifecycle.Last_activity = LAST(TO_DATETIME(user.entity.lifecycle.Last_activity), @timestamp) WHERE user.entity.lifecycle.Last_activity IS NOT NULL, - recent.entity.behaviors.Brute_force_victim = LAST(TO_BOOLEAN(user.entity.behaviors.Brute_force_victim), @timestamp) WHERE user.entity.behaviors.Brute_force_victim IS NOT NULL, - recent.entity.behaviors.New_country_login = LAST(TO_BOOLEAN(user.entity.behaviors.New_country_login), @timestamp) WHERE user.entity.behaviors.New_country_login IS NOT NULL, - recent.entity.behaviors.Used_usb_device = LAST(TO_BOOLEAN(user.entity.behaviors.Used_usb_device), @timestamp) WHERE user.entity.behaviors.Used_usb_device IS NOT NULL, + recent.entity.attributes.privileged = LAST(TO_BOOLEAN(user.entity.attributes.privileged), @timestamp) WHERE user.entity.attributes.privileged IS NOT NULL, + recent.entity.attributes.asset = LAST(TO_BOOLEAN(user.entity.attributes.asset), @timestamp) WHERE user.entity.attributes.asset IS NOT NULL, + recent.entity.attributes.managed = LAST(TO_BOOLEAN(user.entity.attributes.managed), @timestamp) WHERE user.entity.attributes.managed IS NOT NULL, + recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(user.entity.attributes.mfa_enabled), @timestamp) WHERE user.entity.attributes.mfa_enabled IS NOT NULL, + recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(user.entity.lifecycle.first_seen), @timestamp) WHERE user.entity.lifecycle.first_seen IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(user.entity.lifecycle.last_activity), @timestamp) WHERE user.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.behaviors.brute_force_victim = LAST(TO_BOOLEAN(user.entity.behaviors.brute_force_victim), @timestamp) WHERE user.entity.behaviors.brute_force_victim IS NOT NULL, + recent.entity.behaviors.new_country_login = LAST(TO_BOOLEAN(user.entity.behaviors.new_country_login), @timestamp) WHERE user.entity.behaviors.new_country_login IS NOT NULL, + recent.entity.behaviors.used_usb_device = LAST(TO_BOOLEAN(user.entity.behaviors.used_usb_device), @timestamp) WHERE user.entity.behaviors.used_usb_device IS NOT NULL, recent.entity.risk.calculated_level = LAST(TO_STRING(user.entity.risk.calculated_level), @timestamp) WHERE user.entity.risk.calculated_level IS NOT NULL, recent.entity.risk.calculated_score = LAST(user.entity.risk.calculated_score, @timestamp) WHERE user.entity.risk.calculated_score IS NOT NULL, recent.entity.risk.calculated_score_norm = LAST(user.entity.risk.calculated_score_norm, @timestamp) WHERE user.entity.risk.calculated_score_norm IS NOT NULL, recent.host.entity.id = TOP(MV_DEDUPE(TO_STRING(host.entity.id)), 10) WHERE host.entity.id IS NOT NULL, recent.host.id = TOP(MV_DEDUPE(TO_STRING(host.id)), 10) WHERE host.id IS NOT NULL, recent.host.name = TOP(MV_DEDUPE(TO_STRING(host.name)), 10) WHERE host.name IS NOT NULL, - recent.entity.relationships.Accesses_frequently = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.Accesses_frequently)), 10) WHERE user.entity.relationships.Accesses_frequently IS NOT NULL, - recent.entity.relationships.Owns = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.Owns)), 10) WHERE user.entity.relationships.Owns IS NOT NULL, - recent.entity.relationships.Supervises = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.Supervises)), 10) WHERE user.entity.relationships.Supervises IS NOT NULL, - recent.entity.relationships.Supervised_by = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.Supervised_by)), 10) WHERE user.entity.relationships.Supervised_by IS NOT NULL + recent.entity.relationships.accesses_frequently = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.accesses_frequently)), 10) WHERE user.entity.relationships.accesses_frequently IS NOT NULL, + recent.entity.relationships.owns = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.owns)), 10) WHERE user.entity.relationships.owns IS NOT NULL, + recent.entity.relationships.supervises = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.supervises)), 10) WHERE user.entity.relationships.supervises IS NOT NULL, + recent.entity.relationships.supervised_by = TOP(MV_DEDUPE(TO_STRING(user.entity.relationships.supervised_by)), 10) WHERE user.entity.relationships.supervised_by IS NOT NULL BY recent.entity.EngineMetadata.UntypedId | SORT entity.EngineMetadata.FirstSeenLogInPage ASC, recent.entity.EngineMetadata.UntypedId ASC @@ -1076,25 +1076,25 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti entity.type = COALESCE(recent.entity.type, entity.type), entity.sub_type = COALESCE(recent.entity.sub_type, entity.sub_type), entity.url = COALESCE(recent.entity.url, entity.url), - entity.attributes.Privileged = COALESCE(recent.entity.attributes.Privileged, entity.attributes.Privileged), - entity.attributes.Asset = COALESCE(recent.entity.attributes.Asset, entity.attributes.Asset), - entity.attributes.Managed = COALESCE(recent.entity.attributes.Managed, entity.attributes.Managed), - entity.attributes.Mfa_enabled = COALESCE(recent.entity.attributes.Mfa_enabled, entity.attributes.Mfa_enabled), - entity.lifecycle.First_seen = COALESCE(recent.entity.lifecycle.First_seen, entity.lifecycle.First_seen), - entity.lifecycle.Last_activity = COALESCE(recent.entity.lifecycle.Last_activity, entity.lifecycle.Last_activity), - entity.behaviors.Brute_force_victim = COALESCE(recent.entity.behaviors.Brute_force_victim, entity.behaviors.Brute_force_victim), - entity.behaviors.New_country_login = COALESCE(recent.entity.behaviors.New_country_login, entity.behaviors.New_country_login), - entity.behaviors.Used_usb_device = COALESCE(recent.entity.behaviors.Used_usb_device, entity.behaviors.Used_usb_device), + entity.attributes.privileged = COALESCE(recent.entity.attributes.privileged, entity.attributes.privileged), + entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), + entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), + entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), + entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), + entity.behaviors.brute_force_victim = COALESCE(recent.entity.behaviors.brute_force_victim, entity.behaviors.brute_force_victim), + entity.behaviors.new_country_login = COALESCE(recent.entity.behaviors.new_country_login, entity.behaviors.new_country_login), + entity.behaviors.used_usb_device = COALESCE(recent.entity.behaviors.used_usb_device, entity.behaviors.used_usb_device), entity.risk.calculated_level = COALESCE(recent.entity.risk.calculated_level, entity.risk.calculated_level), entity.risk.calculated_score = COALESCE(recent.entity.risk.calculated_score, entity.risk.calculated_score), entity.risk.calculated_score_norm = COALESCE(recent.entity.risk.calculated_score_norm, entity.risk.calculated_score_norm), host.entity.id = MV_SLICE(MV_UNION(recent.host.entity.id, host.entity.id), 0, 9), host.id = MV_SLICE(MV_UNION(recent.host.id, host.id), 0, 9), host.name = MV_SLICE(MV_UNION(recent.host.name, host.name), 0, 9), - entity.relationships.Accesses_frequently = MV_SLICE(MV_UNION(recent.entity.relationships.Accesses_frequently, entity.relationships.Accesses_frequently), 0, 9), - entity.relationships.Owns = MV_SLICE(MV_UNION(recent.entity.relationships.Owns, entity.relationships.Owns), 0, 9), - entity.relationships.Supervises = MV_SLICE(MV_UNION(recent.entity.relationships.Supervises, entity.relationships.Supervises), 0, 9), - entity.relationships.Supervised_by = MV_SLICE(MV_UNION(recent.entity.relationships.Supervised_by, entity.relationships.Supervised_by), 0, 9) + entity.relationships.accesses_frequently = MV_SLICE(MV_UNION(recent.entity.relationships.accesses_frequently, entity.relationships.accesses_frequently), 0, 9), + entity.relationships.owns = MV_SLICE(MV_UNION(recent.entity.relationships.owns, entity.relationships.owns), 0, 9), + entity.relationships.supervises = MV_SLICE(MV_UNION(recent.entity.relationships.supervises, entity.relationships.supervises), 0, 9), + entity.relationships.supervised_by = MV_SLICE(MV_UNION(recent.entity.relationships.supervised_by, entity.relationships.supervised_by), 0, 9) | EVAL @timestamp = recent.timestamp, entity.name = CASE(entity.name IS NOT NULL AND entity.name != \\"\\", entity.name, recent.entity.EngineMetadata.UntypedId), entity.EngineMetadata.Type = \\"user\\", @@ -1129,25 +1129,25 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti entity.type, entity.sub_type, entity.url, - entity.attributes.Privileged, - entity.attributes.Asset, - entity.attributes.Managed, - entity.attributes.Mfa_enabled, - entity.lifecycle.First_seen, - entity.lifecycle.Last_activity, - entity.behaviors.Brute_force_victim, - entity.behaviors.New_country_login, - entity.behaviors.Used_usb_device, + entity.attributes.privileged, + entity.attributes.asset, + entity.attributes.managed, + entity.attributes.mfa_enabled, + entity.lifecycle.first_seen, + entity.lifecycle.last_activity, + entity.behaviors.brute_force_victim, + entity.behaviors.new_country_login, + entity.behaviors.used_usb_device, entity.risk.calculated_level, entity.risk.calculated_score, entity.risk.calculated_score_norm, host.entity.id, host.id, host.name, - entity.relationships.Accesses_frequently, - entity.relationships.Owns, - entity.relationships.Supervises, - entity.relationships.Supervised_by, + entity.relationships.accesses_frequently, + entity.relationships.owns, + entity.relationships.supervises, + entity.relationships.supervised_by, @timestamp, entity.id, entity.name, From efc0f5104822b57083801ca7ae2dcb355941ce7b Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 20 Feb 2026 15:40:41 +0000 Subject: [PATCH 51/69] Changes from node scripts/eslint_all_files --no-cache --fix --- .../plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts index 3718d378f6d9c..724048b1e6769 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts @@ -67,7 +67,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { }); } catch (error) { if (error instanceof BadCRUDRequestError) { - return res.badRequest({ body: error }); + return res.badRequest({ body: error }); } logger.error(error); From e1499afab722e9ca11107deec49b31bc3f8e5881 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 09:49:36 +0100 Subject: [PATCH 52/69] fix eslint issue --- .../plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts index 3718d378f6d9c..724048b1e6769 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts @@ -67,7 +67,7 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { }); } catch (error) { if (error instanceof BadCRUDRequestError) { - return res.badRequest({ body: error }); + return res.badRequest({ body: error }); } logger.error(error); From 2b4f0b02fb5937135bc60175493f45bbb53061b5 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 10:11:27 +0100 Subject: [PATCH 53/69] crud_client: extract util funcs --- .../{crud_client.ts => crud_client/index.ts} | 132 ++---------------- .../server/domain/crud_client/utils.ts | 131 +++++++++++++++++ 2 files changed, 139 insertions(+), 124 deletions(-) rename x-pack/solutions/security/plugins/entity_store/server/domain/{crud_client.ts => crud_client/index.ts} (55%) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts similarity index 55% rename from x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts rename to x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index 1cef82b35d7d5..e155807d0fcb5 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -13,20 +13,14 @@ import type { } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { createHash } from 'crypto'; -import { unset } from 'lodash'; -import { getFlattenedObject } from '@kbn/std'; -import { ENTITY_ID_FIELD } from '../../common/domain/definitions/common_fields'; -import { getLatestEntitiesIndexName } from './assets/latest_index'; -import { BadCRUDRequestError, EntityNotFoundError } from './errors'; -import type { Entity } from '../../common/domain/definitions/entity.gen'; -import { getEntityDefinition } from '../../common/domain/definitions/registry'; -import type { EntityType } from '../../common'; -import type { - EntityField, - ManagedEntityDefinition, -} from '../../common/domain/definitions/entity_schema'; -import { getEuidFromObject } from '../../common/domain/euid'; -import { getUpdatesEntitiesDataStreamName } from './assets/updates_data_stream'; +import type { Entity } from '../../../common/domain/definitions/entity.gen'; +import { getEntityDefinition } from '../../../common/domain/definitions/registry'; +import type { EntityType } from '../../../common'; +import { getEuidFromObject } from '../../../common/domain/euid'; +import { getLatestEntitiesIndexName } from '../assets/latest_index'; +import { BadCRUDRequestError, EntityNotFoundError } from '../errors'; +import { getUpdatesEntitiesDataStreamName } from '../assets/updates_data_stream'; +import { removeEUIDFields, validateAndTransformDoc } from './utils'; interface CRUDClientDependencies { logger: Logger; @@ -46,21 +40,6 @@ interface BulkObjectResponse { reason: string; } -function validateAndTransformDoc( - entityType: EntityType, - namespace: string, - doc: Entity, - force: boolean -): Record { - const definition = getEntityDefinition(entityType, namespace); - if (!force) { - const flat = getFlattenedObject(doc); - const fieldDescriptions = getFieldDescriptions(flat, definition); - assertOnlyNonForcedAttributesInReq(fieldDescriptions); - } - return transformDocForUpsert(entityType, doc); -} - export class CRUDClient { private readonly logger: Logger; private readonly esClient: ElasticsearchClient; @@ -186,98 +165,3 @@ export class CRUDClient { } } } - -function getFieldDescriptions( - flatProps: Record, - description: ManagedEntityDefinition -): Record { - const allFieldDescriptions = description.fields.reduce((obj, field) => { - obj[field.destination || field.source] = field; - return obj; - }, {} as Record); - - const invalid: string[] = []; - const descriptions: Record = {}; - - for (const [key, value] of Object.entries(flatProps)) { - if (key === ENTITY_ID_FIELD || description.identityField.requiresOneOfFields.includes(key)) { - continue; - } - - if (!allFieldDescriptions[key]) { - invalid.push(key); - } else { - descriptions[key] = { - ...allFieldDescriptions[key], - value, - }; - } - } - - // This will catch differences between - // API and entity store definition - if (invalid.length > 0) { - const invalidString = invalid.join(', '); - throw new BadCRUDRequestError( - `The following attributes are not allowed to be updated: [${invalidString}]` - ); - } - - return descriptions; -} - -function assertOnlyNonForcedAttributesInReq(fields: Record) { - const notAllowedProps = []; - - for (const [name, description] of Object.entries(fields)) { - if (!description.allowAPIUpdate) { - notAllowedProps.push(name); - } - } - - if (notAllowedProps.length > 0) { - const notAllowedPropsString = notAllowedProps.join(', '); - throw new BadCRUDRequestError( - `The following attributes are not allowed to be ` + - `updated without forcing it (?force=true): ${notAllowedPropsString}` - ); - } -} - -function transformDocForUpsert(type: EntityType, data: Partial): Record { - const now = new Date().toISOString(); - if (type === 'generic') { - return { - '@timestamp': now, - ...data, - }; - } - - // Get host, user, service field - const typeData = (data[type as keyof typeof data] || {}) as Record; - - // Force name to be picked by the store - typeData.name = data.entity?.id; - // Nest entity under type data - typeData.entity = data.entity; - - const doc: Record = { - '@timestamp': now, - ...data, - }; - - // Remove entity from root - delete doc.entity; - - // override the host, user service - // field with the built value - doc[type as keyof typeof doc] = typeData; - - return doc; -} - -function removeEUIDFields(definition: ManagedEntityDefinition, doc: Record) { - for (const euidField of definition.identityField.requiresOneOfFields) { - unset(doc, euidField); - } -} diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts new file mode 100644 index 0000000000000..ed7073c9e4633 --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { unset } from 'lodash'; +import { getFlattenedObject } from '@kbn/std'; +import { ENTITY_ID_FIELD } from '../../../common/domain/definitions/common_fields'; +import { BadCRUDRequestError } from '../errors'; +import type { Entity } from '../../../common/domain/definitions/entity.gen'; +import { getEntityDefinition } from '../../../common/domain/definitions/registry'; +import type { EntityType } from '../../../common'; +import type { + EntityField, + ManagedEntityDefinition, +} from '../../../common/domain/definitions/entity_schema'; + +export function validateAndTransformDoc( + entityType: EntityType, + namespace: string, + doc: Entity, + force: boolean +): Record { + const definition = getEntityDefinition(entityType, namespace); + if (!force) { + const flat = getFlattenedObject(doc); + const fieldDescriptions = getFieldDescriptions(flat, definition); + assertOnlyNonForcedAttributesInReq(fieldDescriptions); + } + return transformDocForUpsert(entityType, doc); +} + +function getFieldDescriptions( + flatProps: Record, + description: ManagedEntityDefinition +): Record { + const allFieldDescriptions = description.fields.reduce((obj, field) => { + obj[field.destination || field.source] = field; + return obj; + }, {} as Record); + + const invalid: string[] = []; + const descriptions: Record = {}; + + for (const [key, value] of Object.entries(flatProps)) { + if (key === ENTITY_ID_FIELD || description.identityField.requiresOneOfFields.includes(key)) { + continue; + } + + if (!allFieldDescriptions[key]) { + invalid.push(key); + } else { + descriptions[key] = { + ...allFieldDescriptions[key], + value, + }; + } + } + + // This will catch differences between + // API and entity store definition + if (invalid.length > 0) { + const invalidString = invalid.join(', '); + throw new BadCRUDRequestError( + `The following attributes are not allowed to be updated: [${invalidString}]` + ); + } + + return descriptions; +} + +function assertOnlyNonForcedAttributesInReq(fields: Record) { + const notAllowedProps = []; + + for (const [name, description] of Object.entries(fields)) { + if (!description.allowAPIUpdate) { + notAllowedProps.push(name); + } + } + + if (notAllowedProps.length > 0) { + const notAllowedPropsString = notAllowedProps.join(', '); + throw new BadCRUDRequestError( + `The following attributes are not allowed to be ` + + `updated without forcing it (?force=true): ${notAllowedPropsString}` + ); + } +} + +function transformDocForUpsert(type: EntityType, data: Partial): Record { + const now = new Date().toISOString(); + if (type === 'generic') { + return { + '@timestamp': now, + ...data, + }; + } + + // Get host, user, service field + const typeData = (data[type as keyof typeof data] || {}) as Record; + + // Force name to be picked by the store + typeData.name = data.entity?.id; + // Nest entity under type data + typeData.entity = data.entity; + + const doc: Record = { + '@timestamp': now, + ...data, + }; + + // Remove entity from root + delete doc.entity; + + // override the host, user service + // field with the built value + doc[type as keyof typeof doc] = typeData; + + return doc; +} + +export function removeEUIDFields( + definition: ManagedEntityDefinition, + doc: Record +) { + for (const euidField of definition.identityField.requiresOneOfFields) { + unset(doc, euidField); + } +} From ffb61f045aae4b5f3112646c367e7edf21d5e080 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 10:31:20 +0100 Subject: [PATCH 54/69] crud_client: add utils.ts tests --- .../server/domain/crud_client/utils.test.ts | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts new file mode 100644 index 0000000000000..b4b8ea20e3e8b --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Entity } from '../../../common/domain/definitions/entity.gen'; +import { + type EntityField, + type EntityType, + type ManagedEntityDefinition, +} from '../../../common/domain/definitions/entity_schema'; +import { getEntityDefinition } from '../../../common/domain/definitions/registry'; +import { BadCRUDRequestError } from '../errors'; +import { removeEUIDFields, validateAndTransformDoc } from './utils'; + +jest.mock('../../../common/domain/definitions/registry'); + +const mockGetEntityDefinition = getEntityDefinition as jest.MockedFunction< + typeof getEntityDefinition +>; + +const createField = (source: string, allowAPIUpdate = true): EntityField => ({ + allowAPIUpdate, + source, + destination: source, + mapping: { type: 'keyword' }, + retention: { operation: 'prefer_newest_value' }, +}); + +const createDefinition = ( + type: EntityType, + fields: EntityField[], + requiresOneOfFields: string[] = [] +): ManagedEntityDefinition => ({ + id: `security_${type}_default`, + name: `${type} definition`, + type, + fields, + identityField: { + requiresOneOfFields, + euidFields: [[{ field: 'entity.id' }]], + }, + indexPatterns: ['logs-*'], +}); + +describe('crud_client utils', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('validateAndTransformDoc: returns generic document with timestamp', () => { + mockGetEntityDefinition.mockReturnValue(createDefinition('generic', [])); + + const doc: Entity = { + entity: { id: 'entity-generic' }, + }; + const result = validateAndTransformDoc('generic', 'default', doc, true); + + expect(result).toEqual( + expect.objectContaining({ + '@timestamp': expect.any(String), + entity: { id: 'entity-generic' }, + }) + ); + }); + + it('transformDocForUpsert: nests entity under typed field and rewrites name', () => { + mockGetEntityDefinition.mockReturnValue(createDefinition('host', [createField('host.name')])); + + const doc: Entity = { + entity: { id: 'entity-host' }, + host: { name: 'original-host-name' }, + }; + const result = validateAndTransformDoc('host', 'default', doc, false); + + expect(result).toEqual(expect.objectContaining({ '@timestamp': expect.any(String) })); + expect(result).not.toHaveProperty('entity'); + expect(result).toHaveProperty('host.entity.id', 'entity-host'); + expect(result).toHaveProperty('host.name', 'entity-host'); + }); + + it('getFieldDescriptions: ignores identity source fields from validation', () => { + mockGetEntityDefinition.mockReturnValue(createDefinition('host', [], ['host.name'])); + + const doc: Entity = { + entity: { id: 'entity-host-identity' }, + host: { name: 'identity-host' }, + }; + + expect(() => validateAndTransformDoc('host', 'default', doc, false)).not.toThrow(); + }); + + it('assertOnlyNonForcedAttributesInReq: allows updates for allowAPIUpdate fields', () => { + mockGetEntityDefinition.mockReturnValue( + createDefinition('generic', [createField('entity.attributes.privileged', true)]) + ); + + const doc: Entity = { + entity: { + id: 'entity-allow-update', + attributes: { + privileged: true, + }, + }, + }; + const result = validateAndTransformDoc('generic', 'default', doc, false); + + expect(result).toHaveProperty('entity.attributes.privileged', true); + }); + + it('getFieldDescriptions: throws for fields missing from definition', () => { + mockGetEntityDefinition.mockReturnValue(createDefinition('generic', [])); + + const doc: Entity = { + entity: { + id: 'entity-invalid-field', + attributes: { + managed: true, + }, + }, + }; + + expect(() => validateAndTransformDoc('generic', 'default', doc, false)).toThrow( + new BadCRUDRequestError( + 'The following attributes are not allowed to be updated: [entity.attributes.managed]' + ) + ); + }); + + it('assertOnlyNonForcedAttributesInReq: throws when non-force request updates restricted fields', () => { + mockGetEntityDefinition.mockReturnValue( + createDefinition('generic', [createField('entity.attributes.managed', false)]) + ); + + const doc: Entity = { + entity: { + id: 'entity-restricted-update', + attributes: { + managed: true, + }, + }, + }; + + expect(() => validateAndTransformDoc('generic', 'default', doc, false)).toThrow( + new BadCRUDRequestError( + 'The following attributes are not allowed to be updated without forcing it (?force=true): entity.attributes.managed' + ) + ); + }); + + it('validateAndTransformDoc: bypasses validation when force=true', () => { + mockGetEntityDefinition.mockReturnValue( + createDefinition('generic', [createField('entity.attributes.managed', false)]) + ); + + const doc: Entity = { + entity: { + id: 'entity-force-update', + attributes: { + managed: true, + }, + }, + }; + + expect(() => validateAndTransformDoc('generic', 'default', doc, true)).not.toThrow(); + }); + + it('removeEUIDFields: removes identity fields from update doc', () => { + const definition = createDefinition('host', [], ['host.name']); + const doc: Record = { + host: { + name: 'host-to-remove', + id: ['host-id'], + }, + }; + + removeEUIDFields(definition, doc); + + expect(doc).toEqual({ + host: { + id: ['host-id'], + }, + }); + }); + + it('removeEUIDFields: is a no-op with no identity fields', () => { + const definition = createDefinition('host', [], []); + const doc: Record = { + host: { + name: 'host-unchanged', + }, + }; + + removeEUIDFields(definition, doc); + + expect(doc).toEqual({ + host: { + name: 'host-unchanged', + }, + }); + }); +}); From 4cc1b0830493825d5448571bed30ef72e557381c Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 11:11:57 +0100 Subject: [PATCH 55/69] crud_client: use update+doc_as_upsert --- .../server/domain/crud_client/index.ts | 67 ++++++------------- .../server/domain/crud_client/utils.test.ts | 37 +--------- .../server/domain/crud_client/utils.ts | 10 --- 3 files changed, 21 insertions(+), 93 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index e155807d0fcb5..7e86c97adf3cc 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -14,13 +14,12 @@ import type { import type { ElasticsearchClient } from '@kbn/core/server'; import { createHash } from 'crypto'; import type { Entity } from '../../../common/domain/definitions/entity.gen'; -import { getEntityDefinition } from '../../../common/domain/definitions/registry'; import type { EntityType } from '../../../common'; import { getEuidFromObject } from '../../../common/domain/euid'; import { getLatestEntitiesIndexName } from '../assets/latest_index'; import { BadCRUDRequestError, EntityNotFoundError } from '../errors'; import { getUpdatesEntitiesDataStreamName } from '../assets/updates_data_stream'; -import { removeEUIDFields, validateAndTransformDoc } from './utils'; +import { validateAndTransformDoc } from './utils'; interface CRUDClientDependencies { logger: Logger; @@ -51,42 +50,6 @@ export class CRUDClient { this.namespace = deps.namespace; } - private async createEntity(hashedId: string, doc: Record): Promise { - this.logger.debug(`Creating entity ID: ${hashedId}`); - await this.esClient.create({ - index: getLatestEntitiesIndexName(this.namespace), - id: hashedId, - document: doc, - refresh: 'wait_for', - }); - this.logger.debug(`Created entity ID ${hashedId}`); - } - - private async updateEntity( - hashedId: string, - entityType: EntityType, - doc: Record - ): Promise { - this.logger.debug(`Updating entity ID: ${hashedId}`); - const definition = getEntityDefinition(entityType, this.namespace); - removeEUIDFields(definition, doc); - const { result } = await this.esClient.update({ - index: getLatestEntitiesIndexName(this.namespace), - id: hashedId, - doc, - retry_on_conflict: 3, - }); - - switch (result as Result) { - case 'updated': - this.logger.debug(`Updated entity ID ${hashedId}`); - break; - case 'noop': - this.logger.debug(`Updated entity ID ${hashedId} (no change)`); - break; - } - } - public async upsertEntity(entityType: EntityType, doc: Entity, force: boolean): Promise { const id = getEuidFromObject(entityType, doc); if (id === undefined) { @@ -100,18 +63,28 @@ export class CRUDClient { if (!doc.entity?.id) { doc.entity.id = id; } + const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); - try { - await this.createEntity(hashedId, readyDoc); - } catch (error) { - if (error.statusCode !== 409) { - throw error; - } - this.logger.debug(`Conflict while creating entity ID ${id}, updating instead`); - } + const { result } = await this.esClient.update({ + index: getLatestEntitiesIndexName(this.namespace), + id: hashedId, + doc: readyDoc, + doc_as_upsert: true, + retry_on_conflict: 3, + }); - await this.updateEntity(hashedId, entityType, readyDoc); + switch (result as Result) { + case 'created': + this.logger.debug(`Created entity ID ${hashedId}`); + break; + case 'updated': + this.logger.debug(`Updated entity ID ${hashedId}`); + break; + case 'noop': + this.logger.debug(`Updated entity ID ${hashedId} (no change)`); + break; + } return; } diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts index b4b8ea20e3e8b..02af912ee3033 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts @@ -13,7 +13,7 @@ import { } from '../../../common/domain/definitions/entity_schema'; import { getEntityDefinition } from '../../../common/domain/definitions/registry'; import { BadCRUDRequestError } from '../errors'; -import { removeEUIDFields, validateAndTransformDoc } from './utils'; +import { validateAndTransformDoc } from './utils'; jest.mock('../../../common/domain/definitions/registry'); @@ -166,39 +166,4 @@ describe('crud_client utils', () => { expect(() => validateAndTransformDoc('generic', 'default', doc, true)).not.toThrow(); }); - - it('removeEUIDFields: removes identity fields from update doc', () => { - const definition = createDefinition('host', [], ['host.name']); - const doc: Record = { - host: { - name: 'host-to-remove', - id: ['host-id'], - }, - }; - - removeEUIDFields(definition, doc); - - expect(doc).toEqual({ - host: { - id: ['host-id'], - }, - }); - }); - - it('removeEUIDFields: is a no-op with no identity fields', () => { - const definition = createDefinition('host', [], []); - const doc: Record = { - host: { - name: 'host-unchanged', - }, - }; - - removeEUIDFields(definition, doc); - - expect(doc).toEqual({ - host: { - name: 'host-unchanged', - }, - }); - }); }); diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts index ed7073c9e4633..4d0a48ea346db 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { unset } from 'lodash'; import { getFlattenedObject } from '@kbn/std'; import { ENTITY_ID_FIELD } from '../../../common/domain/definitions/common_fields'; import { BadCRUDRequestError } from '../errors'; @@ -120,12 +119,3 @@ function transformDocForUpsert(type: EntityType, data: Partial): Record< return doc; } - -export function removeEUIDFields( - definition: ManagedEntityDefinition, - doc: Record -) { - for (const euidField of definition.identityField.requiresOneOfFields) { - unset(doc, euidField); - } -} From 690395b925fbb0033a3026e7a8a64d2436363ec6 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 11:29:47 +0100 Subject: [PATCH 56/69] crud_api: throw 503 if not started for bulk --- ...store_not_installed.ts => entity_store_not_running.ts} | 4 ++-- .../plugins/entity_store/server/domain/errors/index.ts | 2 +- .../entity_store/server/routes/apis/crud/delete.ts | 7 ++----- .../entity_store/server/routes/apis/crud/upsert.ts | 7 ++----- .../entity_store/server/routes/apis/crud/upsert_bulk.ts | 8 +++++--- 5 files changed, 12 insertions(+), 16 deletions(-) rename x-pack/solutions/security/plugins/entity_store/server/domain/errors/{entity_store_not_installed.ts => entity_store_not_running.ts} (72%) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_running.ts similarity index 72% rename from x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts rename to x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_running.ts index fa38ba4e24b74..7a30f1720c80c 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_installed.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/entity_store_not_running.ts @@ -5,8 +5,8 @@ * 2.0. */ -export class EntityStoreNotInstalledError extends Error { +export class EntityStoreNotRunningError extends Error { constructor() { - super(`No EntityStore engine is running`); + super(`Entity Store has not been started`); } } diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts index 485936c56d5dc..ef0242fe5d9b5 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/errors/index.ts @@ -7,5 +7,5 @@ export { EntityNotFoundError } from './entity_not_found'; export { DocumentVersionConflictError } from './document_version_conflict'; -export { EntityStoreNotInstalledError } from './entity_store_not_installed'; +export { EntityStoreNotRunningError } from './entity_store_not_running'; export { BadCRUDRequestError } from './bad_crud_request_error'; diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts index bc7b3bef041f1..3ea8c30816858 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts @@ -11,7 +11,7 @@ import type { IKibanaResponse } from '@kbn/core-http-server'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../../constants'; import type { EntityStorePluginRouter } from '../../../types'; import { wrapMiddlewares } from '../../middleware'; -import { EntityNotFoundError, EntityStoreNotInstalledError } from '../../../domain/errors'; +import { EntityNotFoundError } from '../../../domain/errors'; const paramsSchema = z.object({ id: z.string(), @@ -38,12 +38,9 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { }, wrapMiddlewares(async (ctx, req, res): Promise => { const entityStoreCtx = await ctx.entityStore; - const { logger, assetManager, crudClient } = entityStoreCtx; + const { logger, crudClient } = entityStoreCtx; logger.debug('CRUD Delete api called'); - if (!(await assetManager.isInstalled())) { - return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); - } try { await crudClient.deleteEntity(req.params.id); diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts index a97cf60c88132..93333f25e02f7 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert.ts @@ -11,7 +11,7 @@ import { z } from '@kbn/zod'; import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../../constants'; import type { EntityStorePluginRouter } from '../../../types'; import { wrapMiddlewares } from '../../middleware'; -import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../../domain/errors'; +import { BadCRUDRequestError } from '../../../domain/errors'; import { Entity } from '../../../../common/domain/definitions/entity.gen'; import { EntityType } from '../../../../common/domain/definitions/entity_schema'; @@ -48,12 +48,9 @@ export function registerCRUDUpsert(router: EntityStorePluginRouter) { }, wrapMiddlewares(async (ctx, req, res): Promise => { const entityStoreCtx = await ctx.entityStore; - const { logger, assetManager, crudClient } = entityStoreCtx; + const { logger, crudClient } = entityStoreCtx; logger.debug('CRUD Upsert api called'); - if (!(await assetManager.isInstalled())) { - return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); - } try { await crudClient.upsertEntity(req.params.entityType, req.body, req.query.force); diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts index 724048b1e6769..894b8b69ef6f0 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/upsert_bulk.ts @@ -12,7 +12,8 @@ import { EntityType } from '../../../../common/domain/definitions/entity_schema' import { API_VERSIONS, DEFAULT_ENTITY_STORE_PERMISSIONS } from '../../constants'; import type { EntityStorePluginRouter } from '../../../types'; import { wrapMiddlewares } from '../../middleware'; -import { BadCRUDRequestError, EntityStoreNotInstalledError } from '../../../domain/errors'; +import { ENGINE_STATUS } from '../../../domain/constants'; +import { BadCRUDRequestError, EntityStoreNotRunningError } from '../../../domain/errors'; import { Entity } from '../../../../common/domain/definitions/entity.gen'; const bodySchema = z.object({ @@ -53,8 +54,9 @@ export function registerCRUDUpsertBulk(router: EntityStorePluginRouter) { const { logger, assetManager, crudClient } = entityStoreCtx; logger.debug('CRUD Upsert Bulk api called'); - if (!(await assetManager.isInstalled())) { - return res.customError({ statusCode: 503, body: new EntityStoreNotInstalledError() }); + const { engines } = await assetManager.getStatus(); + if (engines.some((engine) => engine.status !== ENGINE_STATUS.STARTED)) { + return res.customError({ statusCode: 503, body: new EntityStoreNotRunningError() }); } try { From 6114f349f11c2d912016f060e7014320f263c67c Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 12:07:18 +0100 Subject: [PATCH 57/69] crud_api: wait for shard sync on update --- .../plugins/entity_store/server/domain/crud_client/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index 7e86c97adf3cc..11f3d11a03909 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -72,6 +72,7 @@ export class CRUDClient { doc: readyDoc, doc_as_upsert: true, retry_on_conflict: 3, + refresh: 'wait_for', }); switch (result as Result) { From 6ed1c022b1dc72e49da3be70674ab8994471125e Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 12:36:24 +0100 Subject: [PATCH 58/69] crud_api: handle 503 for bulk api --- .../scout/api/tests/crud_api_stopped.spec.ts | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api_stopped.spec.ts diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api_stopped.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api_stopped.spec.ts new file mode 100644 index 0000000000000..fbd20a8ad320f --- /dev/null +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api_stopped.spec.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { apiTest } from '@kbn/scout-security'; +import { expect } from '@kbn/scout-security/api'; +import { COMMON_HEADERS, ENTITY_STORE_ROUTES, ENTITY_STORE_TAGS } from '../fixtures/constants'; +import { FF_ENABLE_ENTITY_STORE_V2 } from '../../../../common'; + +apiTest.describe( + 'Entity Store CRUD API tests: 503 errors on stopped engine', + { tag: ENTITY_STORE_TAGS }, + () => { + let defaultHeaders: Record; + + apiTest.beforeAll(async ({ apiClient, kbnClient, samlAuth }) => { + const credentials = await samlAuth.asInteractiveUser('admin'); + defaultHeaders = { + ...credentials.cookieHeader, + ...COMMON_HEADERS, + }; + + // enable feature flag + await kbnClient.uiSettings.update({ + [FF_ENABLE_ENTITY_STORE_V2]: true, + }); + + // Install the entity store + const response = await apiClient.post(ENTITY_STORE_ROUTES.INSTALL, { + headers: defaultHeaders, + responseType: 'json', + body: {}, + }); + expect(response.statusCode).toBe(201); + + const entityTypesBody = { entityTypes: ['generic'] }; + + const stopResponse = await apiClient.put(ENTITY_STORE_ROUTES.STOP, { + headers: defaultHeaders, + responseType: 'json', + body: entityTypesBody, + }); + expect(stopResponse.statusCode).toBe(200); + }); + + apiTest.afterAll(async ({ apiClient }) => { + const response = await apiClient.post(ENTITY_STORE_ROUTES.UNINSTALL, { + headers: defaultHeaders, + responseType: 'json', + body: {}, + }); + expect(response.statusCode).toBe(200); + }); + + apiTest( + 'Should return 503 for bulk upsert when generic engine is stopped', + async ({ apiClient }) => { + const bulkBody = { + entities: [ + { + type: 'generic', + doc: { + entity: { + id: 'required-id-1-bulk-stopped', + }, + }, + }, + { + type: 'generic', + doc: { + entity: { + id: 'required-id-2-bulk-stopped', + }, + }, + }, + ], + }; + + const bulkUpsert = await apiClient.put(ENTITY_STORE_ROUTES.CRUD_UPSERT_BULK, { + headers: defaultHeaders, + responseType: 'json', + body: bulkBody, + }); + expect(bulkUpsert.statusCode).toBe(503); + } + ); + } +); From 86ddefa8a89806a6c175c29d33d8c38f572630d8 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 12:55:20 +0100 Subject: [PATCH 59/69] crud_client: update transformDocForUpsert to be more readable and preserve name --- .../server/domain/crud_client/utils.test.ts | 4 +-- .../server/domain/crud_client/utils.ts | 35 ++++++++----------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts index 02af912ee3033..e55b517c6fbf2 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts @@ -66,7 +66,7 @@ describe('crud_client utils', () => { ); }); - it('transformDocForUpsert: nests entity under typed field and rewrites name', () => { + it('transformDocForUpsert: nests entity under typed field and preserves name', () => { mockGetEntityDefinition.mockReturnValue(createDefinition('host', [createField('host.name')])); const doc: Entity = { @@ -78,7 +78,7 @@ describe('crud_client utils', () => { expect(result).toEqual(expect.objectContaining({ '@timestamp': expect.any(String) })); expect(result).not.toHaveProperty('entity'); expect(result).toHaveProperty('host.entity.id', 'entity-host'); - expect(result).toHaveProperty('host.name', 'entity-host'); + expect(result).toHaveProperty('host.name', 'original-host-name'); }); it('getFieldDescriptions: ignores identity source fields from validation', () => { diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts index 4d0a48ea346db..b465e88abf307 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts @@ -89,33 +89,28 @@ function assertOnlyNonForcedAttributesInReq(fields: Record) } function transformDocForUpsert(type: EntityType, data: Partial): Record { - const now = new Date().toISOString(); + const doc: Record = { + '@timestamp': new Date().toISOString(), + ...data, + }; + if (type === 'generic') { - return { - '@timestamp': now, - ...data, - }; + return doc; } - // Get host, user, service field - const typeData = (data[type as keyof typeof data] || {}) as Record; - - // Force name to be picked by the store - typeData.name = data.entity?.id; - // Nest entity under type data - typeData.entity = data.entity; + const typeKey = type as keyof typeof doc; + if (!doc[typeKey] || typeof doc[typeKey] !== 'object') { + doc[typeKey] = {}; + } + const typeDoc = doc[typeKey] as Record; - const doc: Record = { - '@timestamp': now, - ...data, - }; + if (!typeDoc.name) { + typeDoc.name = data.entity?.id; + } + typeDoc.entity = data.entity; // Remove entity from root delete doc.entity; - // override the host, user service - // field with the built value - doc[type as keyof typeof doc] = typeData; - return doc; } From 322a612b4e934fbdb0fc3db104265975d45e1eb7 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 13:01:13 +0100 Subject: [PATCH 60/69] crud_client: document sync/async upsert methods --- .../entity_store/server/domain/crud_client/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index 11f3d11a03909..3def3e25eb33e 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -50,6 +50,9 @@ export class CRUDClient { this.namespace = deps.namespace; } + // upsertEntity takes a single entity and tries to either create or update + // (if an entity with the same EUID already exists) it directly in the LATEST + // index. This is considered a single synchronous upsert. public async upsertEntity(entityType: EntityType, doc: Entity, force: boolean): Promise { const id = getEuidFromObject(entityType, doc); if (id === undefined) { @@ -89,6 +92,10 @@ export class CRUDClient { return; } + // upsertEntitiesBulk takes one or more entities and creates documents in + // UPDATES index for log extraction task to pick up. This will result in + // appropriate Entities being created or updated on next log extraction run. + // This is considered a bulk asynchronous upsert. public async upsertEntitiesBulk( objects: BulkObject[], force: boolean From 0188a41e4037adcd7b91929462c03bab2b47fb6e Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 13:04:10 +0100 Subject: [PATCH 61/69] openapi: comment with script command --- .../entity_store/common/domain/definitions/entity.schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml index bd6af8587f646..31501fe456349 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml @@ -1,3 +1,4 @@ +# Generated by node scripts/generate_openapi.js --rootDir x-pack/solutions/security/plugins/entity_store/ openapi: 3.0.0 info: title: Common Entities Schemas From bbdbdc61519c661f9718a2817ac0bdb49cb4e133 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 13:08:15 +0100 Subject: [PATCH 62/69] openapi: make the comment clearer --- .../entity_store/common/domain/definitions/entity.schema.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml index 31501fe456349..af5e08f00dcf6 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml @@ -1,4 +1,5 @@ -# Generated by node scripts/generate_openapi.js --rootDir x-pack/solutions/security/plugins/entity_store/ +# Generate schema by calling: +# node scripts/generate_openapi.js --rootDir x-pack/solutions/security/plugins/entity_store/ openapi: 3.0.0 info: title: Common Entities Schemas From bc40d88602f0647e401e810d562c41b86162e9f1 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 14:49:00 +0100 Subject: [PATCH 63/69] crud_client: apply review remarks --- .../entity_store/server/domain/crud_client/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index 3def3e25eb33e..a893206bcb272 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -21,6 +21,8 @@ import { BadCRUDRequestError, EntityNotFoundError } from '../errors'; import { getUpdatesEntitiesDataStreamName } from '../assets/updates_data_stream'; import { validateAndTransformDoc } from './utils'; +const RETRY_ON_CONFLICT = 3; + interface CRUDClientDependencies { logger: Logger; esClient: ElasticsearchClient; @@ -74,19 +76,19 @@ export class CRUDClient { id: hashedId, doc: readyDoc, doc_as_upsert: true, - retry_on_conflict: 3, + retry_on_conflict: RETRY_ON_CONFLICT, refresh: 'wait_for', }); switch (result as Result) { case 'created': - this.logger.debug(`Created entity ID ${hashedId}`); + this.logger.debug(`Created entity ID ${id}`); break; case 'updated': - this.logger.debug(`Updated entity ID ${hashedId}`); + this.logger.debug(`Updated entity ID ${id}`); break; case 'noop': - this.logger.debug(`Updated entity ID ${hashedId} (no change)`); + this.logger.debug(`Updated entity ID ${id} (no change)`); break; } return; From 0d85afce51ed390c57d2398fce95a19aac85b4da Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 23 Feb 2026 14:58:02 +0100 Subject: [PATCH 64/69] crud_client: make generic a const --- .../plugins/entity_store/server/domain/crud_client/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts index b465e88abf307..e789dea788803 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts @@ -16,6 +16,8 @@ import type { ManagedEntityDefinition, } from '../../../common/domain/definitions/entity_schema'; +const GENERIC_TYPE = 'generic' as EntityType; + export function validateAndTransformDoc( entityType: EntityType, namespace: string, @@ -94,7 +96,7 @@ function transformDocForUpsert(type: EntityType, data: Partial): Record< ...data, }; - if (type === 'generic') { + if (type === GENERIC_TYPE) { return doc; } From d6b2e4a45254afaf132901558007ca6dbf7ff536 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 24 Feb 2026 09:55:47 +0100 Subject: [PATCH 65/69] asset_manager: remove unused isInstalled function --- .../entity_store/server/domain/asset_manager.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts index da6fcc1d530bf..e1d926dff6ea4 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager.ts @@ -46,7 +46,6 @@ import { import { getUpdatesEntitiesDataStreamName } from './assets/updates_data_stream'; import type { LogsExtractionClient } from './logs_extraction_client'; import type { ManagedEntityDefinition } from '../../common/domain/definitions/entity_schema'; -import { ALL_ENTITY_TYPES } from '../../common/domain/definitions/entity_schema'; import { getEntityDefinition } from '../../common/domain/definitions/registry'; interface AssetManagerDependencies { @@ -170,17 +169,6 @@ export class AssetManager { } } - public async isInstalled(): Promise { - for (const type of ALL_ENTITY_TYPES) { - try { - await this.engineDescriptorClient.findOrThrow(type); - } catch { - return false; - } - } - return true; - } - public async getStatus(withComponents: boolean = false): Promise { try { const engines = await this.engineDescriptorClient.getAll(); From 5e00170d247252deda52f7eee0a9c5a40d977438 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 24 Feb 2026 09:58:38 +0100 Subject: [PATCH 66/69] crud_client: rename util func for clarity --- .../server/domain/crud_client/index.ts | 6 +++--- .../server/domain/crud_client/utils.test.ts | 20 +++++++++---------- .../server/domain/crud_client/utils.ts | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index a893206bcb272..a65d59f4d7b41 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -19,7 +19,7 @@ import { getEuidFromObject } from '../../../common/domain/euid'; import { getLatestEntitiesIndexName } from '../assets/latest_index'; import { BadCRUDRequestError, EntityNotFoundError } from '../errors'; import { getUpdatesEntitiesDataStreamName } from '../assets/updates_data_stream'; -import { validateAndTransformDoc } from './utils'; +import { validateAndTransformDocForUpsert } from './utils'; const RETRY_ON_CONFLICT = 3; @@ -69,7 +69,7 @@ export class CRUDClient { doc.entity.id = id; } - const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); + const readyDoc = validateAndTransformDocForUpsert(entityType, this.namespace, doc, force); const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), @@ -106,7 +106,7 @@ export class CRUDClient { this.logger.debug(`Preparing ${objects.length} entities for bulk upsert`); for (const { type: entityType, doc } of objects) { - const readyDoc = validateAndTransformDoc(entityType, this.namespace, doc, force); + const readyDoc = validateAndTransformDocForUpsert(entityType, this.namespace, doc, force); operations.push({ create: {} }, readyDoc); } diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts index e55b517c6fbf2..258d60179bce0 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts @@ -13,7 +13,7 @@ import { } from '../../../common/domain/definitions/entity_schema'; import { getEntityDefinition } from '../../../common/domain/definitions/registry'; import { BadCRUDRequestError } from '../errors'; -import { validateAndTransformDoc } from './utils'; +import { validateAndTransformDocForUpsert } from './utils'; jest.mock('../../../common/domain/definitions/registry'); @@ -50,13 +50,13 @@ describe('crud_client utils', () => { jest.clearAllMocks(); }); - it('validateAndTransformDoc: returns generic document with timestamp', () => { + it('validateAndTransformDocForUpsert: returns generic document with timestamp', () => { mockGetEntityDefinition.mockReturnValue(createDefinition('generic', [])); const doc: Entity = { entity: { id: 'entity-generic' }, }; - const result = validateAndTransformDoc('generic', 'default', doc, true); + const result = validateAndTransformDocForUpsert('generic', 'default', doc, true); expect(result).toEqual( expect.objectContaining({ @@ -73,7 +73,7 @@ describe('crud_client utils', () => { entity: { id: 'entity-host' }, host: { name: 'original-host-name' }, }; - const result = validateAndTransformDoc('host', 'default', doc, false); + const result = validateAndTransformDocForUpsert('host', 'default', doc, false); expect(result).toEqual(expect.objectContaining({ '@timestamp': expect.any(String) })); expect(result).not.toHaveProperty('entity'); @@ -89,7 +89,7 @@ describe('crud_client utils', () => { host: { name: 'identity-host' }, }; - expect(() => validateAndTransformDoc('host', 'default', doc, false)).not.toThrow(); + expect(() => validateAndTransformDocForUpsert('host', 'default', doc, false)).not.toThrow(); }); it('assertOnlyNonForcedAttributesInReq: allows updates for allowAPIUpdate fields', () => { @@ -105,7 +105,7 @@ describe('crud_client utils', () => { }, }, }; - const result = validateAndTransformDoc('generic', 'default', doc, false); + const result = validateAndTransformDocForUpsert('generic', 'default', doc, false); expect(result).toHaveProperty('entity.attributes.privileged', true); }); @@ -122,7 +122,7 @@ describe('crud_client utils', () => { }, }; - expect(() => validateAndTransformDoc('generic', 'default', doc, false)).toThrow( + expect(() => validateAndTransformDocForUpsert('generic', 'default', doc, false)).toThrow( new BadCRUDRequestError( 'The following attributes are not allowed to be updated: [entity.attributes.managed]' ) @@ -143,14 +143,14 @@ describe('crud_client utils', () => { }, }; - expect(() => validateAndTransformDoc('generic', 'default', doc, false)).toThrow( + expect(() => validateAndTransformDocForUpsert('generic', 'default', doc, false)).toThrow( new BadCRUDRequestError( 'The following attributes are not allowed to be updated without forcing it (?force=true): entity.attributes.managed' ) ); }); - it('validateAndTransformDoc: bypasses validation when force=true', () => { + it('validateAndTransformDocForUpsert: bypasses validation when force=true', () => { mockGetEntityDefinition.mockReturnValue( createDefinition('generic', [createField('entity.attributes.managed', false)]) ); @@ -164,6 +164,6 @@ describe('crud_client utils', () => { }, }; - expect(() => validateAndTransformDoc('generic', 'default', doc, true)).not.toThrow(); + expect(() => validateAndTransformDocForUpsert('generic', 'default', doc, true)).not.toThrow(); }); }); diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts index e789dea788803..8d221865e342c 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts @@ -18,7 +18,7 @@ import type { const GENERIC_TYPE = 'generic' as EntityType; -export function validateAndTransformDoc( +export function validateAndTransformDocForUpsert( entityType: EntityType, namespace: string, doc: Entity, From 6edeb1281ef7baf5a1cc86267db00bbe536ac315 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 24 Feb 2026 10:08:11 +0100 Subject: [PATCH 67/69] crud_client: filter bulk_upsert errors --- .../server/domain/crud_client/index.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index a65d59f4d7b41..f47f26b566737 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -122,15 +122,17 @@ export class CRUDClient { return []; } this.logger.debug(`Bulk upserted ${objects.length} entities with errors`); - return resp.items.map((item) => { - const [, value] = Object.entries(item)[0]; - return { - _id: value._id, - status: value.status, - type: value.error?.type, - reason: value.error?.reason, - } as BulkObjectResponse; - }); + return resp.items + .map((item) => Object.entries(item)[0][1]) + .filter((value) => value.error !== undefined || value.status >= 400) + .map((value) => { + return { + _id: value._id, + status: value.status, + type: value.error?.type, + reason: value.error?.reason, + } as BulkObjectResponse; + }); } public async deleteEntity(id: string): Promise { From 5beb7cb0d75767914076c240eb4b28d22e8a3e37 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 24 Feb 2026 10:23:15 +0100 Subject: [PATCH 68/69] crud_client: delete based on EUID rather than hash --- .../entity_store/server/domain/crud_client/index.ts | 10 +++------- .../server/domain/crud_client/utils.test.ts | 9 ++++++++- .../entity_store/server/domain/crud_client/utils.ts | 7 +++++++ .../entity_store/server/routes/apis/crud/delete.ts | 10 +++++----- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts index f47f26b566737..685abde22cae6 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/index.ts @@ -12,14 +12,13 @@ import type { Result, } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; -import { createHash } from 'crypto'; import type { Entity } from '../../../common/domain/definitions/entity.gen'; import type { EntityType } from '../../../common'; import { getEuidFromObject } from '../../../common/domain/euid'; import { getLatestEntitiesIndexName } from '../assets/latest_index'; import { BadCRUDRequestError, EntityNotFoundError } from '../errors'; import { getUpdatesEntitiesDataStreamName } from '../assets/updates_data_stream'; -import { validateAndTransformDocForUpsert } from './utils'; +import { hashEuid, validateAndTransformDocForUpsert } from './utils'; const RETRY_ON_CONFLICT = 3; @@ -60,9 +59,6 @@ export class CRUDClient { if (id === undefined) { throw new BadCRUDRequestError(`Could not derive entity EUID from document`); } - // EUID generation uses MD5. It is not a security-related feature. - // eslint-disable-next-line @kbn/eslint/no_unsafe_hash - const hashedId: string = createHash('md5').update(id).digest('hex'); this.logger.debug(`Upserting entity ID ${id}`); if (!doc.entity?.id) { @@ -73,7 +69,7 @@ export class CRUDClient { const { result } = await this.esClient.update({ index: getLatestEntitiesIndexName(this.namespace), - id: hashedId, + id: hashEuid(id), doc: readyDoc, doc_as_upsert: true, retry_on_conflict: RETRY_ON_CONFLICT, @@ -140,7 +136,7 @@ export class CRUDClient { this.logger.debug(`Deleting Entity ID ${id}`); await this.esClient.delete({ index: getLatestEntitiesIndexName(this.namespace), - id, + id: hashEuid(id), }); } catch (error) { if (error.statusCode === 404) { diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts index 258d60179bce0..b09ba8eb6e532 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.test.ts @@ -13,7 +13,7 @@ import { } from '../../../common/domain/definitions/entity_schema'; import { getEntityDefinition } from '../../../common/domain/definitions/registry'; import { BadCRUDRequestError } from '../errors'; -import { validateAndTransformDocForUpsert } from './utils'; +import { hashEuid, validateAndTransformDocForUpsert } from './utils'; jest.mock('../../../common/domain/definitions/registry'); @@ -50,6 +50,13 @@ describe('crud_client utils', () => { jest.clearAllMocks(); }); + it('hashEuid: returns a valid MD5 hash', () => { + const hashedId = hashEuid('entity-id'); + + expect(hashedId).toMatch(/^[a-f0-9]{32}$/); + expect(hashedId).toBe('169fbe0cb705d8d8811b5098d0cf4588'); + }); + it('validateAndTransformDocForUpsert: returns generic document with timestamp', () => { mockGetEntityDefinition.mockReturnValue(createDefinition('generic', [])); diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts index 8d221865e342c..b8eb5906200f7 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/crud_client/utils.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { createHash } from 'crypto'; import { getFlattenedObject } from '@kbn/std'; import { ENTITY_ID_FIELD } from '../../../common/domain/definitions/common_fields'; import { BadCRUDRequestError } from '../errors'; @@ -18,6 +19,12 @@ import type { const GENERIC_TYPE = 'generic' as EntityType; +export function hashEuid(id: string): string { + // EUID generation uses MD5. It is not a security-related feature. + // eslint-disable-next-line @kbn/eslint/no_unsafe_hash + return createHash('md5').update(id).digest('hex'); +} + export function validateAndTransformDocForUpsert( entityType: EntityType, namespace: string, diff --git a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts index 3ea8c30816858..88b83aaa3ed5d 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/routes/apis/crud/delete.ts @@ -13,14 +13,14 @@ import type { EntityStorePluginRouter } from '../../../types'; import { wrapMiddlewares } from '../../middleware'; import { EntityNotFoundError } from '../../../domain/errors'; -const paramsSchema = z.object({ - id: z.string(), +const bodySchema = z.object({ + entityId: z.string(), }); export function registerCRUDDelete(router: EntityStorePluginRouter) { router.versioned .delete({ - path: '/internal/security/entity-store/entities/{id}', + path: '/internal/security/entity-store/entities/', access: 'internal', security: { authz: DEFAULT_ENTITY_STORE_PERMISSIONS, @@ -32,7 +32,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { version: API_VERSIONS.internal.v2, validate: { request: { - params: buildRouteValidationWithZod(paramsSchema), + body: buildRouteValidationWithZod(bodySchema), }, }, }, @@ -43,7 +43,7 @@ export function registerCRUDDelete(router: EntityStorePluginRouter) { logger.debug('CRUD Delete api called'); try { - await crudClient.deleteEntity(req.params.id); + await crudClient.deleteEntity(req.body.entityId); } catch (error) { if (error instanceof EntityNotFoundError) { return res.customError({ From 21e38e39107dd1295cc7df3ac64b668743b0aa9a Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 24 Feb 2026 10:45:23 +0100 Subject: [PATCH 69/69] crud_client: update DELETE tests and check ids --- .../test/scout/api/fixtures/constants.ts | 2 +- .../test/scout/api/tests/crud_api.spec.ts | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts index 6107b78377465..fb6e1f8778513 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/fixtures/constants.ts @@ -26,7 +26,7 @@ export const ENTITY_STORE_ROUTES = { `internal/security/entity-store/${entityType}/force-log-extraction`, CRUD_UPSERT: (entityType: string) => `internal/security/entity-store/entities/${entityType}`, CRUD_UPSERT_BULK: 'internal/security/entity-store/entities/bulk', - CRUD_DELETE: (id: string) => `internal/security/entity-store/entities/${id}`, + CRUD_DELETE: 'internal/security/entity-store/entities/', } as const; export const ENTITY_STORE_TAGS = [...tags.stateful.classic, ...tags.serverless.security.complete]; diff --git a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts index e71d9fab58d22..fb333eb7e9968 100644 --- a/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts +++ b/x-pack/solutions/security/plugins/entity_store/test/scout/api/tests/crud_api.spec.ts @@ -8,6 +8,8 @@ import { apiTest } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/api'; import type { Client } from '@elastic/elasticsearch'; +import { hashEuid } from '../../../../server/domain/crud_client/utils'; +import { getEuidFromObject } from '../../../../common/domain/euid'; import type { Entity, HostEntity } from '../../../../common/domain/definitions/entity.gen'; import { COMMON_HEADERS, @@ -66,6 +68,9 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = expect(create.body).toStrictEqual({ ok: true }); expect(await countEntitiesByID(esClient, LATEST_INDEX, entityObj.entity.id)).toBe(1); + const euid = getEuidFromObject('generic', entityObj) as string; + const check = await esClient.get({ index: LATEST_INDEX, id: hashEuid(euid) }); + expect(check.found).toBe(true); }); apiTest('Should require a force flag', async ({ apiClient, esClient }) => { @@ -234,11 +239,19 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = }); expect(resp.hits.hits).toHaveLength(1); expect(resp.hits.hits[0]._id).toBeDefined(); - const entityId = resp.hits.hits[0]._id as string; - const del = await apiClient.delete(ENTITY_STORE_ROUTES.CRUD_DELETE(entityId), { + const euid = getEuidFromObject('generic', entityObj) as string; + + const expectedHashedEntityId = hashEuid(euid); + const hashedEntityId = resp.hits.hits[0]._id as string; + expect(hashedEntityId).toBe(expectedHashedEntityId); + + const del = await apiClient.delete(ENTITY_STORE_ROUTES.CRUD_DELETE, { headers: defaultHeaders, responseType: 'json', + body: { + entityId: euid, + }, }); expect(del.body).toStrictEqual({ deleted: true }); expect(del.statusCode).toBe(200); @@ -246,13 +259,16 @@ apiTest.describe('Entity Store CRUD API tests', { tag: ENTITY_STORE_TAGS }, () = await expect( esClient.get({ index: LATEST_INDEX, - id: entityId, + id: hashedEntityId, }) ).rejects.toThrow(`"found":false`); - const apiNotFound = await apiClient.delete(ENTITY_STORE_ROUTES.CRUD_DELETE(entityId), { + const apiNotFound = await apiClient.delete(ENTITY_STORE_ROUTES.CRUD_DELETE, { headers: defaultHeaders, responseType: 'json', + body: { + entityId: entityObj.entity.id, + }, }); expect(apiNotFound.body.statusCode).toBe(404); });