From 776712d8bd6bbcc135657a48750f6548e5845a59 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Thu, 19 Mar 2026 10:30:21 +0100 Subject: [PATCH 1/5] common fields: update first_seen & last_seen to use @timestamp --- .../common/domain/definitions/common_fields.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts index 5a60088ba7ce1..96cf5255c7645 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts @@ -7,7 +7,7 @@ import type { Condition } from '@kbn/streamlang'; import type { EntityType, EntityField } from './entity_schema'; -import { collectValues, newestValue } from './field_retention_operations'; +import { collectValues, newestValue, oldestValue } from './field_retention_operations'; export const ENTITY_ID_FIELD = 'entity.id'; // Copied from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts @@ -84,13 +84,13 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => { }), // LIFECYCLE ------------------------------------------------------------ - newestValue({ - source: `${prefix}.lifecycle.first_seen`, + oldestValue({ + source: '@timestamp', destination: 'entity.lifecycle.first_seen', mapping: { type: 'date' }, }), newestValue({ - source: `${prefix}.lifecycle.last_activity`, + source: '@timestamp', destination: 'entity.lifecycle.last_activity', mapping: { type: 'date' }, }), From 67a6dc883918bd095815b74d12acc6fc804a18ee Mon Sep 17 00:00:00 2001 From: kubasobon Date: Thu, 19 Mar 2026 10:30:48 +0100 Subject: [PATCH 2/5] test: update snapshots --- ...logs_extraction_query_builder.test.ts.snap | 12 +++---- ...logs_extraction_query_builder.test.ts.snap | 36 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap index cac867a5b816d..6725656dc4c5f 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap @@ -72,8 +72,8 @@ FROM remote:metrics-* entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, - entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, - entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -154,8 +154,8 @@ FROM remote:logs-* entity.attributes.asset = LAST(TO_BOOLEAN(user.entity.attributes.asset), @timestamp) WHERE user.entity.attributes.asset IS NOT NULL, entity.attributes.managed = LAST(TO_BOOLEAN(user.entity.attributes.managed), @timestamp) WHERE user.entity.attributes.managed IS NOT NULL, entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(user.entity.attributes.mfa_enabled), @timestamp) WHERE user.entity.attributes.mfa_enabled IS NOT NULL, - entity.lifecycle.first_seen = LAST(TO_DATETIME(user.entity.lifecycle.first_seen), @timestamp) WHERE user.entity.lifecycle.first_seen IS NOT NULL, - entity.lifecycle.last_activity = LAST(TO_DATETIME(user.entity.lifecycle.last_activity), @timestamp) WHERE user.entity.lifecycle.last_activity IS NOT NULL, + entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.rule_names), 100)) WHERE user.entity.behaviors.rule_names IS NOT NULL, entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.anomaly_job_ids), 100)) WHERE user.entity.behaviors.anomaly_job_ids IS NOT NULL, entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(user.entity.relationships.communicates_with), 10)) WHERE user.entity.relationships.communicates_with IS NOT NULL, @@ -209,8 +209,8 @@ FROM remote_cluster:logs-* entity.attributes.asset = LAST(TO_BOOLEAN(entity.attributes.asset), @timestamp) WHERE entity.attributes.asset IS NOT NULL, entity.attributes.managed = LAST(TO_BOOLEAN(entity.attributes.managed), @timestamp) WHERE entity.attributes.managed IS NOT NULL, entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(entity.attributes.mfa_enabled), @timestamp) WHERE entity.attributes.mfa_enabled IS NOT NULL, - entity.lifecycle.first_seen = LAST(TO_DATETIME(entity.lifecycle.first_seen), @timestamp) WHERE entity.lifecycle.first_seen IS NOT NULL, - entity.lifecycle.last_activity = LAST(TO_DATETIME(entity.lifecycle.last_activity), @timestamp) WHERE entity.lifecycle.last_activity IS NOT NULL, + entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.rule_names), 100)) WHERE entity.behaviors.rule_names IS NOT NULL, entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.anomaly_job_ids), 100)) WHERE entity.behaviors.anomaly_job_ids IS NOT NULL, entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(entity.relationships.communicates_with), 10)) WHERE entity.relationships.communicates_with IS NOT NULL, diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap index ceb79176fc6a0..0f22e2e2f5d04 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap @@ -21,8 +21,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e recent.entity.attributes.asset = LAST(TO_BOOLEAN(entity.attributes.asset), @timestamp) WHERE entity.attributes.asset IS NOT NULL, recent.entity.attributes.managed = LAST(TO_BOOLEAN(entity.attributes.managed), @timestamp) WHERE entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(entity.attributes.mfa_enabled), @timestamp) WHERE entity.attributes.mfa_enabled IS NOT NULL, - recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(entity.lifecycle.first_seen), @timestamp) WHERE entity.lifecycle.first_seen IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(entity.lifecycle.last_activity), @timestamp) WHERE entity.lifecycle.last_activity IS NOT NULL, + recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.rule_names), 100)) WHERE entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.anomaly_job_ids), 100)) WHERE entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(entity.relationships.communicates_with), 10)) WHERE entity.relationships.communicates_with IS NOT NULL, @@ -90,7 +90,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), - entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -229,8 +229,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti recent.entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, - recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -308,7 +308,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), - entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -408,8 +408,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with recent.entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, - recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -490,7 +490,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), - entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -590,8 +590,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with recent.entity.attributes.asset = LAST(TO_BOOLEAN(host.entity.attributes.asset), @timestamp) WHERE host.entity.attributes.asset IS NOT NULL, recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, - recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(host.entity.lifecycle.first_seen), @timestamp) WHERE host.entity.lifecycle.first_seen IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(host.entity.lifecycle.last_activity), @timestamp) WHERE host.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -672,7 +672,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), - entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -746,8 +746,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e recent.entity.attributes.asset = LAST(TO_BOOLEAN(service.entity.attributes.asset), @timestamp) WHERE service.entity.attributes.asset IS NOT NULL, recent.entity.attributes.managed = LAST(TO_BOOLEAN(service.entity.attributes.managed), @timestamp) WHERE service.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(service.entity.attributes.mfa_enabled), @timestamp) WHERE service.entity.attributes.mfa_enabled IS NOT NULL, - recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(service.entity.lifecycle.first_seen), @timestamp) WHERE service.entity.lifecycle.first_seen IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(service.entity.lifecycle.last_activity), @timestamp) WHERE service.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(service.entity.behaviors.rule_names), 100)) WHERE service.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(service.entity.behaviors.anomaly_job_ids), 100)) WHERE service.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(service.entity.relationships.communicates_with), 10)) WHERE service.entity.relationships.communicates_with IS NOT NULL, @@ -801,7 +801,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), - entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -887,8 +887,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti recent.entity.attributes.asset = LAST(TO_BOOLEAN(user.entity.attributes.asset), @timestamp) WHERE user.entity.attributes.asset IS NOT NULL, recent.entity.attributes.managed = LAST(TO_BOOLEAN(user.entity.attributes.managed), @timestamp) WHERE user.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(user.entity.attributes.mfa_enabled), @timestamp) WHERE user.entity.attributes.mfa_enabled IS NOT NULL, - recent.entity.lifecycle.first_seen = LAST(TO_DATETIME(user.entity.lifecycle.first_seen), @timestamp) WHERE user.entity.lifecycle.first_seen IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(user.entity.lifecycle.last_activity), @timestamp) WHERE user.entity.lifecycle.last_activity IS NOT NULL, + recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.rule_names), 100)) WHERE user.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.anomaly_job_ids), 100)) WHERE user.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(user.entity.relationships.communicates_with), 10)) WHERE user.entity.relationships.communicates_with IS NOT NULL, @@ -951,7 +951,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti entity.attributes.asset = COALESCE(recent.entity.attributes.asset, entity.attributes.asset), entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), - entity.lifecycle.first_seen = COALESCE(recent.entity.lifecycle.first_seen, entity.lifecycle.first_seen), + entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), From 899e99b64f3e8c107400e64b01cb982eb89d4675 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Thu, 19 Mar 2026 12:42:36 +0100 Subject: [PATCH 3/5] common fields: restore last_activity and add last_seen --- .../domain/definitions/common_fields.ts | 5 ++++ .../domain/definitions/entity.schema.yaml | 29 ++++++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts index 96cf5255c7645..bbdb4e40570bd 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts @@ -91,6 +91,11 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => { }), newestValue({ source: '@timestamp', + destination: 'entity.lifecycle.last_seen', + mapping: { type: 'date' }, + }), + newestValue({ + source: '${prefix}.lifecycle.last_activity', destination: 'entity.lifecycle.last_activity', mapping: { type: 'date' }, }), diff --git a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml index 89d84a9d7fe59..9a1b0893b4654 100644 --- a/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml +++ b/x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml @@ -8,10 +8,10 @@ info: paths: {} components: schemas: - EngineMetadata: + EngineMetadata: type: object additionalProperties: false - required: + required: - Type properties: Type: @@ -71,6 +71,9 @@ components: first_seen: type: string format: date-time + last_seen: + type: string + format: date-time last_activity: type: string format: date-time @@ -220,7 +223,7 @@ components: ingested: type: string format: date-time - + HostEntity: type: object additionalProperties: false @@ -282,7 +285,7 @@ components: ingested: type: string format: date-time - + ServiceEntity: type: object additionalProperties: false @@ -318,11 +321,11 @@ components: format: date-time # The Generic Entity definition maps more than just entity. - # however I don't see a reason to duplicate the definition - # of all the fields just for the sake of doing. - # Thus the current mapping maps entity and asset only - # (used in code). If you end up needing the fields mapped - # in the schema just add it. + # however I don't see a reason to duplicate the definition + # of all the fields just for the sake of doing. + # Thus the current mapping maps entity and asset only + # (used in code). If you end up needing the fields mapped + # in the schema just add it. GenericEntity: type: object additionalProperties: false @@ -337,10 +340,10 @@ components: asset: $ref: '#/components/schemas/Asset' additionalProperties: false - + # These entities represent the API of Entity Store # where entity is a root field, as it's in the final - # index for entities. Note that this is different from the + # index for entities. Note that this is different from the # source logs mapping where `entity` will be nested under # `host`, `user` and `service` Entity: @@ -349,7 +352,7 @@ components: - $ref: '#/components/schemas/HostEntity' - $ref: '#/components/schemas/ServiceEntity' - $ref: '#/components/schemas/GenericEntity' - + # Temporary addition of risk score and asset criticality assets AssetCriticalityLevel: type: string @@ -497,4 +500,4 @@ components: additionalProperties: true required: - type - - contribution \ No newline at end of file + - contribution From 43dd5c1c48ef16cb04cce24733a100991dcafe4b Mon Sep 17 00:00:00 2001 From: kubasobon Date: Thu, 19 Mar 2026 12:43:54 +0100 Subject: [PATCH 4/5] test: update snapshots --- ...logs_extraction_query_builder.test.ts.snap | 9 ++++--- ...logs_extraction_query_builder.test.ts.snap | 24 ++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap index 6725656dc4c5f..f93770c358952 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/ccs_logs_extraction_query_builder.test.ts.snap @@ -73,7 +73,8 @@ FROM remote:metrics-* entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -155,7 +156,8 @@ FROM remote:logs-* entity.attributes.managed = LAST(TO_BOOLEAN(user.entity.attributes.managed), @timestamp) WHERE user.entity.attributes.managed IS NOT NULL, entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(user.entity.attributes.mfa_enabled), @timestamp) WHERE user.entity.attributes.mfa_enabled IS NOT NULL, entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.rule_names), 100)) WHERE user.entity.behaviors.rule_names IS NOT NULL, entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.anomaly_job_ids), 100)) WHERE user.entity.behaviors.anomaly_job_ids IS NOT NULL, entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(user.entity.relationships.communicates_with), 10)) WHERE user.entity.relationships.communicates_with IS NOT NULL, @@ -210,7 +212,8 @@ FROM remote_cluster:logs-* entity.attributes.managed = LAST(TO_BOOLEAN(entity.attributes.managed), @timestamp) WHERE entity.attributes.managed IS NOT NULL, entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(entity.attributes.mfa_enabled), @timestamp) WHERE entity.attributes.mfa_enabled IS NOT NULL, entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.rule_names), 100)) WHERE entity.behaviors.rule_names IS NOT NULL, entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.anomaly_job_ids), 100)) WHERE entity.behaviors.anomaly_job_ids IS NOT NULL, entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(entity.relationships.communicates_with), 10)) WHERE entity.relationships.communicates_with IS NOT NULL, diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap index 0f22e2e2f5d04..55274e5ca00dc 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/__snapshots__/logs_extraction_query_builder.test.ts.snap @@ -22,7 +22,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e recent.entity.attributes.managed = LAST(TO_BOOLEAN(entity.attributes.managed), @timestamp) WHERE entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(entity.attributes.mfa_enabled), @timestamp) WHERE entity.attributes.mfa_enabled IS NOT NULL, recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.rule_names), 100)) WHERE entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(entity.behaviors.anomaly_job_ids), 100)) WHERE entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(entity.relationships.communicates_with), 10)) WHERE entity.relationships.communicates_with IS NOT NULL, @@ -91,6 +92,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for generic e entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), + entity.lifecycle.last_seen = COALESCE(recent.entity.lifecycle.last_seen, entity.lifecycle.last_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -230,7 +232,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -309,6 +312,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host enti entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), + entity.lifecycle.last_seen = COALESCE(recent.entity.lifecycle.last_seen, entity.lifecycle.last_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -409,7 +413,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -491,6 +496,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), + entity.lifecycle.last_seen = COALESCE(recent.entity.lifecycle.last_seen, entity.lifecycle.last_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -591,7 +597,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with recent.entity.attributes.managed = LAST(TO_BOOLEAN(host.entity.attributes.managed), @timestamp) WHERE host.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(host.entity.attributes.mfa_enabled), @timestamp) WHERE host.entity.attributes.mfa_enabled IS NOT NULL, recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.rule_names), 100)) WHERE host.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(host.entity.behaviors.anomaly_job_ids), 100)) WHERE host.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(host.entity.relationships.communicates_with), 10)) WHERE host.entity.relationships.communicates_with IS NOT NULL, @@ -673,6 +680,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for host with entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), + entity.lifecycle.last_seen = COALESCE(recent.entity.lifecycle.last_seen, entity.lifecycle.last_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -747,7 +755,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e recent.entity.attributes.managed = LAST(TO_BOOLEAN(service.entity.attributes.managed), @timestamp) WHERE service.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(service.entity.attributes.mfa_enabled), @timestamp) WHERE service.entity.attributes.mfa_enabled IS NOT NULL, recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(service.entity.behaviors.rule_names), 100)) WHERE service.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(service.entity.behaviors.anomaly_job_ids), 100)) WHERE service.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(service.entity.relationships.communicates_with), 10)) WHERE service.entity.relationships.communicates_with IS NOT NULL, @@ -802,6 +811,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for service e entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), + entity.lifecycle.last_seen = COALESCE(recent.entity.lifecycle.last_seen, entity.lifecycle.last_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), @@ -888,7 +898,8 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti recent.entity.attributes.managed = LAST(TO_BOOLEAN(user.entity.attributes.managed), @timestamp) WHERE user.entity.attributes.managed IS NOT NULL, recent.entity.attributes.mfa_enabled = LAST(TO_BOOLEAN(user.entity.attributes.mfa_enabled), @timestamp) WHERE user.entity.attributes.mfa_enabled IS NOT NULL, recent.entity.lifecycle.first_seen = FIRST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, - recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_seen = LAST(TO_DATETIME(@timestamp), @timestamp) WHERE @timestamp IS NOT NULL, + recent.entity.lifecycle.last_activity = LAST(TO_DATETIME(\${prefix}.lifecycle.last_activity), @timestamp) WHERE \${prefix}.lifecycle.last_activity IS NOT NULL, recent.entity.behaviors.rule_names = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.rule_names), 100)) WHERE user.entity.behaviors.rule_names IS NOT NULL, recent.entity.behaviors.anomaly_job_ids = MV_DEDUPE(TOP(TO_STRING(user.entity.behaviors.anomaly_job_ids), 100)) WHERE user.entity.behaviors.anomaly_job_ids IS NOT NULL, recent.entity.relationships.communicates_with = MV_DEDUPE(TOP(TO_STRING(user.entity.relationships.communicates_with), 10)) WHERE user.entity.relationships.communicates_with IS NOT NULL, @@ -952,6 +963,7 @@ exports[`buildLogsExtractionEsqlQuery generates the expected query for user enti entity.attributes.managed = COALESCE(recent.entity.attributes.managed, entity.attributes.managed), entity.attributes.mfa_enabled = COALESCE(recent.entity.attributes.mfa_enabled, entity.attributes.mfa_enabled), entity.lifecycle.first_seen = COALESCE(entity.lifecycle.first_seen, recent.entity.lifecycle.first_seen), + entity.lifecycle.last_seen = COALESCE(recent.entity.lifecycle.last_seen, entity.lifecycle.last_seen), entity.lifecycle.last_activity = COALESCE(recent.entity.lifecycle.last_activity, entity.lifecycle.last_activity), entity.behaviors.rule_names = MV_SLICE(MV_UNION(recent.entity.behaviors.rule_names, entity.behaviors.rule_names), 0, 99), entity.behaviors.anomaly_job_ids = MV_SLICE(MV_UNION(recent.entity.behaviors.anomaly_job_ids, entity.behaviors.anomaly_job_ids), 0, 99), From 2e83a1907861b4d38a12bbc848a1637fc3c31ab0 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Thu, 19 Mar 2026 16:43:04 +0100 Subject: [PATCH 5/5] Use entity.lifecycle.last_seen for automated resolution watermark Switch watermark field from @timestamp to entity.lifecycle.last_seen to avoid daily full re-scans caused by the history snapshot operation resetting @timestamp on all entities. --- .../automated_resolution/__tests__/run.test.ts | 14 +++++++++----- .../server/maintainers/automated_resolution/run.ts | 6 ++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/__tests__/run.test.ts b/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/__tests__/run.test.ts index c9f1d158942f6..0757bd9fcc31d 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/__tests__/run.test.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/__tests__/run.test.ts @@ -126,8 +126,10 @@ describe('Automated Resolution', () => { const step1Query = mockEsClient.search.mock.calls[0][0] as any; const filters = step1Query.query.bool.filter; - const hasTimestampFilter = filters.some((f: any) => f.range && f.range['@timestamp']); - expect(hasTimestampFilter).toBe(false); + const hasLastSeenFilter = filters.some( + (f: any) => f.range && f.range['entity.lifecycle.last_seen'] + ); + expect(hasLastSeenFilter).toBe(false); }); it('should include timestamp range filter for incremental scan', async () => { @@ -153,9 +155,11 @@ describe('Automated Resolution', () => { const step1Query = mockEsClient.search.mock.calls[0][0] as any; const filters = step1Query.query.bool.filter; - const timestampFilter = filters.find((f: any) => f.range && f.range['@timestamp']); - expect(timestampFilter).toEqual({ - range: { '@timestamp': { gt: '2026-03-09T00:00:00Z' } }, + const lastSeenFilter = filters.find( + (f: any) => f.range && f.range['entity.lifecycle.last_seen'] + ); + expect(lastSeenFilter).toEqual({ + range: { 'entity.lifecycle.last_seen': { gt: '2026-03-09T00:00:00Z' } }, }); }); diff --git a/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/run.ts b/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/run.ts index 66ac68504316a..bc9d17d629986 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/run.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/maintainers/automated_resolution/run.ts @@ -118,7 +118,9 @@ async function collectNewEmailValues( ]; if (state.lastProcessedTimestamp) { - filters.push({ range: { '@timestamp': { gt: state.lastProcessedTimestamp } } }); + filters.push({ + range: { 'entity.lifecycle.last_seen': { gt: state.lastProcessedTimestamp } }, + }); } do { @@ -137,7 +139,7 @@ async function collectNewEmailValues( ...(afterKey ? { after: afterKey } : {}), }, }, - max_timestamp: { max: { field: '@timestamp' } }, + max_timestamp: { max: { field: 'entity.lifecycle.last_seen' } }, }, });