diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/entity_client.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/entity_client.ts index 0d13dbe3dd95c..1cc83c0b27560 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/entity_client.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/entity_client.ts @@ -8,7 +8,6 @@ import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { IScopedClusterClient, Logger } from '@kbn/core/server'; import { EntityV2 } from '@kbn/entities-schema'; -import { without } from 'lodash'; import { ReadSourceDefinitionOptions, readSourceDefinitions, @@ -16,7 +15,7 @@ import { } from './definitions/source_definition'; import { readTypeDefinitions, storeTypeDefinition } from './definitions/type_definition'; import { getEntityInstancesQuery } from './queries'; -import { mergeEntitiesList } from './queries/utils'; +import { mergeEntitiesList, sortEntitiesList } from './queries/utils'; import { EntitySourceDefinition, EntityTypeDefinition, @@ -25,6 +24,7 @@ import { } from './types'; import { UnknownEntityType } from './errors/unknown_entity_type'; import { runESQLQuery } from './run_esql_query'; +import { validateFields } from './validate_fields'; export class EntityClient { constructor( @@ -59,72 +59,63 @@ export class EntityClient { sort, limit, }: SearchBySources) { - const entities = await Promise.all( - sources.map(async (source) => { - const mandatoryFields = [ - ...source.identity_fields, - ...(source.timestamp_field ? [source.timestamp_field] : []), - ...(source.display_name ? [source.display_name] : []), - ]; - const metaFields = [...metadataFields, ...source.metadata_fields]; - - // operations on an unmapped field result in a failing query so we verify - // field capabilities beforehand - const { fields } = await this.options.clusterClient.asCurrentUser.fieldCaps({ - index: source.index_patterns, - fields: [...mandatoryFields, ...metaFields], - }); - - const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]); - if (!sourceHasMandatoryFields) { - // we can't build entities without id fields so we ignore the source. - // TODO filters should likely behave similarly. we should also throw - const missingFields = mandatoryFields.filter((field) => !fields[field]); - this.options.logger.info( - `Ignoring source for type [${source.type_id}] with index_patterns [${ - source.index_patterns - }] because some mandatory fields [${missingFields.join(', ')}] are not mapped` - ); - return []; - } - - // but metadata field not being available is fine - const availableMetadataFields = metaFields.filter((field) => fields[field]); - if (availableMetadataFields.length < metaFields.length) { - this.options.logger.info( - `Ignoring unmapped fields [${without(metaFields, ...availableMetadataFields).join( - ', ' - )}]` - ); - } - - const { query, filter } = getEntityInstancesQuery({ - source: { - ...source, - metadata_fields: availableMetadataFields, - filters: [...source.filters, ...filters], - }, - start, - end, - sort, - limit, - }); - this.options.logger.debug( - () => `Entity query: ${query}\nfilter: ${JSON.stringify(filter, null, 2)}` - ); - - const rawEntities = await runESQLQuery('resolve entities', { - query, - filter, - esClient: this.options.clusterClient.asCurrentUser, - logger: this.options.logger, - }); - - return rawEntities; - }) - ).then((results) => results.flat()); - - return mergeEntitiesList(sources, entities).slice(0, limit); + const searches = sources.map(async (source) => { + const availableMetadataFields = await validateFields({ + source, + metadataFields, + esClient: this.options.clusterClient.asCurrentUser, + logger: this.options.logger, + }); + + const { query, filter } = getEntityInstancesQuery({ + source: { + ...source, + metadata_fields: availableMetadataFields, + filters: [...source.filters, ...filters], + }, + start, + end, + sort, + limit, + }); + this.options.logger.debug( + () => `Entity query: ${query}\nfilter: ${JSON.stringify(filter, null, 2)}` + ); + + const rawEntities = await runESQLQuery('resolve entities', { + query, + filter, + esClient: this.options.clusterClient.asCurrentUser, + logger: this.options.logger, + }); + + return rawEntities; + }); + + const results = await Promise.allSettled(searches); + const entities = ( + results.filter((result) => result.status === 'fulfilled') as Array< + PromiseFulfilledResult + > + ).flatMap((result) => result.value); + const errors = ( + results.filter((result) => result.status === 'rejected') as PromiseRejectedResult[] + ).map((result) => result.reason.message); + + if (sources.length === 1) { + return { entities, errors }; + } + + // we have to manually merge, sort and limit entities since we run + // independant queries for each source + return { + errors, + entities: sortEntitiesList({ + sources, + sort, + entities: mergeEntitiesList({ entities, sources, metadataFields }), + }).slice(0, limit), + }; } async storeTypeDefinition(type: EntityTypeDefinition) { diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.test.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.test.ts index 9bc475d031923..3ec87b220f84c 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.test.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.test.ts @@ -29,7 +29,7 @@ describe('getEntityInstancesQuery', () => { expect(query).toEqual( 'FROM logs-*, metrics-* | ' + - 'STATS host.name = VALUES(host.name::keyword), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id::keyword) BY service.name::keyword | ' + + 'STATS host.name = TOP(host.name::keyword, 10, "asc"), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id::keyword) BY service.name::keyword | ' + 'RENAME `service.name::keyword` AS service.name | ' + 'EVAL entity.type = "service", entity.id = service.name, entity.display_name = COALESCE(service.id, entity.id) | ' + 'SORT entity.id DESC | ' + diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.ts index 5ce7a54eb1d1c..8814b907f7d38 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/index.ts @@ -6,7 +6,7 @@ */ import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; -import { asKeyword } from './utils'; +import { asKeyword, defaultSort } from './utils'; import { EntitySourceDefinition, SortBy } from '../types'; const sourceCommand = ({ source }: { source: EntitySourceDefinition }) => { @@ -46,7 +46,7 @@ const dslFilter = ({ const statsCommand = ({ source }: { source: EntitySourceDefinition }) => { const aggs = source.metadata_fields .filter((field) => !source.identity_fields.some((idField) => idField === field)) - .map((field) => `${field} = VALUES(${asKeyword(field)})`); + .map((field) => `${field} = TOP(${asKeyword(field)}, 10, "asc")`); if (source.timestamp_field) { aggs.push(`entity.last_seen_timestamp = MAX(${source.timestamp_field})`); @@ -84,15 +84,11 @@ const evalCommand = ({ source }: { source: EntitySourceDefinition }) => { }; const sortCommand = ({ source, sort }: { source: EntitySourceDefinition; sort?: SortBy }) => { - if (sort) { - return `SORT ${sort.field} ${sort.direction}`; + if (!sort) { + sort = defaultSort([source]); } - if (source.timestamp_field) { - return `SORT entity.last_seen_timestamp DESC`; - } - - return `SORT entity.id ASC`; + return `SORT ${sort.field} ${sort.direction}`; }; export function getEntityInstancesQuery({ diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.test.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.test.ts index 295ab7796585c..2aee58828defa 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.test.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.test.ts @@ -17,22 +17,32 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', }, { 'entity.id': 'foo', 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', }, ]; - const mergedEntities = mergeEntitiesList([], entities); + const mergedEntities = mergeEntitiesList({ + entities, + metadataFields: [], + sources: [ + { identity_fields: ['service.name'], metadata_fields: ['only_in_record_1'] }, + { identity_fields: ['service.name'], metadata_fields: ['only_in_record_2'] }, + ] as EntitySourceDefinition[], + }); expect(mergedEntities.length).toEqual(1); expect(mergedEntities[0]).toEqual({ 'entity.id': 'foo', 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', }); }); @@ -43,6 +53,7 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-1', 'agent.name': 'agent-1', 'service.environment': ['dev', 'staging'], @@ -53,6 +64,7 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': ['host-2', 'host-3'], 'agent.name': 'agent-2', 'service.environment': 'prod', @@ -60,23 +72,21 @@ describe('mergeEntitiesList', () => { }, ]; - const mergedEntities = mergeEntitiesList( - [ - { - metadata_fields: ['host.name', 'agent.name', 'service.environment', 'only_in_record_1'], - }, - { - metadata_fields: ['host.name', 'agent.name', 'service.environment', 'only_in_record_2'], - }, + const mergedEntities = mergeEntitiesList({ + entities, + metadataFields: ['host.name', 'agent.name', 'service.environment'], + sources: [ + { identity_fields: ['service.name'], metadata_fields: ['only_in_record_1'] }, + { identity_fields: ['service.name'], metadata_fields: ['only_in_record_2'] }, ] as EntitySourceDefinition[], - entities - ); + }); expect(mergedEntities.length).toEqual(1); expect(mergedEntities[0]).toEqual({ 'entity.id': 'foo', 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': ['host-1', 'host-2', 'host-3'], 'agent.name': ['agent-1', 'agent-2'], 'service.environment': ['dev', 'staging', 'prod'], @@ -92,6 +102,7 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-1', }, { @@ -99,6 +110,7 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-2', }, { @@ -106,33 +118,33 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-3', }, { 'entity.id': 'foo', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-3', }, ]; - const mergedEntities = mergeEntitiesList( - [ - { - metadata_fields: ['host.name'], - }, - { - metadata_fields: ['host.name'], - }, + const mergedEntities = mergeEntitiesList({ + entities, + metadataFields: ['host.name'], + sources: [ + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, ] as EntitySourceDefinition[], - entities - ); + }); expect(mergedEntities.length).toEqual(1); expect(mergedEntities[0]).toEqual({ 'entity.id': 'foo', 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': ['host-1', 'host-2', 'host-3'], }); }); @@ -143,28 +155,32 @@ describe('mergeEntitiesList', () => { 'entity.id': 'foo', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-1', }, { 'entity.id': 'foo', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-2', }, ]; - const mergedEntities = mergeEntitiesList( - [ - { metadata_fields: ['host.name'] }, - { metadata_fields: ['host.name'] }, + const mergedEntities = mergeEntitiesList({ + entities, + metadataFields: ['host.name'], + sources: [ + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, ] as EntitySourceDefinition[], - entities - ); + }); expect(mergedEntities.length).toEqual(1); expect(mergedEntities[0]).toEqual({ 'entity.id': 'foo', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': ['host-1', 'host-2'], }); }); @@ -176,6 +192,7 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-1', }, { @@ -183,6 +200,7 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': 'host-2', }, { @@ -190,29 +208,74 @@ describe('mergeEntitiesList', () => { 'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': ['host-1', 'host-2'], }, ]; - const mergedEntities = mergeEntitiesList( - [ - { - metadata_fields: ['host.name'], - }, - { - metadata_fields: ['host.name'], - }, + const mergedEntities = mergeEntitiesList({ + entities, + metadataFields: ['host.name'], + sources: [ + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, ] as EntitySourceDefinition[], - entities - ); + }); expect(mergedEntities.length).toEqual(1); expect(mergedEntities[0]).toEqual({ 'entity.id': 'foo', 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', 'entity.type': 'service', 'entity.display_name': 'foo', + 'service.name': 'foo', 'host.name': ['host-1', 'host-2'], }); }); + + it('assigns all identity fields to the merged entity', () => { + const entities = [ + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + 'entity.display_name': 'foo', + 'service.name': 'foo', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', + 'entity.type': 'service', + 'entity.display_name': 'foo', + servicename_field: 'foo', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z', + 'entity.type': 'service', + 'entity.display_name': 'foo', + service_name: 'foo', + }, + ]; + + const mergedEntities = mergeEntitiesList({ + entities, + metadataFields: [], + sources: [ + { identity_fields: ['service.name'], metadata_fields: [] as string[] }, + { identity_fields: ['servicename_field'], metadata_fields: [] as string[] }, + { identity_fields: ['service_name'], metadata_fields: [] as string[] }, + ] as EntitySourceDefinition[], + }); + expect(mergedEntities.length).toEqual(1); + expect(mergedEntities[0]).toEqual({ + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', + 'entity.type': 'service', + 'entity.display_name': 'foo', + 'service.name': 'foo', + servicename_field: 'foo', + service_name: 'foo', + }); + }); }); }); diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.ts index 5d1ebf3001434..c08d5adb3ce4d 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.ts @@ -6,8 +6,8 @@ */ import { EntityV2 } from '@kbn/entities-schema'; -import { compact, uniq } from 'lodash'; -import { EntitySourceDefinition } from '../types'; +import { orderBy, uniq } from 'lodash'; +import { EntitySourceDefinition, SortBy } from '../types'; function getLatestDate(date1?: string, date2?: string) { if (!date1 && !date2) return; @@ -17,7 +17,12 @@ function getLatestDate(date1?: string, date2?: string) { ).toISOString(); } -function mergeEntities(metadataFields: string[], entity1: EntityV2, entity2: EntityV2): EntityV2 { +function mergeEntities( + identityFields: string[], + mergeableFields: string[], + entity1: EntityV2, + entity2: EntityV2 +): EntityV2 { const merged: EntityV2 = { ...entity1 }; const latestTimestamp = getLatestDate( @@ -29,13 +34,17 @@ function mergeEntities(metadataFields: string[], entity1: EntityV2, entity2: Ent } for (const [key, value] of Object.entries(entity2).filter(([_key]) => - metadataFields.includes(_key) + mergeableFields.includes(_key) )) { if (merged[key]) { - merged[key] = uniq([ - ...(Array.isArray(merged[key]) ? merged[key] : [merged[key]]), - ...(Array.isArray(value) ? value : [value]), - ]); + // we want to keep identity fields as single-value properties. + // this can happen if two sources group by the same identity + if (!identityFields.includes(key)) { + merged[key] = uniq([ + ...(Array.isArray(merged[key]) ? merged[key] : [merged[key]]), + ...(Array.isArray(value) ? value : [value]), + ]); + } } else { merged[key] = value; } @@ -43,13 +52,21 @@ function mergeEntities(metadataFields: string[], entity1: EntityV2, entity2: Ent return merged; } -export function mergeEntitiesList( - sources: EntitySourceDefinition[], - entities: EntityV2[] -): EntityV2[] { - const metadataFields = uniq( - sources.flatMap((source) => compact([source.timestamp_field, ...source.metadata_fields])) - ); +export function mergeEntitiesList({ + entities, + sources, + metadataFields, +}: { + entities: EntityV2[]; + sources: EntitySourceDefinition[]; + metadataFields: string[]; +}): EntityV2[] { + const identityFields = uniq([...sources.flatMap((source) => source.identity_fields)]); + const mergeableFields = uniq([ + ...identityFields, + ...metadataFields, + ...sources.flatMap((source) => source.metadata_fields), + ]); const instances: { [key: string]: EntityV2 } = {}; for (let i = 0; i < entities.length; i++) { @@ -57,7 +74,7 @@ export function mergeEntitiesList( const id = entity['entity.id']; if (instances[id]) { - instances[id] = mergeEntities(metadataFields, instances[id], entity); + instances[id] = mergeEntities(identityFields, mergeableFields, instances[id], entity); } else { instances[id] = entity; } @@ -66,6 +83,30 @@ export function mergeEntitiesList( return Object.values(instances); } +export function sortEntitiesList({ + entities, + sources, + sort, +}: { + entities: EntityV2[]; + sources: EntitySourceDefinition[]; + sort?: SortBy; +}) { + if (!sort) { + sort = defaultSort(sources); + } + + return orderBy(entities, sort.field, sort.direction.toLowerCase() as 'asc' | 'desc'); +} + export function asKeyword(field: string) { return `${field}::keyword`; } + +export function defaultSort(sources: EntitySourceDefinition[]): SortBy { + if (sources.some((source) => source.timestamp_field)) { + return { field: 'entity.last_seen_timestamp', direction: 'DESC' }; + } + + return { field: 'entity.id', direction: 'ASC' }; +} diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/types.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/types.ts index 202a32233d198..f3db5840e7a6a 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/types.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/types.ts @@ -83,6 +83,8 @@ const searchCommonRt = z.object({ filters: z.optional(z.array(z.string())).default([]), }); +export type SearchCommon = z.output; + export const searchByTypeRt = z.intersection( searchCommonRt, z.object({ diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/validate_fields.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/validate_fields.ts new file mode 100644 index 0000000000000..f24cee745ffd7 --- /dev/null +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/v2/validate_fields.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { without } from 'lodash'; +import { Logger } from '@kbn/logging'; +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { EntitySourceDefinition } from './types'; + +// verifies field capabilities of the provided source. +// we map source fields in two categories: +// - mandatory: those are necessary for building entities according to the +// source definition (identity_fields, timestamp_field and display_name). +// unmapped mandatory field throws an error +// - optional: the requested metadata fields. unmapped metadata field is not +// fatal, we simply ignore it +// returns the available metadata fields. +export async function validateFields({ + esClient, + source, + metadataFields, + logger, +}: { + esClient: ElasticsearchClient; + source: EntitySourceDefinition; + metadataFields: string[]; + logger: Logger; +}) { + const mandatoryFields = [ + ...source.identity_fields, + ...(source.timestamp_field ? [source.timestamp_field] : []), + ...(source.display_name ? [source.display_name] : []), + ]; + const metaFields = [...metadataFields, ...source.metadata_fields]; + + const { fields } = await esClient + .fieldCaps({ + index: source.index_patterns, + fields: [...mandatoryFields, ...metaFields], + }) + .catch((err) => { + if (err.meta?.statusCode === 404) { + throw new Error( + `No index found for source [${ + source.id + }] with index patterns [${source.index_patterns.join(', ')}]` + ); + } + throw err; + }); + + const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]); + if (!sourceHasMandatoryFields) { + // TODO filters should likely behave similarly + const missingFields = mandatoryFields.filter((field) => !fields[field]); + throw new Error( + `Mandatory fields [${missingFields.join(', ')}] are not mapped for source [${ + source.id + }] with index patterns [${source.index_patterns.join(', ')}]` + ); + } + + // operations on an unmapped field result in a failing query + const availableMetadataFields = metaFields.filter((field) => fields[field]); + if (availableMetadataFields.length < metaFields.length) { + logger.info( + `Ignoring unmapped metadata fields [${without(metaFields, ...availableMetadataFields).join( + ', ' + )}] for source [${source.id}]` + ); + } + return availableMetadataFields; +} diff --git a/x-pack/platform/plugins/shared/entity_manager/server/routes/v2/search.ts b/x-pack/platform/plugins/shared/entity_manager/server/routes/v2/search.ts index 82d0e5e84f2da..a2ebb26da715f 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/routes/v2/search.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/routes/v2/search.ts @@ -24,9 +24,9 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({ handler: async ({ request, response, params, logger, getScopedClient }) => { try { const client = await getScopedClient({ request }); - const entities = await client.v2.searchEntities(params.body); + const result = await client.v2.searchEntities(params.body); - return response.ok({ body: { entities } }); + return response.ok({ body: result }); } catch (e) { logger.error(e); @@ -51,9 +51,9 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({ }), handler: async ({ request, response, params, getScopedClient }) => { const client = await getScopedClient({ request }); - const entities = await client.v2.searchEntitiesBySources(params.body); + const result = await client.v2.searchEntitiesBySources(params.body); - return response.ok({ body: { entities } }); + return response.ok({ body: result }); }, }); diff --git a/x-pack/test/api_integration/apis/entity_manager/helpers/clear_entity_definitions.ts b/x-pack/test/api_integration/apis/entity_manager/helpers/clear_entity_definitions.ts new file mode 100644 index 0000000000000..796761011e7ac --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/helpers/clear_entity_definitions.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { DEFINITIONS_ALIAS } from '@kbn/entityManager-plugin/server/lib/v2/constants'; + +export async function clearEntityDefinitions(esClient: ElasticsearchClient) { + await esClient.deleteByQuery({ + index: DEFINITIONS_ALIAS, + query: { + match_all: {}, + }, + }); +} diff --git a/x-pack/test/api_integration/apis/entity_manager/helpers/data_generation.ts b/x-pack/test/api_integration/apis/entity_manager/helpers/data_generation.ts new file mode 100644 index 0000000000000..3af84e894b7ff --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/helpers/data_generation.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; +import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'; + +export async function createIndexWithDocuments( + client: Client, + options: { + index: string; + properties: Record; + documents: Array>; + } +) { + await client.indices.create({ + index: options.index, + mappings: { + dynamic: false, + properties: options.properties, + }, + }); + + const bulkActions = options.documents.flatMap((doc) => { + return [{ create: { _index: options.index } }, doc]; + }); + + await client.bulk({ + body: bulkActions, + refresh: 'wait_for', + }); + + return () => client.indices.delete({ index: options.index }); +} diff --git a/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts b/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts index c21f33cc8793a..4c09ecb664ab3 100644 --- a/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts +++ b/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts @@ -8,6 +8,7 @@ import { Agent } from 'supertest'; import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types'; +import { EntitySourceDefinition } from '@kbn/entityManager-plugin/server/lib/v2/types'; export interface Auth { username: string; @@ -89,3 +90,51 @@ export const upgradeBuiltinDefinitions = async ( .expect(200); return response.body; }; + +export const createEntityTypeDefinition = ( + supertest: Agent, + params: { + type: { + id: string; + display_name: string; + }; + } +) => { + return supertest + .post('/internal/entities/v2/definitions/types') + .set('kbn-xsrf', 'xxx') + .send({ type: params.type }) + .expect(201); +}; + +export const createEntitySourceDefinition = ( + supertest: Agent, + params: { + source: EntitySourceDefinition; + } +) => { + return supertest + .post('/internal/entities/v2/definitions/sources') + .set('kbn-xsrf', 'xxx') + .send({ source: params.source }) + .expect(201); +}; + +export const searchEntities = async ( + supertest: Agent, + params: { + type: string; + start?: string; + end?: string; + metadata_fields?: string[]; + filters?: string[]; + }, + expectedCode?: number +) => { + const response = await supertest + .post('/internal/entities/v2/_search') + .set('kbn-xsrf', 'xxx') + .send(params) + .expect(expectedCode ?? 200); + return response.body; +}; diff --git a/x-pack/test/api_integration/apis/entity_manager/index.ts b/x-pack/test/api_integration/apis/entity_manager/index.ts index 8f7af35e5c8ad..9b3ecfa550c0d 100644 --- a/x-pack/test/api_integration/apis/entity_manager/index.ts +++ b/x-pack/test/api_integration/apis/entity_manager/index.ts @@ -13,5 +13,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./builtin_definitions')); loadTestFile(require.resolve('./definitions')); + loadTestFile(require.resolve('./search')); }); } diff --git a/x-pack/test/api_integration/apis/entity_manager/search.ts b/x-pack/test/api_integration/apis/entity_manager/search.ts new file mode 100644 index 0000000000000..778682ff9008d --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/search.ts @@ -0,0 +1,814 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import expect from 'expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { + createEntitySourceDefinition, + createEntityTypeDefinition, + searchEntities, +} from './helpers/request'; +import { createIndexWithDocuments } from './helpers/data_generation'; +import { clearEntityDefinitions } from './helpers/clear_entity_definitions'; + +export default function ({ getService }: FtrProviderContext) { + const esClient = getService('es'); + const supertest = getService('supertest'); + + describe('_search API', () => { + let cleanup: Function[] = []; + + afterEach(async () => { + await Promise.all([clearEntityDefinitions(esClient), ...cleanup.map((fn) => fn())]); + cleanup = []; + }); + + it('returns 404 when no matching sources', async () => { + await searchEntities(supertest, { type: 'undefined-type' }, 404); + }); + + it('resolves entities from sources with timestamp', async () => { + const now = moment(); + + cleanup.push( + await createIndexWithDocuments(esClient, { + index: 'index-1-with-services', + properties: { + custom_timestamp: { type: 'date' }, + 'service.name': { type: 'keyword' }, + }, + documents: [ + { + custom_timestamp: moment(now).subtract(1, 'minute').toISOString(), + 'service.name': 'service-one', + }, + { + custom_timestamp: moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-two', + }, + { + custom_timestamp: moment(now).subtract(1, 'hour').toISOString(), + 'service.name': 'service-three', + }, + ], + }) + ); + + await createEntityTypeDefinition(supertest, { + type: { id: 'services-with-timestamp', display_name: 'services-with-timestamp' }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'source-1-with-services', + type_id: 'services-with-timestamp', + index_patterns: ['index-1-with-services'], + identity_fields: ['service.name'], + metadata_fields: [], + filters: [], + timestamp_field: 'custom_timestamp', + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'services-with-timestamp', + start: moment(now).subtract(10, 'minute').toISOString(), + end: now.toISOString(), + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.last_seen_timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'entity.id': 'service-one', + 'entity.display_name': 'service-one', + 'entity.type': 'services-with-timestamp', + 'service.name': 'service-one', + }, + { + 'entity.last_seen_timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'entity.id': 'service-two', + 'entity.display_name': 'service-two', + 'entity.type': 'services-with-timestamp', + 'service.name': 'service-two', + }, + ]); + }); + + it('resolves entities from sources without timestamp', async () => { + cleanup.push( + await createIndexWithDocuments(esClient, { + index: 'index-1-with-home-appliances', + properties: { 'appliance.name': { type: 'keyword' } }, + documents: [ + { 'appliance.name': 'rice cooker' }, + { 'appliance.name': 'kettle' }, + { 'appliance.name': 'dishwasher' }, + ], + }) + ); + + await createEntityTypeDefinition(supertest, { + type: { id: 'home-appliances', display_name: 'home-appliances' }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'appliances-no-timestamp', + type_id: 'home-appliances', + index_patterns: ['index-1-with-home-appliances'], + identity_fields: ['appliance.name'], + metadata_fields: [], + filters: [], + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'home-appliances', + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.id': 'dishwasher', + 'entity.display_name': 'dishwasher', + 'entity.type': 'home-appliances', + 'appliance.name': 'dishwasher', + }, + { + 'entity.id': 'kettle', + 'entity.display_name': 'kettle', + 'entity.type': 'home-appliances', + 'appliance.name': 'kettle', + }, + { + 'entity.id': 'rice cooker', + 'entity.display_name': 'rice cooker', + 'entity.type': 'home-appliances', + 'appliance.name': 'rice cooker', + }, + ]); + }); + + it('merges entities from different sources', async () => { + const now = moment(); + + cleanup = await Promise.all([ + createIndexWithDocuments(esClient, { + index: 'index-1-with-hosts', + properties: { + '@timestamp': { type: 'date' }, + 'host.name': { type: 'keyword' }, + 'agent.name': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'host.name': 'host-uno', + 'agent.name': 'agent-a', + }, + { + '@timestamp': moment(now).subtract(3, 'minute').toISOString(), + 'host.name': 'host-uno', + 'agent.name': 'agent-b', + }, + { + '@timestamp': moment(now).subtract(3, 'minute').toISOString(), + 'host.name': 'host-dos', + 'agent.name': 'agent-3', + }, + { + '@timestamp': moment(now).subtract(4, 'minute').toISOString(), + 'host.name': 'host-tres', + }, + ], + }), + + createIndexWithDocuments(esClient, { + index: 'index-2-with-hosts', + properties: { + timestamp: { type: 'date' }, + hostname_field: { type: 'keyword' }, + 'agent.name': { type: 'keyword' }, + }, + documents: [ + { + timestamp: moment(now).subtract(2, 'minute').toISOString(), + hostname_field: 'host-uno', + 'agent.name': 'agent-1', + }, + { + timestamp: moment(now).subtract(2, 'minute').toISOString(), + hostname_field: 'host-dos', + 'agent.name': 'agent-k', + }, + { + timestamp: moment(now).subtract(5, 'minute').toISOString(), + hostname_field: 'host-four', + }, + ], + }), + ]); + + await createEntityTypeDefinition(supertest, { + type: { id: 'hosts-with-agents', display_name: 'hosts-with-agents' }, + }); + await Promise.all([ + createEntitySourceDefinition(supertest, { + source: { + id: 'hosts-with-agents-1', + type_id: 'hosts-with-agents', + index_patterns: ['index-1-with-hosts'], + identity_fields: ['host.name'], + metadata_fields: [], + filters: [], + timestamp_field: '@timestamp', + }, + }), + createEntitySourceDefinition(supertest, { + source: { + id: 'hosts-with-agents-2', + type_id: 'hosts-with-agents', + index_patterns: ['index-2-with-hosts'], + identity_fields: ['hostname_field'], + metadata_fields: [], + filters: [], + timestamp_field: 'timestamp', + }, + }), + ]); + + const { entities, errors } = await searchEntities(supertest, { + type: 'hosts-with-agents', + start: moment(now).subtract(10, 'minute').toISOString(), + end: moment(now).toISOString(), + metadata_fields: ['agent.name'], + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.id': 'host-uno', + 'entity.display_name': 'host-uno', + 'entity.type': 'hosts-with-agents', + 'entity.last_seen_timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'host.name': 'host-uno', + hostname_field: 'host-uno', + 'agent.name': expect.arrayContaining(['agent-a', 'agent-b', 'agent-1']), + }, + { + 'entity.id': 'host-dos', + 'entity.display_name': 'host-dos', + 'entity.type': 'hosts-with-agents', + 'entity.last_seen_timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'host.name': 'host-dos', + hostname_field: 'host-dos', + 'agent.name': expect.arrayContaining(['agent-3', 'agent-k']), + }, + { + 'entity.id': 'host-tres', + 'entity.display_name': 'host-tres', + 'entity.type': 'hosts-with-agents', + 'entity.last_seen_timestamp': moment(now).subtract(4, 'minute').toISOString(), + 'host.name': 'host-tres', + 'agent.name': null, + }, + { + 'entity.id': 'host-four', + 'entity.display_name': 'host-four', + 'entity.type': 'hosts-with-agents', + 'entity.last_seen_timestamp': moment(now).subtract(5, 'minute').toISOString(), + hostname_field: 'host-four', + 'agent.name': null, + }, + ]); + }); + + it('includes source and additional metadata fields', async () => { + const now = moment(); + + cleanup = await Promise.all([ + createIndexWithDocuments(esClient, { + index: 'index-1-with-services', + properties: { + '@timestamp': { type: 'date' }, + 'service.name': { type: 'keyword' }, + 'agent.name': { type: 'keyword' }, + 'host.name': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'service.name': 'service-one', + 'agent.name': 'agent-one', + 'host.name': 'host-one', + }, + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-one', + 'agent.name': 'agent-one', + 'host.name': 'host-one', + }, + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-one', + 'host.name': 'host-two', + }, + { + '@timestamp': moment(now).subtract(3, 'minute').toISOString(), + 'service.name': 'service-two', + 'host.name': 'host-three', + }, + ], + }), + + createIndexWithDocuments(esClient, { + index: 'index-2-with-services', + properties: { + '@timestamp': { type: 'date' }, + 'service.name': { type: 'keyword' }, + 'agent.name': { type: 'keyword' }, + 'host.name': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'service.name': 'service-one', + 'agent.name': 'agent-one', + 'host.name': 'host-one', + }, + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-one', + 'agent.name': 'agent-ten', + 'host.name': 'host-ten', + }, + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-two', + 'agent.name': 'agent-nine', + 'host.name': 'host-nine', + }, + ], + }), + ]); + + await createEntityTypeDefinition(supertest, { + type: { id: 'services-with-hosts', display_name: 'services-with-hosts' }, + }); + await Promise.all([ + createEntitySourceDefinition(supertest, { + source: { + id: 'source-1-with-services', + type_id: 'services-with-hosts', + index_patterns: ['index-1-with-services'], + identity_fields: ['service.name'], + metadata_fields: ['host.name'], + filters: [], + timestamp_field: '@timestamp', + }, + }), + createEntitySourceDefinition(supertest, { + source: { + id: 'source-2-with-services', + type_id: 'services-with-hosts', + index_patterns: ['index-2-with-services'], + identity_fields: ['service.name'], + metadata_fields: ['host.name'], + filters: [], + timestamp_field: '@timestamp', + }, + }), + ]); + + const { entities, errors } = await searchEntities(supertest, { + type: 'services-with-hosts', + metadata_fields: ['agent.name', 'non.existing.metadata.field'], + start: moment(now).subtract(5, 'minute').toISOString(), + end: moment(now).toISOString(), + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.last_seen_timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'entity.id': 'service-one', + 'entity.display_name': 'service-one', + 'entity.type': 'services-with-hosts', + 'service.name': 'service-one', + 'agent.name': expect.arrayContaining(['agent-one', 'agent-ten']), + 'host.name': expect.arrayContaining(['host-one', 'host-two', 'host-ten']), + }, + { + 'entity.last_seen_timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'entity.id': 'service-two', + 'entity.display_name': 'service-two', + 'entity.type': 'services-with-hosts', + 'service.name': 'service-two', + 'agent.name': 'agent-nine', + 'host.name': expect.arrayContaining(['host-three', 'host-nine']), + }, + ]); + }); + + it('respects filters', async () => { + const now = moment(); + + cleanup.push( + await createIndexWithDocuments(esClient, { + index: 'index-1-with-services', + properties: { + '@timestamp': { type: 'date' }, + 'service.name': { type: 'keyword' }, + 'service.environment': { type: 'keyword' }, + 'service.url': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'service.name': 'service-one', + 'service.environment': 'prod', + 'service.url': 'http://prod.service-one.com', + }, + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-one', + 'service.environment': 'staging', + 'service.url': 'http://staging.service-one.com', + }, + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'service.name': 'service-two', + 'service.environment': 'prod', + 'service.url': 'http://prod.service-two.com', + }, + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'service.name': 'service-two', + 'service.environment': 'staging', + 'service.url': 'http://staging.service-two.com', + }, + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'service.name': 'service-three', + 'service.environment': 'prod', + 'service.url': 'http://prod.service-three.com', + }, + ], + }) + ); + + await createEntityTypeDefinition(supertest, { + type: { id: 'service-one-type', display_name: 'service-one-type' }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'source-1-with-services', + type_id: 'service-one-type', + index_patterns: ['index-1-with-services'], + identity_fields: ['service.name'], + metadata_fields: [], + filters: ['service.name: service-one'], + timestamp_field: '@timestamp', + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'service-one-type', + start: moment(now).subtract(5, 'minute').toISOString(), + end: moment(now).toISOString(), + filters: ['service.environment: prod'], + metadata_fields: ['service.environment', 'service.url'], + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.last_seen_timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'entity.id': 'service-one', + 'entity.display_name': 'service-one', + 'entity.type': 'service-one-type', + 'service.name': 'service-one', + 'service.environment': 'prod', + 'service.url': 'http://prod.service-one.com', + }, + ]); + }); + + it('resolves entities with multiple identity fields', async () => { + const now = moment(); + + cleanup.push( + await createIndexWithDocuments(esClient, { + index: 'index-1-with-cars', + properties: { + '@timestamp': { type: 'date' }, + 'car.brand': { type: 'keyword' }, + 'car.model': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'car.brand': 'Fiat', + 'car.model': 'Multipla', + }, + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'car.brand': 'Citroen', + 'car.model': 'ZX break', + }, + ], + }) + ); + + await createEntityTypeDefinition(supertest, { + type: { id: 'most-refined-cars', display_name: 'most-refined-cars' }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'source-1-with-cars', + type_id: 'most-refined-cars', + index_patterns: ['index-1-with-cars'], + identity_fields: ['car.brand', 'car.model'], + metadata_fields: [], + filters: [], + timestamp_field: '@timestamp', + display_name: 'car.model', + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'most-refined-cars', + start: moment(now).subtract(5, 'minute').toISOString(), + end: moment(now).toISOString(), + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.last_seen_timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'entity.id': 'Citroen:ZX break', + 'entity.display_name': 'ZX break', + 'entity.type': 'most-refined-cars', + 'car.brand': 'Citroen', + 'car.model': 'ZX break', + }, + { + 'entity.last_seen_timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'entity.id': 'Fiat:Multipla', + 'entity.display_name': 'Multipla', + 'entity.type': 'most-refined-cars', + 'car.brand': 'Fiat', + 'car.model': 'Multipla', + }, + ]); + }); + + it('resolves entities with multiple identity fields across sources', async () => { + const now = moment(); + + cleanup = await Promise.all([ + createIndexWithDocuments(esClient, { + index: 'index-1-with-cars', + properties: { + '@timestamp': { type: 'date' }, + 'car.brand': { type: 'keyword' }, + 'car.model': { type: 'keyword' }, + 'car.color': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'car.brand': 'Fiat', + 'car.model': 'Multipla', + 'car.color': 'cyan', + }, + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'car.brand': 'Citroen', + 'car.model': 'ZX break', + 'car.color': 'white', + }, + ], + }), + + createIndexWithDocuments(esClient, { + index: 'index-2-with-cars', + properties: { + '@timestamp': { type: 'date' }, + car_brand: { type: 'keyword' }, + car_model: { type: 'keyword' }, + 'car.color': { type: 'keyword' }, + }, + documents: [ + { + '@timestamp': moment(now).subtract(2, 'minute').toISOString(), + car_brand: 'Fiat', + car_model: 'Multipla', + 'car.color': 'purple', + }, + { + '@timestamp': moment(now).subtract(1, 'minute').toISOString(), + car_brand: 'Citroen', + car_model: 'ZX break', + 'car.color': 'orange', + }, + ], + }), + ]); + + await createEntityTypeDefinition(supertest, { + type: { id: 'most-refined-cars', display_name: 'most-refined-cars' }, + }); + await Promise.all([ + createEntitySourceDefinition(supertest, { + source: { + id: 'source-1-with-cars', + type_id: 'most-refined-cars', + index_patterns: ['index-1-with-cars'], + identity_fields: ['car.brand', 'car.model'], + metadata_fields: [], + filters: [], + timestamp_field: '@timestamp', + }, + }), + createEntitySourceDefinition(supertest, { + source: { + id: 'source-2-with-cars', + type_id: 'most-refined-cars', + index_patterns: ['index-2-with-cars'], + identity_fields: ['car_brand', 'car_model'], + metadata_fields: [], + filters: [], + timestamp_field: '@timestamp', + }, + }), + ]); + + const { entities, errors } = await searchEntities(supertest, { + type: 'most-refined-cars', + start: moment(now).subtract(5, 'minute').toISOString(), + end: moment(now).toISOString(), + metadata_fields: ['car.color'], + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.last_seen_timestamp': moment(now).subtract(1, 'minute').toISOString(), + 'entity.id': 'Citroen:ZX break', + 'entity.display_name': 'Citroen:ZX break', + 'entity.type': 'most-refined-cars', + 'car.brand': 'Citroen', + 'car.model': 'ZX break', + car_brand: 'Citroen', + car_model: 'ZX break', + 'car.color': expect.arrayContaining(['white', 'orange']), + }, + { + 'entity.last_seen_timestamp': moment(now).subtract(2, 'minute').toISOString(), + 'entity.id': 'Fiat:Multipla', + 'entity.display_name': 'Fiat:Multipla', + 'entity.type': 'most-refined-cars', + 'car.brand': 'Fiat', + 'car.model': 'Multipla', + car_brand: 'Fiat', + car_model: 'Multipla', + 'car.color': expect.arrayContaining(['cyan', 'purple']), + }, + ]); + }); + + it('casts conflicting mappings to keywords', async () => { + cleanup = await Promise.all([ + await createIndexWithDocuments(esClient, { + index: 'index-service-name-as-keyword', + properties: { 'service.name': { type: 'keyword' } }, + documents: [{ 'service.name': 'service-name-as-keyword' }], + }), + + await createIndexWithDocuments(esClient, { + index: 'index-service-name-as-text', + properties: { 'service.name': { type: 'text' } }, + documents: [{ 'service.name': 'service-name-as-text' }], + }), + + await createIndexWithDocuments(esClient, { + index: 'index-service-name-as-long', + properties: { 'service.name': { type: 'long' } }, + documents: [{ 'service.name': 123 }], + }), + ]); + + await createEntityTypeDefinition(supertest, { + type: { + id: 'type-with-conflicting-mappings', + display_name: 'type-with-conflicting-mappings', + }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'conflicting-mappings', + type_id: 'type-with-conflicting-mappings', + index_patterns: ['index-service-name-as-*'], + identity_fields: ['service.name'], + metadata_fields: [], + filters: [], + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'type-with-conflicting-mappings', + }); + + expect(errors).toEqual([]); + expect(entities).toEqual([ + { + 'entity.id': '123', + 'entity.display_name': '123', + 'entity.type': 'type-with-conflicting-mappings', + 'service.name': '123', + }, + { + 'entity.id': 'service-name-as-keyword', + 'entity.display_name': 'service-name-as-keyword', + 'entity.type': 'type-with-conflicting-mappings', + 'service.name': 'service-name-as-keyword', + }, + { + 'entity.id': 'service-name-as-text', + 'entity.display_name': 'service-name-as-text', + 'entity.type': 'type-with-conflicting-mappings', + 'service.name': 'service-name-as-text', + }, + ]); + }); + + it('returns error if index does not exist', async () => { + await createEntityTypeDefinition(supertest, { + type: { id: 'type-with-non-existing-index', display_name: 'type-with-non-existing-index' }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'non-existing-index', + type_id: 'type-with-non-existing-index', + index_patterns: ['non-existing-index-pattern*', 'non-existing-index'], + identity_fields: ['service.name'], + metadata_fields: [], + filters: [], + timestamp_field: '@timestamp', + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'type-with-non-existing-index', + }); + expect(errors).toEqual([ + 'No index found for source [non-existing-index] with index patterns [non-existing-index-pattern*, non-existing-index]', + ]); + expect(entities).toEqual([]); + }); + + it('returns error if mandatory fields are not mapped', async () => { + cleanup.push( + await createIndexWithDocuments(esClient, { + index: 'unmapped-id-fields', + properties: { 'service.environment': { type: 'keyword' } }, + documents: [ + { + '@timestamp': moment().toISOString(), + 'service.name': 'service-one', + 'service.environment': 'prod', + }, + ], + }) + ); + + await createEntityTypeDefinition(supertest, { + type: { id: 'type-with-unmapped-id-fields', display_name: 'type-with-unmapped-id-fields' }, + }); + await createEntitySourceDefinition(supertest, { + source: { + id: 'unmapped-fields', + type_id: 'type-with-unmapped-id-fields', + index_patterns: ['unmapped-id-fields'], + identity_fields: ['service.name', 'service.environment'], + metadata_fields: [], + filters: [], + timestamp_field: '@timestamp', + }, + }); + + const { entities, errors } = await searchEntities(supertest, { + type: 'type-with-unmapped-id-fields', + }); + expect(errors).toEqual([ + 'Mandatory fields [service.name, @timestamp] are not mapped for source [unmapped-fields] with index patterns [unmapped-id-fields]', + ]); + expect(entities).toEqual([]); + }); + }); +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 95178eca83172..4bfe01a721695 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -190,6 +190,7 @@ "@kbn/usage-collection-plugin", "@kbn/sse-utils-server", "@kbn/gen-ai-functional-testing", - "@kbn/integration-assistant-plugin" + "@kbn/integration-assistant-plugin", + "@kbn/core-elasticsearch-server" ] }