From 73598053d558fefaff592b4cfd2e5c12a8d176bf Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 25 Sep 2024 15:17:34 +0200 Subject: [PATCH 01/21] remove history transform --- .../src/schema/entity_definition.ts | 12 +++ .../built_in/services_from_ecs_data.ts | 8 +- .../create_and_install_ingest_pipeline.ts | 3 +- .../entities/create_and_install_transform.ts | 43 +-------- .../lib/entities/delete_ingest_pipeline.ts | 31 ++----- .../server/lib/entities/delete_transforms.ts | 48 +++------- .../lib/entities/find_entity_definition.ts | 33 +++---- .../lib/entities/install_entity_definition.ts | 89 ++++--------------- .../server/lib/entities/start_transforms.ts | 37 ++------ .../server/lib/entities/stop_transforms.ts | 54 +++-------- .../transform/generate_history_transform.ts | 50 ----------- .../entities/uninstall_entity_definition.ts | 18 +--- .../server/lib/manage_index_templates.ts | 39 +++++++- .../server/saved_objects/entity_definition.ts | 33 +++++++ 14 files changed, 167 insertions(+), 331 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index 74be36cc5d802..23d78e84f01dd 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -57,6 +57,18 @@ export const entityDefinitionSchema = z.object({ ]) ), installStartedAt: z.optional(z.string()), + installedComponents: z.optional( + z.array( + z.object({ + type: z.union([ + z.literal('transform'), + z.literal('ingest_pipeline'), + z.literal('template'), + ]), + id: z.string(), + }) + ) + ), }); export const entityDefinitionUpdateSchema = entityDefinitionSchema diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index 96667fb4d0af4..fcb6f6136ef98 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -28,10 +28,10 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition = type: 'service', managed: true, indexPatterns: [ - 'logs-*', - 'filebeat*', - 'metrics-apm.service_transaction.1m*', - 'metrics-apm.service_summary.1m*', + 'remote_cluster:logs-*', + 'remote_cluster:filebeat*', + 'remote_cluster:metrics-apm.service_transaction.1m*', + 'remote_cluster:metrics-apm.service_summary.1m*', ], history: { timestampField: '@timestamp', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index 360f416cd5a00..ceb95cb9a1ec7 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -19,7 +19,7 @@ export async function createAndInstallHistoryIngestPipeline( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger -) { +): Promise> { try { const historyProcessors = generateHistoryProcessors(definition); const historyId = generateHistoryIngestPipelineId(definition); @@ -35,6 +35,7 @@ export async function createAndInstallHistoryIngestPipeline( }), { logger } ); + return [{ type: 'ingest_pipeline', id: historyId }]; } catch (e) { logger.error( `Cannot create entity history ingest pipelines for [${definition.id}] entity defintion` diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts index d6379773479fc..c143754aa4374 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts @@ -8,58 +8,21 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; -import { generateLatestTransform } from './transform/generate_latest_transform'; -import { - generateBackfillHistoryTransform, - generateHistoryTransform, -} from './transform/generate_history_transform'; +import { generateHistoryTransform } from './transform/generate_history_transform'; export async function createAndInstallHistoryTransform( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger -) { +): Promise> { try { const historyTransform = generateHistoryTransform(definition); await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), { logger, }); + return [{ type: 'transform', id: historyTransform.transform_id }]; } catch (e) { logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`); throw e; } } - -export async function createAndInstallHistoryBackfillTransform( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { - try { - const historyTransform = generateBackfillHistoryTransform(definition); - await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), { - logger, - }); - } catch (e) { - logger.error( - `Cannot create entity history backfill transform for [${definition.id}] entity definition` - ); - throw e; - } -} - -export async function createAndInstallLatestTransform( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { - try { - const latestTransform = generateLatestTransform(definition); - await retryTransientEsErrors(() => esClient.transform.putTransform(latestTransform), { - logger, - }); - } catch (e) { - logger.error(`Cannot create entity latest transform for [${definition.id}] entity definition`); - throw e; - } -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts index f4c46d8447d8f..c3a55f9721ad1 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts @@ -7,40 +7,23 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryIngestPipelineId, - generateLatestIngestPipelineId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -export async function deleteHistoryIngestPipeline( +export async function deleteIngestPipelines( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger ) { try { - const historyPipelineId = generateHistoryIngestPipelineId(definition); - await retryTransientEsErrors(() => - esClient.ingest.deletePipeline({ id: historyPipelineId }, { ignore: [404] }) + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'ingest_pipeline') + .map(({ id }) => + retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] })) + ) ); } catch (e) { logger.error(`Unable to delete history ingest pipeline [${definition.id}]: ${e}`); throw e; } } - -export async function deleteLatestIngestPipeline( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { - try { - const latestPipelineId = generateLatestIngestPipelineId(definition); - await retryTransientEsErrors(() => - esClient.ingest.deletePipeline({ id: latestPipelineId }, { ignore: [404] }) - ); - } catch (e) { - logger.error(`Unable to delete latest ingest pipeline [${definition.id}]: ${e}`); - throw e; - } -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts index a66c0998c014d..03809c7736be7 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts @@ -7,14 +7,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; - -import { - generateHistoryTransformId, - generateHistoryBackfillTransformId, - generateLatestTransformId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function deleteTransforms( esClient: ElasticsearchClient, @@ -22,34 +15,19 @@ export async function deleteTransforms( logger: Logger ) { try { - const historyTransformId = generateHistoryTransformId(definition); - const latestTransformId = generateLatestTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.deleteTransform( - { transform_id: historyTransformId, force: true }, - { ignore: [404] } - ), - { logger } - ); - if (isBackfillEnabled(definition)) { - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.deleteTransform( - { transform_id: historyBackfillTransformId, force: true }, - { ignore: [404] } - ), - { logger } - ); - } - await retryTransientEsErrors( - () => - esClient.transform.deleteTransform( - { transform_id: latestTransformId, force: true }, - { ignore: [404] } - ), - { logger } + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => + retryTransientEsErrors( + () => + esClient.transform.deleteTransform( + { transform_id: id, force: true }, + { ignore: [404] } + ), + { logger } + ) + ) ); } catch (e) { logger.error(`Cannot delete history transform [${definition.id}]: ${e}`); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts index 6932394a14e69..b4fa37f900e23 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts @@ -10,18 +10,8 @@ import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/serve import { EntityDefinition } from '@kbn/entities-schema'; import { NodesIngestTotal } from '@elastic/elasticsearch/lib/api/types'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; -import { - generateHistoryTransformId, - generateHistoryBackfillTransformId, - generateHistoryIngestPipelineId, - generateHistoryIndexTemplateId, - generateLatestTransformId, - generateLatestIngestPipelineId, - generateLatestIndexTemplateId, -} from './helpers/generate_component_id'; import { BUILT_IN_ID_PREFIX } from './built_in'; import { EntityDefinitionState, EntityDefinitionWithState } from './types'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function findEntityDefinitions({ soClient, @@ -117,11 +107,9 @@ async function getTransformState({ definition: EntityDefinition; esClient: ElasticsearchClient; }) { - const transformIds = [ - generateHistoryTransformId(definition), - generateLatestTransformId(definition), - ...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []), - ]; + const transformIds = (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => id); const transformStats = await Promise.all( transformIds.map((id) => esClient.transform.getTransformStats({ transform_id: id })) @@ -149,10 +137,10 @@ async function getIngestPipelineState({ definition: EntityDefinition; esClient: ElasticsearchClient; }) { - const ingestPipelineIds = [ - generateHistoryIngestPipelineId(definition), - generateLatestIngestPipelineId(definition), - ]; + const ingestPipelineIds = (definition.installedComponents ?? []) + .filter(({ type }) => type === 'ingest_pipeline') + .map(({ id }) => id); + const [ingestPipelines, ingestPipelinesStats] = await Promise.all([ esClient.ingest.getPipeline({ id: ingestPipelineIds.join(',') }, { ignore: [404] }), esClient.nodes.stats({ @@ -190,10 +178,9 @@ async function getIndexTemplatesState({ definition: EntityDefinition; esClient: ElasticsearchClient; }) { - const indexTemplatesIds = [ - generateLatestIndexTemplateId(definition), - generateHistoryIndexTemplateId(definition), - ]; + const indexTemplatesIds = (definition.installedComponents ?? []) + .filter(({ type }) => type === 'template') + .map(({ id }) => id); const templates = await Promise.all( indexTemplatesIds.map((id) => esClient.indices diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts index 7d6dee4fb2ced..de681efadebcf 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts @@ -10,33 +10,19 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; -import { - generateHistoryIndexTemplateId, - generateLatestIndexTemplateId, -} from './helpers/generate_component_id'; -import { - createAndInstallHistoryIngestPipeline, - createAndInstallLatestIngestPipeline, -} from './create_and_install_ingest_pipeline'; -import { - createAndInstallHistoryBackfillTransform, - createAndInstallHistoryTransform, - createAndInstallLatestTransform, -} from './create_and_install_transform'; +import { generateHistoryIndexTemplateId } from './helpers/generate_component_id'; +import { createAndInstallHistoryIngestPipeline } from './create_and_install_ingest_pipeline'; +import { createAndInstallHistoryTransform } from './create_and_install_transform'; import { validateDefinitionCanCreateValidTransformIds } from './transform/validate_transform_ids'; import { deleteEntityDefinition } from './delete_entity_definition'; -import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline'; +import { deleteIngestPipelines } from './delete_ingest_pipeline'; import { findEntityDefinitionById } from './find_entity_definition'; import { entityDefinitionExists, saveEntityDefinition, updateEntityDefinition, } from './save_entity_definition'; - -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; -import { deleteTemplate, upsertTemplate } from '../manage_index_templates'; -import { generateEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template'; -import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template'; +import { createAndInstallTemplates, deleteTemplate } from '../manage_index_templates'; import { EntityIdConflict } from './errors/entity_id_conflict_error'; import { EntityDefinitionNotFound } from './errors/entity_not_found'; import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update'; @@ -51,16 +37,6 @@ export interface InstallDefinitionParams { logger: Logger; } -const throwIfRejected = (values: Array | PromiseRejectedResult>) => { - const rejectedPromise = values.find( - (value) => value.status === 'rejected' - ) as PromiseRejectedResult; - if (rejectedPromise) { - throw new Error(rejectedPromise.reason); - } - return values; -}; - // install an entity definition from scratch with all its required components // after verifying that the definition id is valid and available. // attempt to remove all installed components if the installation fails. @@ -91,23 +67,13 @@ export async function installEntityDefinition({ logger.error(`Failed to install entity definition ${definition.id}: ${e}`); await stopAndDeleteTransforms(esClient, definition, logger); - await Promise.all([ - deleteHistoryIngestPipeline(esClient, definition, logger), - deleteLatestIngestPipeline(esClient, definition, logger), - ]); + await deleteIngestPipelines(esClient, definition, logger); - await Promise.all([ - deleteTemplate({ - esClient, - logger, - name: generateHistoryIndexTemplateId(definition), - }), - deleteTemplate({ - esClient, - logger, - name: generateLatestIndexTemplateId(definition), - }), - ]); + await deleteTemplate({ + esClient, + logger, + name: generateHistoryIndexTemplateId(definition), + }); await deleteEntityDefinition(soClient, definition).catch((err) => { if (err instanceof EntityDefinitionNotFound) { @@ -191,35 +157,18 @@ async function install({ ); logger.debug(`Installing index templates for definition ${definition.id}`); - await Promise.allSettled([ - upsertTemplate({ - esClient, - logger, - template: generateEntitiesHistoryIndexTemplateConfig(definition), - }), - upsertTemplate({ - esClient, - logger, - template: generateEntitiesLatestIndexTemplateConfig(definition), - }), - ]).then(throwIfRejected); + const templates = await createAndInstallTemplates(esClient, definition, logger); logger.debug(`Installing ingest pipelines for definition ${definition.id}`); - await Promise.allSettled([ - createAndInstallHistoryIngestPipeline(esClient, definition, logger), - createAndInstallLatestIngestPipeline(esClient, definition, logger), - ]).then(throwIfRejected); + const pipelines = await createAndInstallHistoryIngestPipeline(esClient, definition, logger); logger.debug(`Installing transforms for definition ${definition.id}`); - await Promise.allSettled([ - createAndInstallHistoryTransform(esClient, definition, logger), - isBackfillEnabled(definition) - ? createAndInstallHistoryBackfillTransform(esClient, definition, logger) - : Promise.resolve(), - createAndInstallLatestTransform(esClient, definition, logger), - ]).then(throwIfRejected); - - await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed' }); + const transforms = await createAndInstallHistoryTransform(esClient, definition, logger); + + await updateEntityDefinition(soClient, definition.id, { + installStatus: 'installed', + installedComponents: [...templates, ...pipelines, ...transforms], + }); return { ...definition, installStatus: 'installed' }; } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts index ea2ec7adb5ddc..f4cd8fc89dd11 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts @@ -7,13 +7,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryBackfillTransformId, - generateHistoryTransformId, - generateLatestTransformId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function startTransforms( esClient: ElasticsearchClient, @@ -21,28 +15,15 @@ export async function startTransforms( logger: Logger ) { try { - const historyTransformId = generateHistoryTransformId(definition); - const latestTransformId = generateLatestTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.startTransform({ transform_id: historyTransformId }, { ignore: [409] }), - { logger } - ); - if (isBackfillEnabled(definition)) { - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.startTransform( - { transform_id: historyBackfillTransformId }, - { ignore: [409] } - ), - { logger } - ); - } - await retryTransientEsErrors( - () => - esClient.transform.startTransform({ transform_id: latestTransformId }, { ignore: [409] }), - { logger } + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'transform') + .map(({ id }) => + retryTransientEsErrors( + () => esClient.transform.startTransform({ transform_id: id }, { ignore: [409] }), + { logger } + ) + ) ); } catch (err) { logger.error(`Cannot start entity transforms [${definition.id}]: ${err}`); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts index 98f9ad351e377..055f17f43ce2c 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts @@ -8,55 +8,27 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryTransformId, - generateHistoryBackfillTransformId, - generateLatestTransformId, -} from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { isBackfillEnabled } from './helpers/is_backfill_enabled'; - export async function stopTransforms( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger ) { try { - const historyTransformId = generateHistoryTransformId(definition); - const latestTransformId = generateLatestTransformId(definition); - - await retryTransientEsErrors( - () => - esClient.transform.stopTransform( - { transform_id: historyTransformId, wait_for_completion: true, force: true }, - { ignore: [409, 404] } - ), - { logger } - ); - - if (isBackfillEnabled(definition)) { - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - await retryTransientEsErrors( - () => - esClient.transform.stopTransform( - { - transform_id: historyBackfillTransformId, - wait_for_completion: true, - force: true, - }, - { ignore: [409, 404] } - ), - { logger } - ); - } - await retryTransientEsErrors( - () => - esClient.transform.stopTransform( - { transform_id: latestTransformId, wait_for_completion: true, force: true }, - { ignore: [409, 404] } - ), - { logger } + await Promise.all( + definition.installedComponents + .filter(({ type }) => type === 'transform') + .map(({ id }) => + retryTransientEsErrors( + () => + esClient.transform.stopTransform( + { transform_id: id, wait_for_completion: true, force: true }, + { ignore: [409, 404] } + ), + { logger } + ) + ) ); } catch (e) { logger.error(`Cannot stop entity transforms [${definition.id}]: ${e}`); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts index 239359738624c..64f692cc671c9 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts @@ -21,9 +21,7 @@ import { generateHistoryTransformId, generateHistoryIngestPipelineId, generateHistoryIndexName, - generateHistoryBackfillTransformId, } from '../helpers/generate_component_id'; -import { isBackfillEnabled } from '../helpers/is_backfill_enabled'; export function generateHistoryTransform( definition: EntityDefinition @@ -59,48 +57,6 @@ export function generateHistoryTransform( }); } -export function generateBackfillHistoryTransform( - definition: EntityDefinition -): TransformPutTransformRequest { - if (!isBackfillEnabled(definition)) { - throw new Error( - 'generateBackfillHistoryTransform called without history.settings.backfillSyncDelay set' - ); - } - - const filter: QueryDslQueryContainer[] = []; - - if (definition.filter) { - filter.push(getElasticsearchQueryOrThrow(definition.filter)); - } - - if (definition.history.settings.backfillLookbackPeriod) { - filter.push({ - range: { - [definition.history.timestampField]: { - gte: `now-${definition.history.settings.backfillLookbackPeriod}`, - }, - }, - }); - } - - if (definition.identityFields.some(({ optional }) => !optional)) { - definition.identityFields - .filter(({ optional }) => !optional) - .forEach(({ field }) => { - filter.push({ exists: { field } }); - }); - } - - return generateTransformPutRequest({ - definition, - filter, - transformId: generateHistoryBackfillTransformId(definition), - frequency: definition.history.settings.backfillFrequency, - syncDelay: definition.history.settings.backfillSyncDelay, - }); -} - const generateTransformPutRequest = ({ definition, filter, @@ -157,12 +113,6 @@ const generateTransformPutRequest = ({ }), {} ), - ['@timestamp']: { - date_histogram: { - field: definition.history.timestampField, - fixed_interval: definition.history.interval, - }, - }, }, aggs: { ...generateHistoryMetricAggregations(definition), diff --git a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts index 8bc8efa3870aa..d0e0410b6e422 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts @@ -11,14 +11,10 @@ import { EntityDefinition } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; import { deleteEntityDefinition } from './delete_entity_definition'; import { deleteIndices } from './delete_index'; -import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline'; +import { deleteIngestPipelines } from './delete_ingest_pipeline'; import { findEntityDefinitions } from './find_entity_definition'; -import { - generateHistoryIndexTemplateId, - generateLatestIndexTemplateId, -} from './helpers/generate_component_id'; -import { deleteTemplate } from '../manage_index_templates'; +import { deleteTemplates } from '../manage_index_templates'; import { stopTransforms } from './stop_transforms'; @@ -40,19 +36,13 @@ export async function uninstallEntityDefinition({ await stopTransforms(esClient, definition, logger); await deleteTransforms(esClient, definition, logger); - await Promise.all([ - deleteHistoryIngestPipeline(esClient, definition, logger), - deleteLatestIngestPipeline(esClient, definition, logger), - ]); + await deleteIngestPipelines(esClient, definition, logger); if (deleteData) { await deleteIndices(esClient, definition, logger); } - await Promise.all([ - deleteTemplate({ esClient, logger, name: generateHistoryIndexTemplateId(definition) }), - deleteTemplate({ esClient, logger, name: generateLatestIndexTemplateId(definition) }), - ]); + await deleteTemplates(esClient, definition, logger); await deleteEntityDefinition(soClient, definition); } diff --git a/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts index b0789b6cf2769..92ca7e586bbf3 100644 --- a/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { EntityDefinition } from '@kbn/entities-schema'; import { ClusterPutComponentTemplateRequest, IndicesPutIndexTemplateRequest, @@ -15,6 +16,7 @@ import { entitiesLatestBaseComponentTemplateConfig } from '../templates/componen import { entitiesEntityComponentTemplateConfig } from '../templates/components/entity'; import { entitiesEventComponentTemplateConfig } from '../templates/components/event'; import { retryTransientEsErrors } from './entities/helpers/retry'; +import { generateEntitiesHistoryIndexTemplateConfig } from './entities/templates/entities_history_template'; interface TemplateManagementOptions { esClient: ElasticsearchClient; @@ -67,14 +69,27 @@ interface DeleteTemplateOptions { export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { try { - await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + const result = await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { + logger, + }); logger.debug(() => `Installed entity manager index template: ${JSON.stringify(template)}`); + return result; } catch (error: any) { logger.error(`Error updating entity manager index template: ${error.message}`); throw error; } } +export async function createAndInstallTemplates( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +): Promise> { + const template = generateEntitiesHistoryIndexTemplateConfig(definition); + await upsertTemplate({ esClient, template, logger }); + return [{ type: 'template', id: template.name }]; +} + export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { try { await retryTransientEsErrors( @@ -87,6 +102,28 @@ export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateO } } +export async function deleteTemplates( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + await Promise.all( + (definition.installedComponents ?? []) + .filter(({ type }) => type === 'template') + .map(({ id }) => + retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name: id }, { ignore: [404] }), + { logger } + ) + ) + ); + } catch (error: any) { + logger.error(`Error deleting entity manager index template: ${error.message}`); + throw error; + } +} + export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { try { await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { diff --git a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts index fdf2510e8627e..bdea2b71e4141 100644 --- a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts @@ -5,11 +5,36 @@ * 2.0. */ +import { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server'; import { SavedObject, SavedObjectsType } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; +import { + generateHistoryIndexTemplateId, + generateHistoryIngestPipelineId, + generateHistoryTransformId, + generateLatestIndexTemplateId, + generateLatestIngestPipelineId, + generateLatestTransformId, +} from '../lib/entities/helpers/generate_component_id'; export const SO_ENTITY_DEFINITION_TYPE = 'entity-definition'; +export const backfillInstalledComponents: SavedObjectModelDataBackfillFn< + EntityDefinition, + EntityDefinition +> = (savedObject) => { + const definition = savedObject.attributes; + definition.installedComponents = [ + { type: 'transform', id: generateHistoryTransformId(definition) }, + { type: 'transform', id: generateLatestTransformId(definition) }, + { type: 'ingest_pipeline', id: generateHistoryIngestPipelineId(definition) }, + { type: 'ingest_pipeline', id: generateLatestIngestPipelineId(definition) }, + { type: 'template', id: generateHistoryIndexTemplateId(definition) }, + { type: 'template', id: generateLatestIndexTemplateId(definition) }, + ]; + return savedObject; +}; + export const entityDefinition: SavedObjectsType = { name: SO_ENTITY_DEFINITION_TYPE, hidden: false, @@ -64,5 +89,13 @@ export const entityDefinition: SavedObjectsType = { }, ], }, + '3': { + changes: [ + { + type: 'data_backfill', + backfillFn: backfillInstalledComponents, + }, + ], + }, }, }; From 46423e5920dc65af992446d306ffb03ac17e5bce Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:30:00 +0000 Subject: [PATCH 02/21] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- x-pack/plugins/entity_manager/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json index 29c100ee4c9d2..34c57a27dd829 100644 --- a/x-pack/plugins/entity_manager/tsconfig.json +++ b/x-pack/plugins/entity_manager/tsconfig.json @@ -34,5 +34,6 @@ "@kbn/zod-helpers", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", + "@kbn/core-saved-objects-server", ] } From be50f9d33de3af7d246d09f2292ccfad2c4de417 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 08:36:20 +0000 Subject: [PATCH 03/21] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index a8b31ffdd90fa..6896eca6a4b2e 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -91,7 +91,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", - "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9", + "entity-definition": "e3811fd5fbb878d170067c0d6897a2e63010af36", "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88", "entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927", "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd", From 87f2632e2f45c18b7d3b2b25e2d16bac80a1c461 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 13:10:16 +0200 Subject: [PATCH 04/21] fix tests --- .../create_and_install_ingest_pipeline.ts | 39 +-- .../entities/create_and_install_transform.ts | 10 +- .../lib/entities/delete_ingest_pipeline.ts | 21 +- .../server/lib/entities/delete_transforms.ts | 23 +- .../generate_history_processors.test.ts.snap | 327 ------------------ .../generate_latest_processors.test.ts.snap | 156 ++++++--- .../generate_history_processors.test.ts | 21 -- .../generate_history_processors.ts | 222 ------------ .../generate_latest_processors.ts | 92 +++-- .../install_entity_definition.test.ts | 95 ++--- .../lib/entities/install_entity_definition.ts | 39 ++- .../server/lib/entities/stop_transforms.ts | 29 +- .../generate_history_transform.test.ts.snap | 305 ---------------- .../generate_latest_transform.test.ts.snap | 202 ++++------- .../generate_history_transform.test.ts | 24 -- .../transform/generate_history_transform.ts | 128 ------- .../transform/generate_latest_transform.ts | 107 ++++-- .../server/lib/manage_index_templates.ts | 4 +- 18 files changed, 459 insertions(+), 1385 deletions(-) delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index ceb95cb9a1ec7..0b3900363c0c8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -7,47 +7,15 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { - generateHistoryIngestPipelineId, - generateLatestIngestPipelineId, -} from './helpers/generate_component_id'; +import { generateLatestIngestPipelineId } from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; -import { generateHistoryProcessors } from './ingest_pipeline/generate_history_processors'; import { generateLatestProcessors } from './ingest_pipeline/generate_latest_processors'; -export async function createAndInstallHistoryIngestPipeline( +export async function createAndInstallIngestPipelines( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger ): Promise> { - try { - const historyProcessors = generateHistoryProcessors(definition); - const historyId = generateHistoryIngestPipelineId(definition); - await retryTransientEsErrors( - () => - esClient.ingest.putPipeline({ - id: historyId, - processors: historyProcessors, - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - }), - { logger } - ); - return [{ type: 'ingest_pipeline', id: historyId }]; - } catch (e) { - logger.error( - `Cannot create entity history ingest pipelines for [${definition.id}] entity defintion` - ); - throw e; - } -} -export async function createAndInstallLatestIngestPipeline( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { try { const latestProcessors = generateLatestProcessors(definition); const latestId = generateLatestIngestPipelineId(definition); @@ -63,9 +31,10 @@ export async function createAndInstallLatestIngestPipeline( }), { logger } ); + return [{ type: 'ingest_pipeline', id: latestId }]; } catch (e) { logger.error( - `Cannot create entity latest ingest pipelines for [${definition.id}] entity defintion` + `Cannot create entity latest ingest pipelines for [${definition.id}] entity definition` ); throw e; } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts index c143754aa4374..779e0994a33b8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts @@ -8,19 +8,19 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; -import { generateHistoryTransform } from './transform/generate_history_transform'; +import { generateLatestTransform } from './transform/generate_latest_transform'; -export async function createAndInstallHistoryTransform( +export async function createAndInstallTransforms( esClient: ElasticsearchClient, definition: EntityDefinition, logger: Logger ): Promise> { try { - const historyTransform = generateHistoryTransform(definition); - await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), { + const latestTransform = generateLatestTransform(definition); + await retryTransientEsErrors(() => esClient.transform.putTransform(latestTransform), { logger, }); - return [{ type: 'transform', id: historyTransform.transform_id }]; + return [{ type: 'transform', id: latestTransform.transform_id }]; } catch (e) { logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`); throw e; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts index c3a55f9721ad1..a3b910dd4cb5e 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts @@ -8,6 +8,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; +import { generateLatestIngestPipelineId } from './helpers/generate_component_id'; export async function deleteIngestPipelines( esClient: ElasticsearchClient, @@ -23,7 +24,25 @@ export async function deleteIngestPipelines( ) ); } catch (e) { - logger.error(`Unable to delete history ingest pipeline [${definition.id}]: ${e}`); + logger.error(`Unable to delete ingest pipelines for definition [${definition.id}]: ${e}`); + throw e; + } +} + +export async function deleteLatestIngestPipeline( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + await retryTransientEsErrors(() => + esClient.ingest.deletePipeline( + { id: generateLatestIngestPipelineId(definition) }, + { ignore: [404] } + ) + ); + } catch (e) { + logger.error(`Unable to delete latest ingest pipeline [${definition.id}]: ${e}`); throw e; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts index 03809c7736be7..79b83998d38db 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts @@ -8,6 +8,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; +import { generateLatestTransformId } from './helpers/generate_component_id'; export async function deleteTransforms( esClient: ElasticsearchClient, @@ -30,7 +31,27 @@ export async function deleteTransforms( ) ); } catch (e) { - logger.error(`Cannot delete history transform [${definition.id}]: ${e}`); + logger.error(`Cannot delete transforms for definition [${definition.id}]: ${e}`); + throw e; + } +} + +export async function deleteLatestTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + await retryTransientEsErrors( + () => + esClient.transform.deleteTransform( + { transform_id: generateLatestTransformId(definition), force: true }, + { ignore: [404] } + ), + { logger } + ); + } catch (e) { + logger.error(`Cannot delete latest transform for definition [${definition.id}]: ${e}`); throw e; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap deleted file mode 100644 index c2e4605e5f909..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap +++ /dev/null @@ -1,327 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`generateHistoryProcessors(definition) should generate a valid pipeline for builtin definition 1`] = ` -Array [ - Object { - "set": Object { - "field": "event.ingested", - "value": "{{{_ingest.timestamp}}}", - }, - }, - Object { - "set": Object { - "field": "entity.type", - "value": "service", - }, - }, - Object { - "set": Object { - "field": "entity.definitionId", - "value": "builtin_mock_entity_definition", - }, - }, - Object { - "set": Object { - "field": "entity.definitionVersion", - "value": "1.0.0", - }, - }, - Object { - "set": Object { - "field": "entity.schemaVersion", - "value": "v1", - }, - }, - Object { - "set": Object { - "field": "entity.identityFields", - "value": Array [ - "log.logger", - "event.category", - ], - }, - }, - Object { - "script": Object { - "description": "Generated the entity.id field", - "source": "// This function will recursively collect all the values of a HashMap of HashMaps -Collection collectValues(HashMap subject) { - Collection values = new ArrayList(); - // Iterate through the values - for(Object value: subject.values()) { - // If the value is a HashMap, recurse - if (value instanceof HashMap) { - values.addAll(collectValues((HashMap) value)); - } else { - values.add(String.valueOf(value)); - } - } - return values; -} -// Create the string builder -StringBuilder entityId = new StringBuilder(); -if (ctx[\\"entity\\"][\\"identity\\"] != null) { - // Get the values as a collection - Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); - // Convert to a list and sort - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - // Create comma delimited string - for(String instanceValue: sortedValues) { - entityId.append(instanceValue); - entityId.append(\\":\\"); - } - // Assign the entity.id - ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; -}", - }, - }, - Object { - "fingerprint": Object { - "fields": Array [ - "entity.id", - ], - "method": "MurmurHash3", - "target_field": "entity.id", - }, - }, - Object { - "script": Object { - "source": "if (ctx.entity?.metadata?.tags != null) { - ctx.tags = ctx.entity.metadata.tags.keySet(); -} -if (ctx.entity?.metadata?.host?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - ctx.host.name = ctx.entity.metadata.host.name.keySet(); -} -if (ctx.entity?.metadata?.host?.os?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - if (ctx.host.os == null) { - ctx.host.os = new HashMap(); - } - ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); -} -if (ctx.entity?.metadata?.sourceIndex != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); -}", - }, - }, - Object { - "remove": Object { - "field": "entity.metadata", - "ignore_missing": true, - }, - }, - Object { - "set": Object { - "field": "log.logger", - "if": "ctx.entity?.identity?.log?.logger != null", - "value": "{{entity.identity.log.logger}}", - }, - }, - Object { - "set": Object { - "field": "event.category", - "if": "ctx.entity?.identity?.event?.category != null", - "value": "{{entity.identity.event.category}}", - }, - }, - Object { - "remove": Object { - "field": "entity.identity", - "ignore_missing": true, - }, - }, - Object { - "date_index_name": Object { - "date_formats": Array [ - "UNIX_MS", - "ISO8601", - "yyyy-MM-dd'T'HH:mm:ss.SSSXX", - ], - "date_rounding": "M", - "field": "@timestamp", - "index_name_prefix": ".entities.v1.history.builtin_mock_entity_definition.", - }, - }, -] -`; - -exports[`generateHistoryProcessors(definition) should generate a valid pipeline for custom definition 1`] = ` -Array [ - Object { - "set": Object { - "field": "event.ingested", - "value": "{{{_ingest.timestamp}}}", - }, - }, - Object { - "set": Object { - "field": "entity.type", - "value": "service", - }, - }, - Object { - "set": Object { - "field": "entity.definitionId", - "value": "admin-console-services", - }, - }, - Object { - "set": Object { - "field": "entity.definitionVersion", - "value": "1.0.0", - }, - }, - Object { - "set": Object { - "field": "entity.schemaVersion", - "value": "v1", - }, - }, - Object { - "set": Object { - "field": "entity.identityFields", - "value": Array [ - "log.logger", - "event.category", - ], - }, - }, - Object { - "script": Object { - "description": "Generated the entity.id field", - "source": "// This function will recursively collect all the values of a HashMap of HashMaps -Collection collectValues(HashMap subject) { - Collection values = new ArrayList(); - // Iterate through the values - for(Object value: subject.values()) { - // If the value is a HashMap, recurse - if (value instanceof HashMap) { - values.addAll(collectValues((HashMap) value)); - } else { - values.add(String.valueOf(value)); - } - } - return values; -} -// Create the string builder -StringBuilder entityId = new StringBuilder(); -if (ctx[\\"entity\\"][\\"identity\\"] != null) { - // Get the values as a collection - Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); - // Convert to a list and sort - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - // Create comma delimited string - for(String instanceValue: sortedValues) { - entityId.append(instanceValue); - entityId.append(\\":\\"); - } - // Assign the entity.id - ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; -}", - }, - }, - Object { - "fingerprint": Object { - "fields": Array [ - "entity.id", - ], - "method": "MurmurHash3", - "target_field": "entity.id", - }, - }, - Object { - "script": Object { - "source": "if (ctx.entity?.metadata?.tags != null) { - ctx.tags = ctx.entity.metadata.tags.keySet(); -} -if (ctx.entity?.metadata?.host?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - ctx.host.name = ctx.entity.metadata.host.name.keySet(); -} -if (ctx.entity?.metadata?.host?.os?.name != null) { - if (ctx.host == null) { - ctx.host = new HashMap(); - } - if (ctx.host.os == null) { - ctx.host.os = new HashMap(); - } - ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); -} -if (ctx.entity?.metadata?.sourceIndex != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); -}", - }, - }, - Object { - "remove": Object { - "field": "entity.metadata", - "ignore_missing": true, - }, - }, - Object { - "set": Object { - "field": "log.logger", - "if": "ctx.entity?.identity?.log?.logger != null", - "value": "{{entity.identity.log.logger}}", - }, - }, - Object { - "set": Object { - "field": "event.category", - "if": "ctx.entity?.identity?.event?.category != null", - "value": "{{entity.identity.event.category}}", - }, - }, - Object { - "remove": Object { - "field": "entity.identity", - "ignore_missing": true, - }, - }, - Object { - "date_index_name": Object { - "date_formats": Array [ - "UNIX_MS", - "ISO8601", - "yyyy-MM-dd'T'HH:mm:ss.SSSXX", - ], - "date_rounding": "M", - "field": "@timestamp", - "index_name_prefix": ".entities.v1.history.admin-console-services.", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services@platform", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services-history@platform", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services@custom", - }, - }, - Object { - "pipeline": Object { - "ignore_missing_pipeline": true, - "name": "admin-console-services-history@custom", - }, - }, -] -`; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap index f277b3ac84ab8..7e3bd58b307e5 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap @@ -43,26 +43,70 @@ Array [ }, Object { "script": Object { - "source": "if (ctx.entity?.metadata?.tags.data != null) { - ctx.tags = ctx.entity.metadata.tags.data.keySet(); + "description": "Generated the entity.id field", + "source": "// This function will recursively collect all the values of a HashMap of HashMaps +Collection collectValues(HashMap subject) { + Collection values = new ArrayList(); + // Iterate through the values + for(Object value: subject.values()) { + // If the value is a HashMap, recurse + if (value instanceof HashMap) { + values.addAll(collectValues((HashMap) value)); + } else { + values.add(String.valueOf(value)); + } + } + return values; +} +// Create the string builder +StringBuilder entityId = new StringBuilder(); +if (ctx[\\"entity\\"][\\"identity\\"] != null) { + // Get the values as a collection + Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); + // Convert to a list and sort + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + // Create comma delimited string + for(String instanceValue: sortedValues) { + entityId.append(instanceValue); + entityId.append(\\":\\"); + } + // Assign the entity.id + ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; +}", + }, + }, + Object { + "fingerprint": Object { + "fields": Array [ + "entity.id", + ], + "method": "MurmurHash3", + "target_field": "entity.id", + }, + }, + Object { + "script": Object { + "source": "if (ctx.entity?.metadata?.tags != null) { + ctx.tags = ctx.entity.metadata.tags.keySet(); } -if (ctx.entity?.metadata?.host?.name.data != null) { +if (ctx.entity?.metadata?.host?.name != null) { if (ctx.host == null) { ctx.host = new HashMap(); } - ctx.host.name = ctx.entity.metadata.host.name.data.keySet(); + ctx.host.name = ctx.entity.metadata.host.name.keySet(); } -if (ctx.entity?.metadata?.host?.os?.name.data != null) { +if (ctx.entity?.metadata?.host?.os?.name != null) { if (ctx.host == null) { ctx.host = new HashMap(); } if (ctx.host.os == null) { ctx.host.os = new HashMap(); } - ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet(); + ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); } -if (ctx.entity?.metadata?.sourceIndex.data != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet(); +if (ctx.entity?.metadata?.sourceIndex != null) { + ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); }", }, }, @@ -72,28 +116,18 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) { "ignore_missing": true, }, }, - Object { - "dot_expander": Object { - "field": "log.logger", - "path": "entity.identity.log.logger.top_metric", - }, - }, Object { "set": Object { "field": "log.logger", - "value": "{{entity.identity.log.logger.top_metric.log.logger}}", - }, - }, - Object { - "dot_expander": Object { - "field": "event.category", - "path": "entity.identity.event.category.top_metric", + "if": "ctx.entity?.identity?.log?.logger != null", + "value": "{{entity.identity.log.logger}}", }, }, Object { "set": Object { "field": "event.category", - "value": "{{entity.identity.event.category.top_metric.event.category}}", + "if": "ctx.entity?.identity?.event?.category != null", + "value": "{{entity.identity.event.category}}", }, }, Object { @@ -160,26 +194,70 @@ Array [ }, Object { "script": Object { - "source": "if (ctx.entity?.metadata?.tags.data != null) { - ctx.tags = ctx.entity.metadata.tags.data.keySet(); + "description": "Generated the entity.id field", + "source": "// This function will recursively collect all the values of a HashMap of HashMaps +Collection collectValues(HashMap subject) { + Collection values = new ArrayList(); + // Iterate through the values + for(Object value: subject.values()) { + // If the value is a HashMap, recurse + if (value instanceof HashMap) { + values.addAll(collectValues((HashMap) value)); + } else { + values.add(String.valueOf(value)); + } + } + return values; +} +// Create the string builder +StringBuilder entityId = new StringBuilder(); +if (ctx[\\"entity\\"][\\"identity\\"] != null) { + // Get the values as a collection + Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]); + // Convert to a list and sort + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + // Create comma delimited string + for(String instanceValue: sortedValues) { + entityId.append(instanceValue); + entityId.append(\\":\\"); + } + // Assign the entity.id + ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\"; +}", + }, + }, + Object { + "fingerprint": Object { + "fields": Array [ + "entity.id", + ], + "method": "MurmurHash3", + "target_field": "entity.id", + }, + }, + Object { + "script": Object { + "source": "if (ctx.entity?.metadata?.tags != null) { + ctx.tags = ctx.entity.metadata.tags.keySet(); } -if (ctx.entity?.metadata?.host?.name.data != null) { +if (ctx.entity?.metadata?.host?.name != null) { if (ctx.host == null) { ctx.host = new HashMap(); } - ctx.host.name = ctx.entity.metadata.host.name.data.keySet(); + ctx.host.name = ctx.entity.metadata.host.name.keySet(); } -if (ctx.entity?.metadata?.host?.os?.name.data != null) { +if (ctx.entity?.metadata?.host?.os?.name != null) { if (ctx.host == null) { ctx.host = new HashMap(); } if (ctx.host.os == null) { ctx.host.os = new HashMap(); } - ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet(); + ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); } -if (ctx.entity?.metadata?.sourceIndex.data != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet(); +if (ctx.entity?.metadata?.sourceIndex != null) { + ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); }", }, }, @@ -189,28 +267,18 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) { "ignore_missing": true, }, }, - Object { - "dot_expander": Object { - "field": "log.logger", - "path": "entity.identity.log.logger.top_metric", - }, - }, Object { "set": Object { "field": "log.logger", - "value": "{{entity.identity.log.logger.top_metric.log.logger}}", - }, - }, - Object { - "dot_expander": Object { - "field": "event.category", - "path": "entity.identity.event.category.top_metric", + "if": "ctx.entity?.identity?.log?.logger != null", + "value": "{{entity.identity.log.logger}}", }, }, Object { "set": Object { "field": "event.category", - "value": "{{entity.identity.event.category.top_metric.event.category}}", + "if": "ctx.entity?.identity?.event?.category != null", + "value": "{{entity.identity.event.category}}", }, }, Object { diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts deleted file mode 100644 index 717241b89143d..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinition, builtInEntityDefinition } from '../helpers/fixtures'; -import { generateHistoryProcessors } from './generate_history_processors'; - -describe('generateHistoryProcessors(definition)', () => { - it('should generate a valid pipeline for custom definition', () => { - const processors = generateHistoryProcessors(entityDefinition); - expect(processors).toMatchSnapshot(); - }); - - it('should generate a valid pipeline for builtin definition', () => { - const processors = generateHistoryProcessors(builtInEntityDefinition); - expect(processors).toMatchSnapshot(); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts deleted file mode 100644 index d51ab0be75db1..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema'; -import { - initializePathScript, - cleanScript, -} from '../helpers/ingest_pipeline_script_processor_helpers'; -import { generateHistoryIndexName } from '../helpers/generate_component_id'; -import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; - -function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { - if (aggregation.type === 'terms') { - return `ctx.entity.metadata.${destination}.keySet()`; - } else if (aggregation.type === 'top_value') { - return `ctx.entity.metadata.${destination}.top_value["${source}"]`; - } -} - -function mapDestinationToPainless(metadata: MetadataField) { - const field = metadata.destination; - return ` - ${initializePathScript(field)} - ctx.${field} = ${getMetadataSourceField(metadata)}; - `; -} - -function createMetadataPainlessScript(definition: EntityDefinition) { - if (!definition.metadata) { - return ''; - } - - return definition.metadata.reduce((acc, metadata) => { - const { destination, source } = metadata; - const optionalFieldPath = destination.replaceAll('.', '?.'); - - if (metadata.aggregation.type === 'terms') { - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath} != null) { - ${mapDestinationToPainless(metadata)} - } - `; - return `${acc}\n${next}`; - } else if (metadata.aggregation.type === 'top_value') { - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) { - ${mapDestinationToPainless(metadata)} - } - `; - return `${acc}\n${next}`; - } - - return acc; - }, ''); -} - -function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) { - return definition.identityFields.map((key) => ({ - set: { - if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`, - field: key.field, - value: `{{entity.identity.${key.field}}}`, - }, - })); -} - -function getCustomIngestPipelines(definition: EntityDefinition) { - if (isBuiltinDefinition(definition)) { - return []; - } - - return [ - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}@platform`, - }, - }, - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}-history@platform`, - }, - }, - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}@custom`, - }, - }, - { - pipeline: { - ignore_missing_pipeline: true, - name: `${definition.id}-history@custom`, - }, - }, - ]; -} - -export function generateHistoryProcessors(definition: EntityDefinition) { - return [ - { - set: { - field: 'event.ingested', - value: '{{{_ingest.timestamp}}}', - }, - }, - { - set: { - field: 'entity.type', - value: definition.type, - }, - }, - { - set: { - field: 'entity.definitionId', - value: definition.id, - }, - }, - { - set: { - field: 'entity.definitionVersion', - value: definition.version, - }, - }, - { - set: { - field: 'entity.schemaVersion', - value: ENTITY_SCHEMA_VERSION_V1, - }, - }, - { - set: { - field: 'entity.identityFields', - value: definition.identityFields.map((identityField) => identityField.field), - }, - }, - { - script: { - description: 'Generated the entity.id field', - source: cleanScript(` - // This function will recursively collect all the values of a HashMap of HashMaps - Collection collectValues(HashMap subject) { - Collection values = new ArrayList(); - // Iterate through the values - for(Object value: subject.values()) { - // If the value is a HashMap, recurse - if (value instanceof HashMap) { - values.addAll(collectValues((HashMap) value)); - } else { - values.add(String.valueOf(value)); - } - } - return values; - } - - // Create the string builder - StringBuilder entityId = new StringBuilder(); - - if (ctx["entity"]["identity"] != null) { - // Get the values as a collection - Collection values = collectValues(ctx["entity"]["identity"]); - - // Convert to a list and sort - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - - // Create comma delimited string - for(String instanceValue: sortedValues) { - entityId.append(instanceValue); - entityId.append(":"); - } - - // Assign the entity.id - ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown"; - } - `), - }, - }, - { - fingerprint: { - fields: ['entity.id'], - target_field: 'entity.id', - method: 'MurmurHash3', - }, - }, - ...(definition.staticFields != null - ? Object.keys(definition.staticFields).map((field) => ({ - set: { field, value: definition.staticFields![field] }, - })) - : []), - ...(definition.metadata != null - ? [{ script: { source: cleanScript(createMetadataPainlessScript(definition)) } }] - : []), - { - remove: { - field: 'entity.metadata', - ignore_missing: true, - }, - }, - ...liftIdentityFieldsToDocumentRoot(definition), - { - remove: { - field: 'entity.identity', - ignore_missing: true, - }, - }, - { - date_index_name: { - field: '@timestamp', - index_name_prefix: `${generateHistoryIndexName(definition)}.`, - date_rounding: 'M', - date_formats: ['UNIX_MS', 'ISO8601', "yyyy-MM-dd'T'HH:mm:ss.SSSXX"], - }, - }, - ...getCustomIngestPipelines(definition), - ]; -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 16823221fffb3..64af189fa7059 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -15,9 +15,9 @@ import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { if (aggregation.type === 'terms') { - return `ctx.entity.metadata.${destination}.data.keySet()`; + return `ctx.entity.metadata.${destination}.keySet()`; } else if (aggregation.type === 'top_value') { - return `ctx.entity.metadata.${destination}.top_value["${destination}"]`; + return `ctx.entity.metadata.${destination}.top_value["${source}"]`; } } @@ -35,19 +35,19 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } return definition.metadata.reduce((acc, metadata) => { - const destination = metadata.destination; + const { destination, source } = metadata; const optionalFieldPath = destination.replaceAll('.', '?.'); if (metadata.aggregation.type === 'terms') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) { + if (ctx.entity?.metadata?.${optionalFieldPath} != null) { ${mapDestinationToPainless(metadata)} } `; return `${acc}\n${next}`; } else if (metadata.aggregation.type === 'top_value') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${destination}"] != null) { + if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) { ${mapDestinationToPainless(metadata)} } `; @@ -59,30 +59,13 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) { - return definition.identityFields - .map((identityField) => { - const setProcessor = { - set: { - field: identityField.field, - value: `{{entity.identity.${identityField.field}.top_metric.${identityField.field}}}`, - }, - }; - - if (!identityField.field.includes('.')) { - return [setProcessor]; - } - - return [ - { - dot_expander: { - field: identityField.field, - path: `entity.identity.${identityField.field}.top_metric`, - }, - }, - setProcessor, - ]; - }) - .flat(); + return definition.identityFields.map((key) => ({ + set: { + if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`, + field: key.field, + value: `{{entity.identity.${key.field}}}`, + }, + })); } function getCustomIngestPipelines(definition: EntityDefinition) { @@ -156,6 +139,55 @@ export function generateLatestProcessors(definition: EntityDefinition) { value: definition.identityFields.map((identityField) => identityField.field), }, }, + { + script: { + description: 'Generated the entity.id field', + source: cleanScript(` + // This function will recursively collect all the values of a HashMap of HashMaps + Collection collectValues(HashMap subject) { + Collection values = new ArrayList(); + // Iterate through the values + for(Object value: subject.values()) { + // If the value is a HashMap, recurse + if (value instanceof HashMap) { + values.addAll(collectValues((HashMap) value)); + } else { + values.add(String.valueOf(value)); + } + } + return values; + } + + // Create the string builder + StringBuilder entityId = new StringBuilder(); + + if (ctx["entity"]["identity"] != null) { + // Get the values as a collection + Collection values = collectValues(ctx["entity"]["identity"]); + + // Convert to a list and sort + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + + // Create comma delimited string + for(String instanceValue: sortedValues) { + entityId.append(instanceValue); + entityId.append(":"); + } + + // Assign the entity.id + ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown"; + } + `), + }, + }, + { + fingerprint: { + fields: ['entity.id'], + target_field: 'entity.id', + method: 'MurmurHash3', + }, + }, ...(definition.staticFields != null ? Object.keys(definition.staticFields).map((field) => ({ set: { field, value: definition.staticFields![field] }, @@ -177,8 +209,8 @@ export function generateLatestProcessors(definition: EntityDefinition) { ignore_missing: true, }, }, + // This must happen AFTER we lift the identity fields into the root of the document { - // This must happen AFTER we lift the identity fields into the root of the document set: { field: 'entity.displayName', value: definition.displayNameTemplate, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts index 5cee21dc43a07..66db454f43005 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -19,19 +19,23 @@ import { } from './install_entity_definition'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; import { - generateHistoryIndexTemplateId, - generateHistoryIngestPipelineId, - generateHistoryTransformId, generateLatestIndexTemplateId, generateLatestIngestPipelineId, generateLatestTransformId, } from './helpers/generate_component_id'; -import { generateHistoryTransform } from './transform/generate_history_transform'; import { generateLatestTransform } from './transform/generate_latest_transform'; import { entityDefinition as mockEntityDefinition } from './helpers/fixtures/entity_definition'; import { EntityDefinitionIdInvalid } from './errors/entity_definition_id_invalid'; import { EntityIdConflict } from './errors/entity_id_conflict_error'; +const getExpectedInstalledComponents = (definition: EntityDefinition) => { + return [ + { type: 'template', id: generateLatestIndexTemplateId(definition) }, + { type: 'ingest_pipeline', id: generateLatestIngestPipelineId(definition) }, + { type: 'transform', id: generateLatestTransformId(definition) }, + ]; +}; + const assertHasCreatedDefinition = ( definition: EntityDefinition, soClient: SavedObjectsClientContract, @@ -44,6 +48,7 @@ const assertHasCreatedDefinition = ( ...definition, installStatus: 'installing', installStartedAt: expect.any(String), + installedComponents: [], }, { id: definition.id, @@ -54,29 +59,17 @@ const assertHasCreatedDefinition = ( expect(soClient.update).toBeCalledTimes(1); expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, { installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(definition), }); - expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.putIndexTemplate).toBeCalledWith( - expect.objectContaining({ - name: `entities_v1_history_${definition.id}_index_template`, - }) - ); + expect(esClient.indices.putIndexTemplate).toBeCalledTimes(1); expect(esClient.indices.putIndexTemplate).toBeCalledWith( expect.objectContaining({ name: `entities_v1_latest_${definition.id}_index_template`, }) ); - expect(esClient.ingest.putPipeline).toBeCalledTimes(2); - expect(esClient.ingest.putPipeline).toBeCalledWith({ - id: generateHistoryIngestPipelineId(definition), - processors: expect.anything(), - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - }); + expect(esClient.ingest.putPipeline).toBeCalledTimes(1); expect(esClient.ingest.putPipeline).toBeCalledWith({ id: generateLatestIngestPipelineId(definition), processors: expect.anything(), @@ -86,8 +79,7 @@ const assertHasCreatedDefinition = ( }, }); - expect(esClient.transform.putTransform).toBeCalledTimes(2); - expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition)); + expect(esClient.transform.putTransform).toBeCalledTimes(1); expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition)); }; @@ -101,32 +93,21 @@ const assertHasUpgradedDefinition = ( ...definition, installStatus: 'upgrading', installStartedAt: expect.any(String), + installedComponents: getExpectedInstalledComponents(definition), }); expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, { installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(definition), }); - expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.putIndexTemplate).toBeCalledWith( - expect.objectContaining({ - name: `entities_v1_history_${definition.id}_index_template`, - }) - ); + expect(esClient.indices.putIndexTemplate).toBeCalledTimes(1); expect(esClient.indices.putIndexTemplate).toBeCalledWith( expect.objectContaining({ name: `entities_v1_latest_${definition.id}_index_template`, }) ); - expect(esClient.ingest.putPipeline).toBeCalledTimes(2); - expect(esClient.ingest.putPipeline).toBeCalledWith({ - id: generateHistoryIngestPipelineId(definition), - processors: expect.anything(), - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - }); + expect(esClient.ingest.putPipeline).toBeCalledTimes(1); expect(esClient.ingest.putPipeline).toBeCalledWith({ id: generateLatestIngestPipelineId(definition), processors: expect.anything(), @@ -136,8 +117,7 @@ const assertHasUpgradedDefinition = ( }, }); - expect(esClient.transform.putTransform).toBeCalledTimes(2); - expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition)); + expect(esClient.transform.putTransform).toBeCalledTimes(1); expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition)); }; @@ -148,13 +128,7 @@ const assertHasDeletedDefinition = ( ) => { assertHasDeletedTransforms(definition, esClient); - expect(esClient.ingest.deletePipeline).toBeCalledTimes(2); - expect(esClient.ingest.deletePipeline).toBeCalledWith( - { - id: generateHistoryIngestPipelineId(definition), - }, - { ignore: [404] } - ); + expect(esClient.ingest.deletePipeline).toBeCalledTimes(1); expect(esClient.ingest.deletePipeline).toBeCalledWith( { id: generateLatestIngestPipelineId(definition), @@ -162,13 +136,7 @@ const assertHasDeletedDefinition = ( { ignore: [404] } ); - expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.deleteIndexTemplate).toBeCalledWith( - { - name: generateHistoryIndexTemplateId(definition), - }, - { ignore: [404] } - ); + expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(1); expect(esClient.indices.deleteIndexTemplate).toBeCalledWith( { name: generateLatestIndexTemplateId(definition), @@ -184,33 +152,21 @@ const assertHasDeletedTransforms = ( definition: EntityDefinition, esClient: ElasticsearchClient ) => { - expect(esClient.transform.stopTransform).toBeCalledTimes(2); - expect(esClient.transform.stopTransform).toBeCalledWith( - expect.objectContaining({ - transform_id: generateHistoryTransformId(definition), - }), - expect.anything() - ); - expect(esClient.transform.deleteTransform).toBeCalledWith( - expect.objectContaining({ - transform_id: generateHistoryTransformId(definition), - }), - expect.anything() - ); + expect(esClient.transform.stopTransform).toBeCalledTimes(1); expect(esClient.transform.stopTransform).toBeCalledWith( expect.objectContaining({ transform_id: generateLatestTransformId(definition), }), expect.anything() ); + + expect(esClient.transform.deleteTransform).toBeCalledTimes(1); expect(esClient.transform.deleteTransform).toBeCalledWith( expect.objectContaining({ transform_id: generateLatestTransformId(definition), }), expect.anything() ); - - expect(esClient.transform.deleteTransform).toBeCalledTimes(2); }; describe('install_entity_definition', () => { @@ -242,6 +198,7 @@ describe('install_entity_definition', () => { attributes: { ...mockEntityDefinition, installStatus: 'installed', + installedComponents: [], }, }, ], @@ -329,6 +286,7 @@ describe('install_entity_definition', () => { attributes: { ...mockEntityDefinition, installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -367,6 +325,7 @@ describe('install_entity_definition', () => { attributes: { ...mockEntityDefinition, installStatus: 'installed', + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -407,6 +366,7 @@ describe('install_entity_definition', () => { // upgrading for 1h installStatus: 'upgrading', installStartedAt: moment().subtract(1, 'hour').toISOString(), + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], @@ -442,6 +402,7 @@ describe('install_entity_definition', () => { ...mockEntityDefinition, installStatus: 'failed', installStartedAt: new Date().toISOString(), + installedComponents: getExpectedInstalledComponents(mockEntityDefinition), }, }, ], diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts index de681efadebcf..5a8ec504a1d16 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts @@ -10,12 +10,12 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; -import { generateHistoryIndexTemplateId } from './helpers/generate_component_id'; -import { createAndInstallHistoryIngestPipeline } from './create_and_install_ingest_pipeline'; -import { createAndInstallHistoryTransform } from './create_and_install_transform'; +import { generateLatestIndexTemplateId } from './helpers/generate_component_id'; +import { createAndInstallIngestPipelines } from './create_and_install_ingest_pipeline'; +import { createAndInstallTransforms } from './create_and_install_transform'; import { validateDefinitionCanCreateValidTransformIds } from './transform/validate_transform_ids'; import { deleteEntityDefinition } from './delete_entity_definition'; -import { deleteIngestPipelines } from './delete_ingest_pipeline'; +import { deleteLatestIngestPipeline } from './delete_ingest_pipeline'; import { findEntityDefinitionById } from './find_entity_definition'; import { entityDefinitionExists, @@ -27,8 +27,8 @@ import { EntityIdConflict } from './errors/entity_id_conflict_error'; import { EntityDefinitionNotFound } from './errors/entity_not_found'; import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update'; import { EntityDefinitionWithState } from './types'; -import { stopTransforms } from './stop_transforms'; -import { deleteTransforms } from './delete_transforms'; +import { stopLatestTransform, stopTransforms } from './stop_transforms'; +import { deleteLatestTransform, deleteTransforms } from './delete_transforms'; export interface InstallDefinitionParams { esClient: ElasticsearchClient; @@ -48,31 +48,34 @@ export async function installEntityDefinition({ }: InstallDefinitionParams): Promise { validateDefinitionCanCreateValidTransformIds(definition); - try { - if (await entityDefinitionExists(soClient, definition.id)) { - throw new EntityIdConflict( - `Entity definition with [${definition.id}] already exists.`, - definition - ); - } + if (await entityDefinitionExists(soClient, definition.id)) { + throw new EntityIdConflict( + `Entity definition with [${definition.id}] already exists.`, + definition + ); + } + try { const entityDefinition = await saveEntityDefinition(soClient, { ...definition, installStatus: 'installing', installStartedAt: new Date().toISOString(), + installedComponents: [], }); return await install({ esClient, soClient, logger, definition: entityDefinition }); } catch (e) { logger.error(`Failed to install entity definition ${definition.id}: ${e}`); - await stopAndDeleteTransforms(esClient, definition, logger); - await deleteIngestPipelines(esClient, definition, logger); + await stopLatestTransform(esClient, definition, logger); + await deleteLatestTransform(esClient, definition, logger); + + await deleteLatestIngestPipeline(esClient, definition, logger); await deleteTemplate({ esClient, logger, - name: generateHistoryIndexTemplateId(definition), + name: generateLatestIndexTemplateId(definition), }); await deleteEntityDefinition(soClient, definition).catch((err) => { @@ -160,10 +163,10 @@ async function install({ const templates = await createAndInstallTemplates(esClient, definition, logger); logger.debug(`Installing ingest pipelines for definition ${definition.id}`); - const pipelines = await createAndInstallHistoryIngestPipeline(esClient, definition, logger); + const pipelines = await createAndInstallIngestPipelines(esClient, definition, logger); logger.debug(`Installing transforms for definition ${definition.id}`); - const transforms = await createAndInstallHistoryTransform(esClient, definition, logger); + const transforms = await createAndInstallTransforms(esClient, definition, logger); await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts b/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts index 055f17f43ce2c..9aabad926b239 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts @@ -9,6 +9,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; +import { generateLatestTransformId } from './helpers/generate_component_id'; export async function stopTransforms( esClient: ElasticsearchClient, @@ -17,7 +18,7 @@ export async function stopTransforms( ) { try { await Promise.all( - definition.installedComponents + (definition.installedComponents ?? []) .filter(({ type }) => type === 'transform') .map(({ id }) => retryTransientEsErrors( @@ -31,7 +32,31 @@ export async function stopTransforms( ) ); } catch (e) { - logger.error(`Cannot stop entity transforms [${definition.id}]: ${e}`); + logger.error(`Cannot stop transforms for definition [${definition.id}]: ${e}`); + throw e; + } +} + +export async function stopLatestTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + await retryTransientEsErrors( + () => + esClient.transform.stopTransform( + { + transform_id: generateLatestTransformId(definition), + wait_for_completion: true, + force: true, + }, + { ignore: [409, 404] } + ), + { logger } + ); + } catch (e) { + logger.error(`Cannot stop latest transform for definition [${definition.id}]: ${e}`); throw e; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap deleted file mode 100644 index b19a805b24b12..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap +++ /dev/null @@ -1,305 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`generateHistoryTransform(definition) should generate a valid history backfill transform 1`] = ` -Object { - "_meta": Object { - "definitionVersion": "999.999.999", - "managed": false, - }, - "defer_validation": true, - "dest": Object { - "index": ".entities.v1.history.noop", - "pipeline": "entities-v1-history-admin-console-services-backfill", - }, - "frequency": "5m", - "pivot": Object { - "aggs": Object { - "_errorRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "match_phrase": Object { - "log.level": "ERROR", - }, - }, - ], - }, - }, - }, - "_logRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "exists": Object { - "field": "log.level", - }, - }, - ], - }, - }, - }, - "entity.lastSeenTimestamp": Object { - "max": Object { - "field": "@timestamp", - }, - }, - "entity.metadata.host.name": Object { - "terms": Object { - "field": "host.name", - "size": 1000, - }, - }, - "entity.metadata.host.os.name": Object { - "terms": Object { - "field": "host.os.name", - "size": 1000, - }, - }, - "entity.metadata.sourceIndex": Object { - "terms": Object { - "field": "_index", - "size": 1000, - }, - }, - "entity.metadata.tags": Object { - "terms": Object { - "field": "tags", - "size": 1000, - }, - }, - "entity.metrics.errorRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_errorRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - "entity.metrics.logRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_logRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - }, - "group_by": Object { - "@timestamp": Object { - "date_histogram": Object { - "field": "@timestamp", - "fixed_interval": "1m", - }, - }, - "entity.identity.event.category": Object { - "terms": Object { - "field": "event.category", - "missing_bucket": true, - }, - }, - "entity.identity.log.logger": Object { - "terms": Object { - "field": "log.logger", - "missing_bucket": false, - }, - }, - }, - }, - "settings": Object { - "deduce_mappings": false, - "unattended": true, - }, - "source": Object { - "index": Array [ - "kbn-data-forge-fake_stack.*", - ], - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "range": Object { - "@timestamp": Object { - "gte": "now-72h", - }, - }, - }, - Object { - "exists": Object { - "field": "log.logger", - }, - }, - ], - }, - }, - }, - "sync": Object { - "time": Object { - "delay": "15m", - "field": "@timestamp", - }, - }, - "transform_id": "entities-v1-history-backfill-admin-console-services-backfill", -} -`; - -exports[`generateHistoryTransform(definition) should generate a valid history transform 1`] = ` -Object { - "_meta": Object { - "definitionVersion": "1.0.0", - "managed": false, - }, - "defer_validation": true, - "dest": Object { - "index": ".entities.v1.history.noop", - "pipeline": "entities-v1-history-admin-console-services", - }, - "frequency": "2m", - "pivot": Object { - "aggs": Object { - "_errorRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "match_phrase": Object { - "log.level": "ERROR", - }, - }, - ], - }, - }, - }, - "_logRate_A": Object { - "filter": Object { - "bool": Object { - "minimum_should_match": 1, - "should": Array [ - Object { - "exists": Object { - "field": "log.level", - }, - }, - ], - }, - }, - }, - "entity.lastSeenTimestamp": Object { - "max": Object { - "field": "@timestamp", - }, - }, - "entity.metadata.host.name": Object { - "terms": Object { - "field": "host.name", - "size": 1000, - }, - }, - "entity.metadata.host.os.name": Object { - "terms": Object { - "field": "host.os.name", - "size": 1000, - }, - }, - "entity.metadata.sourceIndex": Object { - "terms": Object { - "field": "_index", - "size": 1000, - }, - }, - "entity.metadata.tags": Object { - "terms": Object { - "field": "tags", - "size": 1000, - }, - }, - "entity.metrics.errorRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_errorRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - "entity.metrics.logRate": Object { - "bucket_script": Object { - "buckets_path": Object { - "A": "_logRate_A>_count", - }, - "script": Object { - "lang": "painless", - "source": "params.A", - }, - }, - }, - }, - "group_by": Object { - "@timestamp": Object { - "date_histogram": Object { - "field": "@timestamp", - "fixed_interval": "1m", - }, - }, - "entity.identity.event.category": Object { - "terms": Object { - "field": "event.category", - "missing_bucket": true, - }, - }, - "entity.identity.log.logger": Object { - "terms": Object { - "field": "log.logger", - "missing_bucket": false, - }, - }, - }, - }, - "settings": Object { - "deduce_mappings": false, - "unattended": true, - }, - "source": Object { - "index": Array [ - "kbn-data-forge-fake_stack.*", - ], - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "exists": Object { - "field": "log.logger", - }, - }, - Object { - "range": Object { - "@timestamp": Object { - "gte": "now-10m", - }, - }, - }, - ], - }, - }, - }, - "sync": Object { - "time": Object { - "delay": "2m", - "field": "@timestamp", - }, - }, - "transform_id": "entities-v1-history-admin-console-services", -} -`; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap index ab1224525f4d7..c3c19397ff9a9 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap @@ -11,170 +11,100 @@ Object { "index": ".entities.v1.latest.noop", "pipeline": "entities-v1-latest-admin-console-services", }, - "frequency": "30s", + "frequency": "2m", "pivot": Object { "aggs": Object { - "_errorRate": Object { - "top_metrics": Object { - "metrics": Array [ - Object { - "field": "entity.metrics.errorRate", - }, - ], - "sort": Array [ - Object { - "@timestamp": "desc", - }, - ], - }, - }, - "_logRate": Object { - "top_metrics": Object { - "metrics": Array [ - Object { - "field": "entity.metrics.logRate", - }, - ], - "sort": Array [ - Object { - "@timestamp": "desc", - }, - ], - }, - }, - "entity.firstSeenTimestamp": Object { - "min": Object { - "field": "@timestamp", - }, - }, - "entity.identity.event.category": Object { - "aggs": Object { - "top_metric": Object { - "top_metrics": Object { - "metrics": Object { - "field": "event.category", - }, - "sort": "_score", - }, - }, - }, + "_errorRate_A": Object { "filter": Object { - "exists": Object { - "field": "event.category", - }, - }, - }, - "entity.identity.log.logger": Object { - "aggs": Object { - "top_metric": Object { - "top_metrics": Object { - "metrics": Object { - "field": "log.logger", + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "log.level": "ERROR", + }, }, - "sort": "_score", - }, + ], }, }, + }, + "_logRate_A": Object { "filter": Object { - "exists": Object { - "field": "log.logger", + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "log.level", + }, + }, + ], }, }, }, "entity.lastSeenTimestamp": Object { "max": Object { - "field": "entity.lastSeenTimestamp", + "field": "@timestamp", }, }, "entity.metadata.host.name": Object { - "aggs": Object { - "data": Object { - "terms": Object { - "field": "host.name", - "size": 1000, - }, - }, - }, - "filter": Object { - "range": Object { - "@timestamp": Object { - "gte": "now-360s", - }, - }, + "terms": Object { + "field": "host.name", + "size": 1000, }, }, "entity.metadata.host.os.name": Object { - "aggs": Object { - "data": Object { - "terms": Object { - "field": "host.os.name", - "size": 1000, - }, - }, - }, - "filter": Object { - "range": Object { - "@timestamp": Object { - "gte": "now-360s", - }, - }, + "terms": Object { + "field": "host.os.name", + "size": 1000, }, }, "entity.metadata.sourceIndex": Object { - "aggs": Object { - "data": Object { - "terms": Object { - "field": "sourceIndex", - "size": 1000, - }, - }, - }, - "filter": Object { - "range": Object { - "@timestamp": Object { - "gte": "now-360s", - }, - }, + "terms": Object { + "field": "_index", + "size": 1000, }, }, "entity.metadata.tags": Object { - "aggs": Object { - "data": Object { - "terms": Object { - "field": "tags", - "size": 1000, - }, - }, - }, - "filter": Object { - "range": Object { - "@timestamp": Object { - "gte": "now-360s", - }, - }, + "terms": Object { + "field": "tags", + "size": 1000, }, }, "entity.metrics.errorRate": Object { "bucket_script": Object { "buckets_path": Object { - "value": "_errorRate[entity.metrics.errorRate]", + "A": "_errorRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", }, - "script": "params.value", }, }, "entity.metrics.logRate": Object { "bucket_script": Object { "buckets_path": Object { - "value": "_logRate[entity.metrics.logRate]", + "A": "_logRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", }, - "script": "params.value", }, }, }, "group_by": Object { - "entity.id": Object { + "entity.identity.event.category": Object { + "terms": Object { + "field": "event.category", + "missing_bucket": true, + }, + }, + "entity.identity.log.logger": Object { "terms": Object { - "field": "entity.id", + "field": "log.logger", + "missing_bucket": false, }, }, }, @@ -184,12 +114,32 @@ Object { "unattended": true, }, "source": Object { - "index": ".entities.v1.history.admin-console-services.*", + "index": Array [ + "kbn-data-forge-fake_stack.*", + ], + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "exists": Object { + "field": "log.logger", + }, + }, + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-10m", + }, + }, + }, + ], + }, + }, }, "sync": Object { "time": Object { - "delay": "1s", - "field": "event.ingested", + "delay": "2m", + "field": "@timestamp", }, }, "transform_id": "entities-v1-latest-admin-console-services", diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts deleted file mode 100644 index f49ec0cd88a37..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinition } from '../helpers/fixtures/entity_definition'; -import { entityDefinitionWithBackfill } from '../helpers/fixtures/entity_definition_with_backfill'; -import { - generateBackfillHistoryTransform, - generateHistoryTransform, -} from './generate_history_transform'; - -describe('generateHistoryTransform(definition)', () => { - it('should generate a valid history transform', () => { - const transform = generateHistoryTransform(entityDefinition); - expect(transform).toMatchSnapshot(); - }); - it('should generate a valid history backfill transform', () => { - const transform = generateBackfillHistoryTransform(entityDefinitionWithBackfill); - expect(transform).toMatchSnapshot(); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts deleted file mode 100644 index 64f692cc671c9..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import { - QueryDslQueryContainer, - TransformPutTransformRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; -import { generateHistoryMetricAggregations } from './generate_metric_aggregations'; -import { - ENTITY_DEFAULT_HISTORY_FREQUENCY, - ENTITY_DEFAULT_HISTORY_SYNC_DELAY, -} from '../../../../common/constants_entities'; -import { generateHistoryMetadataAggregations } from './generate_metadata_aggregations'; -import { - generateHistoryTransformId, - generateHistoryIngestPipelineId, - generateHistoryIndexName, -} from '../helpers/generate_component_id'; - -export function generateHistoryTransform( - definition: EntityDefinition -): TransformPutTransformRequest { - const filter: QueryDslQueryContainer[] = []; - - if (definition.filter) { - filter.push(getElasticsearchQueryOrThrow(definition.filter)); - } - - if (definition.identityFields.some(({ optional }) => !optional)) { - definition.identityFields - .filter(({ optional }) => !optional) - .forEach(({ field }) => { - filter.push({ exists: { field } }); - }); - } - - filter.push({ - range: { - [definition.history.timestampField]: { - gte: `now-${definition.history.settings.lookbackPeriod}`, - }, - }, - }); - - return generateTransformPutRequest({ - definition, - filter, - transformId: generateHistoryTransformId(definition), - frequency: definition.history.settings.frequency, - syncDelay: definition.history.settings.syncDelay, - }); -} - -const generateTransformPutRequest = ({ - definition, - filter, - transformId, - frequency, - syncDelay, -}: { - definition: EntityDefinition; - transformId: string; - filter: QueryDslQueryContainer[]; - frequency?: string; - syncDelay?: string; -}) => { - return { - transform_id: transformId, - _meta: { - definitionVersion: definition.version, - managed: definition.managed, - }, - defer_validation: true, - source: { - index: definition.indexPatterns, - ...(filter.length > 0 && { - query: { - bool: { - filter, - }, - }, - }), - }, - dest: { - index: `${generateHistoryIndexName({ id: 'noop' } as EntityDefinition)}`, - pipeline: generateHistoryIngestPipelineId(definition), - }, - frequency: frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY, - sync: { - time: { - field: definition.history.settings.syncField || definition.history.timestampField, - delay: syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY, - }, - }, - settings: { - deduce_mappings: false, - unattended: true, - }, - pivot: { - group_by: { - ...definition.identityFields.reduce( - (acc, id) => ({ - ...acc, - [`entity.identity.${id.field}`]: { - terms: { field: id.field, missing_bucket: id.optional }, - }, - }), - {} - ), - }, - aggs: { - ...generateHistoryMetricAggregations(definition), - ...generateHistoryMetadataAggregations(definition), - 'entity.lastSeenTimestamp': { - max: { - field: definition.history.timestampField, - }, - }, - }, - }, - }; -}; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts index 85ee57fefea2c..abb9901df9b9d 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts @@ -5,44 +5,97 @@ * 2.0. */ -import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; import { EntityDefinition } from '@kbn/entities-schema'; import { - ENTITY_DEFAULT_LATEST_FREQUENCY, - ENTITY_DEFAULT_LATEST_SYNC_DELAY, + QueryDslQueryContainer, + TransformPutTransformRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; +import { generateHistoryMetricAggregations } from './generate_metric_aggregations'; +import { + ENTITY_DEFAULT_HISTORY_FREQUENCY, + ENTITY_DEFAULT_HISTORY_SYNC_DELAY, } from '../../../../common/constants_entities'; +import { generateHistoryMetadataAggregations } from './generate_metadata_aggregations'; import { - generateHistoryIndexName, - generateLatestIndexName, - generateLatestIngestPipelineId, generateLatestTransformId, + generateLatestIngestPipelineId, + generateLatestIndexName, } from '../helpers/generate_component_id'; -import { generateIdentityAggregations } from './generate_identity_aggregations'; -import { generateLatestMetadataAggregations } from './generate_metadata_aggregations'; -import { generateLatestMetricAggregations } from './generate_metric_aggregations'; export function generateLatestTransform( definition: EntityDefinition ): TransformPutTransformRequest { + const filter: QueryDslQueryContainer[] = []; + + if (definition.filter) { + filter.push(getElasticsearchQueryOrThrow(definition.filter)); + } + + if (definition.identityFields.some(({ optional }) => !optional)) { + definition.identityFields + .filter(({ optional }) => !optional) + .forEach(({ field }) => { + filter.push({ exists: { field } }); + }); + } + + filter.push({ + range: { + [definition.history.timestampField]: { + gte: `now-${definition.history.settings.lookbackPeriod}`, + }, + }, + }); + + return generateTransformPutRequest({ + definition, + filter, + transformId: generateLatestTransformId(definition), + frequency: definition.history.settings.frequency, + syncDelay: definition.history.settings.syncDelay, + }); +} + +const generateTransformPutRequest = ({ + definition, + filter, + transformId, + frequency, + syncDelay, +}: { + definition: EntityDefinition; + transformId: string; + filter: QueryDslQueryContainer[]; + frequency?: string; + syncDelay?: string; +}) => { return { - transform_id: generateLatestTransformId(definition), + transform_id: transformId, _meta: { definitionVersion: definition.version, managed: definition.managed, }, defer_validation: true, source: { - index: `${generateHistoryIndexName(definition)}.*`, + index: definition.indexPatterns, + ...(filter.length > 0 && { + query: { + bool: { + filter, + }, + }, + }), }, dest: { index: `${generateLatestIndexName({ id: 'noop' } as EntityDefinition)}`, pipeline: generateLatestIngestPipelineId(definition), }, - frequency: definition.latest?.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY, + frequency: frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY, sync: { time: { - field: definition.latest?.settings?.syncField ?? 'event.ingested', - delay: definition.latest?.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY, + field: definition.history.settings.syncField || definition.history.timestampField, + delay: syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY, }, }, settings: { @@ -51,25 +104,25 @@ export function generateLatestTransform( }, pivot: { group_by: { - ['entity.id']: { - terms: { field: 'entity.id' }, - }, + ...definition.identityFields.reduce( + (acc, id) => ({ + ...acc, + [`entity.identity.${id.field}`]: { + terms: { field: id.field, missing_bucket: id.optional }, + }, + }), + {} + ), }, aggs: { - ...generateLatestMetricAggregations(definition), - ...generateLatestMetadataAggregations(definition), - ...generateIdentityAggregations(definition), + ...generateHistoryMetricAggregations(definition), + ...generateHistoryMetadataAggregations(definition), 'entity.lastSeenTimestamp': { max: { - field: 'entity.lastSeenTimestamp', - }, - }, - 'entity.firstSeenTimestamp': { - min: { - field: '@timestamp', + field: definition.history.timestampField, }, }, }, }, }; -} +}; diff --git a/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts index 92ca7e586bbf3..ffa58cd9c0145 100644 --- a/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts @@ -16,7 +16,7 @@ import { entitiesLatestBaseComponentTemplateConfig } from '../templates/componen import { entitiesEntityComponentTemplateConfig } from '../templates/components/entity'; import { entitiesEventComponentTemplateConfig } from '../templates/components/event'; import { retryTransientEsErrors } from './entities/helpers/retry'; -import { generateEntitiesHistoryIndexTemplateConfig } from './entities/templates/entities_history_template'; +import { generateEntitiesLatestIndexTemplateConfig } from './entities/templates/entities_latest_template'; interface TemplateManagementOptions { esClient: ElasticsearchClient; @@ -85,7 +85,7 @@ export async function createAndInstallTemplates( definition: EntityDefinition, logger: Logger ): Promise> { - const template = generateEntitiesHistoryIndexTemplateConfig(definition); + const template = generateEntitiesLatestIndexTemplateConfig(definition); await upsertTemplate({ esClient, template, logger }); return [{ type: 'template', id: template.name }]; } From f01a8a00315b381fbd922e0af0d5c8dfd7b23a46 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 14:04:52 +0200 Subject: [PATCH 05/21] fix aggs --- .../kbn-entities-schema/src/schema/common.ts | 10 ++-- .../src/schema/entity_definition.ts | 24 ++++----- .../common/constants_entities.ts | 8 ++- .../built_in/containers_from_ecs_data.ts | 2 +- .../entities/built_in/hosts_from_ecs_data.ts | 2 +- .../built_in/services_from_ecs_data.ts | 2 +- .../generate_latest_processors.ts | 4 +- .../transform/generate_latest_transform.ts | 36 +++++-------- .../generate_metadata_aggregations.ts | 53 ++----------------- .../transform/generate_metric_aggregations.ts | 30 +---------- 10 files changed, 45 insertions(+), 126 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index aa54dbd16c9aa..39b9c9d9bec26 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -85,7 +85,11 @@ export const keyMetricSchema = z.object({ export type KeyMetric = z.infer; export const metadataAggregation = z.union([ - z.object({ type: z.literal('terms'), limit: z.number().default(1000) }), + z.object({ + type: z.literal('terms'), + limit: z.number().default(10), + lookbackPeriod: z.optional(durationSchema), + }), z.object({ type: z.literal('top_value'), sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), @@ -99,13 +103,13 @@ export const metadataSchema = z destination: z.optional(z.string()), aggregation: z .optional(metadataAggregation) - .default({ type: z.literal('terms').value, limit: 1000 }), + .default({ type: z.literal('terms').value, limit: 10, lookbackPeriod: '1h' }), }) .or( z.string().transform((value) => ({ source: value, destination: value, - aggregation: { type: z.literal('terms').value, limit: 1000 }, + aggregation: { type: z.literal('terms').value, limit: 10, lookbackPeriod: '1h' }, })) ) .transform((metadata) => ({ diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index 23d78e84f01dd..9670869b3237c 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -14,7 +14,6 @@ import { durationSchema, identityFieldsSchema, semVerSchema, - historySettingsSchema, durationSchemaWithMinimum, } from './common'; @@ -32,22 +31,17 @@ export const entityDefinitionSchema = z.object({ metrics: z.optional(z.array(keyMetricSchema)), staticFields: z.optional(z.record(z.string(), z.string())), managed: z.optional(z.boolean()).default(false), - history: z.object({ + latest: z.object({ timestampField: z.string(), interval: durationSchemaWithMinimum(1), - settings: historySettingsSchema, + settings: z.optional( + z.object({ + syncField: z.optional(z.string()), + syncDelay: z.optional(durationSchema), + frequency: z.optional(durationSchema), + }) + ), }), - latest: z.optional( - z.object({ - settings: z.optional( - z.object({ - syncField: z.optional(z.string()), - syncDelay: z.optional(durationSchema), - frequency: z.optional(durationSchema), - }) - ), - }) - ), installStatus: z.optional( z.union([ z.literal('installing'), @@ -81,7 +75,7 @@ export const entityDefinitionUpdateSchema = entityDefinitionSchema .partial() .merge( z.object({ - history: z.optional(entityDefinitionSchema.shape.history.partial()), + latest: z.optional(entityDefinitionSchema.shape.latest.partial()), version: semVerSchema, }) ); diff --git a/x-pack/plugins/entity_manager/common/constants_entities.ts b/x-pack/plugins/entity_manager/common/constants_entities.ts index c53847afbb548..c17e6f33918c6 100644 --- a/x-pack/plugins/entity_manager/common/constants_entities.ts +++ b/x-pack/plugins/entity_manager/common/constants_entities.ts @@ -33,8 +33,6 @@ export const ENTITY_LATEST_PREFIX_V1 = `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_LATEST}` as const; // Transform constants -export const ENTITY_DEFAULT_HISTORY_FREQUENCY = '1m'; -export const ENTITY_DEFAULT_HISTORY_SYNC_DELAY = '60s'; -export const ENTITY_DEFAULT_LATEST_FREQUENCY = '30s'; -export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '1s'; -export const ENTITY_DEFAULT_METADATA_LIMIT = 1000; +export const ENTITY_DEFAULT_LATEST_FREQUENCY = '1m'; +export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '60s'; +export const ENTITY_DEFAULT_METADATA_LIMIT = 10; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts index b9119143ec37e..7109856ff1d11 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts @@ -20,7 +20,7 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition = indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], identityFields: ['container.id'], displayNameTemplate: '{{container.id}}', - history: { + latest: { timestampField: '@timestamp', interval: '5m', settings: { diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts index 5fead32f5c0e8..f93300f5908c5 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts @@ -19,7 +19,7 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], identityFields: ['host.name'], displayNameTemplate: '{{host.name}}', - history: { + latest: { timestampField: '@timestamp', interval: '5m', settings: { diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index fcb6f6136ef98..75cb747813bc8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -33,7 +33,7 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition = 'remote_cluster:metrics-apm.service_transaction.1m*', 'remote_cluster:metrics-apm.service_summary.1m*', ], - history: { + latest: { timestampField: '@timestamp', interval: '1m', settings: { diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 64af189fa7059..0e3812de2e320 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -15,7 +15,7 @@ import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { if (aggregation.type === 'terms') { - return `ctx.entity.metadata.${destination}.keySet()`; + return `ctx.entity.metadata.${destination}.data.keySet()`; } else if (aggregation.type === 'top_value') { return `ctx.entity.metadata.${destination}.top_value["${source}"]`; } @@ -40,7 +40,7 @@ function createMetadataPainlessScript(definition: EntityDefinition) { if (metadata.aggregation.type === 'terms') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath} != null) { + if (ctx.entity?.metadata?.${optionalFieldPath}?.data != null) { ${mapDestinationToPainless(metadata)} } `; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts index abb9901df9b9d..861486be218c4 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts @@ -11,17 +11,17 @@ import { TransformPutTransformRequest, } from '@elastic/elasticsearch/lib/api/types'; import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; -import { generateHistoryMetricAggregations } from './generate_metric_aggregations'; +import { generateLatestMetricAggregations } from './generate_metric_aggregations'; import { - ENTITY_DEFAULT_HISTORY_FREQUENCY, - ENTITY_DEFAULT_HISTORY_SYNC_DELAY, + ENTITY_DEFAULT_LATEST_FREQUENCY, + ENTITY_DEFAULT_LATEST_SYNC_DELAY, } from '../../../../common/constants_entities'; -import { generateHistoryMetadataAggregations } from './generate_metadata_aggregations'; import { generateLatestTransformId, generateLatestIngestPipelineId, generateLatestIndexName, } from '../helpers/generate_component_id'; +import { generateLatestMetadataAggregations } from './generate_metadata_aggregations'; export function generateLatestTransform( definition: EntityDefinition @@ -40,20 +40,12 @@ export function generateLatestTransform( }); } - filter.push({ - range: { - [definition.history.timestampField]: { - gte: `now-${definition.history.settings.lookbackPeriod}`, - }, - }, - }); - return generateTransformPutRequest({ definition, filter, transformId: generateLatestTransformId(definition), - frequency: definition.history.settings.frequency, - syncDelay: definition.history.settings.syncDelay, + frequency: definition.latest.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY, + syncDelay: definition.latest.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY, }); } @@ -67,8 +59,8 @@ const generateTransformPutRequest = ({ definition: EntityDefinition; transformId: string; filter: QueryDslQueryContainer[]; - frequency?: string; - syncDelay?: string; + frequency: string; + syncDelay: string; }) => { return { transform_id: transformId, @@ -91,11 +83,11 @@ const generateTransformPutRequest = ({ index: `${generateLatestIndexName({ id: 'noop' } as EntityDefinition)}`, pipeline: generateLatestIngestPipelineId(definition), }, - frequency: frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY, + frequency, sync: { time: { - field: definition.history.settings.syncField || definition.history.timestampField, - delay: syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY, + field: definition.latest.settings?.syncField || definition.latest.timestampField, + delay: syncDelay, }, }, settings: { @@ -115,11 +107,11 @@ const generateTransformPutRequest = ({ ), }, aggs: { - ...generateHistoryMetricAggregations(definition), - ...generateHistoryMetadataAggregations(definition), + ...generateLatestMetricAggregations(definition), + ...generateLatestMetadataAggregations(definition), 'entity.lastSeenTimestamp': { max: { - field: definition.history.timestampField, + field: definition.latest.timestampField, }, }, }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 0fc4464672219..c6a24807b1f69 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -6,55 +6,12 @@ */ import { EntityDefinition } from '@kbn/entities-schema'; -import { calculateOffset } from '../helpers/calculate_offset'; - -export function generateHistoryMetadataAggregations(definition: EntityDefinition) { - if (!definition.metadata) { - return {}; - } - return definition.metadata.reduce((aggs, metadata) => { - let agg; - if (metadata.aggregation.type === 'terms') { - agg = { - terms: { - field: metadata.source, - size: metadata.aggregation.limit, - }, - }; - } else if (metadata.aggregation.type === 'top_value') { - agg = { - filter: { - exists: { - field: metadata.source, - }, - }, - aggs: { - top_value: { - top_metrics: { - metrics: { - field: metadata.source, - }, - sort: metadata.aggregation.sort, - }, - }, - }, - }; - } - - return { - ...aggs, - [`entity.metadata.${metadata.destination}`]: agg, - }; - }, {}); -} export function generateLatestMetadataAggregations(definition: EntityDefinition) { if (!definition.metadata) { return {}; } - const offsetInSeconds = `${calculateOffset(definition)}s`; - return definition.metadata.reduce((aggs, metadata) => { let agg; if (metadata.aggregation.type === 'terms') { @@ -62,14 +19,14 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) filter: { range: { '@timestamp': { - gte: `now-${offsetInSeconds}`, + gte: `now-${metadata.aggregation.lookbackPeriod ?? '1h'}`, }, }, }, aggs: { data: { terms: { - field: metadata.destination, + field: metadata.source, size: metadata.aggregation.limit, }, }, @@ -83,13 +40,13 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) { range: { '@timestamp': { - gte: `now-${metadata.aggregation.lookbackPeriod ?? offsetInSeconds}`, + gte: `now-${metadata.aggregation.lookbackPeriod ?? '1h'}`, }, }, }, { exists: { - field: metadata.destination, + field: metadata.source, }, }, ], @@ -99,7 +56,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) top_value: { top_metrics: { metrics: { - field: metadata.destination, + field: metadata.source, }, sort: metadata.aggregation.sort, }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts index bd1af365116cb..d42dd69b37eff 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts @@ -104,41 +104,15 @@ function buildMetricEquation(keyMetric: KeyMetric) { }; } -export function generateHistoryMetricAggregations(definition: EntityDefinition) { - if (!definition.metrics) { - return {}; - } - return definition.metrics.reduce((aggs, keyMetric) => { - return { - ...aggs, - ...buildMetricAggregations(keyMetric, definition.history.timestampField), - [`entity.metrics.${keyMetric.name}`]: buildMetricEquation(keyMetric), - }; - }, {}); -} - export function generateLatestMetricAggregations(definition: EntityDefinition) { if (!definition.metrics) { return {}; } - return definition.metrics.reduce((aggs, keyMetric) => { return { ...aggs, - [`_${keyMetric.name}`]: { - top_metrics: { - metrics: [{ field: `entity.metrics.${keyMetric.name}` }], - sort: [{ '@timestamp': 'desc' }], - }, - }, - [`entity.metrics.${keyMetric.name}`]: { - bucket_script: { - buckets_path: { - value: `_${keyMetric.name}[entity.metrics.${keyMetric.name}]`, - }, - script: 'params.value', - }, - }, + ...buildMetricAggregations(keyMetric, definition.latest.timestampField), + [`entity.metrics.${keyMetric.name}`]: buildMetricEquation(keyMetric), }; }, {}); } From 8d0df932939fbe6b9aac8177f8717a2e82af4250 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 17:23:31 +0200 Subject: [PATCH 06/21] latest lookbackPeriod --- .../src/schema/entity_definition.ts | 1 + .../entities/transform/generate_latest_transform.ts | 8 ++++++++ .../transform/generate_metadata_aggregations.ts | 5 +++-- .../server/templates/components/helpers.ts | 13 ------------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index 9670869b3237c..4f18635d790bb 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -34,6 +34,7 @@ export const entityDefinitionSchema = z.object({ latest: z.object({ timestampField: z.string(), interval: durationSchemaWithMinimum(1), + lookbackPeriod: z.optional(durationSchema).default('24h'), settings: z.optional( z.object({ syncField: z.optional(z.string()), diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts index 861486be218c4..573bb2225f183 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts @@ -40,6 +40,14 @@ export function generateLatestTransform( }); } + filter.push({ + range: { + [definition.latest.timestampField]: { + gte: `now-${definition.latest.lookbackPeriod}`, + }, + }, + }); + return generateTransformPutRequest({ definition, filter, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index c6a24807b1f69..796d1e25b55ec 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -13,13 +13,14 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) } return definition.metadata.reduce((aggs, metadata) => { + const lookbackPeriod = metadata.aggregation.lookbackPeriod || definition.latest.lookbackPeriod; let agg; if (metadata.aggregation.type === 'terms') { agg = { filter: { range: { '@timestamp': { - gte: `now-${metadata.aggregation.lookbackPeriod ?? '1h'}`, + gte: `now-${lookbackPeriod}`, }, }, }, @@ -40,7 +41,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) { range: { '@timestamp': { - gte: `now-${metadata.aggregation.lookbackPeriod ?? '1h'}`, + gte: `now-${lookbackPeriod}`, }, }, }, diff --git a/x-pack/plugins/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/entity_manager/server/templates/components/helpers.ts index 23cc7cccb6a13..a890fca4f107f 100644 --- a/x-pack/plugins/entity_manager/server/templates/components/helpers.ts +++ b/x-pack/plugins/entity_manager/server/templates/components/helpers.ts @@ -20,16 +20,3 @@ export const getCustomLatestTemplateComponents = (definition: EntityDefinition) `${definition.id}-latest@custom`, ]; }; - -export const getCustomHistoryTemplateComponents = (definition: EntityDefinition) => { - if (isBuiltinDefinition(definition)) { - return []; - } - - return [ - `${definition.id}@platform`, // @platform goes before so it can be overwritten by custom - `${definition.id}-history@platform`, - `${definition.id}@custom`, - `${definition.id}-history@custom`, - ]; -}; From eef049e41abb7fd0027d6071613e9e958a7aa4df Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 17:29:01 +0200 Subject: [PATCH 07/21] remove ccs index --- .../lib/entities/built_in/services_from_ecs_data.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index 75cb747813bc8..4bba9911ef0b2 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -28,10 +28,10 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition = type: 'service', managed: true, indexPatterns: [ - 'remote_cluster:logs-*', - 'remote_cluster:filebeat*', - 'remote_cluster:metrics-apm.service_transaction.1m*', - 'remote_cluster:metrics-apm.service_summary.1m*', + 'logs-*', + 'filebeat*', + 'metrics-apm.service_transaction.1m*', + 'metrics-apm.service_summary.1m*', ], latest: { timestampField: '@timestamp', From 238a3c928739f10881fd88311af55093f18ffec5 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 19:31:13 +0200 Subject: [PATCH 08/21] fix tests --- .../src/schema/__snapshots__/common.test.ts.snap | 12 ++++++++---- .../kbn-entities-schema/src/schema/common.test.ts | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap index 9210d3b9991cf..55173b95623b4 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap +++ b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap @@ -78,7 +78,8 @@ exports[`schemas metadataSchema should parse successfully with a source and desi Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": "1h", "type": "terms", }, "destination": "hostName", @@ -92,7 +93,8 @@ exports[`schemas metadataSchema should parse successfully with an valid string 1 Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": "1h", "type": "terms", }, "destination": "host.name", @@ -106,7 +108,8 @@ exports[`schemas metadataSchema should parse successfully with just a source 1`] Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": "1h", "type": "terms", }, "destination": "host.name", @@ -120,7 +123,8 @@ exports[`schemas metadataSchema should parse successfully with valid object 1`] Object { "data": Object { "aggregation": Object { - "limit": 1000, + "limit": 10, + "lookbackPeriod": "1h", "type": "terms", }, "destination": "hostName", diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts index 1a737ac3f4d9b..bbba6f1d36d87 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -66,7 +66,7 @@ describe('schemas', () => { expect(result.data).toEqual({ source: 'host.name', destination: 'hostName', - aggregation: { type: 'terms', limit: 1000 }, + aggregation: { type: 'terms', limit: 10, lookbackPeriod: '1h' }, }); }); From 1d9441ece1f05a9378a8543cff7d741e486dfd58 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 19:50:23 +0200 Subject: [PATCH 09/21] fix tests --- .../kbn-entities-schema/src/schema/common.ts | 4 +- .../fixtures/builtin_entity_definition.ts | 2 +- .../helpers/fixtures/entity_definition.ts | 4 +- .../entity_definition_with_backfill.ts | 51 ------ .../lib/entities/helpers/fixtures/index.ts | 1 - .../generate_latest_processors.test.ts.snap | 32 ++-- .../entities_history_template.test.ts.snap | 152 ------------------ .../entities_history_template.test.ts | 21 --- .../templates/entities_history_template.ts | 96 ----------- .../templates/entities_latest_template.ts | 15 +- .../generate_latest_transform.test.ts.snap | 68 ++++++-- .../generate_metadata_aggregations.test.ts | 144 ++--------------- .../templates/components/helpers.test.ts | 31 ---- .../server/templates/components/helpers.ts | 22 --- 14 files changed, 108 insertions(+), 535 deletions(-) delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts delete mode 100644 x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts delete mode 100644 x-pack/plugins/entity_manager/server/templates/components/helpers.ts diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index 39b9c9d9bec26..caecf48d88aac 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -103,13 +103,13 @@ export const metadataSchema = z destination: z.optional(z.string()), aggregation: z .optional(metadataAggregation) - .default({ type: z.literal('terms').value, limit: 10, lookbackPeriod: '1h' }), + .default({ type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined }), }) .or( z.string().transform((value) => ({ source: value, destination: value, - aggregation: { type: z.literal('terms').value, limit: 10, lookbackPeriod: '1h' }, + aggregation: { type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined }, })) ) .transform((metadata) => ({ diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts index 5092e2caa5d78..561fb4c354e33 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts @@ -13,7 +13,7 @@ export const builtInEntityDefinition = entityDefinitionSchema.parse({ type: 'service', indexPatterns: ['kbn-data-forge-fake_stack.*'], managed: true, - history: { + latest: { timestampField: '@timestamp', interval: '1m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index 940e209260c54..4c0fd57b83ba2 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -12,11 +12,11 @@ export const rawEntityDefinition = { name: 'Services for Admin Console', type: 'service', indexPatterns: ['kbn-data-forge-fake_stack.*'], - history: { + latest: { timestampField: '@timestamp', interval: '1m', + lookbackPeriod: '10m', settings: { - lookbackPeriod: '10m', frequency: '2m', syncDelay: '2m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts deleted file mode 100644 index 66a79825fbfb0..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinitionSchema } from '@kbn/entities-schema'; -export const entityDefinitionWithBackfill = entityDefinitionSchema.parse({ - id: 'admin-console-services-backfill', - version: '999.999.999', - name: 'Services for Admin Console', - type: 'service', - indexPatterns: ['kbn-data-forge-fake_stack.*'], - history: { - timestampField: '@timestamp', - interval: '1m', - settings: { - backfillSyncDelay: '15m', - backfillLookbackPeriod: '72h', - backfillFrequency: '5m', - }, - }, - identityFields: ['log.logger', { field: 'event.category', optional: true }], - displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}', - metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }], - metrics: [ - { - name: 'logRate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'log.level: *', - }, - ], - }, - { - name: 'errorRate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'log.level: "ERROR"', - }, - ], - }, - ], -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts index c24dcee1f8cf7..e841b1c8e23dd 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts @@ -6,5 +6,4 @@ */ export { entityDefinition } from './entity_definition'; -export { entityDefinitionWithBackfill } from './entity_definition_with_backfill'; export { builtInEntityDefinition } from './builtin_entity_definition'; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap index 7e3bd58b307e5..218deda422fe2 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap @@ -87,26 +87,26 @@ if (ctx[\\"entity\\"][\\"identity\\"] != null) { }, Object { "script": Object { - "source": "if (ctx.entity?.metadata?.tags != null) { - ctx.tags = ctx.entity.metadata.tags.keySet(); + "source": "if (ctx.entity?.metadata?.tags?.data != null) { + ctx.tags = ctx.entity.metadata.tags.data.keySet(); } -if (ctx.entity?.metadata?.host?.name != null) { +if (ctx.entity?.metadata?.host?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } - ctx.host.name = ctx.entity.metadata.host.name.keySet(); + ctx.host.name = ctx.entity.metadata.host.name.data.keySet(); } -if (ctx.entity?.metadata?.host?.os?.name != null) { +if (ctx.entity?.metadata?.host?.os?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } if (ctx.host.os == null) { ctx.host.os = new HashMap(); } - ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); + ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet(); } -if (ctx.entity?.metadata?.sourceIndex != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); +if (ctx.entity?.metadata?.sourceIndex?.data != null) { + ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet(); }", }, }, @@ -238,26 +238,26 @@ if (ctx[\\"entity\\"][\\"identity\\"] != null) { }, Object { "script": Object { - "source": "if (ctx.entity?.metadata?.tags != null) { - ctx.tags = ctx.entity.metadata.tags.keySet(); + "source": "if (ctx.entity?.metadata?.tags?.data != null) { + ctx.tags = ctx.entity.metadata.tags.data.keySet(); } -if (ctx.entity?.metadata?.host?.name != null) { +if (ctx.entity?.metadata?.host?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } - ctx.host.name = ctx.entity.metadata.host.name.keySet(); + ctx.host.name = ctx.entity.metadata.host.name.data.keySet(); } -if (ctx.entity?.metadata?.host?.os?.name != null) { +if (ctx.entity?.metadata?.host?.os?.name?.data != null) { if (ctx.host == null) { ctx.host = new HashMap(); } if (ctx.host.os == null) { ctx.host.os = new HashMap(); } - ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet(); + ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet(); } -if (ctx.entity?.metadata?.sourceIndex != null) { - ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet(); +if (ctx.entity?.metadata?.sourceIndex?.data != null) { + ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet(); }", }, }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap deleted file mode 100644 index fd4ed11f8cb94..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap +++ /dev/null @@ -1,152 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template for builtin definition 1`] = ` -Object { - "_meta": Object { - "description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", - "ecs_version": "8.0.0", - "managed": true, - "managed_by": "elastic_entity_model", - }, - "composed_of": Array [ - "entities_v1_history_base", - "entities_v1_entity", - "entities_v1_event", - ], - "ignore_missing_component_templates": Array [], - "index_patterns": Array [ - ".entities.v1.history.builtin_mock_entity_definition.*", - ], - "name": "entities_v1_history_builtin_mock_entity_definition_index_template", - "priority": 200, - "template": Object { - "aliases": Object { - "entities-service-history": Object {}, - }, - "mappings": Object { - "_meta": Object { - "version": "1.6.0", - }, - "date_detection": false, - "dynamic_templates": Array [ - Object { - "strings_as_keyword": Object { - "mapping": Object { - "fields": Object { - "text": Object { - "type": "text", - }, - }, - "ignore_above": 1024, - "type": "keyword", - }, - "match_mapping_type": "string", - }, - }, - Object { - "entity_metrics": Object { - "mapping": Object { - "type": "{dynamic_type}", - }, - "match_mapping_type": Array [ - "long", - "double", - ], - "path_match": "entity.metrics.*", - }, - }, - ], - }, - "settings": Object { - "index": Object { - "codec": "best_compression", - "mapping": Object { - "total_fields": Object { - "limit": 2000, - }, - }, - }, - }, - }, -} -`; - -exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template for custom definition 1`] = ` -Object { - "_meta": Object { - "description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", - "ecs_version": "8.0.0", - "managed": true, - "managed_by": "elastic_entity_model", - }, - "composed_of": Array [ - "entities_v1_history_base", - "entities_v1_entity", - "entities_v1_event", - "admin-console-services@platform", - "admin-console-services-history@platform", - "admin-console-services@custom", - "admin-console-services-history@custom", - ], - "ignore_missing_component_templates": Array [ - "admin-console-services@platform", - "admin-console-services-history@platform", - "admin-console-services@custom", - "admin-console-services-history@custom", - ], - "index_patterns": Array [ - ".entities.v1.history.admin-console-services.*", - ], - "name": "entities_v1_history_admin-console-services_index_template", - "priority": 200, - "template": Object { - "aliases": Object { - "entities-service-history": Object {}, - }, - "mappings": Object { - "_meta": Object { - "version": "1.6.0", - }, - "date_detection": false, - "dynamic_templates": Array [ - Object { - "strings_as_keyword": Object { - "mapping": Object { - "fields": Object { - "text": Object { - "type": "text", - }, - }, - "ignore_above": 1024, - "type": "keyword", - }, - "match_mapping_type": "string", - }, - }, - Object { - "entity_metrics": Object { - "mapping": Object { - "type": "{dynamic_type}", - }, - "match_mapping_type": Array [ - "long", - "double", - ], - "path_match": "entity.metrics.*", - }, - }, - ], - }, - "settings": Object { - "index": Object { - "codec": "best_compression", - "mapping": Object { - "total_fields": Object { - "limit": 2000, - }, - }, - }, - }, - }, -} -`; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts deleted file mode 100644 index 72e8d8591ab2d..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entityDefinition, builtInEntityDefinition } from '../helpers/fixtures'; -import { generateEntitiesHistoryIndexTemplateConfig } from './entities_history_template'; - -describe('generateEntitiesHistoryIndexTemplateConfig(definition)', () => { - it('should generate a valid index template for custom definition', () => { - const template = generateEntitiesHistoryIndexTemplateConfig(entityDefinition); - expect(template).toMatchSnapshot(); - }); - - it('should generate a valid index template for builtin definition', () => { - const template = generateEntitiesHistoryIndexTemplateConfig(builtInEntityDefinition); - expect(template).toMatchSnapshot(); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts deleted file mode 100644 index b1539d8108a6d..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { - ENTITY_HISTORY, - EntityDefinition, - entitiesIndexPattern, - entitiesAliasPattern, - ENTITY_SCHEMA_VERSION_V1, -} from '@kbn/entities-schema'; -import { generateHistoryIndexTemplateId } from '../helpers/generate_component_id'; -import { - ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, - ENTITY_EVENT_COMPONENT_TEMPLATE_V1, - ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, -} from '../../../../common/constants_entities'; -import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers'; - -export const generateEntitiesHistoryIndexTemplateConfig = ( - definition: EntityDefinition -): IndicesPutIndexTemplateRequest => ({ - name: generateHistoryIndexTemplateId(definition), - _meta: { - description: - "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", - ecs_version: '8.0.0', - managed: true, - managed_by: 'elastic_entity_model', - }, - ignore_missing_component_templates: getCustomHistoryTemplateComponents(definition), - composed_of: [ - ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, - ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, - ENTITY_EVENT_COMPONENT_TEMPLATE_V1, - ...getCustomHistoryTemplateComponents(definition), - ], - index_patterns: [ - `${entitiesIndexPattern({ - schemaVersion: ENTITY_SCHEMA_VERSION_V1, - dataset: ENTITY_HISTORY, - definitionId: definition.id, - })}.*`, - ], - priority: 200, - template: { - aliases: { - [entitiesAliasPattern({ type: definition.type, dataset: ENTITY_HISTORY })]: {}, - }, - mappings: { - _meta: { - version: '1.6.0', - }, - date_detection: false, - dynamic_templates: [ - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - fields: { - text: { - type: 'text', - }, - }, - }, - match_mapping_type: 'string', - }, - }, - { - entity_metrics: { - mapping: { - type: '{dynamic_type}', - }, - match_mapping_type: ['long', 'double'], - path_match: 'entity.metrics.*', - }, - }, - ], - }, - settings: { - index: { - codec: 'best_compression', - mapping: { - total_fields: { - limit: 2000, - }, - }, - }, - }, - }, -}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts index ea476cf769644..e0c02c7471217 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts @@ -19,7 +19,7 @@ import { ENTITY_EVENT_COMPONENT_TEMPLATE_V1, ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1, } from '../../../../common/constants_entities'; -import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers'; +import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; export const generateEntitiesLatestIndexTemplateConfig = ( definition: EntityDefinition @@ -94,3 +94,16 @@ export const generateEntitiesLatestIndexTemplateConfig = ( }, }, }); + +function getCustomLatestTemplateComponents(definition: EntityDefinition) { + if (isBuiltinDefinition(definition)) { + return []; + } + + return [ + `${definition.id}@platform`, // @platform goes before so it can be overwritten by custom + `${definition.id}-latest@platform`, + `${definition.id}@custom`, + `${definition.id}-latest@custom`, + ]; +} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap index c3c19397ff9a9..2f904114a657b 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap @@ -48,27 +48,71 @@ Object { }, }, "entity.metadata.host.name": Object { - "terms": Object { - "field": "host.name", - "size": 1000, + "aggs": Object { + "data": Object { + "terms": Object { + "field": "host.name", + "size": 10, + }, + }, + }, + "filter": Object { + "range": Object { + "@timestamp": Object { + "gte": "now-10m", + }, + }, }, }, "entity.metadata.host.os.name": Object { - "terms": Object { - "field": "host.os.name", - "size": 1000, + "aggs": Object { + "data": Object { + "terms": Object { + "field": "host.os.name", + "size": 10, + }, + }, + }, + "filter": Object { + "range": Object { + "@timestamp": Object { + "gte": "now-10m", + }, + }, }, }, "entity.metadata.sourceIndex": Object { - "terms": Object { - "field": "_index", - "size": 1000, + "aggs": Object { + "data": Object { + "terms": Object { + "field": "_index", + "size": 10, + }, + }, + }, + "filter": Object { + "range": Object { + "@timestamp": Object { + "gte": "now-10m", + }, + }, }, }, "entity.metadata.tags": Object { - "terms": Object { - "field": "tags", - "size": 1000, + "aggs": Object { + "data": Object { + "terms": Object { + "field": "tags", + "size": 10, + }, + }, + }, + "filter": Object { + "range": Object { + "@timestamp": Object { + "gte": "now-10m", + }, + }, }, }, "entity.metrics.errorRate": Object { diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts index 7746be66f5033..12535d313143b 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -7,134 +7,22 @@ import { entityDefinitionSchema } from '@kbn/entities-schema'; import { rawEntityDefinition } from '../helpers/fixtures/entity_definition'; -import { - generateHistoryMetadataAggregations, - generateLatestMetadataAggregations, -} from './generate_metadata_aggregations'; +import { generateLatestMetadataAggregations } from './generate_metadata_aggregations'; describe('Generate Metadata Aggregations for history and latest', () => { - describe('generateHistoryMetadataAggregations()', () => { - it('should generate metadata aggregations for string format', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: ['host.name'], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.host.name': { - terms: { - field: 'host.name', - size: 1000, - }, - }, - }); - }); - - it('should generate metadata aggregations for object format with only source', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [{ source: 'host.name' }], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.host.name': { - terms: { - field: 'host.name', - size: 1000, - }, - }, - }); - }); - - it('should generate metadata aggregations for object format with source and aggregation', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.host.name': { - terms: { - field: 'host.name', - size: 10, - }, - }, - }); - }); - - it('should generate metadata aggregations for object format with source, aggregation, and destination', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [ - { - source: 'host.name', - aggregation: { type: 'terms', limit: 20 }, - destination: 'hostName', - }, - ], - }); - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.hostName': { - terms: { - field: 'host.name', - size: 20, - }, - }, - }); - }); - - it('should generate metadata aggregations for terms and top_value', () => { - const definition = entityDefinitionSchema.parse({ - ...rawEntityDefinition, - metadata: [ - { - source: 'host.name', - aggregation: { type: 'terms', limit: 10 }, - destination: 'hostName', - }, - { - source: 'agent.name', - aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, - destination: 'agentName', - }, - ], - }); - - expect(generateHistoryMetadataAggregations(definition)).toEqual({ - 'entity.metadata.hostName': { - terms: { - field: 'host.name', - size: 10, - }, - }, - 'entity.metadata.agentName': { - filter: { - exists: { - field: 'agent.name', - }, - }, - aggs: { - top_value: { - top_metrics: { - metrics: { field: 'agent.name' }, - sort: { '@timestamp': 'desc' }, - }, - }, - }, - }, - }); - }); - }); - describe('generateLatestMetadataAggregations()', () => { it('should generate metadata aggregations for string format', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, metadata: ['host.name'], }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, @@ -142,7 +30,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { data: { terms: { field: 'host.name', - size: 1000, + size: 10, }, }, }, @@ -160,7 +48,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, @@ -168,7 +56,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { data: { terms: { field: 'host.name', - size: 1000, + size: 10, }, }, }, @@ -179,14 +67,16 @@ describe('Generate Metadata Aggregations for history and latest', () => { it('should generate metadata aggregations for object format with source and aggregation', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], + metadata: [ + { source: 'host.name', aggregation: { type: 'terms', limit: 10, lookbackPeriod: '1h' } }, + ], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-1h', }, }, }, @@ -218,14 +108,14 @@ describe('Generate Metadata Aggregations for history and latest', () => { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, aggs: { data: { terms: { - field: 'hostName', + field: 'host.name', size: 10, }, }, @@ -255,14 +145,14 @@ describe('Generate Metadata Aggregations for history and latest', () => { filter: { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, aggs: { data: { terms: { - field: 'hostName', + field: 'host.name', size: 10, }, }, @@ -275,13 +165,13 @@ describe('Generate Metadata Aggregations for history and latest', () => { { range: { '@timestamp': { - gte: 'now-360s', + gte: 'now-10m', }, }, }, { exists: { - field: 'agentName', + field: 'agent.name', }, }, ], @@ -291,7 +181,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { top_value: { top_metrics: { metrics: { - field: 'agentName', + field: 'agent.name', }, sort: { '@timestamp': 'desc', diff --git a/x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts b/x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts deleted file mode 100644 index 90c5e90d43f3a..0000000000000 --- a/x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import { getCustomHistoryTemplateComponents, getCustomLatestTemplateComponents } from './helpers'; - -describe('helpers', () => { - it('getCustomLatestTemplateComponents should return template component in the right sort order', () => { - const result = getCustomLatestTemplateComponents({ id: 'test' } as EntityDefinition); - expect(result).toEqual([ - 'test@platform', - 'test-latest@platform', - 'test@custom', - 'test-latest@custom', - ]); - }); - - it('getCustomHistoryTemplateComponents should return template component in the right sort order', () => { - const result = getCustomHistoryTemplateComponents({ id: 'test' } as EntityDefinition); - expect(result).toEqual([ - 'test@platform', - 'test-history@platform', - 'test@custom', - 'test-history@custom', - ]); - }); -}); diff --git a/x-pack/plugins/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/entity_manager/server/templates/components/helpers.ts deleted file mode 100644 index a890fca4f107f..0000000000000 --- a/x-pack/plugins/entity_manager/server/templates/components/helpers.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import { isBuiltinDefinition } from '../../lib/entities/helpers/is_builtin_definition'; - -export const getCustomLatestTemplateComponents = (definition: EntityDefinition) => { - if (isBuiltinDefinition(definition)) { - return []; - } - - return [ - `${definition.id}@platform`, // @platform goes before so it can be overwritten by custom - `${definition.id}-latest@platform`, - `${definition.id}@custom`, - `${definition.id}-latest@custom`, - ]; -}; From 6cb7e2645b4e4167cec523959ea8199c22cfb93f Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 19:56:08 +0200 Subject: [PATCH 10/21] fix security tests --- .../lib/entity_analytics/entity_store/definition.ts | 4 ++-- .../trial_license_complete_tier/engine.ts | 12 ++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts index a72e00bf7aceb..812e5cab6d9d6 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts @@ -27,7 +27,7 @@ export const buildHostEntityDefinition = (space: string): EntityDefinition => 'host.type', 'host.architecture', ], - history: { + latest: { timestampField: '@timestamp', interval: '1m', }, @@ -44,7 +44,7 @@ export const buildUserEntityDefinition = (space: string): EntityDefinition => identityFields: ['user.name'], displayNameTemplate: '{{user.name}}', metadata: ['user.email', 'user.full_name', 'user.hash', 'user.id', 'user.name', 'user.roles'], - history: { + latest: { timestampField: '@timestamp', interval: '1m', }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 8d57ff428e507..7d7b03d20c2b9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -74,10 +74,7 @@ export default ({ getService }: FtrProviderContext) => { it('should have installed the expected user resources', async () => { await initEntityEngineForEntityType('user'); - const expectedTransforms = [ - 'entities-v1-history-ea_default_user_entity_store', - 'entities-v1-latest-ea_default_user_entity_store', - ]; + const expectedTransforms = ['entities-v1-latest-ea_default_user_entity_store']; await expectTransformsExist(expectedTransforms); }); @@ -85,10 +82,7 @@ export default ({ getService }: FtrProviderContext) => { it('should have installed the expected host resources', async () => { await initEntityEngineForEntityType('host'); - const expectedTransforms = [ - 'entities-v1-history-ea_default_host_entity_store', - 'entities-v1-latest-ea_default_host_entity_store', - ]; + const expectedTransforms = ['entities-v1-latest-ea_default_host_entity_store']; await expectTransformsExist(expectedTransforms); }); @@ -220,7 +214,6 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await expectTransformNotFound('entities-v1-history-ea_host_entity_store'); await expectTransformNotFound('entities-v1-latest-ea_host_entity_store'); }); @@ -234,7 +227,6 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await expectTransformNotFound('entities-v1-history-ea_user_entity_store'); await expectTransformNotFound('entities-v1-latest-ea_user_entity_store'); }); }); From 9f43b89e1962db29d0e019d9dcc53d0d026c9a68 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 26 Sep 2024 19:59:10 +0200 Subject: [PATCH 11/21] update validate_transform_ids --- .../entities/transform/validate_transform_ids.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts b/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts index c16b7f126dded..c703124bdf082 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts @@ -7,26 +7,14 @@ import { EntityDefinition } from '@kbn/entities-schema'; import { EntityDefinitionIdInvalid } from '../errors/entity_definition_id_invalid'; -import { - generateHistoryBackfillTransformId, - generateHistoryTransformId, - generateLatestTransformId, -} from '../helpers/generate_component_id'; +import { generateLatestTransformId } from '../helpers/generate_component_id'; const TRANSFORM_ID_MAX_LENGTH = 64; export function validateDefinitionCanCreateValidTransformIds(definition: EntityDefinition) { - const historyTransformId = generateHistoryTransformId(definition); const latestTransformId = generateLatestTransformId(definition); - const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); - const spareChars = - TRANSFORM_ID_MAX_LENGTH - - Math.max( - historyTransformId.length, - latestTransformId.length, - historyBackfillTransformId.length - ); + const spareChars = TRANSFORM_ID_MAX_LENGTH - latestTransformId.length; if (spareChars < 0) { throw new EntityDefinitionIdInvalid( From 32f33f7b179fa2b969575296d7876cb1eee9db74 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 27 Sep 2024 08:12:50 +0200 Subject: [PATCH 12/21] remove deprecated files --- .../lib/entities/helpers/calculate_offset.ts | 34 ------------------- .../entities/helpers/is_backfill_enabled.ts | 12 ------- .../install_entity_definition.test.ts | 2 +- .../server/routes/entities/reset.ts | 30 ++++------------ 4 files changed, 7 insertions(+), 71 deletions(-) delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts delete mode 100644 x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts deleted file mode 100644 index 3eba710561abf..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; -import moment from 'moment'; -import { - ENTITY_DEFAULT_HISTORY_FREQUENCY, - ENTITY_DEFAULT_HISTORY_SYNC_DELAY, -} from '../../../../common/constants_entities'; - -const durationToSeconds = (dateMath: string) => { - const parts = dateMath.match(/(\d+)([m|s|h|d])/); - if (!parts) { - throw new Error(`Invalid date math supplied: ${dateMath}`); - } - const value = parseInt(parts[1], 10); - const unit = parts[2] as 'm' | 's' | 'h' | 'd'; - return moment.duration(value, unit).asSeconds(); -}; - -export function calculateOffset(definition: EntityDefinition) { - const syncDelay = durationToSeconds( - definition.history.settings.syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY - ); - const frequency = - durationToSeconds(definition.history.settings.frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY) * - 2; - - return syncDelay + frequency; -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts deleted file mode 100644 index 4c34f5d3c0256..0000000000000 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityDefinition } from '@kbn/entities-schema'; - -export function isBackfillEnabled(definition: EntityDefinition) { - return definition.history.settings.backfillSyncDelay != null; -} diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts index 66db454f43005..6d0405e535e07 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -179,7 +179,7 @@ describe('install_entity_definition', () => { installEntityDefinition({ esClient, soClient, - definition: { id: 'a'.repeat(40) } as EntityDefinition, + definition: { id: 'a'.repeat(50) } as EntityDefinition, logger: loggerMock.create(), }) ).rejects.toThrow(EntityDefinitionIdInvalid); diff --git a/x-pack/plugins/entity_manager/server/routes/entities/reset.ts b/x-pack/plugins/entity_manager/server/routes/entities/reset.ts index a59c38b3acf7c..0b6942e335e51 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/reset.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/reset.ts @@ -12,25 +12,13 @@ import { EntitySecurityException } from '../../lib/entities/errors/entity_securi import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; -import { - deleteHistoryIngestPipeline, - deleteLatestIngestPipeline, -} from '../../lib/entities/delete_ingest_pipeline'; +import { deleteIngestPipelines } from '../../lib/entities/delete_ingest_pipeline'; import { deleteIndices } from '../../lib/entities/delete_index'; -import { - createAndInstallHistoryIngestPipeline, - createAndInstallLatestIngestPipeline, -} from '../../lib/entities/create_and_install_ingest_pipeline'; -import { - createAndInstallHistoryBackfillTransform, - createAndInstallHistoryTransform, - createAndInstallLatestTransform, -} from '../../lib/entities/create_and_install_transform'; +import { createAndInstallIngestPipelines } from '../../lib/entities/create_and_install_ingest_pipeline'; +import { createAndInstallTransforms } from '../../lib/entities/create_and_install_transform'; import { startTransforms } from '../../lib/entities/start_transforms'; import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; -import { isBackfillEnabled } from '../../lib/entities/helpers/is_backfill_enabled'; - import { createEntityManagerServerRoute } from '../create_entity_manager_server_route'; import { deleteTransforms } from '../../lib/entities/delete_transforms'; import { stopTransforms } from '../../lib/entities/stop_transforms'; @@ -51,18 +39,12 @@ export const resetEntityDefinitionRoute = createEntityManagerServerRoute({ await stopTransforms(esClient, definition, logger); await deleteTransforms(esClient, definition, logger); - await deleteHistoryIngestPipeline(esClient, definition, logger); - await deleteLatestIngestPipeline(esClient, definition, logger); + await deleteIngestPipelines(esClient, definition, logger); await deleteIndices(esClient, definition, logger); // Recreate everything - await createAndInstallHistoryIngestPipeline(esClient, definition, logger); - await createAndInstallLatestIngestPipeline(esClient, definition, logger); - await createAndInstallHistoryTransform(esClient, definition, logger); - if (isBackfillEnabled(definition)) { - await createAndInstallHistoryBackfillTransform(esClient, definition, logger); - } - await createAndInstallLatestTransform(esClient, definition, logger); + await createAndInstallIngestPipelines(esClient, definition, logger); + await createAndInstallTransforms(esClient, definition, logger); await startTransforms(esClient, definition, logger); return response.ok({ body: { acknowledged: true } }); From 97c02708cd91cce8578cb50b32b1be40a36481f3 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 27 Sep 2024 09:33:22 +0200 Subject: [PATCH 13/21] fix tests --- .../schema/__snapshots__/common.test.ts.snap | 8 ++--- .../src/schema/common.test.ts | 30 ++----------------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap index 55173b95623b4..766ce1c70ac3a 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap +++ b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap @@ -79,7 +79,7 @@ Object { "data": Object { "aggregation": Object { "limit": 10, - "lookbackPeriod": "1h", + "lookbackPeriod": undefined, "type": "terms", }, "destination": "hostName", @@ -94,7 +94,7 @@ Object { "data": Object { "aggregation": Object { "limit": 10, - "lookbackPeriod": "1h", + "lookbackPeriod": undefined, "type": "terms", }, "destination": "host.name", @@ -109,7 +109,7 @@ Object { "data": Object { "aggregation": Object { "limit": 10, - "lookbackPeriod": "1h", + "lookbackPeriod": undefined, "type": "terms", }, "destination": "host.name", @@ -124,7 +124,7 @@ Object { "data": Object { "aggregation": Object { "limit": 10, - "lookbackPeriod": "1h", + "lookbackPeriod": undefined, "type": "terms", }, "destination": "hostName", diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts index bbba6f1d36d87..210e34943bd40 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { durationSchema, metadataSchema, semVerSchema, historySettingsSchema } from './common'; +import { durationSchema, metadataSchema, semVerSchema } from './common'; describe('schemas', () => { describe('metadataSchema', () => { @@ -66,7 +66,7 @@ describe('schemas', () => { expect(result.data).toEqual({ source: 'host.name', destination: 'hostName', - aggregation: { type: 'terms', limit: 10, lookbackPeriod: '1h' }, + aggregation: { type: 'terms', limit: 10, lookbackPeriod: undefined }, }); }); @@ -139,30 +139,4 @@ describe('schemas', () => { expect(result).toMatchSnapshot(); }); }); - - describe('historySettingsSchema', () => { - it('should return default values when not defined', () => { - let result = historySettingsSchema.safeParse(undefined); - expect(result.success).toBeTruthy(); - expect(result.data).toEqual({ lookbackPeriod: '1h' }); - - result = historySettingsSchema.safeParse({ syncDelay: '1m' }); - expect(result.success).toBeTruthy(); - expect(result.data).toEqual({ syncDelay: '1m', lookbackPeriod: '1h' }); - }); - - it('should return user defined values when defined', () => { - const result = historySettingsSchema.safeParse({ - lookbackPeriod: '30m', - syncField: 'event.ingested', - syncDelay: '5m', - }); - expect(result.success).toBeTruthy(); - expect(result.data).toEqual({ - lookbackPeriod: '30m', - syncField: 'event.ingested', - syncDelay: '5m', - }); - }); - }); }); From 7ea323646c7d9580f7c12eea784dcdb71b41f919 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 27 Sep 2024 13:34:31 +0200 Subject: [PATCH 14/21] fix tests --- .../lib/entities/install_entity_definition.ts | 4 ++-- .../lib/entities/save_entity_definition.ts | 2 +- .../entity_manager/server/lib/entity_client.ts | 2 +- .../server/routes/enablement/disable.ts | 5 +++-- .../server/routes/enablement/enable.ts | 9 ++++++--- .../apis/entity_manager/definitions.ts | 18 ++++++++---------- 6 files changed, 21 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts index 5a8ec504a1d16..b4adedaf10374 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts @@ -168,11 +168,11 @@ async function install({ logger.debug(`Installing transforms for definition ${definition.id}`); const transforms = await createAndInstallTransforms(esClient, definition, logger); - await updateEntityDefinition(soClient, definition.id, { + const updatedProps = await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed', installedComponents: [...templates, ...pipelines, ...transforms], }); - return { ...definition, installStatus: 'installed' }; + return { ...definition, ...updatedProps.attributes }; } // stop and delete the current transforms and reinstall all the components diff --git a/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts index 2dff5178aeeaf..d32edfa146917 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts @@ -41,5 +41,5 @@ export async function updateEntityDefinition( id: string, definition: Partial ) { - await soClient.update(SO_ENTITY_DEFINITION_TYPE, id, definition); + return await soClient.update(SO_ENTITY_DEFINITION_TYPE, id, definition); } diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts index 5bd9154ec9daf..a8b7130dad84f 100644 --- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts @@ -41,7 +41,7 @@ export class EntityClient { }); if (!installOnly) { - await startTransforms(this.options.esClient, definition, this.options.logger); + await startTransforms(this.options.esClient, installedDefinition, this.options.logger); } return installedDefinition; diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts index bde68eb85ba9f..9c1c4f403636b 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts @@ -51,8 +51,8 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ }), handler: async ({ context, response, params, logger, server }) => { try { - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const canDisable = await canDisableEntityDiscovery(esClient); + const esClientAsCurrentUser = (await context.core).elasticsearch.client.asCurrentUser; + const canDisable = await canDisableEntityDiscovery(esClientAsCurrentUser); if (!canDisable) { return response.forbidden({ body: { @@ -62,6 +62,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } + const esClient = (await context.core).elasticsearch.client.asSecondaryAuthUser; const soClient = (await context.core).savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts index 9814840d20a0b..1002c1e716df2 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts @@ -80,8 +80,10 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const canEnable = await canEnableEntityDiscovery(esClient); + const core = await context.core; + + const esClientAsCurrentUser = core.elasticsearch.client.asCurrentUser; + const canEnable = await canEnableEntityDiscovery(esClientAsCurrentUser); if (!canEnable) { return response.forbidden({ body: { @@ -91,7 +93,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } - const soClient = (await context.core).savedObjects.getClient({ + const soClient = core.savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); const existingApiKey = await readEntityDiscoveryAPIKey(server); @@ -117,6 +119,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ await saveEntityDiscoveryAPIKey(soClient, apiKey); + const esClient = core.elasticsearch.client.asSecondaryAuthUser; const installedDefinitions = await installBuiltInEntityDefinitions({ esClient, soClient, diff --git a/x-pack/test/api_integration/apis/entity_manager/definitions.ts b/x-pack/test/api_integration/apis/entity_manager/definitions.ts index 466b5e0232bf0..b51a26ad7b5ad 100644 --- a/x-pack/test/api_integration/apis/entity_manager/definitions.ts +++ b/x-pack/test/api_integration/apis/entity_manager/definitions.ts @@ -8,10 +8,7 @@ import semver from 'semver'; import expect from '@kbn/expect'; import { entityLatestSchema } from '@kbn/entities-schema'; -import { - entityDefinition as mockDefinition, - entityDefinitionWithBackfill as mockBackfillDefinition, -} from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures'; +import { entityDefinition as mockDefinition } from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures'; import { PartialConfig, cleanup, generate } from '@kbn/data-forge'; import { generateLatestIndexName } from '@kbn/entityManager-plugin/server/lib/entities/helpers/generate_component_id'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -33,8 +30,9 @@ export default function ({ getService }: FtrProviderContext) { describe('Entity definitions', () => { describe('definitions installations', () => { it('can install multiple definitions', async () => { + const mockDefinitionDup = { ...mockDefinition, id: 'mock_definition_dup' }; await installDefinition(supertest, { definition: mockDefinition }); - await installDefinition(supertest, { definition: mockBackfillDefinition }); + await installDefinition(supertest, { definition: mockDefinitionDup }); const { definitions } = await getInstalledDefinitions(supertest); expect(definitions.length).to.eql(2); @@ -49,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) { expect( definitions.some( (definition) => - definition.id === mockBackfillDefinition.id && + definition.id === mockDefinitionDup.id && definition.state.installed === true && definition.state.running === true ) @@ -57,7 +55,7 @@ export default function ({ getService }: FtrProviderContext) { await Promise.all([ uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }), - uninstallDefinition(supertest, { id: mockBackfillDefinition.id, deleteData: true }), + uninstallDefinition(supertest, { id: mockDefinitionDup.id, deleteData: true }), ]); }); @@ -89,7 +87,7 @@ export default function ({ getService }: FtrProviderContext) { id: mockDefinition.id, update: { version: incVersion!, - history: { + latest: { timestampField: '@updatedTimestampField', }, }, @@ -99,7 +97,7 @@ export default function ({ getService }: FtrProviderContext) { definitions: [updatedDefinition], } = await getInstalledDefinitions(supertest); expect(updatedDefinition.version).to.eql(incVersion); - expect(updatedDefinition.history.timestampField).to.eql('@updatedTimestampField'); + expect(updatedDefinition.latest.timestampField).to.eql('@updatedTimestampField'); await uninstallDefinition(supertest, { id: mockDefinition.id }); }); @@ -114,7 +112,7 @@ export default function ({ getService }: FtrProviderContext) { id: mockDefinition.id, update: { version: '1.0.0', - history: { + latest: { timestampField: '@updatedTimestampField', }, }, From eb3a923f68480411478aa4b176dca711a017e017 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 27 Sep 2024 16:51:13 +0200 Subject: [PATCH 15/21] fix test --- .../install_entity_definition.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts index 6d0405e535e07..e07670c58fd9b 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -221,6 +221,12 @@ describe('install_entity_definition', () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installEntityDefinition({ esClient, @@ -257,6 +263,12 @@ describe('install_entity_definition', () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -294,6 +306,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -333,6 +351,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -374,6 +398,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, @@ -410,6 +440,12 @@ describe('install_entity_definition', () => { page: 1, per_page: 10, }); + soClient.update.mockResolvedValue({ + id: mockEntityDefinition.id, + type: 'entity-definition', + references: [], + attributes: {}, + }); await installBuiltInEntityDefinitions({ esClient, From c2a163dbac0db1dc6171ce2c44157c0579a93892 Mon Sep 17 00:00:00 2001 From: klacabane Date: Sat, 28 Sep 2024 12:07:22 +0200 Subject: [PATCH 16/21] fix test --- x-pack/packages/kbn-entities-schema/src/schema/entity.ts | 1 - .../server/lib/entities/helpers/fixtures/entity_definition.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts index eae6873356c14..3eb87a797ef21 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts @@ -35,7 +35,6 @@ export const entityLatestSchema = z entity: entityBaseSchema.merge( z.object({ lastSeenTimestamp: z.string(), - firstSeenTimestamp: z.string(), }) ), }) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index 4c0fd57b83ba2..47adcbba30d9d 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -17,8 +17,8 @@ export const rawEntityDefinition = { interval: '1m', lookbackPeriod: '10m', settings: { - frequency: '2m', - syncDelay: '2m', + frequency: '30s', + syncDelay: '10s', }, }, identityFields: ['log.logger', { field: 'event.category', optional: true }], From 31cd18adef316275fd8185f157b3a4a0ceaa64ef Mon Sep 17 00:00:00 2001 From: klacabane Date: Sat, 28 Sep 2024 19:15:09 +0200 Subject: [PATCH 17/21] update snapshots --- .../__snapshots__/generate_latest_transform.test.ts.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap index 2f904114a657b..49f8ff4536120 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap +++ b/x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap @@ -11,7 +11,7 @@ Object { "index": ".entities.v1.latest.noop", "pipeline": "entities-v1-latest-admin-console-services", }, - "frequency": "2m", + "frequency": "30s", "pivot": Object { "aggs": Object { "_errorRate_A": Object { @@ -182,7 +182,7 @@ Object { }, "sync": Object { "time": Object { - "delay": "2m", + "delay": "10s", "field": "@timestamp", }, }, From e379a125c982260743884756fcfdd750b3727734 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 4 Oct 2024 12:37:46 +0200 Subject: [PATCH 18/21] remove interval --- .../kbn-entities-schema/src/schema/entity_definition.ts | 2 -- .../server/lib/entities/built_in/containers_from_ecs_data.ts | 2 +- .../server/lib/entities/built_in/hosts_from_ecs_data.ts | 2 +- .../server/lib/entities/built_in/services_from_ecs_data.ts | 3 +-- .../lib/entities/helpers/fixtures/builtin_entity_definition.ts | 1 - .../server/lib/entities/helpers/fixtures/entity_definition.ts | 1 - 6 files changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index 4f18635d790bb..d9d8e6b610013 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -14,7 +14,6 @@ import { durationSchema, identityFieldsSchema, semVerSchema, - durationSchemaWithMinimum, } from './common'; export const entityDefinitionSchema = z.object({ @@ -33,7 +32,6 @@ export const entityDefinitionSchema = z.object({ managed: z.optional(z.boolean()).default(false), latest: z.object({ timestampField: z.string(), - interval: durationSchemaWithMinimum(1), lookbackPeriod: z.optional(durationSchema).default('24h'), settings: z.optional( z.object({ diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts index 7109856ff1d11..1fe0820efe14a 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts @@ -22,7 +22,7 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition = displayNameTemplate: '{{container.id}}', latest: { timestampField: '@timestamp', - interval: '5m', + lookbackPeriod: '10m', settings: { frequency: '5m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts index f93300f5908c5..3389ba876533a 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts @@ -21,7 +21,7 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin displayNameTemplate: '{{host.name}}', latest: { timestampField: '@timestamp', - interval: '5m', + lookbackPeriod: '10m', settings: { frequency: '5m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index 817922a5d4907..e6e52a579b26a 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -35,9 +35,8 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition = ], latest: { timestampField: '@timestamp', - interval: '1m', + lookbackPeriod: '10m', settings: { - lookbackPeriod: '10m', frequency: '2m', syncDelay: '2m', }, diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts index 561fb4c354e33..b1e506150fb60 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts @@ -15,7 +15,6 @@ export const builtInEntityDefinition = entityDefinitionSchema.parse({ managed: true, latest: { timestampField: '@timestamp', - interval: '1m', }, identityFields: ['log.logger', { field: 'event.category', optional: true }], displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}', diff --git a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index 47adcbba30d9d..00ab9ac7759af 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -14,7 +14,6 @@ export const rawEntityDefinition = { indexPatterns: ['kbn-data-forge-fake_stack.*'], latest: { timestampField: '@timestamp', - interval: '1m', lookbackPeriod: '10m', settings: { frequency: '30s', From 6a457c4be969d78acb5eb1678aa0f6e022caa87b Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 7 Oct 2024 11:54:29 +0200 Subject: [PATCH 19/21] remove firstSeenTimestamp investigate_app --- .../investigate_app/server/services/get_entities.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts index 8f3a0abb62b67..00151f2029d21 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -61,7 +61,6 @@ export async function getEntitiesWithSource({ identityFields: entity?.entity.identityFields, id: entity?.entity.id, definitionId: entity?.entity.definitionId, - firstSeenTimestamp: entity?.entity.firstSeenTimestamp, lastSeenTimestamp: entity?.entity.lastSeenTimestamp, displayName: entity?.entity.displayName, metrics: entity?.entity.metrics, From ef2abd2d16b87888ec4d57d930f0679238dd258c Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 9 Oct 2024 09:22:15 +0200 Subject: [PATCH 20/21] fix entity analytics tests --- .../engine_nondefault_spaces.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index 112c8b8b21511..e3ef29d937183 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -38,10 +38,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - const expectedTransforms = [ - `entities-v1-history-ea_${namespace}_user_entity_store`, - `entities-v1-latest-ea_${namespace}_user_entity_store`, - ]; + const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`]; await utils.expectTransformsExist(expectedTransforms); }); @@ -49,10 +46,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - const expectedTransforms = [ - `entities-v1-history-ea_${namespace}_host_entity_store`, - `entities-v1-latest-ea_${namespace}_host_entity_store`, - ]; + const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`]; await utils.expectTransformsExist(expectedTransforms); }); From 6d189dd8ddf112a8ec9347c1d369a80b736e1190 Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 9 Oct 2024 10:54:02 +0200 Subject: [PATCH 21/21] remove interval option --- .../server/lib/entity_analytics/entity_store/definition.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts index 812e5cab6d9d6..09dea151a050a 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts @@ -29,7 +29,6 @@ export const buildHostEntityDefinition = (space: string): EntityDefinition => ], latest: { timestampField: '@timestamp', - interval: '1m', }, version: '1.0.0', managed: true, @@ -46,7 +45,6 @@ export const buildUserEntityDefinition = (space: string): EntityDefinition => metadata: ['user.email', 'user.full_name', 'user.hash', 'user.id', 'user.name', 'user.roles'], latest: { timestampField: '@timestamp', - interval: '1m', }, version: '1.0.0', managed: true,