From 1b9ffa5d2276dc63fdb10e063379135986dea958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 25 Sep 2024 15:29:18 +0200 Subject: [PATCH 01/44] feat(wip): Telemetry index metadata --- .../server/lib/telemetry/async_sender.ts | 17 +- .../lib/telemetry/async_sender.types.ts | 5 +- .../lib/telemetry/event_based/events.ts | 254 ++++++++++++++++-- .../lib/telemetry/indices.metadata.types.ts | 53 ++++ .../server/lib/telemetry/receiver.ts | 145 ++++++++++ .../server/lib/telemetry/sender.ts | 19 +- .../server/lib/telemetry/task.ts | 25 +- .../server/lib/telemetry/tasks/index.ts | 2 + .../lib/telemetry/tasks/indices.metadata.ts | 107 ++++++++ .../security_solution/server/plugin.ts | 3 +- 10 files changed, 583 insertions(+), 47 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts index edbd5d21e3aea..014f1035b9e81 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts @@ -8,7 +8,7 @@ import axios from 'axios'; import * as rx from 'rxjs'; import _, { cloneDeep } from 'lodash'; -import type { Logger, LogMeta } from '@kbn/core/server'; +import type { AnalyticsServiceSetup, EventType, Logger, LogMeta } from '@kbn/core/server'; import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; import type { ITelemetryReceiver } from './receiver'; @@ -17,7 +17,7 @@ import { type QueueConfig, type RetryConfig, } from './async_sender.types'; -import { TelemetryChannel, TelemetryCounter } from './types'; +import { type Nullable, TelemetryChannel, TelemetryCounter } from './types'; import * as collections from './collections_helpers'; import { CachedSubject, retryOnError$ } from './rxjs_helpers'; import { SenderUtils } from './sender_helpers'; @@ -55,6 +55,8 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { private telemetryUsageCounter?: IUsageCounter; private senderUtils: SenderUtils | undefined; + private analytics: Nullable; + constructor(logger: Logger) { this.logger = newTelemetryLogger(logger.get('telemetry_events.async_sender')); } @@ -64,7 +66,8 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { fallbackQueueConfig: QueueConfig, telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, - telemetryUsageCounter?: IUsageCounter + telemetryUsageCounter?: IUsageCounter, + analytics?: AnalyticsServiceSetup ): void { this.logger.l(`Setting up ${AsyncTelemetryEventsSender.name}`); @@ -77,6 +80,7 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { this.telemetryReceiver = telemetryReceiver; this.telemetrySetup = telemetrySetup; this.telemetryUsageCounter = telemetryUsageCounter; + this.analytics = analytics; this.updateStatus(ServiceStatus.CONFIGURED); } @@ -201,6 +205,13 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { } } + public reportEBT(eventType: EventType, eventData: object) { + if (!this.analytics) { + throw Error('analytics is unavailable'); + } + this.analytics.reportEvent(eventType, eventData); + } + // internal methods private queue$( upstream$: rx.Observable, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts index 249493cfdbfc8..e19cc4d7bf0ef 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts @@ -6,6 +6,7 @@ */ import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; +import type { AnalyticsServiceSetup, EventType } from '@kbn/core-analytics-server'; import { type TelemetryChannel } from './types'; import type { ITelemetryReceiver } from './receiver'; @@ -20,7 +21,8 @@ export interface IAsyncTelemetryEventsSender { fallbackQueueConfig: QueueConfig, telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, - telemetryUsageCounter?: IUsageCounter + telemetryUsageCounter?: IUsageCounter, + analytics?: AnalyticsServiceSetup ) => void; start: (telemetryStart?: TelemetryPluginStart) => void; stop: () => Promise; @@ -28,6 +30,7 @@ export interface IAsyncTelemetryEventsSender { simulateSend: (channel: TelemetryChannel, events: unknown[]) => string[]; updateQueueConfig: (channel: TelemetryChannel, config: QueueConfig) => void; updateDefaultQueueConfig: (config: QueueConfig) => void; + reportEBT: (eventType: EventType, eventData: object) => void; } /** diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts index b8a2df85f10ad..b6e7732d01b86 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -11,6 +11,7 @@ import type { ResponseActionsApiCommandNames, } from '../../../../common/endpoint/service/response_actions/constants'; import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics'; +import type { DataStream, IlmPolicy, IlmStats, IndexStats } from '../indices.metadata.types'; export const RISK_SCORE_EXECUTION_SUCCESS_EVENT: EventTypeOpts<{ scoresWritten: number; @@ -98,35 +99,35 @@ interface AssetCriticalitySystemProcessedAssignmentFileEvent { } export const ASSET_CRITICALITY_SYSTEM_PROCESSED_ASSIGNMENT_FILE_EVENT: EventTypeOpts = - { - eventType: 'Asset Criticality Csv Upload Processed', - schema: { - processing: { - properties: { - startTime: { type: 'date', _meta: { description: 'Processing start time' } }, - endTime: { type: 'date', _meta: { description: 'Processing end time' } }, - tookMs: { type: 'long', _meta: { description: 'How long processing took ms' } }, - }, +{ + eventType: 'Asset Criticality Csv Upload Processed', + schema: { + processing: { + properties: { + startTime: { type: 'date', _meta: { description: 'Processing start time' } }, + endTime: { type: 'date', _meta: { description: 'Processing end time' } }, + tookMs: { type: 'long', _meta: { description: 'How long processing took ms' } }, }, - result: { - properties: { - successful: { - type: 'long', - _meta: { description: 'Number of criticality records successfully created or updated' }, - }, - failed: { - type: 'long', - _meta: { description: 'Number of criticality records which had errors' }, - }, - total: { type: 'long', _meta: { description: 'Total number of lines in the file' } }, + }, + result: { + properties: { + successful: { + type: 'long', + _meta: { description: 'Number of criticality records successfully created or updated' }, }, - }, - status: { - type: 'keyword', - _meta: { description: 'Status of the processing either success, partial_success or fail' }, + failed: { + type: 'long', + _meta: { description: 'Number of criticality records which had errors' }, + }, + total: { type: 'long', _meta: { description: 'Total number of lines in the file' } }, }, }, - }; + status: { + type: 'keyword', + _meta: { description: 'Status of the processing either success, partial_success or fail' }, + }, + }, +}; export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ suppressionAlertsCreated: number; @@ -208,6 +209,199 @@ export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ }, }; +export const TELEMETRY_DATA_STREAM_EVENT: EventTypeOpts = { + eventType: 'telemetry_data_stream_event', + schema: { + datastream_name: { + type: 'keyword', + _meta: { description: 'Name of the data stream' }, + }, + indices: { + type: 'array', + items: { + properties: { + index_name: { type: 'date', _meta: { description: 'Index name' } }, + ilm_policy: { type: 'date', _meta: { optional: true, description: 'ILM policy' } }, + }, + }, + _meta: { optional: true, description: 'Indices associated with the data stream' }, + }, + }, +}; + +export const TELEMETRY_INDEX_STATS_EVENT: EventTypeOpts = { + eventType: 'telemetry_index_stats_event', + schema: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index being monitored.' }, + }, + query_total: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of search queries executed on the index.', + }, + }, + query_time_in_millis: { + type: 'long', + _meta: { + optional: true, + description: + 'The total time spent on query execution across all search requests, measured in milliseconds.', + }, + }, + docs_count: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of documents currently stored in the index.', + }, + }, + docs_deleted: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of documents that have been marked as deleted in the index.', + }, + }, + docs_total_size_in_bytes: { + type: 'long', + _meta: { + optional: true, + description: + 'The total size, in bytes, of all documents stored in the index, including storage overhead.', + }, + }, + }, +}; + +export const TELEMETRY_ILM_POLICY_EVENT: EventTypeOpts = { + eventType: 'telemetry_ilm_policy_event', + schema: { + policy_name: { + type: 'keyword', + _meta: { description: 'The name of the ILM policy.' }, + }, + modified_date: { + type: 'date', + _meta: { description: 'The date when the ILM policy was last modified.' }, + }, + phases: { + properties: { + cold: { + properties: { + min_age: { + type: 'text', + _meta: { + description: 'The minimum age before the index transitions to the "cold" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "cold" phase of the ILM policy, applied when data is infrequently accessed.', + }, + }, + delete: { + properties: { + min_age: { + type: 'text', + _meta: { + description: 'The minimum age before the index transitions to the "delete" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "delete" phase of the ILM policy, specifying when the index should be removed.', + }, + }, + frozen: { + properties: { + min_age: { + type: 'text', + _meta: { + description: 'The minimum age before the index transitions to the "frozen" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "frozen" phase of the ILM policy, where data is fully searchable but stored with a reduced resource footprint.', + }, + }, + hot: { + properties: { + min_age: { + type: 'text', + _meta: { + description: 'The minimum age before the index transitions to the "hot" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "hot" phase of the ILM policy, applied to actively written and queried data.', + }, + }, + warm: { + properties: { + min_age: { + type: 'text', + _meta: { + description: 'The minimum age before the index transitions to the "warm" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "warm" phase of the ILM policy, used for read-only data that is less frequently accessed.', + }, + }, + }, + _meta: { + description: + 'The different phases of the ILM policy that define how the index is managed over time.', + }, + }, + }, +}; + +export const TELEMETRY_ILM_STATS_EVENT: EventTypeOpts = { + eventType: 'telemetry_ilm_stats_event', + schema: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index currently managed by the ILM policy.' }, + }, + phase: { + type: 'keyword', + _meta: { + optional: true, + description: + 'The current phase of the ILM policy that the index is in (e.g., hot, warm, cold, frozen, or delete).', + }, + }, + age: { + type: 'text', + _meta: { + optional: true, + description: 'The age of the index since its creation, indicating how long it has existed.', + }, + }, + policy_name: { + type: 'keyword', + _meta: { optional: true, description: 'The name of the ILM policy applied to this index.' }, + }, + }, +}; + interface CreateAssetCriticalityProcessedFileEvent { result?: BulkUpsertAssetCriticalityRecordsResponse['stats']; startTime: Date; @@ -218,9 +412,9 @@ export const createAssetCriticalityProcessedFileEvent = ({ startTime, endTime, }: CreateAssetCriticalityProcessedFileEvent): [ - string, - AssetCriticalitySystemProcessedAssignmentFileEvent -] => { + string, + AssetCriticalitySystemProcessedAssignmentFileEvent + ] => { const status = getUploadStatus(result); const processing = { @@ -390,4 +584,8 @@ export const events = [ ENDPOINT_RESPONSE_ACTION_SENT_EVENT, ENDPOINT_RESPONSE_ACTION_SENT_ERROR_EVENT, ENDPOINT_RESPONSE_ACTION_STATUS_CHANGE_EVENT, + TELEMETRY_DATA_STREAM_EVENT, + TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_ILM_STATS_EVENT, + TELEMETRY_INDEX_STATS_EVENT, ]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts new file mode 100644 index 0000000000000..932015d9bc575 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DateTime } from '@elastic/elasticsearch/lib/api/types'; +import type { Nullable } from './types'; + +export interface IlmPolicy { + policy_name: string; + modified_date: DateTime; + phases: IlmPhases; +} + +export interface IlmPhases { + cold: Nullable; + delete: Nullable; + frozen: Nullable; + hot: Nullable; + warm: Nullable; +} + +export interface IlmPhase { + min_age: string; +} + +export interface IlmStats { + index_name: string; + phase?: string; + age?: string; + policy_name?: string; +} + +export interface IndexStats { + index_name: string; + query_total?: number; + query_time_in_millis?: number; + docs_count?: number; + docs_deleted?: number; + docs_total_size_in_bytes?: number; +} + +export interface Index { + index_name: string; + ilm_policy?: string; +} + +export interface DataStream { + datastream_name: string; + indices?: Index[]; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 22f85d19c83d8..a227f1a711506 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -18,6 +18,7 @@ import type { } from '@kbn/core/server'; import type { AggregationsAggregate, + IlmExplainLifecycleRequest, OpenPointInTimeResponse, SearchRequest, SearchResponse, @@ -38,6 +39,9 @@ import type { SearchHit, SearchRequest as ESSearchRequest, SortResults, + IndicesGetDataStreamRequest, + IndicesStatsRequest, + IlmGetLifecycleRequest, } from '@elastic/elasticsearch/lib/api/types'; import type { TransportResult } from '@elastic/elasticsearch'; import type { AgentPolicy, Installation } from '@kbn/fleet-plugin/common'; @@ -87,6 +91,15 @@ import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants'; import { DEFAULT_DIAGNOSTIC_INDEX } from './constants'; import type { TelemetryLogger } from './telemetry_logger'; +import type { + DataStream, + IlmPhase, + IlmPhases, + IlmPolicy, + IlmStats, + Index, + IndexStats, +} from './indices.metadata.types'; export interface ITelemetryReceiver { start( @@ -238,6 +251,12 @@ export interface ITelemetryReceiver { setMaxPageSizeBytes(bytes: number): void; setNumDocsToSample(n: number): void; + + // ---------- TODO: POC ---------- // + getDataStreams(): Promise; + getIndicesStats(indices: string[]): Promise; + getIlmsStats(indices: string[]): Promise; + getIlmsPolicies(ilms: string[]): Promise; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -1320,4 +1339,130 @@ export class TelemetryReceiver implements ITelemetryReceiver { } return this._esClient; } + + // ---------- TODO: POC ---------- // + public async getDataStreams(): Promise { + const es = this.esClient(); + + const request: IndicesGetDataStreamRequest = { + name: '*', + expand_wildcards: ['open', 'hidden'], + filter_path: ['data_streams.name', 'data_streams.indices'], + }; + + return es.indices.getDataStream(request).then((response) => + response.data_streams.map((ds) => { + return { + datastream_name: ds.name, + indices: + ds.indices?.map((index) => { + return { + index_name: index.index_name, + ilm_policy: index.ilm_policy, + } as Index; + }) ?? [], + } as DataStream; + }) + ); + } + + public async getIndicesStats(indices: string[]): Promise { + const es = this.esClient(); + + this.logger.debug('Fetching indices stats', { indices } as LogMeta); + + const request: IndicesStatsRequest = { + index: indices, + level: 'indices', + metric: ['docs', 'search', 'store'], + expand_wildcards: ['open', 'hidden'], + filter_path: [ + 'indices.*.total.search.query_total', + 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', + 'indices.*.total.docs.deleted', + 'indices.*.total.store.size_in_bytes', + ], + }; + + return es.indices.stats(request).then((response) => + Object.entries(response.indices ?? {}).map(([indexName, stats]) => { + return { + index_name: indexName, + query_total: stats.total?.search?.query_total, + query_time_in_millis: stats.total?.search?.query_time_in_millis, + docs_count: stats.total?.docs?.count, + docs_deleted: stats.total?.docs?.deleted, + docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + } as IndexStats; + }) + ); + } + + public async getIlmsStats(indices: string[]): Promise { + const es = this.esClient(); + + this.logger.debug('Fetching ilms stats', { indices } as LogMeta); + + const request: IlmExplainLifecycleRequest = { + index: indices.join(','), + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + return es.ilm.explainLifecycle(request).then((response) => + Object.entries(response.indices ?? {}).map(([indexName, stats]) => { + return { + index_name: indexName, + phase: ('phase' in stats && stats.phase) || undefined, + age: ('age' in stats && stats.age) || undefined, + policy_name: ('policy' in stats && stats.policy) || undefined, + } as IlmStats; + }) + ); + } + + public async getIlmsPolicies(ilms: string[]): Promise { + const es = this.esClient(); + + this.logger.debug('Fetching ilms policies', { ilms } as LogMeta); + + const request: IlmGetLifecycleRequest = { + name: ilms.join(','), + filter_path: [ + '*.policy.phases.cold.min_age', + '*.policy.phases.delete.min_age', + '*.policy.phases.frozen.min_age', + '*.policy.phases.hot.min_age', + '*.policy.phases.warm.min_age', + '*.modified_date', + ], + }; + + const phase = (obj: unknown): Nullable => { + let value: Nullable; + if (obj !== null && obj !== undefined && typeof obj === 'object' && 'min_age' in obj) { + value = { + min_age: obj.min_age, + } as IlmPhase; + } + return value; + }; + + return es.ilm.getLifecycle(request).then((response) => { + return Object.entries(response ?? {}).map(([policyName, stats]) => { + return { + policy_name: policyName, + modified_date: stats.modified_date, + phases: { + cold: phase(stats.policy.phases.cold), + delete: phase(stats.policy.phases.delete), + frozen: phase(stats.policy.phases.frozen), + hot: phase(stats.policy.phases.hot), + warm: phase(stats.policy.phases.warm), + } as IlmPhases, + } as IlmPolicy; + }); + }); + } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 25dd07d4ef986..975e105f91e35 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -9,7 +9,7 @@ import { cloneDeep } from 'lodash'; import { URL } from 'url'; import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import type { Logger, LogMeta } from '@kbn/core/server'; +import type { EventType, Logger, LogMeta } from '@kbn/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; import type { AxiosInstance } from 'axios'; @@ -88,6 +88,11 @@ export interface ITelemetryEventsSender { * Updates the default queue configuration. */ updateDefaultQueueConfig: (config: QueueConfig) => void; + + /** + * Reports EBT events + */ + reportEBT: (eventType: EventType, eventData: object) => void; } export class TelemetryEventsSender implements ITelemetryEventsSender { @@ -270,12 +275,16 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { const telemetryUrl = await this.fetchTelemetryPingUrl(); const resp = await axios.get(telemetryUrl, { timeout: 3000 }); if (resp.status === 200) { - this.logger.l('[Security Telemetry] elastic telemetry services are reachable'); + this.logger.debug('Elastic telemetry services are reachable'); return true; } return false; - } catch (_err) { + } catch (e) { + this.logger.warn('Error pinging telemetry services', { + error: e.message, + } as LogMeta); + return false; } } @@ -426,6 +435,10 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { this.getAsyncTelemetrySender().updateDefaultQueueConfig(config); } + public reportEBT(eventType: EventType, eventData: object): void { + this.getAsyncTelemetrySender().reportEBT(eventType, eventData); + } + private getAsyncTelemetrySender(): IAsyncTelemetryEventsSender { if (!this.asyncTelemetrySender) { throw new Error('Telemetry Sender V2 not initialized'); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 62d827930e6a9..96cb74dc24ae6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -6,7 +6,7 @@ */ import moment from 'moment'; -import type { Logger } from '@kbn/core/server'; +import type { Logger, LogMeta } from '@kbn/core/server'; import type { ConcreteTaskInstance, TaskManagerSetupContract, @@ -15,8 +15,9 @@ import type { import type { ITelemetryReceiver } from './receiver'; import type { ITelemetryEventsSender } from './sender'; import type { ITaskMetricsService } from './task_metrics.types'; -import { tlog } from './helpers'; import { stateSchemaByVersion, emptyState, type LatestTaskStateSchema } from './task_state'; +import { newTelemetryLogger } from './helpers'; +import { type TelemetryLogger } from './telemetry_logger'; export interface SecurityTelemetryTaskConfig { type: string; @@ -49,7 +50,7 @@ export type LastExecutionTimestampCalculator = ( export class SecurityTelemetryTask { private readonly config: SecurityTelemetryTaskConfig; - private readonly logger: Logger; + private readonly logger: TelemetryLogger; private readonly sender: ITelemetryEventsSender; private readonly receiver: ITelemetryReceiver; private readonly taskMetricsService: ITaskMetricsService; @@ -62,7 +63,7 @@ export class SecurityTelemetryTask { taskMetricsService: ITaskMetricsService ) { this.config = config; - this.logger = logger; + this.logger = newTelemetryLogger(logger.get('task')); this.sender = sender; this.receiver = receiver; this.taskMetricsService = taskMetricsService; @@ -122,7 +123,7 @@ export class SecurityTelemetryTask { public start = async (taskManager: TaskManagerStartContract) => { const taskId = this.getTaskId(); - tlog(this.logger, `[task ${taskId}]: attempting to schedule`); + this.logger.l('Attempting to schedule task', { taskId }); try { await taskManager.ensureScheduled({ id: taskId, @@ -135,30 +136,32 @@ export class SecurityTelemetryTask { params: { version: this.config.version }, }); } catch (e) { - this.logger.error(`[task ${taskId}]: error scheduling task, received ${e.message}`); + this.logger.error('Error scheduling task', { + error: e.message, + } as LogMeta); } }; public runTask = async (taskId: string, executionPeriod: TaskExecutionPeriod) => { - tlog(this.logger, `[task ${taskId}]: attempting to run`); + this.logger.l('Attempting to run', { taskId }); if (taskId !== this.getTaskId()) { - tlog(this.logger, `[task ${taskId}]: outdated task`); + this.logger.l('outdated task', { taskId }); return 0; } const isOptedIn = await this.sender.isTelemetryOptedIn(); if (!isOptedIn) { - tlog(this.logger, `[task ${taskId}]: telemetry is not opted-in`); + this.logger.l('Telemetry is not opted-in', { taskId }); return 0; } const isTelemetryServicesReachable = await this.sender.isTelemetryServicesReachable(); if (!isTelemetryServicesReachable) { - tlog(this.logger, `[task ${taskId}]: cannot reach telemetry services`); + this.logger.l('Cannot reach telemetry services', { taskId }); return 0; } - tlog(this.logger, `[task ${taskId}]: running task`); + this.logger.l('Running task', { taskId }); return this.config.runTask( taskId, this.logger, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts index d237757616f3e..0cf3c610dbc8b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -16,6 +16,7 @@ import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnos import { createTelemetryConfigurationTaskConfig } from './configuration'; import { telemetryConfiguration } from '../configuration'; import { createTelemetryFilterListArtifactTaskConfig } from './filterlists'; +import { createTelemetryIndicesMetadataTaskConfig } from './indices.metadata'; export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { return [ @@ -30,5 +31,6 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { createTelemetryDiagnosticTimelineTaskConfig(), createTelemetryConfigurationTaskConfig(), createTelemetryFilterListArtifactTaskConfig(), + createTelemetryIndicesMetadataTaskConfig(), ]; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts new file mode 100644 index 0000000000000..21d402b60619f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { LogMeta, Logger } from '@kbn/core/server'; +import type { ITelemetryEventsSender } from '../sender'; +import type { ITelemetryReceiver } from '../receiver'; +import type { TaskExecutionPeriod } from '../task'; +import type { ITaskMetricsService } from '../task_metrics.types'; +import { getPreviousDailyTaskTimestamp, newTelemetryLogger } from '../helpers'; +import { + TELEMETRY_DATA_STREAM_EVENT, + TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_ILM_STATS_EVENT, + TELEMETRY_INDEX_STATS_EVENT, +} from '../event_based/events'; + +export function createTelemetryIndicesMetadataTaskConfig() { + const taskType = 'security:indices-metadata-telemetry'; + return { + type: taskType, + title: 'Security Solution Telemetry Indices Metadata task', + interval: '1m', // TODO: update!!! + timeout: '1m', + version: '1.0.0', + getLastExecutionTime: getPreviousDailyTaskTimestamp, + runTask: async ( + taskId: string, + logger: Logger, + receiver: ITelemetryReceiver, + _sender: ITelemetryEventsSender, + taskMetricsService: ITaskMetricsService, + taskExecutionPeriod: TaskExecutionPeriod + ) => { + const mdc = { task_id: taskId, task_execution_period: taskExecutionPeriod }; + const log = newTelemetryLogger(logger.get('indices-metadata'), mdc); + const trace = taskMetricsService.start(taskType); + + log.l('Running indices metadata task'); + + try { + // 1. Get all data streams + const dataStreams = await receiver.getDataStreams(); + + // and calculate index and ilm names + const dsNames = dataStreams.map((stream) => stream.datastream_name); + const indexNames = dataStreams + .map((ds) => ds.indices?.map((i) => i.index_name) ?? []) + .flat(); + const ilmsNames = dataStreams + .map((ds) => + ds.indices?.filter((i) => i.ilm_policy !== undefined)?.map((i) => i.ilm_policy) + ) + .flat() as string[]; + + log.info(`Got data streams`, { + datastreams: dsNames.length, + ilms: ilmsNames.length, + indices: indexNames.length, + } as LogMeta); + + // 2. Get stats + const [indicesStats, ilmsStats, policies] = await Promise.all([ + receiver.getIndicesStats(dsNames), + receiver.getIlmsStats(indexNames), + receiver.getIlmsPolicies(ilmsNames), + ]); + + // TODO: remove this log + log.debug(`Got stats`, { + dataStreams, + indicesStats, + ilmsStats, + policiesStats: policies, + } as LogMeta); + + // 3. Send events + policies.forEach((p) => { + _sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, p); + }); + + ilmsStats.forEach((i) => { + _sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, i); + }); + + dataStreams.forEach((ds) => { + _sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); + }); + + indicesStats.forEach((is) => { + _sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, is); + }); + + return 0; + } catch (err) { + log.warn(`Error running indices metadata task`, { + error: JSON.stringify(err), + } as LogMeta); + await taskMetricsService.end(trace, err); + return 0; + } + }, + }; +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2ac776d37f1e5..d618632e8df4a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -487,7 +487,8 @@ export class Plugin implements ISecuritySolutionPlugin { DEFAULT_QUEUE_CONFIG, this.telemetryReceiver, plugins.telemetry, - this.telemetryUsageCounter + this.telemetryUsageCounter, + core.analytics ); this.telemetryEventsSender.setup( From 30077638d2d65d32e1fd5d5ce6ca68d89366bf5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 26 Sep 2024 16:57:47 +0200 Subject: [PATCH 02/44] Add testing route to trigger new task --- .../server/lib/telemetry/routes/index.ts | 50 +++++++++++++++++++ .../lib/telemetry/tasks/indices.metadata.ts | 14 +++--- .../security_solution/server/routes/index.ts | 4 ++ 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts new file mode 100644 index 0000000000000..af29648227afc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { Logger, IRouter } from '@kbn/core/server'; +import type { ITelemetryReceiver } from '../receiver'; +import type { ITelemetryEventsSender } from '../sender'; +import { TaskMetricsService } from '../task_metrics'; +import { createTelemetryIndicesMetadataTaskConfig } from '../tasks/indices.metadata'; + +// TODO: just to test the POC, remove +export const getTriggerIndicesMetadataTaskRoute = ( + router: IRouter, + logger: Logger, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender +) => { + router.get( + { + path: '/internal/trigger-indices-metadata-task', + validate: false, + }, + async (_context, _request, response) => { + const taskMetricsService = new TaskMetricsService(logger, sender); + const task = createTelemetryIndicesMetadataTaskConfig(); + const timeStart = performance.now(); + let msgSuffix = ''; + if (global.gc) { + global.gc(); + } else { + msgSuffix = ' (Note: Garbage collection is not exposed. Start Node.js with --expose-gc.)'; + } + const initialMemory = process.memoryUsage().heapUsed; + const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { + last: 'last', + current: 'current', + }); + const memoryUsed = process.memoryUsage().heapUsed - initialMemory; + const elapsedTime = performance.now() - timeStart; + + return response.ok({ + body: { + message: `Task processed ${result} datastreams. It took ${elapsedTime} ms to run and required ${memoryUsed} bytes ${msgSuffix}`, + }, + }); + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 21d402b60619f..32a5ac67dbd6f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -23,7 +23,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { return { type: taskType, title: 'Security Solution Telemetry Indices Metadata task', - interval: '1m', // TODO: update!!! + interval: '24h', timeout: '1m', version: '1.0.0', getLastExecutionTime: getPreviousDailyTaskTimestamp, @@ -31,7 +31,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { taskId: string, logger: Logger, receiver: ITelemetryReceiver, - _sender: ITelemetryEventsSender, + sender: ITelemetryEventsSender, taskMetricsService: ITaskMetricsService, taskExecutionPeriod: TaskExecutionPeriod ) => { @@ -79,22 +79,22 @@ export function createTelemetryIndicesMetadataTaskConfig() { // 3. Send events policies.forEach((p) => { - _sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, p); + sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, p); }); ilmsStats.forEach((i) => { - _sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, i); + sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, i); }); dataStreams.forEach((ds) => { - _sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); + sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); }); indicesStats.forEach((is) => { - _sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, is); + sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, is); }); - return 0; + return dataStreams.length; } catch (err) { log.warn(`Error running indices metadata task`, { error: JSON.stringify(err), diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 6f245bd04a02b..ea02dc96df390 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -61,6 +61,7 @@ import { suggestUserProfilesRoute } from '../lib/detection_engine/routes/users/s import { registerTimelineRoutes } from '../lib/timeline/routes'; import { getFleetManagedIndexTemplatesRoute } from '../lib/security_integrations/cribl/routes'; import { registerEntityAnalyticsRoutes } from '../lib/entity_analytics/register_entity_analytics_routes'; +import { getTriggerIndicesMetadataTaskRoute } from '../lib/telemetry/routes'; export const initRoutes = ( router: SecuritySolutionPluginRouter, @@ -147,4 +148,7 @@ export const initRoutes = ( registerEntityAnalyticsRoutes({ router, config, getStartServices, logger }); // Security Integrations getFleetManagedIndexTemplatesRoute(router); + + // TODO: just to test the POC, remove + getTriggerIndicesMetadataTaskRoute(router, logger, previewTelemetryReceiver, telemetrySender); }; From 34cfa87454ab044fed263db5ab15223f9793fa3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 27 Sep 2024 19:17:04 +0200 Subject: [PATCH 03/44] Manual pagination --- .../server/lib/telemetry/receiver.ts | 282 ++++++++++++------ .../lib/telemetry/tasks/indices.metadata.ts | 66 ++-- 2 files changed, 228 insertions(+), 120 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index a227f1a711506..1a42a10636176 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -254,9 +254,9 @@ export interface ITelemetryReceiver { // ---------- TODO: POC ---------- // getDataStreams(): Promise; - getIndicesStats(indices: string[]): Promise; - getIlmsStats(indices: string[]): Promise; - getIlmsPolicies(ilms: string[]): Promise; + getIndicesStats(indices: string[], pageSize: number): AsyncGenerator; + getIlmsStats(indices: string[], pageSize: number): AsyncGenerator; + getIlmsPolicies(ilms: string[], pageSize: number): AsyncGenerator; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -1344,100 +1344,127 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async getDataStreams(): Promise { const es = this.esClient(); + this.logger.debug('Fetching datstreams'); + const request: IndicesGetDataStreamRequest = { name: '*', expand_wildcards: ['open', 'hidden'], filter_path: ['data_streams.name', 'data_streams.indices'], }; - return es.indices.getDataStream(request).then((response) => - response.data_streams.map((ds) => { - return { - datastream_name: ds.name, - indices: - ds.indices?.map((index) => { - return { - index_name: index.index_name, - ilm_policy: index.ilm_policy, - } as Index; - }) ?? [], - } as DataStream; - }) - ); + return es.indices + .getDataStream(request) + .then((response) => + response.data_streams.map((ds) => { + return { + datastream_name: ds.name, + indices: + ds.indices?.map((index) => { + return { + index_name: index.index_name, + ilm_policy: index.ilm_policy, + } as Index; + }) ?? [], + } as DataStream; + }) + ) + .catch((error) => { + this.logger.warn('Error fetching datastreams', { error_message: error } as LogMeta); + throw error; + }); } - public async getIndicesStats(indices: string[]): Promise { + public async *getIndicesStats(indices: string[], pageSize: number) { const es = this.esClient(); - this.logger.debug('Fetching indices stats', { indices } as LogMeta); + this.logger.debug(`Fetching indices stats for ${indices.length} indices`, { + indices, + } as LogMeta); - const request: IndicesStatsRequest = { - index: indices, - level: 'indices', - metric: ['docs', 'search', 'store'], - expand_wildcards: ['open', 'hidden'], - filter_path: [ - 'indices.*.total.search.query_total', - 'indices.*.total.search.query_time_in_millis', - 'indices.*.total.docs.count', - 'indices.*.total.docs.deleted', - 'indices.*.total.store.size_in_bytes', - ], - }; + const chunked = this._chunk(indices, pageSize) + .map((ilms) => this._getCommonPrefixes(ilms)) + .flat(); - return es.indices.stats(request).then((response) => - Object.entries(response.indices ?? {}).map(([indexName, stats]) => { - return { - index_name: indexName, - query_total: stats.total?.search?.query_total, - query_time_in_millis: stats.total?.search?.query_time_in_millis, - docs_count: stats.total?.docs?.count, - docs_deleted: stats.total?.docs?.deleted, - docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, - } as IndexStats; - }) - ); + this.logger.debug(`Splitted indices into ${chunked.length} chunks`, { + chunks: chunked.length, + } as LogMeta); + + for (const name of chunked) { + this.logger.debug(`Fetching indices stats for ${name}`, { name } as LogMeta); + const request: IndicesStatsRequest = { + index: `${name}*`, + level: 'indices', + metric: ['docs', 'search', 'store'], + expand_wildcards: ['open', 'hidden'], + filter_path: [ + 'indices.*.total.search.query_total', + 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', + 'indices.*.total.docs.deleted', + 'indices.*.total.store.size_in_bytes', + ], + }; + + try { + const response = await es.indices.stats(request); + for (const [indexName, stats] of Object.entries(response.indices ?? {})) { + yield { + index_name: indexName, + query_total: stats.total?.search?.query_total, + query_time_in_millis: stats.total?.search?.query_time_in_millis, + docs_count: stats.total?.docs?.count, + docs_deleted: stats.total?.docs?.deleted, + docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + } as IndexStats; + } + } catch (error) { + this.logger.warn('Error fetching indices stats', { error_message: error } as LogMeta); + throw error; + } + } } - public async getIlmsStats(indices: string[]): Promise { + public async *getIlmsStats(indices: string[], pageSize: number) { const es = this.esClient(); - this.logger.debug('Fetching ilms stats', { indices } as LogMeta); + this.logger.debug(`Fetching ilm stats for ${indices.length} indices`, { indices } as LogMeta); - const request: IlmExplainLifecycleRequest = { - index: indices.join(','), - only_managed: false, - filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], - }; + const chunked = this._chunk(indices, pageSize) + .map((ilms) => this._getCommonPrefixes(ilms)) + .flat(); - return es.ilm.explainLifecycle(request).then((response) => - Object.entries(response.indices ?? {}).map(([indexName, stats]) => { - return { - index_name: indexName, - phase: ('phase' in stats && stats.phase) || undefined, - age: ('age' in stats && stats.age) || undefined, - policy_name: ('policy' in stats && stats.policy) || undefined, - } as IlmStats; - }) - ); + for (const name of chunked) { + this.logger.debug(`Fetching ilm stats for ${name}`, { name } as LogMeta); + const request: IlmExplainLifecycleRequest = { + index: `${name}*`, + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + const data = await es.ilm.explainLifecycle(request); + + try { + for (const [indexName, stats] of Object.entries(data.indices ?? {})) { + const entry = { + index_name: indexName, + phase: ('phase' in stats && stats.phase) || undefined, + age: ('age' in stats && stats.age) || undefined, + policy_name: ('policy' in stats && stats.policy) || undefined, + } as IlmStats; + + yield entry; + } + } catch (error) { + this.logger.warn('Error fetching ilm stats', { error_message: error } as LogMeta); + throw error; + } + } } - public async getIlmsPolicies(ilms: string[]): Promise { + public async *getIlmsPolicies(ilms: string[], pageSize: number) { const es = this.esClient(); - this.logger.debug('Fetching ilms policies', { ilms } as LogMeta); - - const request: IlmGetLifecycleRequest = { - name: ilms.join(','), - filter_path: [ - '*.policy.phases.cold.min_age', - '*.policy.phases.delete.min_age', - '*.policy.phases.frozen.min_age', - '*.policy.phases.hot.min_age', - '*.policy.phases.warm.min_age', - '*.modified_date', - ], - }; + this.logger.debug(`Fetching ilms policies for ${ilms.length} ilms`, { ilms } as LogMeta); const phase = (obj: unknown): Nullable => { let value: Nullable; @@ -1449,20 +1476,95 @@ export class TelemetryReceiver implements ITelemetryReceiver { return value; }; - return es.ilm.getLifecycle(request).then((response) => { - return Object.entries(response ?? {}).map(([policyName, stats]) => { - return { - policy_name: policyName, - modified_date: stats.modified_date, - phases: { - cold: phase(stats.policy.phases.cold), - delete: phase(stats.policy.phases.delete), - frozen: phase(stats.policy.phases.frozen), - hot: phase(stats.policy.phases.hot), - warm: phase(stats.policy.phases.warm), - } as IlmPhases, - } as IlmPolicy; - }); - }); + const chunked = this._chunk(ilms, pageSize).map(this._getCommonPrefixes).flat(); + + this.logger.debug(`Splitted ilms into ${chunked.length} chunks`, { + chunks: chunked.length, + } as LogMeta); + + for (const name of chunked) { + this.logger.debug(`Fetching ilm policies for ${name}`, { name } as LogMeta); + const request: IlmGetLifecycleRequest = { + name: `${name}*`, + filter_path: [ + '*.policy.phases.cold.min_age', + '*.policy.phases.delete.min_age', + '*.policy.phases.frozen.min_age', + '*.policy.phases.hot.min_age', + '*.policy.phases.warm.min_age', + '*.modified_date', + ], + }; + + const response = await es.ilm.getLifecycle(request); + try { + for (const [policyName, stats] of Object.entries(response ?? {})) { + yield { + policy_name: policyName, + modified_date: stats.modified_date, + phases: { + cold: phase(stats.policy.phases.cold), + delete: phase(stats.policy.phases.delete), + frozen: phase(stats.policy.phases.frozen), + hot: phase(stats.policy.phases.hot), + warm: phase(stats.policy.phases.warm), + } as IlmPhases, + } as IlmPolicy; + } + } catch (error) { + this.logger.warn('Error fetching ilm policies', { error_message: error } as LogMeta); + throw error; + } + } + } + + // very basic implementation of a common prefix finder + // TODO: add tests to cover edge cases and check performance + private _getCommonPrefixes(names: string[], minLength: number = 10): string[] { + if (names.length === 0) return []; + + names.sort(); + const result: string[] = []; + + let currentPrefix = names[0]; + + for (let i = 1; i < names.length; i++) { + const name = names[i]; + let commonPrefix = ''; + + // Find the common prefix between currentPrefix and the current name + for (let j = 0; j < Math.min(currentPrefix.length, name.length); j++) { + if (currentPrefix[j] === name[j]) { + commonPrefix += currentPrefix[j]; + } else { + break; + } + } + + // Check if the common prefix is long enough + if (commonPrefix.length >= minLength) { + currentPrefix = commonPrefix; + } else { + if (currentPrefix.length >= minLength) { + result.push(currentPrefix); // Save the last valid prefix + } + currentPrefix = name; // Start a new prefix + } + } + + // Add the final prefix if it has at least N characters + if (currentPrefix.length >= minLength) { + result.push(currentPrefix); + } + + return result; + } + + private _chunk(arr: string[], size: number): string[][] { + return arr.reduce( + (result: string[][], _, index) => + index % size === 0 ? [...result, arr.slice(index, index + size)] : result, + [] + ); } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 32a5ac67dbd6f..0ca6ee6bd8572 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -39,11 +39,13 @@ export function createTelemetryIndicesMetadataTaskConfig() { const log = newTelemetryLogger(logger.get('indices-metadata'), mdc); const trace = taskMetricsService.start(taskType); - log.l('Running indices metadata task'); + // config + const pageSize = 500; + const dataStreamsLimit = 500; try { // 1. Get all data streams - const dataStreams = await receiver.getDataStreams(); + const dataStreams = (await receiver.getDataStreams()).slice(0, dataStreamsLimit + 1); // and calculate index and ilm names const dsNames = dataStreams.map((stream) => stream.datastream_name); @@ -62,42 +64,46 @@ export function createTelemetryIndicesMetadataTaskConfig() { indices: indexNames.length, } as LogMeta); - // 2. Get stats - const [indicesStats, ilmsStats, policies] = await Promise.all([ - receiver.getIndicesStats(dsNames), - receiver.getIlmsStats(indexNames), - receiver.getIlmsPolicies(ilmsNames), - ]); + let policyCount = 0; + let indicesCount = 0; + let ilmsCount = 0; + let dsCount = 0; - // TODO: remove this log - log.debug(`Got stats`, { - dataStreams, - indicesStats, - ilmsStats, - policiesStats: policies, - } as LogMeta); - - // 3. Send events - policies.forEach((p) => { - sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, p); - }); + for await (const stat of receiver.getIndicesStats(dsNames, pageSize)) { + sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, stat); + indicesCount++; + } + log.info(`Sent ${indicesCount} indices stats`, { indicesCount } as LogMeta); - ilmsStats.forEach((i) => { - sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, i); - }); + for await (const stat of receiver.getIlmsStats(indexNames, pageSize)) { + sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); + ilmsCount++; + } + log.info(`Sent ${ilmsCount} ILM stats`, { ilmsCount } as LogMeta); - dataStreams.forEach((ds) => { + for (const ds of dataStreams) { sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); - }); + dsCount++; + } + log.info(`Sent ${dsCount} data streams`, { dsCount } as LogMeta); - indicesStats.forEach((is) => { - sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, is); - }); + for await (const policy of receiver.getIlmsPolicies(ilmsNames, pageSize)) { + sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, policy); + policyCount++; + } + log.info(`Sent ${policyCount} ILM policies`, { policyCount } as LogMeta); + + log.info(`Sent EBT events`, { + datastreams: dsCount, + ilms: ilmsCount, + indices: indicesCount, + policies: policyCount, + } as LogMeta); - return dataStreams.length; + return policyCount + indicesCount + ilmsCount + dsCount; } catch (err) { log.warn(`Error running indices metadata task`, { - error: JSON.stringify(err), + error: err.message, } as LogMeta); await taskMetricsService.end(trace, err); return 0; From 4aefe23720cf9957f72e263cae649a58913fbb60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 27 Sep 2024 19:31:43 +0200 Subject: [PATCH 04/44] Parameterize testing api --- .../server/lib/telemetry/routes/index.ts | 21 +++++++++++++++---- .../lib/telemetry/tasks/indices.metadata.ts | 6 +++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts index af29648227afc..74eaf650e5629 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { schema } from '@kbn/config-schema'; import type { Logger, IRouter } from '@kbn/core/server'; import type { ITelemetryReceiver } from '../receiver'; import type { ITelemetryEventsSender } from '../sender'; @@ -20,12 +21,24 @@ export const getTriggerIndicesMetadataTaskRoute = ( router.get( { path: '/internal/trigger-indices-metadata-task', - validate: false, + validate: { + query: schema.object({ + pageSize: schema.maybe(schema.number()), + dataStreamsLimit: schema.maybe(schema.number()), + }), + }, }, - async (_context, _request, response) => { + async (_context, request, response) => { const taskMetricsService = new TaskMetricsService(logger, sender); const task = createTelemetryIndicesMetadataTaskConfig(); const timeStart = performance.now(); + + const { pageSize, dataStreamsLimit } = request.query; + + logger.info( + `Triggering indices metadata task with pageSize: ${pageSize} and dataStreamsLimit: ${dataStreamsLimit}` + ); + let msgSuffix = ''; if (global.gc) { global.gc(); @@ -34,8 +47,8 @@ export const getTriggerIndicesMetadataTaskRoute = ( } const initialMemory = process.memoryUsage().heapUsed; const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { - last: 'last', - current: 'current', + last: `${pageSize || 500}`, + current: `${dataStreamsLimit || 500}`, }); const memoryUsed = process.memoryUsage().heapUsed - initialMemory; const elapsedTime = performance.now() - timeStart; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 0ca6ee6bd8572..6f476050edb8d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -39,9 +39,9 @@ export function createTelemetryIndicesMetadataTaskConfig() { const log = newTelemetryLogger(logger.get('indices-metadata'), mdc); const trace = taskMetricsService.start(taskType); - // config - const pageSize = 500; - const dataStreamsLimit = 500; + // TODO: Taken from taskExecutionPeriod, just for testing purposes + const pageSize = Number(taskExecutionPeriod.last ?? '500'); + const dataStreamsLimit = Number(taskExecutionPeriod.current ?? '500'); try { // 1. Get all data streams From 4ebeb0061af98c05d1644e40a4563879fb4296f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 30 Sep 2024 13:03:18 +0200 Subject: [PATCH 05/44] Update common prefixes logic --- .../telemetry/__mocks__/staging_indices.ts | 1160 +++++++++++++++++ .../lib/telemetry/collections_helpers.test.ts | 45 +- .../lib/telemetry/collections_helpers.ts | 88 ++ .../server/lib/telemetry/receiver.ts | 266 ++-- .../server/lib/telemetry/routes/index.ts | 12 +- .../lib/telemetry/tasks/indices.metadata.ts | 26 +- 6 files changed, 1428 insertions(+), 169 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts new file mode 100644 index 0000000000000..e7fe78336ad06 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts @@ -0,0 +1,1160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const stagingIndices = [ + 'apm-7.14.2-profile-000011', + 'apm-7.14.2-profile-000012', + 'apm-7.14.2-profile-000013', + '.ds-logs-elastic_agent.filebeat-default-2023.07.22-000002', + 'apm-7.16.2-error-000010', + 'apm-7.16.2-error-000011', + '.ds-security_lists_telemetry_elastic-2024.04.15-000052', + 'security_solution_signals_test', + '.ds-v2_enrichment_data_cloud-2023.09.14-000012', + '.internal.alerts-transform.health.alerts-default-000001', + '.ds-security_lists_telemetry_elastic-2024.03.16-000050', + 'apm-7.16.2-span-000001', + 'apm-7.16.2-span-000002', + 'apm-7.16.2-span-000003', + 'apm-7.16.2-span-000004', + 'apm-7.16.2-span-000005', + 'apm-7.16.2-span-000006', + 'apm-7.16.2-span-000007', + 'apm-7.16.2-span-000008', + 'apm-7.16.2-span-000009', + '.ds-metrics-elastic_agent.filebeat-default-2023.08.21-000003', + '.ds-detonate_results_telemetry-2022.08.18-000009', + 'apm-7.14.2-span-000001', + 'apm-7.14.2-span-000002', + 'apm-7.14.2-span-000003', + '.ds-task_metrics_elastic-2023.10.18-000010', + 'apm-7.14.2-span-000004', + 'apm-7.14.2-span-000006', + 'apm-7.14.2-span-000005', + 'apm-7.14.2-span-000008', + 'apm-7.14.2-span-000007', + 'apm-7.14.2-span-000009', + 'apm-7.16.2-span-000010', + 'apm-7.16.2-span-000011', + 'shrink-0gwu-.ds-alert_telemetry_elastic-2023.10.31-000041', + '.ds-security_lists_telemetry_elastic-2023.11.17-000042', + 'shrink-y3pl-.ds-alert_telemetry_elastic-2023.11.30-000044', + '.ds-metrics-elastic_agent.filebeat-default-2023.07.22-000002', + '.ds-detonate_results_telemetry-2022.07.19-000008', + 'apm-7.16.1-profile-000001', + 'apm-7.16.1-profile-000002', + 'apm-7.16.1-profile-000003', + 'apm-7.16.1-profile-000004', + 'apm-7.16.1-profile-000005', + 'apm-7.16.1-profile-000006', + 'apm-7.16.1-profile-000007', + 'apm-7.16.1-profile-000008', + 'apm-7.16.1-profile-000009', + 'apm-7.14.2-span-000011', + '.ds-metrics-apm.app.apm_server-default-2024.06.24-000012', + 'apm-7.14.2-span-000010', + 'apm-7.14.2-span-000013', + 'apm-7.14.2-span-000012', + 'apm-7.14.2-error-000001', + 'apm-7.14.2-error-000002', + 'apm-7.14.2-error-000003', + 'apm-7.14.2-error-000004', + 'apm-7.14.2-error-000005', + 'apm-7.14.2-error-000006', + 'apm-7.14.2-error-000007', + 'apm-7.14.2-error-000008', + 'apm-7.14.2-error-000009', + 'shrink-kbvc-.ds-kibana-snapshot-2024.04.24-000046', + '.kibana-observability-ai-assistant-conversations-000001', + 'apm-7.16.1-profile-000010', + '.kibana_7.12.0_001', + 'apm-7.16.1-profile-000011', + 'apm-7.14.2-error-000010', + 'apm-7.14.2-error-000011', + 'apm-7.14.2-error-000012', + 'apm-7.14.2-error-000013', + '.ds-yaas_alert_telemetry-2022.06.11-000009', + '.ds-cluster_telemetry_elastic-2024.06.05-000058', + '.ds-detections_alert_telemetry_elastic-2024.09.29-000079', + 'apm-7.13.2-profile-000001', + 'apm-7.13.2-profile-000002', + 'apm-7.13.2-profile-000003', + 'apm-7.13.2-profile-000004', + 'apm-7.13.2-profile-000005', + 'apm-7.13.2-profile-000006', + 'apm-7.13.2-profile-000007', + 'apm-7.13.2-profile-000008', + 'apm-7.13.2-profile-000009', + 'model_tracking_latest-1', + 'model_tracking_latest-2', + 'model_tracking_latest-3', + 'model_tracking_latest-6', + '.ds-yaas_alert_telemetry-2022.05.12-000008', + '.ds-cluster_telemetry_elastic-2024.05.06-000056', + 'model_tracking_latest-8', + 'ia-cti-triaged-alerts', + 'apm-7.13.2-profile-000010', + 'apm-7.13.2-profile-000011', + 'apm-7.13.2-profile-000012', + 'apm-7.13.2-profile-000013', + 'apm-7.13.2-profile-000014', + 'apm-7.13.2-profile-000015', + 'apm-7.13.2-profile-000016', + 'apm-7.13.2-profile-000017', + '.ds-alert_timelines_elastic-2024.04.28-000050', + 'shrink-iriq-.ds-kibana-snapshot-2024.03.25-000043', + '.ds-detection_rule_cluster_details-2022.11.05-000010', + 'apm-7.15.1-transaction-000004', + 'apm-7.10.0-transaction-000001', + 'apm-7.10.0-transaction-000002', + 'apm-7.10.0-transaction-000003', + 'apm-7.10.0-transaction-000004', + 'apm-7.10.0-transaction-000005', + 'apm-7.10.0-transaction-000006', + 'apm-7.10.0-transaction-000007', + 'apm-7.10.0-transaction-000008', + 'apm-7.10.0-transaction-000009', + '.ds-metrics-fleet_server.agent_status-default-2024.07.20-000002', + '.ds-yaas_alert_telemetry-2022.09.09-000012', + '.ds-alert_timelines_elastic-2024.03.29-000047', + 'apm-7.15.1-transaction-000009', + 'apm-7.15.1-transaction-000008', + 'apm-7.15.1-transaction-000007', + 'apm-7.15.1-transaction-000006', + 'apm-7.15.1-transaction-000005', + 'apm-7.15.1-transaction-000003', + 'apm-7.15.1-transaction-000002', + 'model_tracking_latest-a', + 'apm-7.15.1-transaction-000001', + 'model_tracking_latest-d', + 'model_tracking_latest-e', + 'test_metadata', + '.ds-endpoint_metadata_telemetry_elastic-2024.02.18-000055', + 'apm-7.15.1-transaction-000011', + 'apm-7.15.1-transaction-000012', + 'apm-7.15.1-transaction-000013', + 'apm-7.10.0-transaction-000010', + 'apm-7.10.0-transaction-000011', + 'apm-7.10.0-transaction-000012', + 'apm-7.10.0-transaction-000013', + 'apm-7.10.0-transaction-000014', + 'apm-7.10.0-transaction-000015', + 'apm-7.10.0-transaction-000016', + 'apm-7.10.0-transaction-000017', + 'apm-7.10.0-transaction-000018', + 'apm-7.10.0-transaction-000019', + 'apm-7.15.1-transaction-000010', + '.ds-endpoint_metadata_telemetry_elastic-2024.01.19-000053', + 'apm-7.10.0-transaction-000020', + 'apm-7.10.0-transaction-000021', + 'apm-7.10.0-transaction-000022', + 'apm-7.10.0-transaction-000023', + 'apm-7.10.0-transaction-000024', + '.ds-alert_telemetry_elastic-2024.05.28-000069', + '.ds-alert_telemetry_elastic-2024.06.27-000072', + 'apm-7.16.1-error-000001', + 'apm-7.16.1-error-000002', + 'apm-7.16.1-error-000003', + 'apm-7.16.1-error-000004', + 'apm-7.16.1-error-000005', + 'apm-7.16.1-error-000006', + 'apm-7.16.1-error-000007', + 'apm-7.16.1-error-000008', + 'apm-7.16.1-error-000009', + '.ds-endpoint_metadata_telemetry_elastic-2024.09.15-000072', + 'apm-7.15.1-profile-000001', + 'apm-7.15.1-profile-000002', + 'apm-7.15.1-profile-000003', + 'apm-7.15.1-profile-000004', + 'apm-7.15.1-profile-000005', + 'apm-7.15.1-profile-000006', + 'apm-7.15.1-profile-000007', + 'apm-7.15.1-profile-000008', + 'apm-7.15.1-profile-000009', + 'shrink-ujrr-.ds-detections_alert_telemetry_elastic-2024.01.03-000046', + 'apm-7.16.1-error-000010', + 'apm-7.16.1-error-000011', + '.internal.alerts-ml.anomaly-detection.alerts-default-000001', + '.ds-endpoint_metadata_telemetry_elastic-2024.08.16-000070', + '.ds-security_lists_telemetry_elastic-2024.02.15-000048', + '.ds-endpoint_metadata_telemetry_elastic-2024.07.17-000068', + '.ds-v2_enrichment_data_cloud-2023.10.14-000013', + 'apm-7.15.1-profile-000010', + 'apm-7.15.1-profile-000011', + 'apm-7.15.1-profile-000012', + 'apm-7.15.1-profile-000013', + 'us-airports', + '.ds-insights_telemetry_elastic-2024.03.10-000042', + 'apm-7.13.2-onboarding-2021.06.15', + '.ds-security_lists_telemetry_elastic-2024.01.16-000046', + '.ds-detonate_results_telemetry-2022.10.17-000011', + '.ds-security_lists_telemetry_elastic-2024.09.12-000062', + 'test-index-1', + 'filebeat-7.12.0-2021.04.08', + '.siem-signals-default-000001', + '.siem-signals-default-000002', + '.siem-signals-default-000003', + '.siem-signals-default-000004', + '.siem-signals-default-000005', + '.siem-signals-default-000006', + '.siem-signals-default-000007', + '.siem-signals-default-000008', + '.siem-signals-default-000009', + 'shrink-b_x8-.ds-detections_alert_telemetry_elastic-2023.10.05-000037', + '.ds-security_lists_telemetry_elastic-2024.08.13-000060', + '.ds-security_lists_telemetry_elastic-2024.07.14-000058', + '.siem-signals-default-000010', + '.siem-signals-default-000011', + '.siem-signals-default-000012', + '.siem-signals-default-000013', + '.siem-signals-default-000014', + '.siem-signals-default-000015', + '.siem-signals-default-000016', + '.siem-signals-default-000017', + '.ds-sample-tags-tracking-2022.06.08-000006', + '.siem-signals-default-000018', + '.siem-signals-default-000019', + '.ds-inline_lookup_decision_tracking-2022.06.05-000006', + '.slm-history-3-000004', + '.kibana_task_manager_7.17.0_001', + 'apm-7.13.2-span-000001', + 'apm-7.13.2-span-000002', + 'apm-7.13.2-span-000003', + 'apm-7.13.2-span-000004', + 'apm-7.13.2-span-000005', + 'apm-7.13.2-span-000006', + '.siem-signals-default-000020', + '.siem-signals-default-000021', + '.siem-signals-default-000022', + '.siem-signals-default-000023', + 'apm-7.13.2-span-000008', + 'apm-7.13.2-span-000009', + 'apm-7.13.2-span-000007', + '.ds-inline_lookup_decision_tracking-2022.05.06-000005', + '.ds-sample-tags-tracking-2022.05.09-000005', + 'latest-cluster_telemetry_elastic', + '.ds-detection_rule_cluster_metrics-2022.11.05-000010', + 'apm-7.11.0-error-000001', + 'apm-7.11.0-error-000002', + 'apm-7.11.0-error-000003', + '.ds-logs-elastic_agent.metricbeat-default-2023.08.21-000003', + 'apm-7.11.0-error-000005', + 'apm-7.11.0-error-000006', + 'apm-7.11.0-error-000007', + 'apm-7.11.0-error-000008', + 'apm-7.11.0-transaction-000001', + '.ds-metrics-fleet_server.agent_status-default-2024.09.18-000004', + 'apm-7.11.0-transaction-000003', + 'apm-7.11.0-transaction-000002', + 'apm-7.11.0-transaction-000005', + 'apm-7.11.0-transaction-000006', + 'apm-7.11.0-transaction-000007', + 'apm-7.11.0-transaction-000008', + '.ds-cluster_telemetry_elastic-2023.10.09-000042', + 'apm-7.11.0-transaction-000009', + 'apm-7.11.0-transaction-000004', + 'apm-7.11.0-error-000009', + 'apm-7.11.0-error-000004', + 'apm-7.12.0-profile-000008', + 'apm-7.12.0-profile-000009', + 'apm-7.12.0-profile-000001', + 'apm-7.12.0-profile-000002', + 'apm-7.12.0-profile-000003', + 'apm-7.12.0-profile-000004', + 'apm-7.12.0-profile-000005', + 'apm-7.12.0-profile-000006', + 'apm-7.12.0-profile-000007', + 'apm-7.11.0-error-000010', + '.kibana_7.16.1_001', + 'apm-7.11.0-error-000012', + 'apm-7.11.0-error-000011', + 'apm-7.11.0-error-000014', + 'apm-7.11.0-error-000015', + '.ds-metrics-fleet_server.agent_status-default-2024.08.19-000003', + 'rollup-cluster_telemetry_elastic', + 'apm-7.11.0-error-000018', + 'apm-7.11.0-error-000019', + 'apm-7.11.0-error-000017', + 'apm-7.11.0-error-000016', + 'apm-7.11.0-error-000013', + 'apm-7.12.0-profile-000010', + 'apm-7.12.0-profile-000011', + '.ds-telemetry_usage-2024.06.01-000046', + 'apm-7.12.0-profile-000012', + 'apm-7.12.0-profile-000013', + 'apm-7.12.0-profile-000014', + 'apm-7.12.0-profile-000015', + 'apm-7.12.0-profile-000016', + 'apm-7.12.0-profile-000019', + 'apm-7.12.0-profile-000017', + 'apm-7.12.0-profile-000018', + 'apm-7.13.2-span-000015', + 'apm-7.13.2-span-000016', + 'apm-7.13.2-span-000013', + 'shrink-h4sx-.ds-detections_alert_telemetry_elastic-2024.03.03-000052', + 'apm-7.13.2-span-000014', + 'apm-7.13.2-span-000011', + 'apm-7.13.2-span-000012', + 'apm-7.11.0-error-000020', + 'apm-7.11.0-error-000021', + 'apm-7.11.0-error-000022', + 'apm-7.13.2-span-000010', + 'apm-7.13.2-span-000017', + 'apm-7.11.0-transaction-000014', + 'apm-7.11.0-transaction-000013', + 'apm-7.11.0-transaction-000012', + 'apm-7.11.0-transaction-000011', + 'apm-7.11.0-transaction-000010', + 'apm-7.11.0-transaction-000019', + 'apm-7.11.0-transaction-000018', + 'apm-7.12.0-profile-000020', + 'apm-7.11.0-transaction-000017', + 'apm-7.11.0-transaction-000016', + '.ds-alert_timelines_elastic-2024.09.25-000066', + 'apm-7.11.0-transaction-000015', + 'apm-7.11.0-transaction-000022', + 'apm-7.11.0-transaction-000021', + 'apm-7.11.0-transaction-000020', + 'shrink-wl9p-.ds-alert_telemetry_elastic-2024.02.28-000053', + '.ds-logs-elastic_agent.metricbeat-default-2023.07.22-000002', + '.ds-telemetry_usage-2023.09.05-000028', + 'apm-7.12.0-metric-000001', + 'apm-7.12.0-metric-000002', + 'apm-7.12.0-metric-000003', + 'apm-7.12.0-metric-000004', + 'apm-7.12.0-metric-000005', + 'apm-7.12.0-metric-000006', + 'apm-7.12.0-metric-000007', + 'apm-7.12.0-metric-000008', + 'apm-7.12.0-metric-000009', + '.ds-alert_timelines_elastic-2024.08.26-000063', + '.ds-logs-elastic_agent-default-2023.08.21-000003', + 'apm-7.17.0-span-000001', + 'apm-7.17.0-span-000002', + 'apm-7.17.0-span-000003', + 'apm-7.17.0-span-000004', + 'apm-7.17.0-span-000005', + 'apm-7.17.0-span-000006', + '.lists-default-000001', + 'apm-7.17.0-span-000008', + 'apm-7.17.0-span-000009', + 'apm-7.17.0-span-000007', + '.ds-download_stats_telemetry_elastic-2023.03.09-000018', + 'apm-7.12.0-metric-000010', + 'apm-7.12.0-metric-000011', + 'apm-7.12.0-metric-000012', + 'apm-7.12.0-metric-000013', + 'apm-7.12.0-metric-000014', + 'apm-7.12.0-metric-000015', + 'apm-7.12.0-metric-000016', + 'apm-7.12.0-metric-000017', + 'apm-7.12.0-metric-000018', + 'apm-7.12.0-metric-000019', + '.ds-alert_timelines_elastic-2024.07.27-000060', + 'detection_alert_telemetry_elastic', + '.kibana_7.14.2_001', + '.ds-logs-elastic_agent-default-2023.07.22-000002', + 'failed-docs-000001', + 'apm-7.12.0-metric-000020', + '.ds-insights_telemetry_elastic-2024.01.10-000038', + 'metrics-endpoint.metadata_current_default', + '.ds-detections_alert_telemetry_elastic-2024.07.01-000067', + 'apm-7.16.2-transaction-000001', + 'apm-7.16.2-transaction-000002', + 'apm-7.16.2-transaction-000003', + 'apm-7.16.2-transaction-000004', + 'apm-7.16.2-transaction-000005', + 'apm-7.16.2-transaction-000006', + 'apm-7.16.2-transaction-000007', + 'apm-7.16.2-transaction-000008', + 'apm-7.16.2-transaction-000009', + 'apm-7.10.2-metric-000001', + 'apm-7.10.2-metric-000002', + 'apm-7.10.2-metric-000003', + 'apm-7.10.2-metric-000004', + 'apm-7.10.2-metric-000005', + 'apm-7.10.2-metric-000006', + 'apm-7.10.2-metric-000007', + 'apm-7.10.2-metric-000008', + 'apm-7.10.2-metric-000009', + '.kibana_task_manager_7.12.0_001', + '.ds-logs-generic-default-2023.09.07-000002', + 'apm-7.13.2-transaction-000001', + 'apm-7.13.2-transaction-000002', + 'apm-7.13.2-transaction-000003', + 'apm-7.13.2-transaction-000004', + 'apm-7.13.2-transaction-000005', + 'apm-7.13.2-transaction-000006', + 'apm-7.13.2-transaction-000007', + 'apm-7.13.2-transaction-000008', + 'apm-7.13.2-transaction-000009', + 'apm-7.16.2-transaction-000011', + 'apm-7.16.2-transaction-000010', + 'apm-7.10.0-metric-000004', + 'apm-7.10.0-metric-000003', + 'apm-7.10.0-metric-000002', + 'apm-7.10.0-metric-000001', + 'apm-7.10.2-metric-000010', + 'apm-7.10.2-metric-000011', + 'apm-7.10.2-metric-000012', + 'apm-7.10.2-metric-000013', + 'apm-7.10.2-metric-000014', + 'apm-7.10.2-metric-000015', + 'apm-7.10.2-metric-000016', + 'apm-7.10.2-metric-000017', + 'apm-7.10.2-metric-000018', + 'apm-7.10.2-metric-000019', + 'apm-7.10.0-metric-000008', + 'apm-7.10.0-metric-000007', + 'apm-7.10.0-metric-000006', + 'shrink-_z0f-.ds-alert_timelines_elastic-2024.02.28-000044', + '.ds-security_lists_telemetry_elastic-2023.12.17-000044', + 'apm-7.13.2-transaction-000010', + 'apm-7.13.2-transaction-000011', + 'apm-7.13.2-transaction-000012', + 'apm-7.13.2-transaction-000013', + 'apm-7.13.2-transaction-000014', + 'apm-7.13.2-transaction-000015', + 'apm-7.13.2-transaction-000016', + 'apm-7.13.2-transaction-000017', + '.ds-metrics-fleet_server.agent_versions-default-2024.09.18-000004', + 'apm-7.10.0-metric-000005', + 'apm-7.10.0-metric-000011', + 'apm-7.10.0-metric-000010', + '.ds-kibana-snapshot-2024.09.21-000059', + 'apm-7.10.0-metric-000015', + 'apm-7.10.0-metric-000014', + 'apm-7.10.0-metric-000013', + 'apm-7.10.2-metric-000020', + 'apm-7.10.2-metric-000021', + 'apm-7.10.2-metric-000022', + 'apm-7.10.2-metric-000023', + 'apm-7.10.0-metric-000012', + 'apm-7.10.0-metric-000019', + 'test-rollup', + 'apm-7.10.0-metric-000018', + 'apm-7.10.0-metric-000017', + 'apm-7.10.0-metric-000016', + 'apm-7.10.0-metric-000009', + 'idx_processed_security-endpoint-metadata', + '.ds-logs-generic-default-2023.08.08-000001', + '.ds-metrics-fleet_server.agent_versions-default-2024.08.19-000003', + '.ds-detonate_results_telemetry-2022.05.20-000006', + 'apm-7.10.0-metric-000020', + 'apm-7.10.0-metric-000021', + 'apm-7.10.0-metric-000022', + 'apm-7.10.0-metric-000023', + 'apm-7.10.0-metric-000024', + '.ds-kibana-snapshot-2024.08.22-000057', + 'shrink-u5p4-.ds-kibana-snapshot-2024.05.24-000049', + '.ds-yaas_alert_telemetry-2022.02.11-000005', + '.ds-kibana-snapshot-2024.07.23-000054', + '.ds-detonate_results_telemetry-2022.09.17-000010', + '.kibana_7.13.2_001', + 'apm-7.15.1-metric-000001', + 'apm-7.15.1-metric-000002', + 'apm-7.15.1-metric-000003', + 'apm-7.15.1-metric-000004', + 'apm-7.15.1-metric-000005', + 'apm-7.15.1-metric-000006', + 'apm-7.15.1-metric-000007', + 'apm-7.15.1-metric-000008', + 'apm-7.15.1-metric-000009', + '.ds-logs-ingest-vt-reports-test-default-2023.06.23-000001', + '.ds-yaas_alert_telemetry-2022.01.12-000004', + '.internal.alerts-default.alerts-default-000001', + 'apm-7.15.1-metric-000010', + 'apm-7.15.1-metric-000011', + 'apm-7.15.1-metric-000012', + 'apm-7.15.1-metric-000013', + '.ds-telemetry_usage-2023.11.04-000032', + 'apm-7.10.2-span-000001', + 'apm-7.10.2-span-000002', + 'apm-7.10.2-span-000003', + 'apm-7.10.2-span-000004', + 'apm-7.10.2-span-000005', + 'apm-7.10.2-span-000006', + 'apm-7.10.2-span-000007', + 'apm-7.10.2-span-000008', + 'apm-7.10.2-span-000009', + '.ds-telemetry_usage-2023.10.05-000030', + 'apm-7.10.2-span-000010', + 'apm-7.10.2-span-000011', + 'apm-7.10.2-span-000012', + '.fleet-enrollment-api-keys-7', + 'apm-7.10.2-span-000013', + 'apm-7.10.2-span-000014', + 'apm-7.10.2-span-000016', + 'apm-7.10.2-span-000017', + 'apm-7.10.2-span-000018', + 'apm-7.10.2-span-000019', + 'apm-7.10.2-span-000015', + 'summary-cluster_telemetry_elastic', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.08.21-000003', + '.ds-detections_alert_telemetry_elastic_v2-2023.01.11-000001', + 'apm-7.10.2-span-000020', + '.ds-v2_alert_telemetry_elastic-2023.01.11-000001', + 'apm-7.10.2-span-000021', + 'apm-7.10.2-span-000023', + 'apm-7.10.2-span-000022', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.07.22-000002', + '.fleet-artifacts-7', + '.kibana_7.16.2_001', + '.ds-machine_learning_cluster_details-2022.11.07-000010', + '.ds-cluster_telemetry_elastic-2023.09.09-000040', + 'threatintel-2021.08.19-000001', + 'ilm-history-3-000004', + 'my-index-permissions', + '.ds-detections_alert_telemetry_elastic-2024.04.02-000058', + '.ds-alert_telemetry_endgame-2022.11.05-000086', + '.ds-endpoint_metadata_telemetry_elastic-2023.09.21-000045', + 'shrink-jwq5-.ds-alert_timelines_elastic-2024.01.29-000041', + '.ds-insights_telemetry_elastic-2024.09.06-000054', + 'kibana_snapshot', + '.ds-insights_telemetry_elastic-2023.12.11-000036', + 'metrics-index_pattern_placeholder', + '.ds-insights_telemetry_elastic-2024.08.07-000052', + 'shrink-48kl-.ds-detections_alert_telemetry_elastic-2023.12.04-000043', + 'apm-7.17.0-metric-000001', + 'failure-alert_telemetry_unified', + 'apm-7.17.0-metric-000002', + 'apm-7.17.0-metric-000003', + 'apm-7.17.0-metric-000004', + 'apm-7.17.0-metric-000005', + '.ds-insights_telemetry_elastic-2024.07.08-000050', + 'apm-7.17.0-metric-000006', + 'shrink-gqzj-.ds-detections_alert_telemetry_elastic-2024.02.02-000049', + 'apm-7.17.0-metric-000007', + 'apm-7.17.0-metric-000008', + 'apm-7.17.0-metric-000009', + '.ds-inline_lookup_decision_tracking-2022.02.05-000002', + '.ds-sample-tags-tracking-2021.12.21-000001', + '.kibana_task_manager_7.16.1_001', + '.ds-sample-tags-tracking-2022.02.08-000002', + '.ds-model_tracking_historical-2021.05.11-000004', + 'my-index-000001', + 'my-index-000002', + 'shrink-cgef-.ds-alert_telemetry_elastic-2024.01.29-000050', + '.ds-inline_lookup_decision_tracking-2022.01.06-000001', + '.security-tokens-7', + '.ds-telemetry_usage-2024.01.03-000036', + 'shrink-ixws-.ds-alert_timelines_elastic-2023.12.21-000037', + 'shrink-ud5m-.ds-alert_timelines_elastic-2023.12.30-000038', + '.ds-sample-tags-tracking-2022.03.10-000003', + 'apm-7.10.2-error-000001', + 'apm-7.10.2-error-000002', + 'apm-7.10.2-error-000003', + 'apm-7.10.2-error-000004', + 'apm-7.10.2-error-000005', + 'apm-7.10.2-error-000006', + 'apm-7.10.2-error-000007', + 'apm-7.10.2-error-000008', + 'apm-7.10.2-error-000009', + '.transform-notifications-000002', + '.ds-cluster_telemetry_elastic-2024.04.06-000054', + 'apm-7.17.0-profile-000001', + 'apm-7.17.0-profile-000002', + 'apm-7.17.0-profile-000003', + 'apm-7.17.0-profile-000004', + 'apm-7.17.0-profile-000005', + 'apm-7.17.0-profile-000006', + 'apm-7.17.0-profile-000007', + 'apm-7.17.0-profile-000008', + 'apm-7.17.0-profile-000009', + 'apm-7.10.2-error-000015', + 'apm-7.10.2-error-000014', + 'apm-7.10.2-error-000017', + 'apm-7.10.2-error-000016', + 'apm-7.10.2-error-000011', + 'apm-7.10.2-error-000010', + 'apm-7.10.2-error-000013', + 'apm-7.10.2-error-000012', + 'apm-7.10.2-error-000019', + 'apm-7.10.2-error-000018', + 'shrink-pcsa-.ds-alert_telemetry_elastic-2023.10.27-000040', + '.ds-yaas_alert_telemetry-2022.03.13-000006', + '.ds-metrics-fleet_server.agent_status-default-2024.06.20-000001', + 'shrink-d-hy-.ds-alert_telemetry_elastic-2023.12.30-000047', + '.ds-cluster_telemetry_elastic-2024.03.07-000052', + 'alert_telemetry_labels', + 'apm-7.10.2-error-000020', + 'apm-7.10.2-error-000021', + 'apm-7.10.2-error-000022', + 'apm-7.10.2-error-000023', + '.ds-yaas_alert_telemetry-2021.12.13-000003', + '.security-7', + '.ds-cluster_telemetry_elastic-2023.11.08-000044', + 'apm-7.16.2-profile-000001', + 'apm-7.16.2-profile-000002', + 'apm-7.16.2-profile-000003', + 'apm-7.16.2-profile-000004', + 'apm-7.16.2-profile-000005', + 'apm-7.16.2-profile-000006', + 'apm-7.16.2-profile-000007', + 'apm-7.16.2-profile-000008', + 'apm-7.16.2-profile-000009', + 'apm-7.16.1-transaction-000001', + 'apm-7.16.1-transaction-000003', + 'apm-7.16.1-transaction-000006', + 'apm-7.16.1-transaction-000007', + 'apm-7.16.1-transaction-000008', + 'apm-7.16.1-transaction-000009', + 'apm-7.16.1-transaction-000002', + 'apm-7.16.1-transaction-000005', + 'apm-7.16.1-transaction-000004', + 'apm-7.10.2-onboarding-2021.02.03', + 'apm-7.10.2-onboarding-2021.02.04', + '.ds-alert_telemetry_elastic-2024.04.28-000066', + 'apm-7.16.2-profile-000010', + 'apm-7.16.2-profile-000011', + '.ds-logs-vt_reports_gold_table_export-default-2023.06.22-000001', + 'apm-7.16.1-transaction-000010', + 'apm-7.16.1-transaction-000011', + '.ds-endpoint_metadata_telemetry_elastic-2023.11.20-000049', + '.internal.alerts-ml.anomaly-detection-health.alerts-default-000001', + '.ds-enrichment_data_cloud-2022.12.02-000013', + '.ds-alert_telemetry_elastic-2024.03.29-000062', + '.kibana_task_manager_7.15.1_001', + '.ds-endpoint_metadata_telemetry_elastic-2023.10.21-000047', + '.ds-endpoint_metadata_telemetry_elastic-2024.06.17-000066', + '.internal.alerts-stack.alerts-default-000001', + 'apm-7.13.2-metric-000001', + 'apm-7.13.2-metric-000002', + 'apm-7.13.2-metric-000003', + 'apm-7.13.2-metric-000004', + 'apm-7.13.2-metric-000005', + 'apm-7.13.2-metric-000006', + 'apm-7.13.2-metric-000007', + 'apm-7.13.2-metric-000008', + 'apm-7.13.2-metric-000009', + '.enrich-enrich-alerts-cust-info-1618854141382', + '.enrich-enrich-alerts-hash-1618853111243', + '.ds-endpoint_metadata_telemetry_elastic-2024.05.18-000064', + 'apm-7.13.2-metric-000010', + 'apm-7.13.2-metric-000011', + 'apm-7.13.2-metric-000012', + 'apm-7.13.2-metric-000013', + 'apm-7.13.2-metric-000014', + 'apm-7.13.2-metric-000015', + 'apm-7.13.2-metric-000016', + 'apm-7.13.2-metric-000017', + 'security_solution_signals_test2', + 'security_solution_signals_test0', + 'security_solution_signals_test9', + '.internal.alerts-observability.apm.alerts-default-000001', + '.kibana_task_manager_7.13.2_001', + 'apm-7.13.2-error-000001', + 'apm-7.13.2-error-000002', + 'apm-7.13.2-error-000003', + 'apm-7.13.2-error-000004', + 'apm-7.13.2-error-000005', + 'apm-7.13.2-error-000006', + 'apm-7.13.2-error-000007', + 'apm-7.13.2-error-000008', + 'apm-7.13.2-error-000009', + '.ds-insights_telemetry_elastic-2024.04.09-000044', + '.ds-security_lists_telemetry_elastic-2024.06.14-000056', + 'apm-7.11.0-span-000001', + 'apm-7.11.0-span-000002', + 'apm-7.11.0-span-000003', + 'apm-7.11.0-span-000004', + 'apm-7.11.0-span-000005', + 'apm-7.11.0-span-000006', + 'apm-7.11.0-span-000007', + 'apm-7.11.0-span-000008', + 'apm-7.11.0-span-000009', + '.internal.alerts-observability.uptime.alerts-default-000001', + 'apm-7.13.2-error-000010', + 'apm-7.13.2-error-000011', + 'apm-7.13.2-error-000012', + 'apm-7.13.2-error-000013', + 'apm-7.13.2-error-000014', + 'apm-7.13.2-error-000015', + 'apm-7.13.2-error-000016', + 'apm-7.13.2-error-000017', + 'apm-7.10.2-onboarding-2021.01.14', + '.fleet-servers-7', + '.ds-security_lists_telemetry_elastic-2024.05.15-000054', + 'apm-7.11.0-span-000010', + 'apm-7.11.0-profile-000001', + 'apm-7.11.0-profile-000002', + 'apm-7.11.0-profile-000003', + 'apm-7.11.0-profile-000004', + 'apm-7.11.0-profile-000005', + 'apm-7.11.0-profile-000006', + 'apm-7.11.0-profile-000007', + 'apm-7.11.0-profile-000008', + 'apm-7.11.0-profile-000009', + 'apm-7.11.0-span-000014', + 'apm-7.11.0-span-000013', + 'apm-7.11.0-span-000016', + 'apm-7.11.0-span-000015', + 'apm-7.11.0-span-000018', + 'apm-7.11.0-span-000017', + 'apm-7.11.0-span-000019', + 'apm-7.11.0-span-000012', + 'apm-7.11.0-span-000011', + '.fleet-policies-leader-7', + '.ds-metrics-fleet_server.agent_versions-default-2024.06.20-000001', + '.ds-sample-tags-tracking-2022.04.09-000004', + 'security-rollups-hourly', + 'test-bq-export', + '.internal.alerts-observability.slo.alerts-default-000001', + 'apm-7.11.0-profile-000010', + 'apm-7.11.0-profile-000011', + 'apm-7.11.0-profile-000012', + 'apm-7.11.0-profile-000013', + 'apm-7.11.0-profile-000014', + 'apm-7.11.0-profile-000015', + 'apm-7.11.0-profile-000016', + 'apm-7.11.0-profile-000017', + 'apm-7.11.0-profile-000018', + 'apm-7.11.0-profile-000019', + 'apm-7.11.0-span-000021', + 'apm-7.11.0-span-000020', + '.ds-inline_lookup_decision_tracking-2022.03.07-000003', + 'apm-7.11.0-span-000022', + 'apm-7.10.2-transaction-000001', + 'apm-7.10.2-profile-000001', + 'apm-7.10.2-profile-000002', + 'apm-7.10.2-profile-000003', + 'apm-7.10.2-profile-000004', + '.ds-cluster_telemetry_elastic-2024.02.06-000050', + 'apm-7.10.2-profile-000005', + 'apm-7.10.2-profile-000006', + 'apm-7.10.2-profile-000008', + 'apm-7.10.2-profile-000009', + '.ds-cluster_telemetry_elastic-2024.01.07-000048', + 'apm-7.10.2-profile-000007', + 'alerts-detections', + 'apm-7.10.2-transaction-000002', + 'apm-7.10.2-transaction-000004', + 'apm-7.11.0-profile-000020', + 'apm-7.11.0-profile-000021', + 'apm-7.11.0-profile-000022', + 'apm-7.10.2-transaction-000003', + 'apm-7.10.2-transaction-000009', + 'apm-7.10.2-transaction-000006', + 'apm-7.10.2-transaction-000005', + 'apm-7.10.2-transaction-000008', + 'apm-7.10.2-transaction-000007', + '.ds-logs-elastic_agent.metricbeat-default-2023.06.22-000001', + '.ds-logs-ingest-vt-reports-test-default-2023.07.23-000002', + 'apm-7.10.2-transaction-000010', + 'apm-7.10.2-profile-000010', + 'apm-7.10.2-profile-000011', + 'apm-7.10.2-profile-000012', + 'apm-7.10.2-profile-000013', + 'apm-7.10.2-profile-000014', + 'apm-7.10.2-profile-000015', + 'apm-7.10.2-profile-000016', + 'apm-7.10.2-profile-000017', + 'apm-7.10.2-profile-000018', + 'apm-7.10.2-profile-000019', + '.kibana_task_manager_7.16.2_001', + 'apm-7.10.2-transaction-000013', + 'apm-7.10.2-transaction-000012', + 'apm-7.10.2-transaction-000015', + 'apm-7.10.2-transaction-000014', + 'apm-7.10.2-transaction-000011', + 'apm-7.10.2-transaction-000017', + 'apm-7.10.2-transaction-000016', + 'apm-7.10.2-transaction-000019', + 'apm-7.10.2-transaction-000018', + '.fleet-policies-7', + 'apm-7.12.0-transaction-000009', + 'apm-7.12.0-transaction-000005', + 'apm-7.12.0-transaction-000006', + 'apm-7.12.0-transaction-000007', + 'apm-7.12.0-transaction-000008', + 'apm-7.12.0-transaction-000001', + '.internal.alerts-observability.metrics.alerts-default-000001', + 'apm-7.15.1-error-000002', + 'apm-7.15.1-error-000001', + '.ds-telemetry_usage-2024.04.02-000042', + 'apm-7.10.2-profile-000020', + 'apm-7.10.2-profile-000021', + 'apm-7.10.2-profile-000022', + 'apm-7.10.2-profile-000023', + '.ds-yaas_alert_telemetry-2022.08.10-000011', + 'apm-7.10.2-transaction-000023', + 'apm-7.10.2-transaction-000020', + 'apm-7.10.2-transaction-000022', + 'apm-7.10.2-transaction-000021', + 'apm-7.15.1-error-000004', + 'apm-7.15.1-error-000003', + 'apm-7.15.1-error-000006', + 'apm-7.15.1-error-000005', + 'apm-7.15.1-error-000008', + 'apm-7.15.1-error-000007', + 'apm-7.15.1-error-000009', + 'apm-7.12.0-transaction-000002', + 'apm-7.12.0-transaction-000003', + 'apm-7.12.0-transaction-000004', + 'apm-7.12.0-transaction-000016', + 'apm-7.12.0-transaction-000017', + 'apm-7.12.0-transaction-000018', + 'apm-7.12.0-transaction-000019', + 'apm-7.12.0-transaction-000012', + 'apm-7.12.0-transaction-000013', + '.ds-telemetry_usage-2024.03.03-000040', + 'apm-7.15.1-error-000010', + 'onweek_rules_browser_index', + 'apm-7.15.1-error-000011', + '.ds-yaas_alert_telemetry-2022.07.11-000010', + 'apm-7.15.1-error-000013', + 'apm-7.15.1-error-000012', + '.ds-alert_timelines_elastic-2024.06.27-000057', + 'apm-7.12.0-transaction-000014', + 'apm-7.12.0-transaction-000015', + 'apm-7.12.0-transaction-000010', + 'apm-7.12.0-transaction-000011', + 'apm-7.16.1-onboarding-2021.12.16', + 'apm-7.16.1-onboarding-2021.12.15', + 'apm-7.12.0-transaction-000020', + '.ds-alert_timelines_elastic-2024.05.28-000053', + 'apm-7.12.0-error-000001', + 'apm-7.12.0-error-000002', + 'apm-7.12.0-error-000003', + 'apm-7.12.0-error-000004', + 'apm-7.12.0-error-000005', + 'apm-7.12.0-error-000006', + 'apm-7.12.0-error-000007', + 'apm-7.12.0-error-000008', + 'apm-7.12.0-error-000009', + 'failure-alert_telemetry_elastic', + '.ds-logs-elastic_agent-default-2023.06.22-000001', + '.ds-metrics-elastic_agent.filebeat-default-2023.11.27-000004', + 'apm-7.10.0-profile-000001', + 'apm-7.10.0-profile-000002', + 'apm-7.10.0-profile-000003', + 'apm-7.10.0-profile-000004', + 'apm-7.10.0-profile-000005', + 'apm-7.10.0-profile-000006', + 'apm-7.10.0-profile-000007', + 'apm-7.10.0-profile-000008', + 'apm-7.10.0-profile-000009', + 'apm-7.12.0-onboarding-2021.03.23', + 'apm-7.12.0-error-000010', + 'apm-7.12.0-error-000011', + 'apm-7.12.0-error-000012', + 'apm-7.12.0-error-000013', + 'apm-7.12.0-error-000014', + 'apm-7.12.0-error-000015', + 'apm-7.12.0-error-000016', + 'apm-7.12.0-error-000017', + 'apm-7.12.0-error-000018', + '.ds-alert_timelines_elastic_v2-2022.10.20-000002', + 'apm-7.12.0-error-000019', + '.ds-alert_telemetry_elastic-2024.09.25-000087', + '.ds-alert_telemetry_elastic-2024.08.26-000078', + 'dead-letter', + '.ds-detections_alert_telemetry_elastic-2024.06.01-000064', + 'apm-7.10.0-profile-000010', + 'apm-7.10.0-profile-000011', + 'apm-7.10.0-profile-000012', + 'apm-7.10.0-profile-000013', + 'apm-7.10.0-profile-000014', + 'apm-7.10.0-profile-000015', + 'apm-7.10.0-profile-000016', + 'apm-7.10.0-profile-000017', + 'apm-7.10.0-profile-000018', + 'apm-7.10.0-profile-000019', + 'data-dictionaries', + 'apm-7.12.0-error-000020', + '.ds-alert_telemetry_elastic-2024.07.27-000075', + '.ds-detections_alert_telemetry_elastic-2024.05.02-000061', + 'apm-7.10.0-profile-000020', + 'apm-7.10.0-profile-000021', + 'apm-7.10.0-profile-000022', + 'apm-7.10.0-profile-000023', + 'apm-7.10.0-profile-000024', + 'apm-7.17.0-error-000001', + 'shrink-e4jt-.ds-alert_timelines_elastic-2023.11.21-000034', + 'apm-7.17.0-error-000003', + 'apm-7.17.0-error-000002', + 'apm-7.17.0-error-000005', + 'apm-7.17.0-error-000006', + 'apm-7.17.0-error-000007', + 'apm-7.17.0-error-000008', + 'apm-7.17.0-error-000009', + 'apm-7.17.0-error-000004', + 'apm-7.16.1-span-000001', + 'apm-7.16.1-span-000002', + 'apm-7.16.1-span-000003', + 'apm-7.16.1-span-000004', + 'apm-7.16.1-span-000005', + 'apm-7.16.1-span-000006', + 'apm-7.16.1-span-000007', + 'apm-7.16.1-span-000008', + 'apm-7.16.1-span-000009', + '.ds-insights_telemetry_elastic-2024.02.09-000040', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000004', + '.ds-logs-elastic_agent.filebeat-default-2023.06.22-000001', + 'ia-signatures', + 'apm-7.16.2-onboarding-2021.12.20', + 'apm-7.14.2-transaction-000001', + 'apm-7.14.2-transaction-000002', + 'apm-7.14.2-transaction-000003', + 'apm-7.14.2-transaction-000005', + 'apm-7.14.2-transaction-000006', + 'apm-7.14.2-transaction-000007', + 'apm-7.14.2-transaction-000008', + 'apm-7.14.2-transaction-000009', + '.ds-security_lists_telemetry_elastic-2023.10.18-000040', + 'apm-7.14.2-transaction-000004', + 'apm-7.16.1-span-000011', + 'apm-7.10.0-error-000001', + 'apm-7.10.0-error-000002', + 'apm-7.10.0-error-000003', + 'apm-7.10.0-error-000004', + 'apm-7.10.0-error-000005', + 'apm-7.10.0-error-000006', + 'apm-7.10.0-error-000007', + 'apm-7.10.0-error-000008', + 'apm-7.10.0-error-000009', + '.ds-insights_telemetry_elastic-2023.09.12-000030', + '.ds-detonate_results_telemetry-2022.04.20-000003', + 'apm-7.12.0-span-000001', + 'apm-7.12.0-span-000005', + 'apm-7.12.0-span-000006', + 'apm-7.12.0-span-000007', + 'apm-7.12.0-span-000008', + 'apm-7.12.0-span-000002', + 'apm-7.12.0-span-000003', + 'apm-7.12.0-span-000004', + 'apm-7.12.0-span-000009', + 'apm-7.14.2-transaction-000012', + 'apm-7.14.2-transaction-000013', + 'apm-7.14.2-transaction-000010', + 'apm-7.14.2-transaction-000011', + 'apm-7.16.1-span-000010', + 'apm-7.10.0-span-000001', + 'apm-7.10.0-span-000002', + '.ds-sample-tags-tracking-2022.09.06-000009', + 'apm-7.10.0-error-000010', + 'apm-7.10.0-error-000011', + 'apm-7.10.0-error-000012', + 'apm-7.10.0-error-000014', + 'apm-7.10.0-error-000015', + 'apm-7.10.0-error-000013', + 'apm-7.10.0-error-000016', + 'apm-7.10.0-error-000018', + 'apm-7.10.0-error-000019', + 'apm-7.10.0-error-000017', + 'apm-7.10.0-span-000009', + 'apm-7.10.0-span-000007', + 'apm-7.10.0-span-000008', + 'apm-7.10.0-span-000005', + 'apm-7.10.0-span-000006', + 'apm-7.10.0-span-000003', + 'apm-7.10.0-span-000004', + 'apm-7.12.0-span-000016', + 'apm-7.12.0-span-000017', + 'apm-7.12.0-span-000018', + 'apm-7.12.0-span-000019', + 'apm-7.12.0-span-000012', + 'apm-7.12.0-span-000013', + 'apm-7.12.0-span-000014', + 'apm-7.12.0-span-000015', + 'apm-7.12.0-span-000010', + '.ds-sample-tags-tracking-2022.08.07-000008', + 'apm-7.10.0-span-000010', + 'apm-7.10.0-span-000011', + 'apm-7.10.0-span-000012', + 'apm-7.10.0-error-000020', + 'apm-7.10.0-error-000021', + 'apm-7.10.0-error-000022', + 'apm-7.10.0-error-000023', + 'apm-7.10.0-error-000024', + 'apm-7.10.0-span-000013', + 'apm-7.12.0-span-000011', + 'apm-7.10.0-span-000018', + 'apm-7.12.0-span-000020', + '.ds-detonate_results_telemetry-2022.06.19-000007', + 'apm-7.10.0-span-000019', + 'apm-7.10.0-span-000016', + 'apm-7.10.0-span-000017', + 'apm-7.10.0-span-000014', + 'apm-7.10.0-span-000015', + '.ds-inline_lookup_decision_tracking-2022.08.04-000008', + '.transform-internal-007', + '.transform-internal-006', + '.transform-internal-005', + '.ds-inline_lookup_decision_tracking-2022.09.03-000009', + '.ds-kibana-snapshot-2024.06.23-000052', + '.ds-inline_lookup_decision_tracking-2022.07.05-000007', + '.ds-metrics-elastic_agent.filebeat-default-2023.06.22-000001', + 'event-type-index', + '.ds-sample-tags-tracking-2022.07.08-000007', + '.ds-telemetry_usage-2024.02.02-000038', + 'apm-7.10.0-span-000020', + 'apm-7.10.0-span-000021', + 'apm-7.10.0-span-000022', + 'apm-7.10.0-span-000023', + 'apm-7.10.0-span-000024', + '.ds-cluster_telemetry_elastic-2023.12.08-000046', + '.ds-yaas_alert_telemetry-2022.04.12-000007', + '.kibana_task_manager_7.14.2_001', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.06.22-000001', + '.ds-logs-vt_reports_gold_table_export-default-2023.07.22-000002', + 'apm-7.16.2-metric-000001', + 'apm-7.16.2-metric-000002', + 'apm-7.16.2-metric-000003', + 'apm-7.16.2-metric-000004', + 'apm-7.16.2-metric-000005', + 'apm-7.16.2-metric-000006', + 'apm-7.16.2-metric-000007', + 'apm-7.16.2-metric-000008', + 'apm-7.16.2-metric-000009', + '.internal.alerts-observability.threshold.alerts-default-000001', + 'apm-7.17.0-onboarding-2022.02.24', + '.ds-endpoint_metadata_telemetry_elastic-2023.12.20-000051', + '.internal.alerts-observability.logs.alerts-default-000001', + '.ds-machine_learning_cluster_metrics-2022.11.07-000010', + 'apm-7.16.2-metric-000010', + 'apm-7.16.2-metric-000011', + '.ds-osquery_packs_telemetry_elastic-2022.03.24-000001', + 'shrink-yvq6-.ds-alert_timelines_elastic-2023.10.22-000031', + 'apm-7.14.2-onboarding-2021.10.13', + 'ia-cs_extract', + 'idx_processed_security-stats', + '.ds-insights_telemetry_elastic-2023.11.11-000034', + 'machine_learning_cluster_metric', + '.ds-insights_telemetry_elastic-2024.06.08-000048', + '.kibana-observability-ai-assistant-kb-000001', + '.ds-insights_telemetry_elastic-2023.10.12-000032', + 'detection_rule_cluster_metric', + '.ds-insights_telemetry_elastic-2024.05.09-000046', + '.ds-inline_lookup_decision_tracking-2022.11.02-000011', + '.internal.alerts-security.alerts-default-000001', + '.ds-sample-tags-tracking-2022.11.05-000011', + 'apm-7.14.2-metric-000001', + 'apm-7.14.2-metric-000002', + 'apm-7.14.2-metric-000003', + 'apm-7.14.2-metric-000004', + 'data-dict-index', + 'apm-7.14.2-metric-000005', + 'apm-7.14.2-metric-000006', + 'apm-7.17.0-transaction-000001', + 'apm-7.17.0-transaction-000002', + 'apm-7.17.0-transaction-000003', + 'apm-7.17.0-transaction-000004', + 'apm-7.17.0-transaction-000005', + 'apm-7.17.0-transaction-000006', + 'apm-7.17.0-transaction-000007', + 'apm-7.17.0-transaction-000008', + 'apm-7.17.0-transaction-000009', + 'apm-7.14.2-metric-000009', + 'apm-7.14.2-metric-000007', + 'apm-7.14.2-metric-000008', + 'idx_processed_security-telemetry-usage', + '.ds-inline_lookup_decision_tracking-2022.10.03-000010', + 'shrink-imua-.ds-alert_timelines_elastic-2023.09.22-000028', + 'apm-7.15.1-span-000003', + 'apm-7.15.1-span-000004', + 'apm-7.15.1-span-000005', + 'apm-7.15.1-span-000006', + 'apm-7.15.1-span-000007', + 'apm-7.15.1-span-000008', + 'apm-7.15.1-span-000009', + 'apm-7.15.1-span-000001', + 'apm-7.15.1-span-000002', + 'apm-7.14.2-metric-000010', + 'apm-7.14.2-metric-000012', + 'apm-7.14.2-metric-000013', + 'apm-7.14.2-metric-000011', + '.ds-sample-tags-tracking-2022.10.06-000010', + '.ds-metrics-fleet_server.agent_versions-default-2024.07.20-000002', + 'apm-7.11.0-onboarding-2021.02.10', + 'apm-7.15.1-span-000010', + 'apm-7.15.1-span-000011', + 'apm-7.15.1-span-000012', + 'apm-7.15.1-span-000013', + 'logs-index_pattern_placeholder', + 'awesome-test', + '.ds-yaas_alert_telemetry-2022.10.09-000013', + 'test-vt-reports-export', + '.ds-task_metrics_elastic-2023.09.18-000005', + '.fleet-agents-7', + '.ds-security_lists_telemetry_elastic-2023.09.18-000038', + '.ds-inline_lookup_decision_tracking-2022.04.06-000004', + 'shrink-abnc-.ds-alert_telemetry_elastic-2023.09.27-000037', + 'shrink-5phn-.ds-detections_alert_telemetry_elastic-2023.11.04-000039', + '.ds-yaas_alert_telemetry-2021.11.13-000002', + '.ds-telemetry_usage-2024.05.02-000044', + 'apm-7.11.0-metric-000001', + 'apm-7.11.0-metric-000002', + 'apm-7.11.0-metric-000003', + 'apm-7.11.0-metric-000004', + 'apm-7.11.0-metric-000005', + 'apm-7.11.0-metric-000006', + 'apm-7.11.0-metric-000007', + 'apm-7.11.0-metric-000008', + 'apm-7.11.0-metric-000009', + '.ds-yaas_alert_telemetry-2021.10.14-000001', + '.ds-detections_alert_telemetry_elastic-2024.08.30-000073', + 'apm-7.15.1-onboarding-2021.10.19', + 'apm-7.11.0-metric-000010', + 'apm-7.11.0-metric-000011', + 'apm-7.11.0-metric-000012', + 'apm-7.11.0-metric-000013', + 'apm-7.11.0-metric-000014', + 'apm-7.11.0-metric-000015', + 'apm-7.11.0-metric-000016', + 'apm-7.11.0-metric-000017', + 'apm-7.11.0-metric-000018', + 'apm-7.11.0-metric-000019', + '.ds-detections_alert_telemetry_elastic-2024.07.31-000070', + '.ds-telemetry_usage-2023.12.04-000034', + '.kibana_7.17.0_001', + 'apm-7.11.0-metric-000021', + 'apm-7.11.0-metric-000022', + 'apm-7.11.0-metric-000020', + 'detection-rules-test', + '.ds-v2_detections_alert_telemetry_elastic-2022.09.11-000002', + 'apm-7.16.1-metric-000001', + 'apm-7.16.1-metric-000002', + 'apm-7.16.1-metric-000003', + 'apm-7.16.1-metric-000004', + 'apm-7.16.1-metric-000005', + 'apm-7.16.1-metric-000006', + 'apm-7.16.1-metric-000007', + 'apm-7.16.1-metric-000008', + 'apm-7.16.1-metric-000009', + '.ds-endpoint_metadata_telemetry_elastic-2024.04.18-000062', + 'machine_learning_cluster_detail', + '.items-default-000001', + 'apm-7.10.0-onboarding-2020.12.03', + 'apm-7.16.1-metric-000010', + 'apm-7.16.1-metric-000011', + '.ds-endpoint_metadata_telemetry_elastic-2024.03.19-000060', + 'apm-7.10.0-onboarding-2020.12.04', + 'apm-7.14.2-profile-000001', + 'apm-7.14.2-profile-000002', + 'apm-7.14.2-profile-000003', + 'apm-7.14.2-profile-000004', + 'apm-7.14.2-profile-000005', + 'apm-7.14.2-profile-000006', + 'apm-7.14.2-profile-000007', + 'apm-7.14.2-profile-000008', + 'apm-7.14.2-profile-000009', + '.ds-logs-elastic_agent.filebeat-default-2023.08.21-000003', + 'detection_rule_cluster_detail', + '.ds-alert_telemetry_unified-2022.11.10-000048', + '.kibana_7.15.1_001', + 'apm-7.16.2-error-000001', + 'apm-7.16.2-error-000002', + 'apm-7.16.2-error-000003', + 'apm-7.16.2-error-000004', + 'apm-7.16.2-error-000005', + 'apm-7.16.2-error-000006', + 'apm-7.16.2-error-000007', + 'apm-7.16.2-error-000008', + 'apm-7.16.2-error-000009', + 'apm-7.14.2-profile-000010', +]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts index 3d67d6cd22b17..da9976599c96f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts @@ -4,7 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { chunked, chunkedBy } from './collections_helpers'; +import { stagingIndices } from './__mocks__/staging_indices'; +import { type QueryConfig, chunked, chunkedBy, findCommonPrefixes } from './collections_helpers'; describe('telemetry.utils.chunked', () => { it('should chunk simple case', async () => { @@ -79,3 +80,45 @@ describe('telemetry.utils.chunkedBy', () => { expect(output).toEqual([['aaaa']]); }); }); + +describe('telemetry.utils.findCommonPrefixes', () => { + it('should find common prefixes in simple case', async () => { + const indices = ['aaa', 'b', 'aa']; + const config: QueryConfig = { + maxPrefixes: 10, + maxGroupSize: 10, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(2); + expect(output.find((v, _) => v[0] === 'b' && v[1] === 1)).not.toBeFalsy(); + expect(output.find((v, _) => v[0] === 'a' && v[1] === 2)).not.toBeFalsy(); + }); + + it('should discard extra indices', async () => { + const indices = ['aaa', 'aaaaaa', 'aa']; + const config: QueryConfig = { + maxPrefixes: 1, + maxGroupSize: 2, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(1); + expect(output.find((v, _) => v[0] === 'aaa' && v[1] === 2)).not.toBeFalsy(); + }); + + it('should group many indices', async () => { + const indices = stagingIndices; + const config: QueryConfig = { + maxPrefixes: 8, + maxGroupSize: 100, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(config.maxPrefixes); + expect(output.map((v, _) => v[1]).reduce((acc, i) => acc + i, 0)).toBe(indices.length); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts index a104ea1d55bf8..f9505428147cd 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts @@ -63,3 +63,91 @@ class Chunked { return this.chunks.filter((chunk) => chunk.length > 0); } } + +export interface QueryConfig { + maxPrefixes: number; + maxGroupSize: number; +} + +interface TrieNode { + char: string; + prefix: string; + children: { [key: string]: TrieNode }; + count: number; + isEnd: boolean; + id: number; +} + +function newTrieNode(char: string = '', prefix: string = '', id: number = 0): TrieNode { + return { + char, + children: {}, + count: 0, + id, + isEnd: false, + prefix, + }; +} + +function* idCounter(): Generator { + let id = 0; + while (true) { + yield id++; + } +} + +export function findCommonPrefixes( + indices: string[], + config: QueryConfig +): Array<[string, number]> { + const idGen = idCounter(); + + const root = newTrieNode('', '', idGen.next().value); + for (const index of indices) { + let node = root; + node.count++; + for (const char of index) { + if (!node.children[char]) { + node.children[char] = newTrieNode(char, node.prefix + char, idGen.next().value); + } + node = node.children[char]; + node.count++; + } + node.isEnd = true; + } + + const nodes = [root]; + const prefixes: Array<[string, number]> = []; + + while (nodes.length > 0) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const node = nodes.pop()!; + if ( + (node.count <= config.maxGroupSize && node.prefix !== '') || + Object.keys(node.children).length === 0 + ) { + prefixes.push([node.prefix, node.count]); + } else { + for (const child of Object.values(node.children)) { + nodes.push(child); + } + } + } + + if (prefixes.length > config.maxPrefixes) { + prefixes.sort((a, b) => a[1] - b[1]); + + while (prefixes.length > config.maxPrefixes) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const [p1, c1] = prefixes.shift()!; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const [p2, c2] = prefixes.shift()!; + const mergedPrefix = `${p1},${p2}`; + const mergedCount = c1 + c2; + prefixes.push([mergedPrefix, mergedCount]); + prefixes.sort((a, b) => a[1] - b[1]); + } + } + + return prefixes; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 1a42a10636176..bf9b2d057d028 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -100,6 +100,7 @@ import type { Index, IndexStats, } from './indices.metadata.types'; +import { type QueryConfig, findCommonPrefixes } from './collections_helpers'; export interface ITelemetryReceiver { start( @@ -254,9 +255,12 @@ export interface ITelemetryReceiver { // ---------- TODO: POC ---------- // getDataStreams(): Promise; - getIndicesStats(indices: string[], pageSize: number): AsyncGenerator; - getIlmsStats(indices: string[], pageSize: number): AsyncGenerator; - getIlmsPolicies(ilms: string[], pageSize: number): AsyncGenerator; + getIndicesStats( + indices: string[], + config: QueryConfig + ): AsyncGenerator; + getIlmsStats(indices: string[], config: QueryConfig): AsyncGenerator; + getIlmsPolicies(ilms: string[], config: QueryConfig): AsyncGenerator; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -1374,98 +1378,102 @@ export class TelemetryReceiver implements ITelemetryReceiver { }); } - public async *getIndicesStats(indices: string[], pageSize: number) { + public async *getIndicesStats(indices: string[], config: QueryConfig) { const es = this.esClient(); this.logger.debug(`Fetching indices stats for ${indices.length} indices`, { indices, } as LogMeta); - const chunked = this._chunk(indices, pageSize) - .map((ilms) => this._getCommonPrefixes(ilms)) - .flat(); + const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v[0].split(',')); - this.logger.debug(`Splitted indices into ${chunked.length} chunks`, { - chunks: chunked.length, + this.logger.debug(`Splitted indices into ${groupedIndices.length} groups`, { + groups: groupedIndices.length, } as LogMeta); - for (const name of chunked) { - this.logger.debug(`Fetching indices stats for ${name}`, { name } as LogMeta); - const request: IndicesStatsRequest = { - index: `${name}*`, - level: 'indices', - metric: ['docs', 'search', 'store'], - expand_wildcards: ['open', 'hidden'], - filter_path: [ - 'indices.*.total.search.query_total', - 'indices.*.total.search.query_time_in_millis', - 'indices.*.total.docs.count', - 'indices.*.total.docs.deleted', - 'indices.*.total.store.size_in_bytes', - ], - }; - - try { - const response = await es.indices.stats(request); - for (const [indexName, stats] of Object.entries(response.indices ?? {})) { - yield { - index_name: indexName, - query_total: stats.total?.search?.query_total, - query_time_in_millis: stats.total?.search?.query_time_in_millis, - docs_count: stats.total?.docs?.count, - docs_deleted: stats.total?.docs?.deleted, - docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, - } as IndexStats; + for (const group of groupedIndices) { + for (const name of group) { + // this.logger.debug(`Fetching indices stats for ${name}`, { name } as LogMeta); + const request: IndicesStatsRequest = { + index: `${name}*`, + level: 'indices', + metric: ['docs', 'search', 'store'], + expand_wildcards: ['open', 'hidden'], + filter_path: [ + 'indices.*.total.search.query_total', + 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', + 'indices.*.total.docs.deleted', + 'indices.*.total.store.size_in_bytes', + ], + }; + + try { + const response = await es.indices.stats(request); + for (const [indexName, stats] of Object.entries(response.indices ?? {})) { + yield { + index_name: indexName, + query_total: stats.total?.search?.query_total, + query_time_in_millis: stats.total?.search?.query_time_in_millis, + docs_count: stats.total?.docs?.count, + docs_deleted: stats.total?.docs?.deleted, + docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + } as IndexStats; + } + } catch (error) { + this.logger.warn('Error fetching indices stats', { error_message: error } as LogMeta); + throw error; } - } catch (error) { - this.logger.warn('Error fetching indices stats', { error_message: error } as LogMeta); - throw error; } } } - public async *getIlmsStats(indices: string[], pageSize: number) { + public async *getIlmsStats(indices: string[], config: QueryConfig) { const es = this.esClient(); this.logger.debug(`Fetching ilm stats for ${indices.length} indices`, { indices } as LogMeta); - const chunked = this._chunk(indices, pageSize) - .map((ilms) => this._getCommonPrefixes(ilms)) - .flat(); - - for (const name of chunked) { - this.logger.debug(`Fetching ilm stats for ${name}`, { name } as LogMeta); - const request: IlmExplainLifecycleRequest = { - index: `${name}*`, - only_managed: false, - filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], - }; + const groupedIndices: string[][] = findCommonPrefixes(indices, config).map((v, _) => + v[0].split(',') + ); - const data = await es.ilm.explainLifecycle(request); + this.logger.debug(`Splitted indices into ${groupedIndices.length} groups`, { + groups: groupedIndices.length, + } as LogMeta); - try { - for (const [indexName, stats] of Object.entries(data.indices ?? {})) { - const entry = { - index_name: indexName, - phase: ('phase' in stats && stats.phase) || undefined, - age: ('age' in stats && stats.age) || undefined, - policy_name: ('policy' in stats && stats.policy) || undefined, - } as IlmStats; - - yield entry; + for (const group of groupedIndices) { + for (const name of group) { + // this.logger.debug(`Fetching ilm stats for ${name}`, { name } as LogMeta); + const request: IlmExplainLifecycleRequest = { + index: `${name}*`, + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + const data = await es.ilm.explainLifecycle(request); + + try { + for (const [indexName, stats] of Object.entries(data.indices ?? {})) { + const entry = { + index_name: indexName, + phase: ('phase' in stats && stats.phase) || undefined, + age: ('age' in stats && stats.age) || undefined, + policy_name: ('policy' in stats && stats.policy) || undefined, + } as IlmStats; + + yield entry; + } + } catch (error) { + this.logger.warn('Error fetching ilm stats', { error_message: error } as LogMeta); + throw error; } - } catch (error) { - this.logger.warn('Error fetching ilm stats', { error_message: error } as LogMeta); - throw error; } } } - public async *getIlmsPolicies(ilms: string[], pageSize: number) { + public async *getIlmsPolicies(ilms: string[], config: QueryConfig) { const es = this.esClient(); - this.logger.debug(`Fetching ilms policies for ${ilms.length} ilms`, { ilms } as LogMeta); - const phase = (obj: unknown): Nullable => { let value: Nullable; if (obj !== null && obj !== undefined && typeof obj === 'object' && 'min_age' in obj) { @@ -1476,95 +1484,49 @@ export class TelemetryReceiver implements ITelemetryReceiver { return value; }; - const chunked = this._chunk(ilms, pageSize).map(this._getCommonPrefixes).flat(); - - this.logger.debug(`Splitted ilms into ${chunked.length} chunks`, { - chunks: chunked.length, - } as LogMeta); - - for (const name of chunked) { - this.logger.debug(`Fetching ilm policies for ${name}`, { name } as LogMeta); - const request: IlmGetLifecycleRequest = { - name: `${name}*`, - filter_path: [ - '*.policy.phases.cold.min_age', - '*.policy.phases.delete.min_age', - '*.policy.phases.frozen.min_age', - '*.policy.phases.hot.min_age', - '*.policy.phases.warm.min_age', - '*.modified_date', - ], - }; - - const response = await es.ilm.getLifecycle(request); - try { - for (const [policyName, stats] of Object.entries(response ?? {})) { - yield { - policy_name: policyName, - modified_date: stats.modified_date, - phases: { - cold: phase(stats.policy.phases.cold), - delete: phase(stats.policy.phases.delete), - frozen: phase(stats.policy.phases.frozen), - hot: phase(stats.policy.phases.hot), - warm: phase(stats.policy.phases.warm), - } as IlmPhases, - } as IlmPolicy; - } - } catch (error) { - this.logger.warn('Error fetching ilm policies', { error_message: error } as LogMeta); - throw error; - } - } - } - - // very basic implementation of a common prefix finder - // TODO: add tests to cover edge cases and check performance - private _getCommonPrefixes(names: string[], minLength: number = 10): string[] { - if (names.length === 0) return []; - - names.sort(); - const result: string[] = []; - - let currentPrefix = names[0]; + this.logger.debug(`Fetching ilms policies for ${ilms.length} ilms`, { ilms } as LogMeta); - for (let i = 1; i < names.length; i++) { - const name = names[i]; - let commonPrefix = ''; + const groupedIlms = findCommonPrefixes(ilms, config).map((v, _) => v[0].split(',')); - // Find the common prefix between currentPrefix and the current name - for (let j = 0; j < Math.min(currentPrefix.length, name.length); j++) { - if (currentPrefix[j] === name[j]) { - commonPrefix += currentPrefix[j]; - } else { - break; - } - } + this.logger.debug(`Splitted ilms into ${groupedIlms.length} groups`, { + groups: groupedIlms.length, + } as LogMeta); - // Check if the common prefix is long enough - if (commonPrefix.length >= minLength) { - currentPrefix = commonPrefix; - } else { - if (currentPrefix.length >= minLength) { - result.push(currentPrefix); // Save the last valid prefix + for (const group of groupedIlms) { + for (const name of group) { + this.logger.debug(`Fetching ilm policies for ${name}`, { name } as LogMeta); + const request: IlmGetLifecycleRequest = { + name: `${name}*`, + filter_path: [ + '*.policy.phases.cold.min_age', + '*.policy.phases.delete.min_age', + '*.policy.phases.frozen.min_age', + '*.policy.phases.hot.min_age', + '*.policy.phases.warm.min_age', + '*.modified_date', + ], + }; + + const response = await es.ilm.getLifecycle(request); + try { + for (const [policyName, stats] of Object.entries(response ?? {})) { + yield { + policy_name: policyName, + modified_date: stats.modified_date, + phases: { + cold: phase(stats.policy.phases.cold), + delete: phase(stats.policy.phases.delete), + frozen: phase(stats.policy.phases.frozen), + hot: phase(stats.policy.phases.hot), + warm: phase(stats.policy.phases.warm), + } as IlmPhases, + } as IlmPolicy; + } + } catch (error) { + this.logger.warn('Error fetching ilm policies', { error_message: error } as LogMeta); + throw error; } - currentPrefix = name; // Start a new prefix } } - - // Add the final prefix if it has at least N characters - if (currentPrefix.length >= minLength) { - result.push(currentPrefix); - } - - return result; - } - - private _chunk(arr: string[], size: number): string[][] { - return arr.reduce( - (result: string[][], _, index) => - index % size === 0 ? [...result, arr.slice(index, index + size)] : result, - [] - ); } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts index 74eaf650e5629..70fbfbcdec546 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -23,8 +23,8 @@ export const getTriggerIndicesMetadataTaskRoute = ( path: '/internal/trigger-indices-metadata-task', validate: { query: schema.object({ - pageSize: schema.maybe(schema.number()), - dataStreamsLimit: schema.maybe(schema.number()), + maxPrefixes: schema.maybe(schema.number()), + maxGroupSize: schema.maybe(schema.number()), }), }, }, @@ -33,10 +33,10 @@ export const getTriggerIndicesMetadataTaskRoute = ( const task = createTelemetryIndicesMetadataTaskConfig(); const timeStart = performance.now(); - const { pageSize, dataStreamsLimit } = request.query; + const { maxPrefixes, maxGroupSize } = request.query; logger.info( - `Triggering indices metadata task with pageSize: ${pageSize} and dataStreamsLimit: ${dataStreamsLimit}` + `Triggering indices metadata task with pageSize: ${maxPrefixes} and dataStreamsLimit: ${maxGroupSize}` ); let msgSuffix = ''; @@ -47,8 +47,8 @@ export const getTriggerIndicesMetadataTaskRoute = ( } const initialMemory = process.memoryUsage().heapUsed; const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { - last: `${pageSize || 500}`, - current: `${dataStreamsLimit || 500}`, + last: `${maxPrefixes || 10}`, + current: `${maxGroupSize || 100}`, }); const memoryUsed = process.memoryUsage().heapUsed - initialMemory; const elapsedTime = performance.now() - timeStart; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 6f476050edb8d..271a839cc663d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -17,6 +17,7 @@ import { TELEMETRY_ILM_STATS_EVENT, TELEMETRY_INDEX_STATS_EVENT, } from '../event_based/events'; +import type { QueryConfig } from '../collections_helpers'; export function createTelemetryIndicesMetadataTaskConfig() { const taskType = 'security:indices-metadata-telemetry'; @@ -40,23 +41,28 @@ export function createTelemetryIndicesMetadataTaskConfig() { const trace = taskMetricsService.start(taskType); // TODO: Taken from taskExecutionPeriod, just for testing purposes - const pageSize = Number(taskExecutionPeriod.last ?? '500'); - const dataStreamsLimit = Number(taskExecutionPeriod.current ?? '500'); + const config: QueryConfig = { + maxPrefixes: Number(taskExecutionPeriod.last ?? '10'), + maxGroupSize: Number(taskExecutionPeriod.current ?? '100'), + }; + const streamLimit = 100; + const indicesLimit = 300; try { // 1. Get all data streams - const dataStreams = (await receiver.getDataStreams()).slice(0, dataStreamsLimit + 1); + const dataStreams = (await receiver.getDataStreams()).slice(0, streamLimit); // and calculate index and ilm names const dsNames = dataStreams.map((stream) => stream.datastream_name); - const indexNames = dataStreams - .map((ds) => ds.indices?.map((i) => i.index_name) ?? []) - .flat(); const ilmsNames = dataStreams .map((ds) => ds.indices?.filter((i) => i.ilm_policy !== undefined)?.map((i) => i.ilm_policy) ) - .flat() as string[]; + .flat() + .slice(0, indicesLimit) as string[]; + const indexNames = dataStreams + .map((ds) => ds.indices?.map((i) => i.index_name) ?? []) + .flat(); log.info(`Got data streams`, { datastreams: dsNames.length, @@ -69,13 +75,13 @@ export function createTelemetryIndicesMetadataTaskConfig() { let ilmsCount = 0; let dsCount = 0; - for await (const stat of receiver.getIndicesStats(dsNames, pageSize)) { + for await (const stat of receiver.getIndicesStats(dsNames, config)) { sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, stat); indicesCount++; } log.info(`Sent ${indicesCount} indices stats`, { indicesCount } as LogMeta); - for await (const stat of receiver.getIlmsStats(indexNames, pageSize)) { + for await (const stat of receiver.getIlmsStats(indexNames, config)) { sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); ilmsCount++; } @@ -87,7 +93,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { } log.info(`Sent ${dsCount} data streams`, { dsCount } as LogMeta); - for await (const policy of receiver.getIlmsPolicies(ilmsNames, pageSize)) { + for await (const policy of receiver.getIlmsPolicies(ilmsNames, config)) { sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, policy); policyCount++; } From 5a7092ca91a88ff43c673d5c5881a8f3015024cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 30 Sep 2024 14:24:29 +0200 Subject: [PATCH 06/44] Code style --- .../lib/telemetry/collections_helpers.test.ts | 14 +++++--- .../lib/telemetry/collections_helpers.ts | 34 ++++++++++++------- .../server/lib/telemetry/receiver.ts | 8 ++--- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts index da9976599c96f..37e1cb2dd65dd 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts @@ -92,8 +92,12 @@ describe('telemetry.utils.findCommonPrefixes', () => { const output = findCommonPrefixes(indices, config); expect(output).toHaveLength(2); - expect(output.find((v, _) => v[0] === 'b' && v[1] === 1)).not.toBeFalsy(); - expect(output.find((v, _) => v[0] === 'a' && v[1] === 2)).not.toBeFalsy(); + expect(output.find((v, _) => v.parts.length === 1 && v.parts[0] === 'a')?.indexCount).toEqual( + 2 + ); + expect(output.find((v, _) => v.parts.length === 1 && v.parts[0] === 'b')?.indexCount).toEqual( + 1 + ); }); it('should discard extra indices', async () => { @@ -106,7 +110,9 @@ describe('telemetry.utils.findCommonPrefixes', () => { const output = findCommonPrefixes(indices, config); expect(output).toHaveLength(1); - expect(output.find((v, _) => v[0] === 'aaa' && v[1] === 2)).not.toBeFalsy(); + expect(output.find((v, _) => v.parts.length === 1 && v.parts[0] === 'aaa')?.indexCount).toEqual( + 2 + ); }); it('should group many indices', async () => { @@ -119,6 +125,6 @@ describe('telemetry.utils.findCommonPrefixes', () => { const output = findCommonPrefixes(indices, config); expect(output).toHaveLength(config.maxPrefixes); - expect(output.map((v, _) => v[1]).reduce((acc, i) => acc + i, 0)).toBe(indices.length); + expect(output.map((v, _) => v.indexCount).reduce((acc, i) => acc + i, 0)).toBe(indices.length); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts index f9505428147cd..4ad39d8095041 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts @@ -96,10 +96,12 @@ function* idCounter(): Generator { } } -export function findCommonPrefixes( - indices: string[], - config: QueryConfig -): Array<[string, number]> { +interface Group { + parts: string[]; + indexCount: number; +} + +export function findCommonPrefixes(indices: string[], config: QueryConfig): Group[] { const idGen = idCounter(); const root = newTrieNode('', '', idGen.next().value); @@ -117,7 +119,7 @@ export function findCommonPrefixes( } const nodes = [root]; - const prefixes: Array<[string, number]> = []; + const prefixes: Group[] = []; while (nodes.length > 0) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -126,7 +128,11 @@ export function findCommonPrefixes( (node.count <= config.maxGroupSize && node.prefix !== '') || Object.keys(node.children).length === 0 ) { - prefixes.push([node.prefix, node.count]); + const group: Group = { + parts: [node.prefix], + indexCount: node.count, + }; + prefixes.push(group); } else { for (const child of Object.values(node.children)) { nodes.push(child); @@ -135,17 +141,19 @@ export function findCommonPrefixes( } if (prefixes.length > config.maxPrefixes) { - prefixes.sort((a, b) => a[1] - b[1]); + prefixes.sort((a, b) => a.indexCount - b.indexCount); while (prefixes.length > config.maxPrefixes) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const [p1, c1] = prefixes.shift()!; + const g1 = prefixes.shift()!; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const [p2, c2] = prefixes.shift()!; - const mergedPrefix = `${p1},${p2}`; - const mergedCount = c1 + c2; - prefixes.push([mergedPrefix, mergedCount]); - prefixes.sort((a, b) => a[1] - b[1]); + const g2 = prefixes.shift()!; + const mergedGroup: Group = { + parts: g1.parts.concat(g2.parts), + indexCount: g1.indexCount + g2.indexCount, + }; + prefixes.push(mergedGroup); + prefixes.sort((a, b) => a.indexCount - b.indexCount); } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index bf9b2d057d028..c38ce17d99e63 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -1385,7 +1385,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { indices, } as LogMeta); - const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v[0].split(',')); + const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); this.logger.debug(`Splitted indices into ${groupedIndices.length} groups`, { groups: groupedIndices.length, @@ -1433,9 +1433,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { this.logger.debug(`Fetching ilm stats for ${indices.length} indices`, { indices } as LogMeta); - const groupedIndices: string[][] = findCommonPrefixes(indices, config).map((v, _) => - v[0].split(',') - ); + const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); this.logger.debug(`Splitted indices into ${groupedIndices.length} groups`, { groups: groupedIndices.length, @@ -1486,7 +1484,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { this.logger.debug(`Fetching ilms policies for ${ilms.length} ilms`, { ilms } as LogMeta); - const groupedIlms = findCommonPrefixes(ilms, config).map((v, _) => v[0].split(',')); + const groupedIlms = findCommonPrefixes(ilms, config).map((v, _) => v.parts); this.logger.debug(`Splitted ilms into ${groupedIlms.length} groups`, { groups: groupedIlms.length, From ee68d1a32b7096cee2c27ded4781e024b7b3fab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 1 Oct 2024 18:40:51 +0200 Subject: [PATCH 07/44] Fix lint error --- .../server/lib/telemetry/preview_sender.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts index fc195098787dd..9d4e0450a7570 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts @@ -7,7 +7,7 @@ import type { AxiosInstance, AxiosResponse } from 'axios'; import axios, { AxiosHeaders } from 'axios'; -import type { Logger } from '@kbn/core/server'; +import type { EventType, Logger } from '@kbn/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; @@ -174,4 +174,8 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { public updateDefaultQueueConfig(config: QueueConfig): void { this.composite.updateDefaultQueueConfig(config); } + + public reportEBT(eventType: EventType, eventData: object): void { + this.composite.reportEBT(eventType, eventData); + } } From ba8ca5f7b23d2665fb1521056f80a4d28b762895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 1 Oct 2024 19:10:51 +0200 Subject: [PATCH 08/44] Fix failing test --- .../security_solution/server/lib/telemetry/task.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 91cb7d881d4c9..7efb2ea1297a6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -16,6 +16,7 @@ import { createMockTaskMetrics, createMockSecurityTelemetryTask, } from './__mocks__'; +import { newTelemetryLogger } from './helpers'; describe('test security telemetry task', () => { let logger: ReturnType; @@ -66,7 +67,7 @@ describe('test security telemetry task', () => { expect(mockTelemetryTaskConfig.runTask).toHaveBeenCalledWith( telemetryTask.getTaskId(), - logger, + newTelemetryLogger(logger.get('task')), mockTelemetryReceiver, mockTelemetryEventsSender, mockTaskMetrics, From bb5e85400f8c9de377c47b836ff635f5f655ca68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 1 Oct 2024 19:15:34 +0200 Subject: [PATCH 09/44] Fix failing test --- .../test_suites/task_manager/check_registered_task_types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 55856f3c80402..4c7d7fc88ff91 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -159,6 +159,7 @@ export default function ({ getService }: FtrProviderContext) { 'security-solution-ea-asset-criticality-ecs-migration', 'security:endpoint-diagnostics', 'security:endpoint-meta-telemetry', + 'security:indices-metadata-telemetry', 'security:telemetry-configuration', 'security:telemetry-detection-rules', 'security:telemetry-diagnostic-timelines', From 0f7707dbe29f5c15b62c3a154a93b85a6cc82144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 2 Oct 2024 15:49:14 +0200 Subject: [PATCH 10/44] Update logging --- .../security_solution/server/lib/telemetry/task.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 96cb74dc24ae6..55c83243adcbc 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -123,7 +123,7 @@ export class SecurityTelemetryTask { public start = async (taskManager: TaskManagerStartContract) => { const taskId = this.getTaskId(); - this.logger.l('Attempting to schedule task', { taskId }); + this.logger.debug('Attempting to schedule task', { taskId } as LogMeta); try { await taskManager.ensureScheduled({ id: taskId, @@ -143,25 +143,25 @@ export class SecurityTelemetryTask { }; public runTask = async (taskId: string, executionPeriod: TaskExecutionPeriod) => { - this.logger.l('Attempting to run', { taskId }); + this.logger.debug('Attempting to run', { taskId } as LogMeta); if (taskId !== this.getTaskId()) { - this.logger.l('outdated task', { taskId }); + this.logger.info('outdated task', { taskId } as LogMeta); return 0; } const isOptedIn = await this.sender.isTelemetryOptedIn(); if (!isOptedIn) { - this.logger.l('Telemetry is not opted-in', { taskId }); + this.logger.info('Telemetry is not opted-in', { taskId } as LogMeta); return 0; } const isTelemetryServicesReachable = await this.sender.isTelemetryServicesReachable(); if (!isTelemetryServicesReachable) { - this.logger.l('Cannot reach telemetry services', { taskId }); + this.logger.info('Cannot reach telemetry services', { taskId } as LogMeta); return 0; } - this.logger.l('Running task', { taskId }); + this.logger.debug('Running task', { taskId } as LogMeta); return this.config.runTask( taskId, this.logger, From 90de95080ac5a61d996110ea84a015d83cc4e620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 3 Oct 2024 12:21:23 +0200 Subject: [PATCH 11/44] make testing api public --- .../security_solution/server/lib/telemetry/routes/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts index 70fbfbcdec546..3cc31bfd5669e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -20,7 +20,12 @@ export const getTriggerIndicesMetadataTaskRoute = ( ) => { router.get( { - path: '/internal/trigger-indices-metadata-task', + path: '/api/trigger-indices-metadata-task', + options: { + tags: ['api'], + access: 'public', + summary: 'Trigger indices metadata task (for testing purposes)', + }, validate: { query: schema.object({ maxPrefixes: schema.maybe(schema.number()), From e0c2ca2558e86dc70b574c62f24261168ec79647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 3 Oct 2024 15:08:33 +0200 Subject: [PATCH 12/44] Add task configuration --- .../server/lib/telemetry/configuration.ts | 24 ++++++++++++++++- .../lib/telemetry/tasks/configuration.ts | 5 ++++ .../lib/telemetry/tasks/indices.metadata.ts | 26 +++++++++++-------- .../server/lib/telemetry/types.ts | 8 ++++++ 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts index 691b0daae8f2c..6b715cafd0b5e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts @@ -6,7 +6,11 @@ */ import os from 'os'; -import type { PaginationConfiguration, TelemetrySenderChannelConfiguration } from './types'; +import type { + IndicesMetadataConfiguration, + PaginationConfiguration, + TelemetrySenderChannelConfiguration, +} from './types'; class TelemetryConfigurationDTO { private readonly DEFAULT_TELEMETRY_MAX_BUFFER_SIZE = 100; @@ -21,6 +25,13 @@ class TelemetryConfigurationDTO { max_page_size_bytes: Math.min(os.totalmem() * 0.02, 80 * 1024 * 1024), num_docs_to_sample: 10, }; + private readonly DEFAULT_INDICES_METADATA_CONFIG = { + indices_threshold: 15000, + datastreams_threshold: 1000, + max_prefixes: 10, + max_group_size: 100, + }; + private _telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; private _max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH; private _max_endpoint_telemetry_batch = this.DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH; @@ -31,6 +42,8 @@ class TelemetryConfigurationDTO { [key: string]: TelemetrySenderChannelConfiguration; } = this.DEFAULT_SENDER_CHANNELS; private _pagination_config: PaginationConfiguration = this.DEFAULT_PAGINATION_CONFIG; + private _indices_metadata_config: IndicesMetadataConfiguration = + this.DEFAULT_INDICES_METADATA_CONFIG; public get telemetry_max_buffer_size(): number { return this._telemetry_max_buffer_size; @@ -96,6 +109,14 @@ class TelemetryConfigurationDTO { return this._pagination_config; } + public set indices_metadata_config(paginationConfiguration: IndicesMetadataConfiguration) { + this._indices_metadata_config = paginationConfiguration; + } + + public get indices_metadata_config(): IndicesMetadataConfiguration { + return this._indices_metadata_config; + } + public resetAllToDefault() { this._telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; this._max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH; @@ -104,6 +125,7 @@ class TelemetryConfigurationDTO { this._max_detection_alerts_batch = this.DEFAULT_MAX_DETECTION_ALERTS_BATCH; this._sender_channels = this.DEFAULT_SENDER_CHANNELS; this._pagination_config = this.DEFAULT_PAGINATION_CONFIG; + this._indices_metadata_config = this.DEFAULT_INDICES_METADATA_CONFIG; } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts index 0558971ebf1e3..1b3f888761115 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts @@ -107,6 +107,11 @@ export function createTelemetryConfigurationTaskConfig() { _receiver.setNumDocsToSample(configArtifact.pagination_config.num_docs_to_sample); } + if (configArtifact.indices_metadata_config) { + log.l('Updating indices metadata configuration'); + telemetryConfiguration.indices_metadata_config = configArtifact.indices_metadata_config; + } + await taskMetricsService.end(trace); log.l('Updated TelemetryConfiguration', { configuration: telemetryConfiguration }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 271a839cc663d..8bd32148ca03e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -18,6 +18,7 @@ import { TELEMETRY_INDEX_STATS_EVENT, } from '../event_based/events'; import type { QueryConfig } from '../collections_helpers'; +import { telemetryConfiguration } from '../configuration'; export function createTelemetryIndicesMetadataTaskConfig() { const taskType = 'security:indices-metadata-telemetry'; @@ -40,17 +41,20 @@ export function createTelemetryIndicesMetadataTaskConfig() { const log = newTelemetryLogger(logger.get('indices-metadata'), mdc); const trace = taskMetricsService.start(taskType); - // TODO: Taken from taskExecutionPeriod, just for testing purposes - const config: QueryConfig = { - maxPrefixes: Number(taskExecutionPeriod.last ?? '10'), - maxGroupSize: Number(taskExecutionPeriod.current ?? '100'), + const taskConfig = telemetryConfiguration.indices_metadata_config; + + // TODO: not use taskExecutionPeriod, it's just to test the task using the temporary API + const queryConfig: QueryConfig = { + maxPrefixes: Number(taskExecutionPeriod.last ?? taskConfig.max_prefixes), + maxGroupSize: Number(taskExecutionPeriod.current ?? taskConfig.max_group_size), }; - const streamLimit = 100; - const indicesLimit = 300; try { // 1. Get all data streams - const dataStreams = (await receiver.getDataStreams()).slice(0, streamLimit); + const dataStreams = (await receiver.getDataStreams()).slice( + 0, + taskConfig.datastreams_threshold + ); // and calculate index and ilm names const dsNames = dataStreams.map((stream) => stream.datastream_name); @@ -59,7 +63,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { ds.indices?.filter((i) => i.ilm_policy !== undefined)?.map((i) => i.ilm_policy) ) .flat() - .slice(0, indicesLimit) as string[]; + .slice(0, taskConfig.indices_threshold) as string[]; const indexNames = dataStreams .map((ds) => ds.indices?.map((i) => i.index_name) ?? []) .flat(); @@ -75,13 +79,13 @@ export function createTelemetryIndicesMetadataTaskConfig() { let ilmsCount = 0; let dsCount = 0; - for await (const stat of receiver.getIndicesStats(dsNames, config)) { + for await (const stat of receiver.getIndicesStats(dsNames, queryConfig)) { sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, stat); indicesCount++; } log.info(`Sent ${indicesCount} indices stats`, { indicesCount } as LogMeta); - for await (const stat of receiver.getIlmsStats(indexNames, config)) { + for await (const stat of receiver.getIlmsStats(indexNames, queryConfig)) { sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); ilmsCount++; } @@ -93,7 +97,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { } log.info(`Sent ${dsCount} data streams`, { dsCount } as LogMeta); - for await (const policy of receiver.getIlmsPolicies(ilmsNames, config)) { + for await (const policy of receiver.getIlmsPolicies(ilmsNames, queryConfig)) { sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, policy); policyCount++; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index eafe04c96a5ab..662e04146d5bb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -465,6 +465,14 @@ export interface TelemetryConfiguration { [key: string]: TelemetrySenderChannelConfiguration; }; pagination_config?: PaginationConfiguration; + indices_metadata_config?: IndicesMetadataConfiguration; +} + +export interface IndicesMetadataConfiguration { + indices_threshold: number; + datastreams_threshold: number; + max_prefixes: number; + max_group_size: number; } export interface PaginationConfiguration { From 8aa3c9c827e5564fd2fbc3441d022becb941b20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 11 Oct 2024 12:06:57 +0200 Subject: [PATCH 13/44] Send cluster stats and compute all indices --- .../lib/telemetry/event_based/events.ts | 38 +++++++++- .../lib/telemetry/indices.metadata.types.ts | 8 +++ .../server/lib/telemetry/receiver.ts | 41 +++++++++++ .../server/lib/telemetry/routes/index.ts | 2 +- .../lib/telemetry/tasks/indices.metadata.ts | 72 +++++++++---------- 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts index b6e7732d01b86..0d4d4dd8b546f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -11,7 +11,13 @@ import type { ResponseActionsApiCommandNames, } from '../../../../common/endpoint/service/response_actions/constants'; import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics'; -import type { DataStream, IlmPolicy, IlmStats, IndexStats } from '../indices.metadata.types'; +import type { + ClusterStats, + DataStream, + IlmPolicy, + IlmStats, + IndexStats, +} from '../indices.metadata.types'; export const RISK_SCORE_EXECUTION_SUCCESS_EVENT: EventTypeOpts<{ scoresWritten: number; @@ -209,6 +215,35 @@ export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ }, }; +export const TELEMETRY_CLUSTER_STATS_EVENT: EventTypeOpts = { + eventType: 'telemetry_cluster_stats_event', + schema: { + num_nodes: { + type: 'long', + _meta: { description: 'How many nodes have the cluster' }, + }, + num_indices: { + type: 'long', + _meta: { description: 'How many indices have the cluster' }, + }, + num_docs: { + type: 'long', + _meta: { description: 'How many nodes have the cluster' }, + }, + num_deleted_docs: { + type: 'long', + _meta: { + description: 'How many nodes have the cluster', + optional: true, + }, + }, + total_size_in_bytes: { + type: 'long', + _meta: { description: 'The total size, in bytes, of all documents stored in the cluster.' }, + }, + }, +}; + export const TELEMETRY_DATA_STREAM_EVENT: EventTypeOpts = { eventType: 'telemetry_data_stream_event', schema: { @@ -584,6 +619,7 @@ export const events = [ ENDPOINT_RESPONSE_ACTION_SENT_EVENT, ENDPOINT_RESPONSE_ACTION_SENT_ERROR_EVENT, ENDPOINT_RESPONSE_ACTION_STATUS_CHANGE_EVENT, + TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, TELEMETRY_ILM_POLICY_EVENT, TELEMETRY_ILM_STATS_EVENT, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts index 932015d9bc575..77a0452728f3c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts @@ -42,6 +42,14 @@ export interface IndexStats { docs_total_size_in_bytes?: number; } +export interface ClusterStats { + num_nodes: number; + num_indices: number; + num_docs: number; + num_deleted_docs?: number; + total_size_in_bytes: number; +} + export interface Index { index_name: string; ilm_policy?: string; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index c38ce17d99e63..6aedfece6662a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -42,6 +42,7 @@ import type { IndicesGetDataStreamRequest, IndicesStatsRequest, IlmGetLifecycleRequest, + CatIndicesRequest, } from '@elastic/elasticsearch/lib/api/types'; import type { TransportResult } from '@elastic/elasticsearch'; import type { AgentPolicy, Installation } from '@kbn/fleet-plugin/common'; @@ -92,6 +93,7 @@ import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/co import { DEFAULT_DIAGNOSTIC_INDEX } from './constants'; import type { TelemetryLogger } from './telemetry_logger'; import type { + ClusterStats, DataStream, IlmPhase, IlmPhases, @@ -254,6 +256,8 @@ export interface ITelemetryReceiver { setNumDocsToSample(n: number): void; // ---------- TODO: POC ---------- // + getClusterStats(): Promise; + getIndices(): Promise; getDataStreams(): Promise; getIndicesStats( indices: string[], @@ -1345,6 +1349,43 @@ export class TelemetryReceiver implements ITelemetryReceiver { } // ---------- TODO: POC ---------- // + public async getClusterStats(): Promise { + const es = this.esClient(); + + this.logger.debug('Fetching cluster stats'); + + return es.cluster.stats({}).then( + (response) => + ({ + num_nodes: response.nodes.count.total, + num_indices: response.indices.count, + num_docs: response.indices.docs.count, + num_deleted_docs: response.indices.docs.deleted, + total_size_in_bytes: response.indices.store.size_in_bytes, + } as ClusterStats) + ); + } + + public async getIndices(): Promise { + const es = this.esClient(); + + this.logger.debug('Fetching indices'); + + const request: CatIndicesRequest = { + expand_wildcards: ['open', 'hidden'], + filter_path: ['index'], + format: 'json', + }; + + return es.cat + .indices(request) + .then((indices) => indices.map(({ index }) => index as string)) + .catch((error) => { + this.logger.warn('Error fetching indices', { error_message: error } as LogMeta); + throw error; + }); + } + public async getDataStreams(): Promise { const es = this.esClient(); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts index 3cc31bfd5669e..aca860d16db32 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -60,7 +60,7 @@ export const getTriggerIndicesMetadataTaskRoute = ( return response.ok({ body: { - message: `Task processed ${result} datastreams. It took ${elapsedTime} ms to run and required ${memoryUsed} bytes ${msgSuffix}`, + message: `Task finished, it processed ${result} indices, took ${elapsedTime} ms to run and required ${memoryUsed} bytes ${msgSuffix} [ v1.1 ]`, }, }); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 8bd32148ca03e..4bd9842f0fff8 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -12,6 +12,7 @@ import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; import { getPreviousDailyTaskTimestamp, newTelemetryLogger } from '../helpers'; import { + TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, TELEMETRY_ILM_POLICY_EVENT, TELEMETRY_ILM_STATS_EVENT, @@ -50,54 +51,53 @@ export function createTelemetryIndicesMetadataTaskConfig() { }; try { - // 1. Get all data streams - const dataStreams = (await receiver.getDataStreams()).slice( - 0, - taskConfig.datastreams_threshold - ); - - // and calculate index and ilm names - const dsNames = dataStreams.map((stream) => stream.datastream_name); - const ilmsNames = dataStreams - .map((ds) => - ds.indices?.filter((i) => i.ilm_policy !== undefined)?.map((i) => i.ilm_policy) - ) - .flat() - .slice(0, taskConfig.indices_threshold) as string[]; - const indexNames = dataStreams - .map((ds) => ds.indices?.map((i) => i.index_name) ?? []) - .flat(); - - log.info(`Got data streams`, { - datastreams: dsNames.length, - ilms: ilmsNames.length, - indices: indexNames.length, - } as LogMeta); - let policyCount = 0; let indicesCount = 0; let ilmsCount = 0; let dsCount = 0; - for await (const stat of receiver.getIndicesStats(dsNames, queryConfig)) { + // 1. Get cluster stats and list of indices and datastreams + const [clusterStats, indices, dataStreams] = await Promise.all([ + receiver.getClusterStats(), + receiver.getIndices(), + receiver.getDataStreams(), + ]); + + sender.reportEBT(TELEMETRY_CLUSTER_STATS_EVENT.eventType, clusterStats); + + // 2. Publish datastreams stats + for (const ds of dataStreams.slice(0, taskConfig.datastreams_threshold)) { + sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); + dsCount++; + } + log.info(`Sent ${dsCount} data streams`, { dsCount } as LogMeta); + + // 3. Get and publish indices stats + for await (const stat of receiver.getIndicesStats( + indices.slice(0, taskConfig.indices_threshold), + queryConfig + )) { sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, stat); indicesCount++; } log.info(`Sent ${indicesCount} indices stats`, { indicesCount } as LogMeta); - for await (const stat of receiver.getIlmsStats(indexNames, queryConfig)) { - sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); - ilmsCount++; + // 4. Get ILM stats and publish them + const ilmNames = new Set(); + for await (const stat of receiver.getIlmsStats(indices, queryConfig)) { + if (stat.policy_name !== undefined) { + ilmNames.add(stat.policy_name); + sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); + ilmsCount++; + } } log.info(`Sent ${ilmsCount} ILM stats`, { ilmsCount } as LogMeta); - for (const ds of dataStreams) { - sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); - dsCount++; - } - log.info(`Sent ${dsCount} data streams`, { dsCount } as LogMeta); - - for await (const policy of receiver.getIlmsPolicies(ilmsNames, queryConfig)) { + // 5. Publish ILM policies + for await (const policy of receiver.getIlmsPolicies( + Array.from(ilmNames.values()), + queryConfig + )) { sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, policy); policyCount++; } @@ -110,7 +110,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { policies: policyCount, } as LogMeta); - return policyCount + indicesCount + ilmsCount + dsCount; + return indicesCount; } catch (err) { log.warn(`Error running indices metadata task`, { error: err.message, From 5846a2a23edaf865683ac9f6619190f2ef2741d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 17 Oct 2024 16:13:49 +0200 Subject: [PATCH 14/44] Add filters option to ftr_helper api --- .../analytics_ftr_helpers/server/plugin.ts | 23 +++++++++++++++---- test/analytics/services/kibana_ebt.ts | 12 +++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/test/analytics/plugins/analytics_ftr_helpers/server/plugin.ts b/test/analytics/plugins/analytics_ftr_helpers/server/plugin.ts index d4bf140aa4e62..f599f5651374b 100644 --- a/test/analytics/plugins/analytics_ftr_helpers/server/plugin.ts +++ b/test/analytics/plugins/analytics_ftr_helpers/server/plugin.ts @@ -14,7 +14,7 @@ import { fetchEvents } from '../common/fetch_events'; import { CustomShipper } from './custom_shipper'; export class AnalyticsFTRHelpers implements Plugin { - public setup({ analytics, http }: CoreSetup, deps: {}) { + public setup({ analytics, http }: CoreSetup, _deps: {}) { const { optIn, registerShipper } = analytics; const events$ = new ReplaySubject(); @@ -31,7 +31,7 @@ export class AnalyticsFTRHelpers implements Plugin { }), }, }, - (context, req, res) => { + (_context, req, res) => { const { consent } = req.query; optIn({ global: { enabled: consent } }); @@ -67,7 +67,7 @@ export class AnalyticsFTRHelpers implements Plugin { }), }, }, - async (context, req, res) => { + async (_context, req, res) => { const { takeNumberOfEvents, ...options } = req.query; const events = await fetchEvents(events$, takeNumberOfEvents, options); return res.ok({ body: events }); @@ -82,10 +82,25 @@ export class AnalyticsFTRHelpers implements Plugin { eventTypes: schema.arrayOf(schema.string()), withTimeoutMs: schema.number(), fromTimestamp: schema.maybe(schema.string()), + filters: schema.maybe( + schema.recordOf( + schema.string(), + schema.recordOf( + schema.oneOf([ + schema.literal('eq'), + schema.literal('gte'), + schema.literal('gt'), + schema.literal('lte'), + schema.literal('lt'), + ]), + schema.any() + ) + ) + ), }), }, }, - async (context, req, res) => { + async (_context, req, res) => { const events = await fetchEvents(events$, Number.MAX_SAFE_INTEGER, req.query); return res.ok({ body: { count: events.length } }); } diff --git a/test/analytics/services/kibana_ebt.ts b/test/analytics/services/kibana_ebt.ts index f2b635dcbdbbe..925bdece221d3 100644 --- a/test/analytics/services/kibana_ebt.ts +++ b/test/analytics/services/kibana_ebt.ts @@ -28,7 +28,7 @@ export function KibanaEBTServerProvider({ getService }: FtrProviderContext): EBT setOptIn, getEvents: async ( takeNumberOfEvents, - { eventTypes = [], withTimeoutMs, fromTimestamp } = {} + { eventTypes = [], withTimeoutMs, fromTimestamp, filters } = {} ) => { await setOptIn(true); const resp = await supertest @@ -38,6 +38,7 @@ export function KibanaEBTServerProvider({ getService }: FtrProviderContext): EBT eventTypes: JSON.stringify(eventTypes), withTimeoutMs, fromTimestamp, + filters: JSON.stringify(filters), }) .set('kbn-xsrf', 'xxx') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') @@ -45,11 +46,16 @@ export function KibanaEBTServerProvider({ getService }: FtrProviderContext): EBT return resp.body; }, - getEventCount: async ({ eventTypes = [], withTimeoutMs, fromTimestamp }) => { + getEventCount: async ({ eventTypes = [], withTimeoutMs, fromTimestamp, filters }) => { await setOptIn(true); const resp = await supertest .get(`/internal/analytics_ftr_helpers/count_events`) - .query({ eventTypes: JSON.stringify(eventTypes), withTimeoutMs, fromTimestamp }) + .query({ + eventTypes: JSON.stringify(eventTypes), + withTimeoutMs, + fromTimestamp, + filters: JSON.stringify(filters), + }) .set('kbn-xsrf', 'xxx') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .expect(200); From 1f395ae123847325fcbf245457e6e67fdbbfe246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 17 Oct 2024 16:21:42 +0200 Subject: [PATCH 15/44] Add analytics_ftr_helpers plugin to kibana server config --- .../config/ess/config.base.ts | 7 +++++++ .../config/serverless/config.base.ts | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index a0d2ee79a7b46..127670ae40e0a 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -5,6 +5,8 @@ * 2.0. */ +import path from 'path'; + import { CA_CERT_PATH } from '@kbn/dev-utils'; import { FtrConfigProviderContext, kbnTestConfig, kibanaTestUser } from '@kbn/test'; import { services as baseServices } from './services'; @@ -85,6 +87,11 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'riskScoringPersistence', 'riskScoringRoutesEnabled', ])}`, + `--plugin-path=${path.resolve( + __dirname, + '../../../../../test/analytics/plugins/analytics_ftr_helpers' + )}`, + '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify(PRECONFIGURED_ACTION_CONNECTORS)}`, ...(ssl diff --git a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts index c2371984512e1..b62f281c4a616 100644 --- a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts @@ -4,6 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import path from 'path'; + import { FtrConfigProviderContext } from '@kbn/test'; import { services } from './services'; import { PRECONFIGURED_ACTION_CONNECTORS } from '../shared'; @@ -32,6 +34,10 @@ export function createTestConfig(options: CreateTestConfigOptions) { '--serverless=security', `--xpack.actions.preconfigured=${JSON.stringify(PRECONFIGURED_ACTION_CONNECTORS)}`, ...(options.kbnTestServerArgs || []), + `--plugin-path=${path.resolve( + __dirname, + '../../../../../test/analytics/plugins/analytics_ftr_helpers' + )}`, ], env: { ...svlSharedConfig.get('kbnTestServer.env'), From 1bc08d5dd5f119e0b9a426da93841cc8a8c786e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 17 Oct 2024 16:22:32 +0200 Subject: [PATCH 16/44] Define kibana_ebt_server service in ftr configs --- .../security_solution_api_integration/config/ess/services.ts | 4 +++- .../config/serverless/services.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/config/ess/services.ts b/x-pack/test/security_solution_api_integration/config/ess/services.ts index e5f66d5c1928a..80d88350e6dd2 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/services.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/services.ts @@ -5,12 +5,14 @@ * 2.0. */ +import { KibanaEBTServerProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; +import { SecuritySolutionESSUtils } from '../services/security_solution_ess_utils'; import { SpacesServiceProvider } from '../../../common/services/spaces'; import { services as essServices } from '../../../api_integration/services'; -import { SecuritySolutionESSUtils } from '../services/security_solution_ess_utils'; export const services = { ...essServices, spaces: SpacesServiceProvider, securitySolutionUtils: SecuritySolutionESSUtils, + kibana_ebt_server: KibanaEBTServerProvider, }; diff --git a/x-pack/test/security_solution_api_integration/config/serverless/services.ts b/x-pack/test/security_solution_api_integration/config/serverless/services.ts index 61b6c484ced3d..372fdaeea5781 100644 --- a/x-pack/test/security_solution_api_integration/config/serverless/services.ts +++ b/x-pack/test/security_solution_api_integration/config/serverless/services.ts @@ -7,6 +7,7 @@ import { BsearchSecureService } from '@kbn/test-suites-serverless/shared/services/bsearch_secure'; import { services as serverlessServices } from '@kbn/test-suites-serverless/api_integration/services'; +import { KibanaEBTServerProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; import { SpacesServiceProvider } from '../../../common/services/spaces'; import { SecuritySolutionServerlessUtils } from '../services/security_solution_serverless_utils'; import { SecuritySolutionServerlessSuperTest } from '../services/security_solution_serverless_supertest'; @@ -17,4 +18,5 @@ export const services = { secureBsearch: BsearchSecureService, securitySolutionUtils: SecuritySolutionServerlessUtils, supertest: SecuritySolutionServerlessSuperTest, + kibana_ebt_server: KibanaEBTServerProvider, }; From cb384a10b2bfc2b6dbab13c7fa838b20814865b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 17 Oct 2024 16:23:07 +0200 Subject: [PATCH 17/44] Add indices_metadata task FTR --- .../trial_license_complete_tier/index.ts | 1 + .../task_based/indices_metadata.ts | 97 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts index 0425b6971d78e..b0e95825579bc 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts @@ -15,6 +15,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./task_based/all_types')); loadTestFile(require.resolve('./task_based/detection_rules')); + loadTestFile(require.resolve('./task_based/indices_metadata')); loadTestFile(require.resolve('./task_based/security_lists')); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts new file mode 100644 index 0000000000000..9c9d6153ae51c --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TaskStatus } from '@kbn/task-manager-plugin/server'; +import { TELEMETRY_DATA_STREAM_EVENT } from '@kbn/security-solution-plugin/server/lib/telemetry/event_based/events'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; +import { waitFor } from '../../../../../../common/utils/security_solution'; + +const TASK_ID = 'security:indices-metadata-telemetry:1.0.0'; + +export default ({ getService }: FtrProviderContext) => { + const ebtServer = getService('kibana_ebt_server'); + const kibanaServer = getService('kibanaServer'); + const logger = getService('log'); + const es = getService('es'); + + const dsPrefix: string = 'test-ds'; + + describe('@ess @serverless Indices metadata task telemetry', () => { + let dsName: string; + + before(async () => {}); + + after(async () => {}); + + beforeEach(async () => { + dsName = `${dsPrefix}-${Date.now()}`; + const indexTemplateBody = { + index_patterns: [`${dsPrefix}-*`], + data_stream: {}, + template: {}, + }; + + await es.indices.putIndexTemplate({ + name: dsPrefix, + body: indexTemplateBody, + }); + + await es.indices.createDataStream({ name: dsName }); + }); + + afterEach(async () => { + es.indices.deleteDataStream({ name: dsName }); + }); + + describe('indices metadata', () => { + it('should publish data stream events', async () => { + await runSoon(TASK_ID); + + const opts = { + eventTypes: [TELEMETRY_DATA_STREAM_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + filters: { + 'properties.datastream_name': { + eq: dsName, + }, + }, + }; + + await waitFor( + async () => { + const eventCount = await ebtServer.getEventCount(opts); + return eventCount === 1; + }, + 'waitForTaskToRun', + logger + ); + }); + }); + }); + + const runSoon = async (taskId: string, delayMillis: number = 1_000) => { + const task = await kibanaServer.savedObjects.get({ + type: 'task', + id: taskId, + }); + + const runAt = new Date(Date.now() + delayMillis).toISOString(); + + await kibanaServer.savedObjects.update({ + type: 'task', + id: taskId, + attributes: { + ...task.attributes, + runAt, + scheduledAt: runAt, + status: TaskStatus.Idle, + }, + }); + }; +}; From 90052d91e7d93d60723de8f08c1d201af36b1cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 17 Oct 2024 17:24:01 +0200 Subject: [PATCH 18/44] Move FTR to its test suite --- .../ftr_security_serverless_configs.yml | 1 + .buildkite/ftr_security_stateful_configs.yml | 1 + .../trial_license_complete_tier/index.ts | 1 - .../telemetry/configs/ess.config.ts | 23 +++++++++++++++++++ .../telemetry/configs/serverless.config.ts | 16 +++++++++++++ .../test_suites/telemetry/index.ts | 13 +++++++++++ .../tasks}/indices_metadata.ts | 4 ++-- 7 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts rename x-pack/test/security_solution_api_integration/test_suites/{detections_response/telemetry/trial_license_complete_tier/task_based => telemetry/tasks}/indices_metadata.ts (94%) diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index eb2b8e3a06873..1e18d2a258e5d 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -74,6 +74,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml index aa37c6f52fb8c..737af6797fe05 100644 --- a/.buildkite/ftr_security_stateful_configs.yml +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -59,6 +59,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts index b0e95825579bc..0425b6971d78e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/index.ts @@ -15,7 +15,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./task_based/all_types')); loadTestFile(require.resolve('./task_based/detection_rules')); - loadTestFile(require.resolve('./task_based/indices_metadata')); loadTestFile(require.resolve('./task_based/security_lists')); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts new file mode 100644 index 0000000000000..5a4bfb39f41f0 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../config/ess/config.base.basic') + ); + + return { + ...functionalConfig.getAll(), + uiSettings: {}, + testFiles: [require.resolve('..')], + junit: { + reportName: 'Security Solution - Telemetry Integration Tests - ESS Env', + }, + }; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts new file mode 100644 index 0000000000000..8c733152e1c68 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../config/serverless/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + uiSettings: {}, + junit: { + reportName: 'Security Solution - Telemetry Integration Tests - Serverless Env', + }, +}); diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts new file mode 100644 index 0000000000000..ff88de12d7124 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default ({ loadTestFile }: FtrProviderContext): void => { + describe('Security Solution - Telemetry', function () { + loadTestFile(require.resolve('./tasks/indices_metadata')); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts similarity index 94% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts rename to x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index 9c9d6153ae51c..2a97fef3c9b94 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -8,8 +8,8 @@ import { TaskStatus } from '@kbn/task-manager-plugin/server'; import { TELEMETRY_DATA_STREAM_EVENT } from '@kbn/security-solution-plugin/server/lib/telemetry/event_based/events'; -import { FtrProviderContext } from '../../../../../ftr_provider_context'; -import { waitFor } from '../../../../../../common/utils/security_solution'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { waitFor } from '../../../../common/utils/security_solution'; const TASK_ID = 'security:indices-metadata-telemetry:1.0.0'; From b82f57b0317f8aff3280cf6ef1535b37d5a8e7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 18 Oct 2024 11:53:11 +0200 Subject: [PATCH 19/44] Add more tests --- .../detections_response/index.ts | 5 +- .../detections_response/tasks/index.ts | 9 ++ .../tasks/indices_metadata.ts | 108 +++++++++++++ .../detections_response/tasks/task_manager.ts | 38 +++++ .../config/serverless/config.base.ts | 2 + .../telemetry/configs/serverless.config.ts | 2 +- .../telemetry/tasks/indices_metadata.ts | 146 ++++++++++++------ 7 files changed, 264 insertions(+), 46 deletions(-) create mode 100644 x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts create mode 100644 x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts create mode 100644 x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts diff --git a/x-pack/test/common/utils/security_solution/detections_response/index.ts b/x-pack/test/common/utils/security_solution/detections_response/index.ts index 43c2a54900c15..45efc2cee3b4c 100644 --- a/x-pack/test/common/utils/security_solution/detections_response/index.ts +++ b/x-pack/test/common/utils/security_solution/detections_response/index.ts @@ -5,9 +5,10 @@ * 2.0. */ -export * from './rules'; export * from './alerts'; -export * from './delete_all_anomalies'; export * from './count_down_test'; +export * from './delete_all_anomalies'; export * from './route_with_namespace'; +export * from './rules'; +export * from './tasks'; export * from './wait_for'; diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts new file mode 100644 index 0000000000000..128f0cfe9b93a --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './indices_metadata'; +export * from './task_manager'; diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts new file mode 100644 index 0000000000000..0c4d90749e423 --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; + +const DS_PREFIX = 'testing-datastream'; +const ILM_PREFIX = 'testing-ilm'; + +export const randomDatastream = async (es: Client, policyName?: string): Promise => { + const name = `${DS_PREFIX}-${Date.now()}`; + + let settings = {}; + + if (policyName) { + settings = { + ...settings, + 'index.lifecycle.name': policyName, + }; + } + + const indexTemplateBody = { + index_patterns: [`${DS_PREFIX}-*`], + data_stream: {}, + template: { + settings, + }, + }; + + await es.indices.putIndexTemplate({ + name: DS_PREFIX, + body: indexTemplateBody, + }); + + await es.indices.createDataStream({ name }); + + return name; +}; + +export const randomIlmPolicy = async (es: Client): Promise => { + const name = `${ILM_PREFIX}-${Date.now()}`; + + const policy = { + phases: { + hot: { + actions: { + rollover: { + max_size: '50gb', + max_age: '30d', + }, + }, + }, + warm: { + min_age: '30d', + actions: { + forcemerge: { + max_num_segments: 1, + }, + shrink: { + number_of_shards: 1, + }, + allocate: { + number_of_replicas: 1, + }, + }, + }, + delete: { + min_age: '90d', + actions: { + delete: {}, + }, + }, + }, + }; + + await es.ilm.putLifecycle({ name, policy }); + + return name; +}; + +export const ensureBackingIndices = async (dsName: string, count: number, es: Client) => { + const stats = await es.indices.dataStreamsStats({ name: dsName }); + if (stats.data_streams.length !== 1) { + throw new Error('Data stream not found'); + } + const current = stats.data_streams[0].backing_indices; + + if (current < count) { + for (let i = current; i < count; i++) { + await es.indices.rollover({ alias: dsName }); + } + } else if (current > count) { + throw new Error('Cannot reduce the number of backing indices'); + } +}; + +export const cleanupDatastreams = async (es: Client) => { + await es.indices.deleteDataStream({ name: `${DS_PREFIX}*` }); +}; + +export const cleanupPolicies = async (es: Client) => { + const policies = await es.ilm.getLifecycle({ name: `${ILM_PREFIX}*` }); + + await Promise.all(Object.entries(policies).map(([name, _]) => es.ilm.deleteLifecycle({ name }))); +}; diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts new file mode 100644 index 0000000000000..c65179239d5da --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TaskStatus } from '@kbn/task-manager-plugin/server'; +import { KbnClient } from '@kbn/test'; +import { ToolingLog } from '@kbn/tooling-log'; + +export const launchTask = async ( + taskId: string, + kbn: KbnClient, + logger: ToolingLog, + delayMillis: number = 1_000 +) => { + logger.info(`Launching task ${taskId}`); + const task = await kbn.savedObjects.get({ + type: 'task', + id: taskId, + }); + + const runAt = new Date(Date.now() + delayMillis).toISOString(); + + await kbn.savedObjects.update({ + type: 'task', + id: taskId, + attributes: { + ...task.attributes, + runAt, + scheduledAt: runAt, + status: TaskStatus.Idle, + }, + }); + + logger.info(`Task ${taskId} launched`); +}; diff --git a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts index b62f281c4a616..8d3bea36a968b 100644 --- a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts @@ -15,6 +15,7 @@ export interface CreateTestConfigOptions { junit: { reportName: string }; kbnTestServerArgs?: string[]; kbnTestServerEnv?: Record; + suiteTags?: { include?: string[]; exclude?: string[] }; } export function createTestConfig(options: CreateTestConfigOptions) { @@ -24,6 +25,7 @@ export function createTestConfig(options: CreateTestConfigOptions) { ); return { ...svlSharedConfig.getAll(), + suiteTags: options.suiteTags, services: { ...services, }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts index 8c733152e1c68..5c47f36439393 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts @@ -9,7 +9,7 @@ import { createTestConfig } from '../../../config/serverless/config.base'; export default createTestConfig({ testFiles: [require.resolve('..')], - uiSettings: {}, + suiteTags: { exclude: ['skipServerless'] }, junit: { reportName: 'Security Solution - Telemetry Integration Tests - Serverless Env', }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index 2a97fef3c9b94..d59024b4bb56f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -5,11 +5,23 @@ * 2.0. */ -import { TaskStatus } from '@kbn/task-manager-plugin/server'; -import { TELEMETRY_DATA_STREAM_EVENT } from '@kbn/security-solution-plugin/server/lib/telemetry/event_based/events'; +import { + TELEMETRY_CLUSTER_STATS_EVENT, + TELEMETRY_DATA_STREAM_EVENT, + TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_INDEX_STATS_EVENT, +} from '@kbn/security-solution-plugin/server/lib/telemetry/event_based/events'; import { FtrProviderContext } from '../../../ftr_provider_context'; -import { waitFor } from '../../../../common/utils/security_solution'; +import { + cleanupDatastreams, + cleanupPolicies, + ensureBackingIndices, + launchTask, + randomDatastream, + randomIlmPolicy, + waitFor, +} from '../../../../common/utils/security_solution'; const TASK_ID = 'security:indices-metadata-telemetry:1.0.0'; @@ -19,38 +31,26 @@ export default ({ getService }: FtrProviderContext) => { const logger = getService('log'); const es = getService('es'); - const dsPrefix: string = 'test-ds'; - - describe('@ess @serverless Indices metadata task telemetry', () => { + describe(' Indices metadata task telemetry', function () { let dsName: string; + let policyName: string; before(async () => {}); after(async () => {}); - beforeEach(async () => { - dsName = `${dsPrefix}-${Date.now()}`; - const indexTemplateBody = { - index_patterns: [`${dsPrefix}-*`], - data_stream: {}, - template: {}, - }; - - await es.indices.putIndexTemplate({ - name: dsPrefix, - body: indexTemplateBody, + describe('@ess @serverless indices metadata', () => { + beforeEach(async () => { + dsName = await randomDatastream(es); + await ensureBackingIndices(dsName, 5, es); }); - await es.indices.createDataStream({ name: dsName }); - }); - - afterEach(async () => { - es.indices.deleteDataStream({ name: dsName }); - }); + afterEach(async () => { + await cleanupDatastreams(es); + }); - describe('indices metadata', () => { it('should publish data stream events', async () => { - await runSoon(TASK_ID); + await launchTask(TASK_ID, kibanaServer, logger); const opts = { eventTypes: [TELEMETRY_DATA_STREAM_EVENT.eventType], @@ -72,26 +72,86 @@ export default ({ getService }: FtrProviderContext) => { logger ); }); - }); - }); - const runSoon = async (taskId: string, delayMillis: number = 1_000) => { - const task = await kibanaServer.savedObjects.get({ - type: 'task', - id: taskId, + it('should publish index stats events', async () => { + await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_INDEX_STATS_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + }; + + const regex = new RegExp(`^\.ds-${dsName}-.*$`); + await waitFor( + async () => { + const events = await ebtServer.getEvents(100, opts); + // .ds--YYYY.MM.DD-NNNNNN + const filtered = events.filter((e) => regex.test(e.properties.index_name as string)); + return filtered.length === 5; + }, + 'waitForTaskToRun', + logger + ); + }); + + it('should publish cluster stats events', async () => { + await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_CLUSTER_STATS_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + }; + + await waitFor( + async () => { + const events = await ebtServer.getEventCount(opts); + return events === 1; + }, + 'waitForTaskToRun', + logger + ); + }); }); - const runAt = new Date(Date.now() + delayMillis).toISOString(); - - await kibanaServer.savedObjects.update({ - type: 'task', - id: taskId, - attributes: { - ...task.attributes, - runAt, - scheduledAt: runAt, - status: TaskStatus.Idle, - }, + describe('@ess indices metadata', function () { + this.tags('skipServerless'); + + beforeEach(async () => { + policyName = await randomIlmPolicy(es); + dsName = await randomDatastream(es, policyName); + await ensureBackingIndices(dsName, 5, es); + }); + + afterEach(async () => { + await cleanupDatastreams(es); + await cleanupPolicies(es); + }); + + it('should publish ilm policy events', async () => { + await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_ILM_POLICY_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + filters: { + 'properties.policy_name': { + eq: policyName, + }, + }, + }; + + await waitFor( + async () => { + const events = await ebtServer.getEventCount(opts); + return events === 1; + }, + 'waitForTaskToRun', + logger + ); + }); }); - }; + }); }; From 920a4440ff9c24c87441d5e99d1ed117261c78f5 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:16:38 +0000 Subject: [PATCH 20/44] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test/security_solution_api_integration/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index 17d5053c05328..1bdde465e3864 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -51,5 +51,6 @@ "@kbn/security-plugin", "@kbn/ftr-common-functional-ui-services", "@kbn/spaces-plugin", + "@kbn/test-suites-src", ] } From cd3ea602922386a091e5a12218a9e50f361bf3d0 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:25:42 +0000 Subject: [PATCH 21/44] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../lib/telemetry/event_based/events.ts | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts index 0d4d4dd8b546f..1e2bcf36ea88a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -105,35 +105,35 @@ interface AssetCriticalitySystemProcessedAssignmentFileEvent { } export const ASSET_CRITICALITY_SYSTEM_PROCESSED_ASSIGNMENT_FILE_EVENT: EventTypeOpts = -{ - eventType: 'Asset Criticality Csv Upload Processed', - schema: { - processing: { - properties: { - startTime: { type: 'date', _meta: { description: 'Processing start time' } }, - endTime: { type: 'date', _meta: { description: 'Processing end time' } }, - tookMs: { type: 'long', _meta: { description: 'How long processing took ms' } }, - }, - }, - result: { - properties: { - successful: { - type: 'long', - _meta: { description: 'Number of criticality records successfully created or updated' }, + { + eventType: 'Asset Criticality Csv Upload Processed', + schema: { + processing: { + properties: { + startTime: { type: 'date', _meta: { description: 'Processing start time' } }, + endTime: { type: 'date', _meta: { description: 'Processing end time' } }, + tookMs: { type: 'long', _meta: { description: 'How long processing took ms' } }, }, - failed: { - type: 'long', - _meta: { description: 'Number of criticality records which had errors' }, + }, + result: { + properties: { + successful: { + type: 'long', + _meta: { description: 'Number of criticality records successfully created or updated' }, + }, + failed: { + type: 'long', + _meta: { description: 'Number of criticality records which had errors' }, + }, + total: { type: 'long', _meta: { description: 'Total number of lines in the file' } }, }, - total: { type: 'long', _meta: { description: 'Total number of lines in the file' } }, + }, + status: { + type: 'keyword', + _meta: { description: 'Status of the processing either success, partial_success or fail' }, }, }, - status: { - type: 'keyword', - _meta: { description: 'Status of the processing either success, partial_success or fail' }, - }, - }, -}; + }; export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ suppressionAlertsCreated: number; @@ -447,9 +447,9 @@ export const createAssetCriticalityProcessedFileEvent = ({ startTime, endTime, }: CreateAssetCriticalityProcessedFileEvent): [ - string, - AssetCriticalitySystemProcessedAssignmentFileEvent - ] => { + string, + AssetCriticalitySystemProcessedAssignmentFileEvent +] => { const status = getUploadStatus(result); const processing = { From 77a14048bafc6b962e013c90bb30ef713ce5472e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 18 Oct 2024 18:30:27 +0200 Subject: [PATCH 22/44] Improve error handling --- .../server/lib/telemetry/task_metrics.ts | 6 +- .../lib/telemetry/tasks/indices.metadata.ts | 133 ++++++++++++++---- 2 files changed, 109 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts index 3306633bcfbb2..966eb0888f45e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts @@ -29,7 +29,11 @@ export class TaskMetricsService implements ITaskMetricsService { public async end(trace: Trace, error?: Error): Promise { const event = this.createTaskMetric(trace, error); - this.logger.l(`Task ${event.name} complete. Task run took ${event.time_executed_in_ms}ms`); + this.logger.l('Task completed', { + task_name: event.name, + time_executed_in_ms: event.time_executed_in_ms, + error_message: event.error_message, + }); if (telemetryConfiguration.use_async_sender) { this.sender.sendAsync(TelemetryChannel.TASK_METRICS, [event]); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 4bd9842f0fff8..55e2a7ab6a362 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -10,7 +10,11 @@ import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; -import { getPreviousDailyTaskTimestamp, newTelemetryLogger } from '../helpers'; +import { + createUsageCounterLabel, + getPreviousDailyTaskTimestamp, + newTelemetryLogger, +} from '../helpers'; import { TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, @@ -20,6 +24,10 @@ import { } from '../event_based/events'; import type { QueryConfig } from '../collections_helpers'; import { telemetryConfiguration } from '../configuration'; +import type { ClusterStats, DataStream } from '../indices.metadata.types'; +import { TelemetryCounter } from '../types'; + +const COUNTER_LABELS = ['security_solution', 'indices-metadata']; export function createTelemetryIndicesMetadataTaskConfig() { const taskType = 'security:indices-metadata-telemetry'; @@ -50,66 +58,133 @@ export function createTelemetryIndicesMetadataTaskConfig() { maxGroupSize: Number(taskExecutionPeriod.current ?? taskConfig.max_group_size), }; - try { - let policyCount = 0; - let indicesCount = 0; - let ilmsCount = 0; - let dsCount = 0; - - // 1. Get cluster stats and list of indices and datastreams - const [clusterStats, indices, dataStreams] = await Promise.all([ - receiver.getClusterStats(), - receiver.getIndices(), - receiver.getDataStreams(), - ]); - - sender.reportEBT(TELEMETRY_CLUSTER_STATS_EVENT.eventType, clusterStats); + const publishClusterStats = (stats: ClusterStats) => { + sender.reportEBT(TELEMETRY_CLUSTER_STATS_EVENT.eventType, stats); + }; - // 2. Publish datastreams stats - for (const ds of dataStreams.slice(0, taskConfig.datastreams_threshold)) { + const publishDatastreamsStats = (stats: DataStream[]): number => { + let counter = 0; + for (const ds of stats) { sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); - dsCount++; + counter++; } - log.info(`Sent ${dsCount} data streams`, { dsCount } as LogMeta); + log.info(`Sent data streams`, { count: counter } as LogMeta); + return counter; + }; - // 3. Get and publish indices stats + const publishIndicesStats = async (indices: string[]): Promise => { + let counter = 0; for await (const stat of receiver.getIndicesStats( indices.slice(0, taskConfig.indices_threshold), queryConfig )) { sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, stat); - indicesCount++; + counter++; } - log.info(`Sent ${indicesCount} indices stats`, { indicesCount } as LogMeta); + log.info(`Sent indices stats`, { count: counter } as LogMeta); + return counter; + }; - // 4. Get ILM stats and publish them + const publishIlmStats = async (indices: string[]): Promise> => { const ilmNames = new Set(); for await (const stat of receiver.getIlmsStats(indices, queryConfig)) { if (stat.policy_name !== undefined) { ilmNames.add(stat.policy_name); sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); - ilmsCount++; } } - log.info(`Sent ${ilmsCount} ILM stats`, { ilmsCount } as LogMeta); + log.info(`Sent ILM stats`, { count: ilmNames.size } as LogMeta); + + return ilmNames; + }; - // 5. Publish ILM policies + const publishIlmPolicies = async (ilmNames: Set): Promise => { + let counter = 0; for await (const policy of receiver.getIlmsPolicies( Array.from(ilmNames.values()), queryConfig )) { sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, policy); - policyCount++; + counter++; } - log.info(`Sent ${policyCount} ILM policies`, { policyCount } as LogMeta); + log.info(`Sent ILM policies`, { count: counter } as LogMeta); + return counter; + }; + + const incrementCounter = (type: TelemetryCounter, name: string, value: number) => { + const telemetryUsageCounter = sender.getTelemetryUsageCluster(); + telemetryUsageCounter?.incrementCounter({ + counterName: createUsageCounterLabel(COUNTER_LABELS.concat(name)), + counterType: type, + incrementBy: value, + }); + }; + + try { + // 1. Get cluster stats and list of indices and datastreams + const [clusterStats, indices, dataStreams] = await Promise.all([ + receiver.getClusterStats(), + receiver.getIndices(), + receiver.getDataStreams(), + ]); + + // 2. Publish cluster stats + publishClusterStats(clusterStats); + incrementCounter(TelemetryCounter.DOCS_SENT, 'cluster-stats', 1); + + // 3. Publish datastreams stats + const dsCount = publishDatastreamsStats( + dataStreams.slice(0, taskConfig.datastreams_threshold) + ); + incrementCounter(TelemetryCounter.DOCS_SENT, 'datastreams-stats', dsCount); + + // 4. Get and publish indices stats + const indicesCount: number = await publishIndicesStats( + indices.slice(0, taskConfig.indices_threshold) + ) + .then((count) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'indices-stats', count); + return count; + }) + .catch((err) => { + log.warn(`Error getting indices stats`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'indices-stats', 1); + return 0; + }); + + // 5. Get ILM stats and publish them + const ilmNames = await publishIlmStats(indices) + .then((names) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-stats', names.size); + return names; + }) + .catch((err) => { + log.warn(`Error getting ILM stats`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'ilm-stats', 1); + return new Set(); + }); + + // 6. Publish ILM policies + const policyCount = await publishIlmPolicies(ilmNames) + .then((count) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-policies', count); + return count; + }) + .catch((err) => { + log.warn(`Error getting ILM policies`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'ilm-policies', 1); + return 0; + }); log.info(`Sent EBT events`, { datastreams: dsCount, - ilms: ilmsCount, + ilms: ilmNames.size, indices: indicesCount, policies: policyCount, } as LogMeta); + await taskMetricsService.end(trace); + return indicesCount; } catch (err) { log.warn(`Error running indices metadata task`, { From 371b49f05f0f41e87560f55b876c6b70eb886d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 18 Oct 2024 15:25:58 +0200 Subject: [PATCH 23/44] Add more tests --- .../telemetry/tasks/indices_metadata.ts | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index d59024b4bb56f..0309eabd54a1a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -9,6 +9,7 @@ import { TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_ILM_STATS_EVENT, TELEMETRY_INDEX_STATS_EVENT, } from '@kbn/security-solution-plugin/server/lib/telemetry/event_based/events'; @@ -24,6 +25,7 @@ import { } from '../../../../common/utils/security_solution'; const TASK_ID = 'security:indices-metadata-telemetry:1.0.0'; +const NUM_INDICES = 5; export default ({ getService }: FtrProviderContext) => { const ebtServer = getService('kibana_ebt_server'); @@ -31,18 +33,14 @@ export default ({ getService }: FtrProviderContext) => { const logger = getService('log'); const es = getService('es'); - describe(' Indices metadata task telemetry', function () { + describe('Indices metadata task telemetry', function () { let dsName: string; let policyName: string; - before(async () => {}); - - after(async () => {}); - describe('@ess @serverless indices metadata', () => { beforeEach(async () => { dsName = await randomDatastream(es); - await ensureBackingIndices(dsName, 5, es); + await ensureBackingIndices(dsName, NUM_INDICES, es); }); afterEach(async () => { @@ -82,13 +80,13 @@ export default ({ getService }: FtrProviderContext) => { fromTimestamp: new Date().toISOString(), }; - const regex = new RegExp(`^\.ds-${dsName}-.*$`); + // .ds--YYYY.MM.DD-NNNNNN + const regex = new RegExp(`^\.ds-${dsName}-\\d{4}.\\d{2}.\\d{2}-\\d{6}$`); await waitFor( async () => { - const events = await ebtServer.getEvents(100, opts); - // .ds--YYYY.MM.DD-NNNNNN - const filtered = events.filter((e) => regex.test(e.properties.index_name as string)); - return filtered.length === 5; + const events = await ebtServer.getEvents(Number.MAX_SAFE_INTEGER, opts); + const filtered = events.filter((ev) => regex.test(ev.properties.index_name as string)); + return filtered.length === NUM_INDICES; }, 'waitForTaskToRun', logger @@ -121,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => { beforeEach(async () => { policyName = await randomIlmPolicy(es); dsName = await randomDatastream(es, policyName); - await ensureBackingIndices(dsName, 5, es); + await ensureBackingIndices(dsName, NUM_INDICES, es); }); afterEach(async () => { @@ -152,6 +150,30 @@ export default ({ getService }: FtrProviderContext) => { logger ); }); + + it('should publish ilm stats events', async () => { + await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_ILM_STATS_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + filters: { + 'properties.policy_name': { + eq: policyName, + }, + }, + }; + + await waitFor( + async () => { + const events = await ebtServer.getEventCount(opts); + return events === NUM_INDICES; + }, + 'waitForTaskToRun', + logger + ); + }); }); }); }; From 57bca8bfd975ee7a472ce002505fb1a85b5271a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 21 Oct 2024 14:19:01 +0200 Subject: [PATCH 24/44] Improve logging --- .../server/lib/telemetry/receiver.ts | 62 +++++++++---------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 6aedfece6662a..e4e1468145649 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -559,7 +559,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { const buckets = endpointMetadataResponse?.aggregations?.endpoint_metadata?.buckets ?? []; return buckets.reduce((cache, endpointAgentId) => { - // const id = endpointAgentId.latest_metadata.hits.hits[0]._id; const doc = endpointAgentId.latest_metadata.hits.hits[0]._source; cache.set(endpointAgentId.key, doc); return cache; @@ -568,7 +567,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async *fetchDiagnosticAlertsBatch(executeFrom: string, executeTo: string) { - this.logger.debug('Searching diagnostic alerts', { + this.logger.l('Searching diagnostic alerts', { from: executeFrom, to: executeTo, } as LogMeta); @@ -612,10 +611,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { fetchMore = false; } - this.logger.debug('Diagnostic alerts to return', { numOfHits } as LogMeta); + this.logger.l('Diagnostic alerts to return', { numOfHits } as LogMeta); fetchMore = numOfHits > 0 && numOfHits < telemetryConfiguration.telemetry_max_buffer_size; } catch (e) { - this.logger.l('Error fetching alerts', { error: JSON.stringify(e) }); + this.logger.warn('Error fetching alerts', { error_message: e.message } as LogMeta); fetchMore = false; } @@ -788,7 +787,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { executeFrom: string, executeTo: string ) { - this.logger.debug('Searching prebuilt rule alerts from', { + this.logger.l('Searching prebuilt rule alerts from', { executeFrom, executeTo, } as LogMeta); @@ -926,14 +925,14 @@ export class TelemetryReceiver implements ITelemetryReceiver { pitId = response?.pit_id; } - this.logger.debug('Prebuilt rule alerts to return', { alerts: alerts.length } as LogMeta); + this.logger.l('Prebuilt rule alerts to return', { alerts: alerts.length } as LogMeta); yield alerts; } } catch (e) { // to keep backward compatibility with the previous implementation, silent return // once we start using `paginate` this error should be managed downstream - this.logger.l('Error fetching alerts', { error: JSON.stringify(e) }); + this.logger.warn('Error fetching alerts', { error_message: e.message } as LogMeta); return; } finally { await this.closePointInTime(pitId); @@ -957,10 +956,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { try { await this.esClient().closePointInTime({ id: pitId }); } catch (error) { - this.logger.l('Error trying to close point in time', { + this.logger.warn('Error trying to close point in time', { pit: pitId, - error: JSON.stringify(error), - }); + error_message: error.message, + } as LogMeta); } } @@ -1046,7 +1045,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { fetchMore = numOfHits > 0; } catch (e) { - this.logger.l('Error fetching alerts', { error: JSON.stringify(e) }); + this.logger.warn('Error fetching alerts', { error_message: e.message } as LogMeta); fetchMore = false; } @@ -1061,11 +1060,11 @@ export class TelemetryReceiver implements ITelemetryReceiver { try { await this.esClient().closePointInTime({ id: pitId }); } catch (error) { - this.logger.l('Error trying to close point in time', { + this.logger.warn('Error trying to close point in time', { pit: pitId, - error: JSON.stringify(error), + error_message: error.message, keepAlive, - }); + } as LogMeta); } this.logger.l('Timeline alerts to return', { alerts: alertsToReturn.length }); @@ -1259,7 +1258,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return ret.license; } catch (err) { - this.logger.l('failed retrieving license', { error: JSON.stringify(err) }); + this.logger.warn('failed retrieving license', { error_message: err.message } as LogMeta); return undefined; } } @@ -1320,7 +1319,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { yield data; } while (esQuery.search_after !== undefined); } catch (e) { - this.logger.l('Error running paginated query', { error: JSON.stringify(e) }); + this.logger.warn('Error running paginated query', { error_message: e.message } as LogMeta); throw e; } finally { await this.closePointInTime(pit.id); @@ -1352,7 +1351,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async getClusterStats(): Promise { const es = this.esClient(); - this.logger.debug('Fetching cluster stats'); + this.logger.l('Fetching cluster stats'); return es.cluster.stats({}).then( (response) => @@ -1369,7 +1368,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async getIndices(): Promise { const es = this.esClient(); - this.logger.debug('Fetching indices'); + this.logger.l('Fetching indices'); const request: CatIndicesRequest = { expand_wildcards: ['open', 'hidden'], @@ -1389,7 +1388,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async getDataStreams(): Promise { const es = this.esClient(); - this.logger.debug('Fetching datstreams'); + this.logger.l('Fetching datstreams'); const request: IndicesGetDataStreamRequest = { name: '*', @@ -1422,19 +1421,17 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async *getIndicesStats(indices: string[], config: QueryConfig) { const es = this.esClient(); - this.logger.debug(`Fetching indices stats for ${indices.length} indices`, { - indices, - } as LogMeta); + this.logger.l('Fetching indices stats', { indices } as LogMeta); const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); - this.logger.debug(`Splitted indices into ${groupedIndices.length} groups`, { + this.logger.l('Splitted indices into groups', { groups: groupedIndices.length, + indices: indices.length, } as LogMeta); for (const group of groupedIndices) { for (const name of group) { - // this.logger.debug(`Fetching indices stats for ${name}`, { name } as LogMeta); const request: IndicesStatsRequest = { index: `${name}*`, level: 'indices', @@ -1472,17 +1469,15 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async *getIlmsStats(indices: string[], config: QueryConfig) { const es = this.esClient(); - this.logger.debug(`Fetching ilm stats for ${indices.length} indices`, { indices } as LogMeta); - const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); - this.logger.debug(`Splitted indices into ${groupedIndices.length} groups`, { + this.logger.l('Splitted indices into groups', { groups: groupedIndices.length, + indices: indices.length, } as LogMeta); for (const group of groupedIndices) { for (const name of group) { - // this.logger.debug(`Fetching ilm stats for ${name}`, { name } as LogMeta); const request: IlmExplainLifecycleRequest = { index: `${name}*`, only_managed: false, @@ -1523,17 +1518,16 @@ export class TelemetryReceiver implements ITelemetryReceiver { return value; }; - this.logger.debug(`Fetching ilms policies for ${ilms.length} ilms`, { ilms } as LogMeta); - const groupedIlms = findCommonPrefixes(ilms, config).map((v, _) => v.parts); - this.logger.debug(`Splitted ilms into ${groupedIlms.length} groups`, { + this.logger.l(`Splitted ilms into groups`, { groups: groupedIlms.length, + ilms: ilms.length, } as LogMeta); for (const group of groupedIlms) { for (const name of group) { - this.logger.debug(`Fetching ilm policies for ${name}`, { name } as LogMeta); + this.logger.l('Fetching ilm policies', { name } as LogMeta); const request: IlmGetLifecycleRequest = { name: `${name}*`, filter_path: [ @@ -1562,7 +1556,9 @@ export class TelemetryReceiver implements ITelemetryReceiver { } as IlmPolicy; } } catch (error) { - this.logger.warn('Error fetching ilm policies', { error_message: error } as LogMeta); + this.logger.warn('Error fetching ilm policies', { + error_message: error.message, + } as LogMeta); throw error; } } From 922f7bc8969a10d28399c8691bf03b81952fe64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 21 Oct 2024 14:25:51 +0200 Subject: [PATCH 25/44] Apply config to index stats --- .../server/lib/telemetry/tasks/indices.metadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 55e2a7ab6a362..d2492e1a3db96 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -153,7 +153,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { }); // 5. Get ILM stats and publish them - const ilmNames = await publishIlmStats(indices) + const ilmNames = await publishIlmStats(indices.slice(0, taskConfig.indices_threshold)) .then((names) => { incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-stats', names.size); return names; From 16a3e29b93a93d25219af2633e5d766f98648ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 21 Oct 2024 16:21:19 +0200 Subject: [PATCH 26/44] More tests --- .../detections_response/tasks/task_manager.ts | 16 +++++++++- .../telemetry/tasks/indices_metadata.ts | 32 +++++++++++++------ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts index c65179239d5da..84015fa7b0b62 100644 --- a/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts @@ -9,12 +9,24 @@ import { TaskStatus } from '@kbn/task-manager-plugin/server'; import { KbnClient } from '@kbn/test'; import { ToolingLog } from '@kbn/tooling-log'; +export const taskHasRun = async (taskId: string, kbn: KbnClient, after: Date): Promise => { + const task = await kbn.savedObjects.get({ + type: 'task', + id: taskId, + }); + + const runAt = new Date(task.attributes.runAt); + const status = task.attributes.status; + + return runAt > after && status === TaskStatus.Idle; +}; + export const launchTask = async ( taskId: string, kbn: KbnClient, logger: ToolingLog, delayMillis: number = 1_000 -) => { +): Promise => { logger.info(`Launching task ${taskId}`); const task = await kbn.savedObjects.get({ type: 'task', @@ -35,4 +47,6 @@ export const launchTask = async ( }); logger.info(`Task ${taskId} launched`); + + return new Date(runAt); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index 0309eabd54a1a..3f45a1868981e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -21,6 +21,7 @@ import { launchTask, randomDatastream, randomIlmPolicy, + taskHasRun, waitFor, } from '../../../../common/utils/security_solution'; @@ -48,7 +49,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should publish data stream events', async () => { - await launchTask(TASK_ID, kibanaServer, logger); + const runAt = await launchTask(TASK_ID, kibanaServer, logger); const opts = { eventTypes: [TELEMETRY_DATA_STREAM_EVENT.eventType], @@ -64,7 +65,10 @@ export default ({ getService }: FtrProviderContext) => { await waitFor( async () => { const eventCount = await ebtServer.getEventCount(opts); - return eventCount === 1; + + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && eventCount === 1; }, 'waitForTaskToRun', logger @@ -72,7 +76,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should publish index stats events', async () => { - await launchTask(TASK_ID, kibanaServer, logger); + const runAt = await launchTask(TASK_ID, kibanaServer, logger); const opts = { eventTypes: [TELEMETRY_INDEX_STATS_EVENT.eventType], @@ -86,7 +90,9 @@ export default ({ getService }: FtrProviderContext) => { async () => { const events = await ebtServer.getEvents(Number.MAX_SAFE_INTEGER, opts); const filtered = events.filter((ev) => regex.test(ev.properties.index_name as string)); - return filtered.length === NUM_INDICES; + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && filtered.length === NUM_INDICES; }, 'waitForTaskToRun', logger @@ -94,7 +100,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should publish cluster stats events', async () => { - await launchTask(TASK_ID, kibanaServer, logger); + const runAt = await launchTask(TASK_ID, kibanaServer, logger); const opts = { eventTypes: [TELEMETRY_CLUSTER_STATS_EVENT.eventType], @@ -105,7 +111,9 @@ export default ({ getService }: FtrProviderContext) => { await waitFor( async () => { const events = await ebtServer.getEventCount(opts); - return events === 1; + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && events === 1; }, 'waitForTaskToRun', logger @@ -128,7 +136,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should publish ilm policy events', async () => { - await launchTask(TASK_ID, kibanaServer, logger); + const runAt = await launchTask(TASK_ID, kibanaServer, logger); const opts = { eventTypes: [TELEMETRY_ILM_POLICY_EVENT.eventType], @@ -144,7 +152,9 @@ export default ({ getService }: FtrProviderContext) => { await waitFor( async () => { const events = await ebtServer.getEventCount(opts); - return events === 1; + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && events === 1; }, 'waitForTaskToRun', logger @@ -152,7 +162,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should publish ilm stats events', async () => { - await launchTask(TASK_ID, kibanaServer, logger); + const runAt = await launchTask(TASK_ID, kibanaServer, logger); const opts = { eventTypes: [TELEMETRY_ILM_STATS_EVENT.eventType], @@ -168,7 +178,9 @@ export default ({ getService }: FtrProviderContext) => { await waitFor( async () => { const events = await ebtServer.getEventCount(opts); - return events === NUM_INDICES; + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && events === NUM_INDICES; }, 'waitForTaskToRun', logger From 9f57dd334314af08e60642be03c1f09f6c0120af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 22 Oct 2024 14:20:12 +0200 Subject: [PATCH 27/44] Update common prefixes function --- .../lib/telemetry/collections_helpers.test.ts | 35 +++++++++-- .../lib/telemetry/collections_helpers.ts | 61 ++++++++++++++----- .../server/lib/telemetry/configuration.ts | 1 + .../server/lib/telemetry/receiver.ts | 20 +++--- .../lib/telemetry/tasks/indices.metadata.ts | 5 +- .../server/lib/telemetry/types.ts | 1 + 6 files changed, 96 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts index 37e1cb2dd65dd..1535344844df7 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts @@ -5,7 +5,12 @@ * 2.0. */ import { stagingIndices } from './__mocks__/staging_indices'; -import { type QueryConfig, chunked, chunkedBy, findCommonPrefixes } from './collections_helpers'; +import { + type CommonPrefixesConfig, + chunked, + chunkedBy, + findCommonPrefixes, +} from './collections_helpers'; describe('telemetry.utils.chunked', () => { it('should chunk simple case', async () => { @@ -84,9 +89,10 @@ describe('telemetry.utils.chunkedBy', () => { describe('telemetry.utils.findCommonPrefixes', () => { it('should find common prefixes in simple case', async () => { const indices = ['aaa', 'b', 'aa']; - const config: QueryConfig = { + const config: CommonPrefixesConfig = { maxPrefixes: 10, maxGroupSize: 10, + minPrefixSize: 1, }; const output = findCommonPrefixes(indices, config); @@ -100,11 +106,31 @@ describe('telemetry.utils.findCommonPrefixes', () => { ); }); + it('should find common prefixes with different minPrefixSize', async () => { + const indices = ['.ds-AA-0001', '.ds-AA-0002', '.ds-BB-0003']; + const config: CommonPrefixesConfig = { + maxPrefixes: 10, + maxGroupSize: 3, + minPrefixSize: 5, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(2); + expect( + output.find((v, _) => v.parts.length === 1 && v.parts[0] === '.ds-A')?.indexCount + ).toEqual(2); + expect( + output.find((v, _) => v.parts.length === 1 && v.parts[0] === '.ds-B')?.indexCount + ).toEqual(1); + }); + it('should discard extra indices', async () => { const indices = ['aaa', 'aaaaaa', 'aa']; - const config: QueryConfig = { + const config: CommonPrefixesConfig = { maxPrefixes: 1, maxGroupSize: 2, + minPrefixSize: 3, }; const output = findCommonPrefixes(indices, config); @@ -117,9 +143,10 @@ describe('telemetry.utils.findCommonPrefixes', () => { it('should group many indices', async () => { const indices = stagingIndices; - const config: QueryConfig = { + const config: CommonPrefixesConfig = { maxPrefixes: 8, maxGroupSize: 100, + minPrefixSize: 3, }; const output = findCommonPrefixes(indices, config); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts index 4ad39d8095041..b3a68fe946a81 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts @@ -64,9 +64,10 @@ class Chunked { } } -export interface QueryConfig { +export interface CommonPrefixesConfig { maxPrefixes: number; maxGroupSize: number; + minPrefixSize: number; } interface TrieNode { @@ -78,6 +79,11 @@ interface TrieNode { id: number; } +interface Group { + parts: string[]; + indexCount: number; +} + function newTrieNode(char: string = '', prefix: string = '', id: number = 0): TrieNode { return { char, @@ -89,19 +95,46 @@ function newTrieNode(char: string = '', prefix: string = '', id: number = 0): Tr }; } -function* idCounter(): Generator { - let id = 0; - while (true) { - yield id++; - } -} +/** + * Finds and groups common prefixes from a list of strings. + * + * @param {string[]} indices - An array of strings from which common prefixes will be extracted. + * @param {CommonPrefixesConfig} config - A configuration object that defines the rules for grouping. + * + * The `config` object contains the following properties: + * - maxGroupSize {number}: The maximum number of indices allowed in a group. + * - maxPrefixes {number}: The maximum number of prefix groups to return. + * - minPrefixSize {number}: The minimum length of a prefix required to form a group. It avoid cases like returning + * a single character prefix, e.g., ['.ds-...1', '.ds-....2', ....] -> returns a single group '.' + * + * @returns {Group[]} - An array of groups where each group contains a list of prefix parts and the count of indices that share that prefix. + * + * Example usage: + * + * ```typescript + * const indices = ['apple', 'appetizer', 'application', 'banana', 'band', 'bandage']; + * const config = { + * maxGroupSize: 5, + * maxPrefixes: 3, + * minPrefixSize: 3 + * }; + * + * const result = findCommonPrefixes(indices, config); + * //result = [ + * // { parts: [ 'ban' ], indexCount: 3 }, + * // { parts: [ 'app' ], indexCount: 3 } + * //] + * ``` + */ -interface Group { - parts: string[]; - indexCount: number; -} +export function findCommonPrefixes(indices: string[], config: CommonPrefixesConfig): Group[] { + const idCounter = function* (): Generator { + let id = 0; + while (true) { + yield id++; + } + }; -export function findCommonPrefixes(indices: string[], config: QueryConfig): Group[] { const idGen = idCounter(); const root = newTrieNode('', '', idGen.next().value); @@ -125,8 +158,8 @@ export function findCommonPrefixes(indices: string[], config: QueryConfig): Grou // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const node = nodes.pop()!; if ( - (node.count <= config.maxGroupSize && node.prefix !== '') || - Object.keys(node.children).length === 0 + (node.count <= config.maxGroupSize && node.prefix.length >= config.minPrefixSize) || + (Object.keys(node.children).length === 0 && node.prefix.length >= config.minPrefixSize) ) { const group: Group = { parts: [node.prefix], diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts index 6b715cafd0b5e..40031574b7489 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts @@ -30,6 +30,7 @@ class TelemetryConfigurationDTO { datastreams_threshold: 1000, max_prefixes: 10, max_group_size: 100, + min_group_size: 5, }; private _telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index e4e1468145649..ebe4f4c53587d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -102,7 +102,7 @@ import type { Index, IndexStats, } from './indices.metadata.types'; -import { type QueryConfig, findCommonPrefixes } from './collections_helpers'; +import { type CommonPrefixesConfig, findCommonPrefixes } from './collections_helpers'; export interface ITelemetryReceiver { start( @@ -261,10 +261,16 @@ export interface ITelemetryReceiver { getDataStreams(): Promise; getIndicesStats( indices: string[], - config: QueryConfig + config: CommonPrefixesConfig ): AsyncGenerator; - getIlmsStats(indices: string[], config: QueryConfig): AsyncGenerator; - getIlmsPolicies(ilms: string[], config: QueryConfig): AsyncGenerator; + getIlmsStats( + indices: string[], + config: CommonPrefixesConfig + ): AsyncGenerator; + getIlmsPolicies( + ilms: string[], + config: CommonPrefixesConfig + ): AsyncGenerator; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -1418,7 +1424,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { }); } - public async *getIndicesStats(indices: string[], config: QueryConfig) { + public async *getIndicesStats(indices: string[], config: CommonPrefixesConfig) { const es = this.esClient(); this.logger.l('Fetching indices stats', { indices } as LogMeta); @@ -1466,7 +1472,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } } - public async *getIlmsStats(indices: string[], config: QueryConfig) { + public async *getIlmsStats(indices: string[], config: CommonPrefixesConfig) { const es = this.esClient(); const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); @@ -1505,7 +1511,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } } - public async *getIlmsPolicies(ilms: string[], config: QueryConfig) { + public async *getIlmsPolicies(ilms: string[], config: CommonPrefixesConfig) { const es = this.esClient(); const phase = (obj: unknown): Nullable => { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index d2492e1a3db96..9a0736d0617ec 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -22,7 +22,7 @@ import { TELEMETRY_ILM_STATS_EVENT, TELEMETRY_INDEX_STATS_EVENT, } from '../event_based/events'; -import type { QueryConfig } from '../collections_helpers'; +import type { CommonPrefixesConfig } from '../collections_helpers'; import { telemetryConfiguration } from '../configuration'; import type { ClusterStats, DataStream } from '../indices.metadata.types'; import { TelemetryCounter } from '../types'; @@ -53,9 +53,10 @@ export function createTelemetryIndicesMetadataTaskConfig() { const taskConfig = telemetryConfiguration.indices_metadata_config; // TODO: not use taskExecutionPeriod, it's just to test the task using the temporary API - const queryConfig: QueryConfig = { + const queryConfig: CommonPrefixesConfig = { maxPrefixes: Number(taskExecutionPeriod.last ?? taskConfig.max_prefixes), maxGroupSize: Number(taskExecutionPeriod.current ?? taskConfig.max_group_size), + minGroupSize: Number(taskExecutionPeriod.current ?? taskConfig.min_group_size), }; const publishClusterStats = (stats: ClusterStats) => { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 662e04146d5bb..7741db5721c78 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -473,6 +473,7 @@ export interface IndicesMetadataConfiguration { datastreams_threshold: number; max_prefixes: number; max_group_size: number; + min_group_size: number; } export interface PaginationConfiguration { From 95905a9fb62798c4316c98490f2cd6b6c78e4e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 22 Oct 2024 15:32:13 +0200 Subject: [PATCH 28/44] Fix typo --- .../server/lib/telemetry/tasks/indices.metadata.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 9a0736d0617ec..93f7351b24bec 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -52,11 +52,10 @@ export function createTelemetryIndicesMetadataTaskConfig() { const taskConfig = telemetryConfiguration.indices_metadata_config; - // TODO: not use taskExecutionPeriod, it's just to test the task using the temporary API const queryConfig: CommonPrefixesConfig = { - maxPrefixes: Number(taskExecutionPeriod.last ?? taskConfig.max_prefixes), - maxGroupSize: Number(taskExecutionPeriod.current ?? taskConfig.max_group_size), - minGroupSize: Number(taskExecutionPeriod.current ?? taskConfig.min_group_size), + maxPrefixes: Number(taskConfig.max_prefixes), + maxGroupSize: Number(taskConfig.max_group_size), + minPrefixSize: Number(taskConfig.min_group_size), }; const publishClusterStats = (stats: ClusterStats) => { From 125ce4965129f6a918d742b2f95913d45cf76d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 23 Oct 2024 19:35:28 +0200 Subject: [PATCH 29/44] Add integration tests --- .../integration_tests/configuration.test.ts | 152 ++++++++++++++++++ .../server/integration_tests/lib/helpers.ts | 46 ++++++ .../lib/telemetry_helpers.ts | 24 ++- .../integration_tests/telemetry.test.ts | 89 +++++----- .../lib/telemetry/tasks/configuration.ts | 2 +- 5 files changed, 265 insertions(+), 48 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts diff --git a/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts b/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts new file mode 100644 index 0000000000000..8629b54c7524d --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import Path from 'path'; +import axios from 'axios'; + +import { cloneDeep } from 'lodash'; + +import { telemetryConfiguration } from '../lib/telemetry/configuration'; +import { + TaskManagerPlugin, + type TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server/plugin'; + +import { + setupTestServers, + removeFile, + mockAxiosPost, + DEFAULT_GET_ROUTES, + mockAxiosGet, + getRandomInt, +} from './lib/helpers'; + +import { + type TestElasticsearchUtils, + type TestKibanaUtils, +} from '@kbn/core-test-helpers-kbn-server'; +import { Plugin as SecuritySolutionPlugin } from '../plugin'; +import { getTelemetryTasks, runSoonConfigTask } from './lib/telemetry_helpers'; +import type { SecurityTelemetryTask } from '../lib/telemetry/task'; + +jest.mock('axios'); + +const logFilePath = Path.join(__dirname, 'config.logs.log'); +const taskManagerStartSpy = jest.spyOn(TaskManagerPlugin.prototype, 'start'); +const securitySolutionStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); + +const mockedAxiosGet = jest.spyOn(axios, 'get'); +const mockedAxiosPost = jest.spyOn(axios, 'post'); + +const securitySolutionPlugin = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); + +describe('configuration', () => { + let esServer: TestElasticsearchUtils; + let kibanaServer: TestKibanaUtils; + let taskManagerPlugin: TaskManagerStartContract; + let tasks: SecurityTelemetryTask[]; + + beforeAll(async () => { + await removeFile(logFilePath); + + const servers = await setupTestServers(logFilePath); + + esServer = servers.esServer; + kibanaServer = servers.kibanaServer; + + expect(taskManagerStartSpy).toHaveBeenCalledTimes(1); + taskManagerPlugin = taskManagerStartSpy.mock.results[0].value; + + expect(securitySolutionStartSpy).toHaveBeenCalledTimes(1); + + tasks = getTelemetryTasks(securitySolutionStartSpy); + + expect(securitySolutionPlugin).toHaveBeenCalledTimes(1); + }); + + afterAll(async () => { + if (kibanaServer) { + await kibanaServer.stop(); + } + if (esServer) { + await esServer.stop(); + } + }); + + beforeEach(async () => { + jest.clearAllMocks(); + mockAxiosPost(mockedAxiosPost); + }); + + afterEach(async () => {}); + + describe('configuration task', () => { + it('should keep default values when no new config was provided', async () => { + const before = cloneDeep(telemetryConfiguration); + + await runSoonConfigTask(tasks, taskManagerPlugin); + + expect(telemetryConfiguration).toEqual(before); + }); + + it('should update values with new manifest', async () => { + const expected = { + telemetry_max_buffer_size: getRandomInt(1, 100), + max_security_list_telemetry_batch: getRandomInt(1, 100), + max_endpoint_telemetry_batch: getRandomInt(1, 100), + max_detection_rule_telemetry_batch: getRandomInt(1, 100), + max_detection_alerts_batch: getRandomInt(1, 100), + use_async_sender: true, + pagination_config: { + max_page_size_bytes: getRandomInt(1, 100), + num_docs_to_sample: getRandomInt(1, 100), + }, + sender_channels: { + default: { + buffer_time_span_millis: getRandomInt(1, 100), + inflight_events_threshold: getRandomInt(1, 100), + max_payload_size_bytes: getRandomInt(1, 100), + }, + }, + indices_metadata_config: { + indices_threshold: getRandomInt(1, 100), + datastreams_threshold: getRandomInt(1, 100), + max_prefixes: getRandomInt(1, 100), + max_group_size: getRandomInt(1, 100), + }, + }; + + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [/.*telemetry-buffer-and-batch-sizes-v1.*/, { status: 200, data: cloneDeep(expected) }], + ]); + + await runSoonConfigTask(tasks, taskManagerPlugin); + + expect(telemetryConfiguration.telemetry_max_buffer_size).toEqual( + expected.telemetry_max_buffer_size + ); + expect(telemetryConfiguration.max_security_list_telemetry_batch).toEqual( + expected.max_security_list_telemetry_batch + ); + expect(telemetryConfiguration.max_endpoint_telemetry_batch).toEqual( + expected.max_endpoint_telemetry_batch + ); + expect(telemetryConfiguration.max_detection_rule_telemetry_batch).toEqual( + expected.max_detection_rule_telemetry_batch + ); + expect(telemetryConfiguration.max_detection_alerts_batch).toEqual( + expected.max_detection_alerts_batch + ); + expect(telemetryConfiguration.use_async_sender).toEqual(expected.use_async_sender); + expect(telemetryConfiguration.sender_channels).toEqual(expected.sender_channels); + expect(telemetryConfiguration.pagination_config).toEqual(expected.pagination_config); + expect(telemetryConfiguration.indices_metadata_config).toEqual( + expected.indices_metadata_config + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts index ccc73435636e9..d6eb3b6d8ca71 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts @@ -10,6 +10,20 @@ import Util from 'util'; import type { ElasticsearchClient } from '@kbn/core/server'; import deepmerge from 'deepmerge'; import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server'; + +export const DEFAULT_GET_ROUTES: Array<[RegExp, unknown]> = [ + [new RegExp('.*/ping$'), { status: 200 }], + [ + /.*kibana\/manifest\/artifacts.*/, + { + status: 200, + data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip', + }, + ], +]; + +export const DEFAULT_POST_ROUTES: Array<[RegExp, unknown]> = [[/.*/, { status: 200 }]]; + const asyncUnlink = Util.promisify(Fs.unlink); /** @@ -127,3 +141,35 @@ export function updateTimestamps(data: object[]): object[] { return { ...d, '@timestamp': new Date(currentTimeMillis + (i + 1) * 100) }; }); } + +export function mockAxiosPost( + postSpy: jest.SpyInstance, + routes: Array<[RegExp, unknown]> = DEFAULT_POST_ROUTES +) { + postSpy.mockImplementation(async (url: string) => { + for (const [route, returnValue] of routes) { + if (route.test(url)) { + return returnValue; + } + } + return { status: 404 }; + }); +} + +export function mockAxiosGet( + getSpy: jest.SpyInstance, + routes: Array<[RegExp, unknown]> = DEFAULT_GET_ROUTES +) { + getSpy.mockImplementation(async (url: string) => { + for (const [route, returnValue] of routes) { + if (route.test(url)) { + return returnValue; + } + } + return { status: 404 }; + }); +} + +export function getRandomInt(min: number, max: number): number { + return Math.floor(Math.random() * (max - min + 1)) + min; +} diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index d83ff1e910ca5..d2576e990ad9a 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -25,12 +25,13 @@ import { deleteExceptionListItem, } from '@kbn/lists-plugin/server/services/exception_lists'; import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants'; +import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { packagePolicyService } from '@kbn/fleet-plugin/server/services'; import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants'; import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock'; -import { bulkInsert, updateTimestamps } from './helpers'; +import { bulkInsert, eventually, updateTimestamps } from './helpers'; import { TelemetryEventsSender } from '../../lib/telemetry/sender'; import type { SecuritySolutionPluginStart, @@ -417,3 +418,24 @@ export function getTelemetryTaskType(task: SecurityTelemetryTask): string { return ''; } } + +export async function runSoonConfigTask( + tasks: SecurityTelemetryTask[], + taskManagerPlugin: TaskManagerStartContract +) { + const configTaskType = 'security:telemetry-configuration'; + const configTask = getTelemetryTask(tasks, configTaskType); + const runAfter = new Date(); + await eventually(async () => { + await taskManagerPlugin.runSoon(configTask.getTaskId()); + }); + + // wait until the task finishes + await eventually(async () => { + const hasRun = await taskManagerPlugin + .get(configTask.getTaskId()) + .then((t) => new Date(t.state.lastExecutionTimestamp) > runAfter) + .catch(() => false); + expect(hasRun).toBe(true); + }); +} diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index 801926adbf948..409eb867e833b 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -20,7 +20,14 @@ import { TELEMETRY_CHANNEL_ENDPOINT_META, } from '../lib/telemetry/constants'; -import { eventually, setupTestServers, removeFile } from './lib/helpers'; +import { + eventually, + setupTestServers, + removeFile, + mockAxiosGet, + mockAxiosPost, + DEFAULT_GET_ROUTES, +} from './lib/helpers'; import { cleanupMockedAlerts, cleanupMockedExceptionLists, @@ -37,6 +44,7 @@ import { mockEndpointData, getTelemetryReceiver, mockPrebuiltRulesData, + runSoonConfigTask, } from './lib/telemetry_helpers'; import { @@ -99,6 +107,7 @@ describe('telemetry tasks', () => { }, }, }); + esServer = servers.esServer; kibanaServer = servers.kibanaServer; @@ -134,7 +143,17 @@ describe('telemetry tasks', () => { beforeEach(async () => { jest.clearAllMocks(); - mockAxiosGet(); + mockAxiosPost(mockedAxiosPost); + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [ + /.*telemetry-buffer-and-batch-sizes-v1.*/, + { + status: 200, + data: fakeBufferAndSizesConfigAsyncDisabled, + }, + ], + ]); deferred = []; }); @@ -208,21 +227,17 @@ describe('telemetry tasks', () => { }); it('should use new sender when configured', async () => { - const configTaskType = 'security:telemetry-configuration'; - const configTask = getTelemetryTask(tasks, configTaskType); + mockAxiosPost(mockedAxiosPost); - mockAxiosGet(fakeBufferAndSizesConfigAsyncEnabled); - await eventually(async () => { - await taskManagerPlugin.runSoon(configTask.getTaskId()); - }); + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [ + /.*telemetry-buffer-and-batch-sizes-v1.*/, + { status: 200, data: fakeBufferAndSizesConfigAsyncEnabled }, + ], + ]); - // wait until the task finishes - await eventually(async () => { - const found = (await taskManagerPlugin.fetch()).docs.find( - (t) => t.taskType === configTaskType - ); - expect(found).toBeFalsy(); - }); + await runSoonConfigTask(tasks, taskManagerPlugin); const [task, started] = await mockAndScheduleDetectionRulesTask(); @@ -238,13 +253,20 @@ describe('telemetry tasks', () => { it('should update sender queue config', async () => { const expectedConfig = fakeBufferAndSizesConfigWithQueues.sender_channels['task-metrics']; - const configTaskType = 'security:telemetry-configuration'; - const configTask = getTelemetryTask(tasks, configTaskType); - mockAxiosGet(fakeBufferAndSizesConfigWithQueues); - await eventually(async () => { - await taskManagerPlugin.runSoon(configTask.getTaskId()); - }); + mockAxiosPost(mockedAxiosPost); + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [ + /.*telemetry-buffer-and-batch-sizes-v1.*/, + { + status: 200, + data: fakeBufferAndSizesConfigWithQueues, + }, + ], + ]); + + await runSoonConfigTask(tasks, taskManagerPlugin); await eventually(async () => { /* eslint-disable dot-notation */ @@ -838,31 +860,6 @@ describe('telemetry tasks', () => { }); } - function mockAxiosGet(bufferConfig: unknown = fakeBufferAndSizesConfigAsyncDisabled) { - mockedAxiosPost.mockImplementation( - async (_url: string, _data?: unknown, _config?: AxiosRequestConfig | undefined) => { - return { status: 200 }; - } - ); - - mockedAxiosGet.mockImplementation(async (url: string) => { - if (url.startsWith(ENDPOINT_STAGING) && url.endsWith('ping')) { - return { status: 200 }; - } else if (url.indexOf('kibana/manifest/artifacts') !== -1) { - return { - status: 200, - data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip', - }; - } else if (url.indexOf('telemetry-buffer-and-batch-sizes-v1') !== -1) { - return { - status: 200, - data: bufferConfig, - }; - } - return { status: 404 }; - }); - } - async function getTaskMetricsRequests( task: SecurityTelemetryTask, olderThan: number diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts index 1b3f888761115..502c27fd3e2a6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts @@ -51,7 +51,7 @@ export function createTelemetryConfigurationTaskConfig() { const configArtifact = manifest.data as unknown as TelemetryConfiguration; log.l('Got telemetry configuration artifact', { - artifact: configArtifact, + artifact: configArtifact ?? '', }); telemetryConfiguration.max_detection_alerts_batch = From acc7731da2ff0ea122977a26cb62e28faf5910d5 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 24 Oct 2024 08:39:28 +0000 Subject: [PATCH 30/44] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test/security_solution_api_integration/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index e13f4bd61520d..26f1ee30916dc 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -52,5 +52,6 @@ "@kbn/ftr-common-functional-ui-services", "@kbn/spaces-plugin", "@kbn/elastic-assistant-plugin", + "@kbn/test-suites-src", ] } From 651c3e90c2a6e19ebc461e5dcbff1567ec409207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 24 Oct 2024 13:07:56 +0200 Subject: [PATCH 31/44] Remove testing route --- .../server/lib/telemetry/routes/index.ts | 68 ------------------- .../security_solution/server/routes/index.ts | 4 -- 2 files changed, 72 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts deleted file mode 100644 index aca860d16db32..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { schema } from '@kbn/config-schema'; -import type { Logger, IRouter } from '@kbn/core/server'; -import type { ITelemetryReceiver } from '../receiver'; -import type { ITelemetryEventsSender } from '../sender'; -import { TaskMetricsService } from '../task_metrics'; -import { createTelemetryIndicesMetadataTaskConfig } from '../tasks/indices.metadata'; - -// TODO: just to test the POC, remove -export const getTriggerIndicesMetadataTaskRoute = ( - router: IRouter, - logger: Logger, - receiver: ITelemetryReceiver, - sender: ITelemetryEventsSender -) => { - router.get( - { - path: '/api/trigger-indices-metadata-task', - options: { - tags: ['api'], - access: 'public', - summary: 'Trigger indices metadata task (for testing purposes)', - }, - validate: { - query: schema.object({ - maxPrefixes: schema.maybe(schema.number()), - maxGroupSize: schema.maybe(schema.number()), - }), - }, - }, - async (_context, request, response) => { - const taskMetricsService = new TaskMetricsService(logger, sender); - const task = createTelemetryIndicesMetadataTaskConfig(); - const timeStart = performance.now(); - - const { maxPrefixes, maxGroupSize } = request.query; - - logger.info( - `Triggering indices metadata task with pageSize: ${maxPrefixes} and dataStreamsLimit: ${maxGroupSize}` - ); - - let msgSuffix = ''; - if (global.gc) { - global.gc(); - } else { - msgSuffix = ' (Note: Garbage collection is not exposed. Start Node.js with --expose-gc.)'; - } - const initialMemory = process.memoryUsage().heapUsed; - const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { - last: `${maxPrefixes || 10}`, - current: `${maxGroupSize || 100}`, - }); - const memoryUsed = process.memoryUsage().heapUsed - initialMemory; - const elapsedTime = performance.now() - timeStart; - - return response.ok({ - body: { - message: `Task finished, it processed ${result} indices, took ${elapsedTime} ms to run and required ${memoryUsed} bytes ${msgSuffix} [ v1.1 ]`, - }, - }); - } - ); -}; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index ea02dc96df390..6f245bd04a02b 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -61,7 +61,6 @@ import { suggestUserProfilesRoute } from '../lib/detection_engine/routes/users/s import { registerTimelineRoutes } from '../lib/timeline/routes'; import { getFleetManagedIndexTemplatesRoute } from '../lib/security_integrations/cribl/routes'; import { registerEntityAnalyticsRoutes } from '../lib/entity_analytics/register_entity_analytics_routes'; -import { getTriggerIndicesMetadataTaskRoute } from '../lib/telemetry/routes'; export const initRoutes = ( router: SecuritySolutionPluginRouter, @@ -148,7 +147,4 @@ export const initRoutes = ( registerEntityAnalyticsRoutes({ router, config, getStartServices, logger }); // Security Integrations getFleetManagedIndexTemplatesRoute(router); - - // TODO: just to test the POC, remove - getTriggerIndicesMetadataTaskRoute(router, logger, previewTelemetryReceiver, telemetrySender); }; From 9d612f9f61e259537da94d8c4dfb2fd03e86c915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 24 Oct 2024 13:08:22 +0200 Subject: [PATCH 32/44] Remove TODOs --- .../plugins/security_solution/server/lib/telemetry/receiver.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index ebe4f4c53587d..de7f26d04769a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -255,7 +255,6 @@ export interface ITelemetryReceiver { setNumDocsToSample(n: number): void; - // ---------- TODO: POC ---------- // getClusterStats(): Promise; getIndices(): Promise; getDataStreams(): Promise; @@ -1353,7 +1352,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this._esClient; } - // ---------- TODO: POC ---------- // public async getClusterStats(): Promise { const es = this.esClient(); From 10be30a0dda247cfca3b3a8ff32990dfa7f3cb80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 24 Oct 2024 13:37:51 +0200 Subject: [PATCH 33/44] Not send cluster stats events --- .../lib/telemetry/event_based/events.ts | 38 +------------------ .../lib/telemetry/indices.metadata.types.ts | 8 ---- .../server/lib/telemetry/receiver.ts | 19 ---------- .../lib/telemetry/tasks/indices.metadata.ts | 22 +++-------- .../telemetry/tasks/indices_metadata.ts | 22 ----------- 5 files changed, 7 insertions(+), 102 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts index 1e2bcf36ea88a..e90fb5f89e38e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -11,13 +11,7 @@ import type { ResponseActionsApiCommandNames, } from '../../../../common/endpoint/service/response_actions/constants'; import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics'; -import type { - ClusterStats, - DataStream, - IlmPolicy, - IlmStats, - IndexStats, -} from '../indices.metadata.types'; +import type { DataStream, IlmPolicy, IlmStats, IndexStats } from '../indices.metadata.types'; export const RISK_SCORE_EXECUTION_SUCCESS_EVENT: EventTypeOpts<{ scoresWritten: number; @@ -215,35 +209,6 @@ export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ }, }; -export const TELEMETRY_CLUSTER_STATS_EVENT: EventTypeOpts = { - eventType: 'telemetry_cluster_stats_event', - schema: { - num_nodes: { - type: 'long', - _meta: { description: 'How many nodes have the cluster' }, - }, - num_indices: { - type: 'long', - _meta: { description: 'How many indices have the cluster' }, - }, - num_docs: { - type: 'long', - _meta: { description: 'How many nodes have the cluster' }, - }, - num_deleted_docs: { - type: 'long', - _meta: { - description: 'How many nodes have the cluster', - optional: true, - }, - }, - total_size_in_bytes: { - type: 'long', - _meta: { description: 'The total size, in bytes, of all documents stored in the cluster.' }, - }, - }, -}; - export const TELEMETRY_DATA_STREAM_EVENT: EventTypeOpts = { eventType: 'telemetry_data_stream_event', schema: { @@ -619,7 +584,6 @@ export const events = [ ENDPOINT_RESPONSE_ACTION_SENT_EVENT, ENDPOINT_RESPONSE_ACTION_SENT_ERROR_EVENT, ENDPOINT_RESPONSE_ACTION_STATUS_CHANGE_EVENT, - TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, TELEMETRY_ILM_POLICY_EVENT, TELEMETRY_ILM_STATS_EVENT, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts index 77a0452728f3c..932015d9bc575 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts @@ -42,14 +42,6 @@ export interface IndexStats { docs_total_size_in_bytes?: number; } -export interface ClusterStats { - num_nodes: number; - num_indices: number; - num_docs: number; - num_deleted_docs?: number; - total_size_in_bytes: number; -} - export interface Index { index_name: string; ilm_policy?: string; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index de7f26d04769a..48df3a9449607 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -93,7 +93,6 @@ import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/co import { DEFAULT_DIAGNOSTIC_INDEX } from './constants'; import type { TelemetryLogger } from './telemetry_logger'; import type { - ClusterStats, DataStream, IlmPhase, IlmPhases, @@ -255,7 +254,6 @@ export interface ITelemetryReceiver { setNumDocsToSample(n: number): void; - getClusterStats(): Promise; getIndices(): Promise; getDataStreams(): Promise; getIndicesStats( @@ -1352,23 +1350,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this._esClient; } - public async getClusterStats(): Promise { - const es = this.esClient(); - - this.logger.l('Fetching cluster stats'); - - return es.cluster.stats({}).then( - (response) => - ({ - num_nodes: response.nodes.count.total, - num_indices: response.indices.count, - num_docs: response.indices.docs.count, - num_deleted_docs: response.indices.docs.deleted, - total_size_in_bytes: response.indices.store.size_in_bytes, - } as ClusterStats) - ); - } - public async getIndices(): Promise { const es = this.esClient(); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 93f7351b24bec..53ef9f09ce8ef 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -16,7 +16,6 @@ import { newTelemetryLogger, } from '../helpers'; import { - TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, TELEMETRY_ILM_POLICY_EVENT, TELEMETRY_ILM_STATS_EVENT, @@ -24,7 +23,7 @@ import { } from '../event_based/events'; import type { CommonPrefixesConfig } from '../collections_helpers'; import { telemetryConfiguration } from '../configuration'; -import type { ClusterStats, DataStream } from '../indices.metadata.types'; +import type { DataStream } from '../indices.metadata.types'; import { TelemetryCounter } from '../types'; const COUNTER_LABELS = ['security_solution', 'indices-metadata']; @@ -58,10 +57,6 @@ export function createTelemetryIndicesMetadataTaskConfig() { minPrefixSize: Number(taskConfig.min_group_size), }; - const publishClusterStats = (stats: ClusterStats) => { - sender.reportEBT(TELEMETRY_CLUSTER_STATS_EVENT.eventType, stats); - }; - const publishDatastreamsStats = (stats: DataStream[]): number => { let counter = 0; for (const ds of stats) { @@ -122,23 +117,18 @@ export function createTelemetryIndicesMetadataTaskConfig() { try { // 1. Get cluster stats and list of indices and datastreams - const [clusterStats, indices, dataStreams] = await Promise.all([ - receiver.getClusterStats(), + const [indices, dataStreams] = await Promise.all([ receiver.getIndices(), receiver.getDataStreams(), ]); - // 2. Publish cluster stats - publishClusterStats(clusterStats); - incrementCounter(TelemetryCounter.DOCS_SENT, 'cluster-stats', 1); - - // 3. Publish datastreams stats + // 2. Publish datastreams stats const dsCount = publishDatastreamsStats( dataStreams.slice(0, taskConfig.datastreams_threshold) ); incrementCounter(TelemetryCounter.DOCS_SENT, 'datastreams-stats', dsCount); - // 4. Get and publish indices stats + // 3. Get and publish indices stats const indicesCount: number = await publishIndicesStats( indices.slice(0, taskConfig.indices_threshold) ) @@ -152,7 +142,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { return 0; }); - // 5. Get ILM stats and publish them + // 4. Get ILM stats and publish them const ilmNames = await publishIlmStats(indices.slice(0, taskConfig.indices_threshold)) .then((names) => { incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-stats', names.size); @@ -164,7 +154,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { return new Set(); }); - // 6. Publish ILM policies + // 5. Publish ILM policies const policyCount = await publishIlmPolicies(ilmNames) .then((count) => { incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-policies', count); diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index 3f45a1868981e..c98cb80182375 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -6,7 +6,6 @@ */ import { - TELEMETRY_CLUSTER_STATS_EVENT, TELEMETRY_DATA_STREAM_EVENT, TELEMETRY_ILM_POLICY_EVENT, TELEMETRY_ILM_STATS_EVENT, @@ -98,27 +97,6 @@ export default ({ getService }: FtrProviderContext) => { logger ); }); - - it('should publish cluster stats events', async () => { - const runAt = await launchTask(TASK_ID, kibanaServer, logger); - - const opts = { - eventTypes: [TELEMETRY_CLUSTER_STATS_EVENT.eventType], - withTimeoutMs: 1000, - fromTimestamp: new Date().toISOString(), - }; - - await waitFor( - async () => { - const events = await ebtServer.getEventCount(opts); - const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); - - return hasRun && events === 1; - }, - 'waitForTaskToRun', - logger - ); - }); }); describe('@ess indices metadata', function () { From 0dcec8ebb0dd430f007bc4db8a5eb2ee8d268c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 4 Nov 2024 16:48:48 +0100 Subject: [PATCH 34/44] Add import --- .../server/integration_tests/lib/telemetry_helpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index 63387c61d2cb4..8dc9f1cbbca49 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -41,6 +41,7 @@ import type { SecurityTelemetryTask } from '../../lib/telemetry/task'; import { Plugin as SecuritySolutionPlugin } from '../../plugin'; import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender'; import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver'; +import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import mockEndpointAlert from '../__mocks__/endpoint-alert.json'; import mockedRule from '../__mocks__/rule.json'; import fleetAgents from '../__mocks__/fleet-agents.json'; From a43ecb8c01bf5041e9a115b133998bcef006725d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 6 Nov 2024 18:20:48 +0100 Subject: [PATCH 35/44] Use indices api instead of _cat --- .../server/lib/telemetry/receiver.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 563fbb9350409..8d9f91be622bb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -42,7 +42,7 @@ import type { IndicesGetDataStreamRequest, IndicesStatsRequest, IlmGetLifecycleRequest, - CatIndicesRequest, + IndicesGetRequest, } from '@elastic/elasticsearch/lib/api/types'; import type { TransportResult } from '@elastic/elasticsearch'; import type { AgentPolicy, Installation } from '@kbn/fleet-plugin/common'; @@ -1355,15 +1355,15 @@ export class TelemetryReceiver implements ITelemetryReceiver { this.logger.l('Fetching indices'); - const request: CatIndicesRequest = { + const request: IndicesGetRequest = { + index: '*', expand_wildcards: ['open', 'hidden'], - filter_path: ['index'], - format: 'json', + filter_path: ['*.settings.index.provided_name'], }; - return es.cat - .indices(request) - .then((indices) => indices.map(({ index }) => index as string)) + return es.indices + .get(request) + .then((indices) => Array.from(Object.keys(indices))) .catch((error) => { this.logger.warn('Error fetching indices', { error_message: error } as LogMeta); throw error; From 342703ca322abf24847e064a34aeb7706cb18769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Thu, 7 Nov 2024 15:59:48 +0100 Subject: [PATCH 36/44] Add preview route --- ...telemetry_detection_rules_preview_route.ts | 8 ++ .../utils/get_indices_metadata_preview.ts | 42 ++++++++++ .../server/lib/telemetry/preview_sender.ts | 11 +++ .../task_based/all_types.ts | 83 ++++++++++++------- ...remove_time_fields_from_telemetry_stats.ts | 38 +++++---- 5 files changed, 138 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts index 271e6e7d27749..013a63d89b7c4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts @@ -15,6 +15,7 @@ import { getDetectionRulesPreview } from './utils/get_detecton_rules_preview'; import { getSecurityListsPreview } from './utils/get_security_lists_preview'; import { getEndpointPreview } from './utils/get_endpoint_preview'; import { getDiagnosticsPreview } from './utils/get_diagnostics_preview'; +import { getIndicesMetadataPreview } from './utils/get_indices_metadata_preview'; export const telemetryDetectionRulesPreviewRoute = ( router: SecuritySolutionPluginRouter, @@ -60,12 +61,19 @@ export const telemetryDetectionRulesPreviewRoute = ( telemetrySender, }); + const indicesMetadata = await getIndicesMetadataPreview({ + logger, + telemetryReceiver, + telemetrySender, + }); + return response.ok({ body: { detection_rules: detectionRules, security_lists: securityLists, endpoints, diagnostics, + indices_metadata: indicesMetadata, }, }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts new file mode 100644 index 0000000000000..7144034f91e71 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; + +import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender'; +import type { ITelemetryReceiver } from '../../../../telemetry/receiver'; +import { PreviewTaskMetricsService } from '../../../../telemetry/preview_task_metrics'; +import type { ITelemetryEventsSender } from '../../../../telemetry/sender'; +import { createTelemetryIndicesMetadataTaskConfig } from '../../../../telemetry/tasks/indices.metadata'; + +export const getIndicesMetadataPreview = async ({ + logger, + telemetryReceiver, + telemetrySender, +}: { + logger: Logger; + telemetryReceiver: ITelemetryReceiver; + telemetrySender: ITelemetryEventsSender; +}): Promise> => { + const taskExecutionPeriod = { + last: new Date(0).toISOString(), + current: new Date().toISOString(), + }; + + const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender); + const taskMetricsService = new PreviewTaskMetricsService(logger, taskSender); + const task = createTelemetryIndicesMetadataTaskConfig(); + await task.runTask( + 'indices-metadata-telemetry', + logger, + telemetryReceiver, + taskSender, + taskMetricsService, + taskExecutionPeriod + ); + return taskSender.getEbtEventsSent(); +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts index 9d4e0450a7570..e304646816635 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts @@ -37,6 +37,9 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { /** Last sent message */ private sentMessages: string[] = []; + /** Last sent EBT events */ + private ebtEventsSent: Array<{ eventType: string; eventData: object }> = []; + /** Logger for this class */ private logger: Logger; @@ -87,6 +90,10 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { return this.sentMessages; } + public getEbtEventsSent(): Array<{ eventType: string; eventData: object }> { + return this.ebtEventsSent; + } + public setup( telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, @@ -176,6 +183,10 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { } public reportEBT(eventType: EventType, eventData: object): void { + this.ebtEventsSent.push({ + eventType, + eventData, + }); this.composite.reportEBT(eventType, eventData); } } diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts index d13e74724c818..2843a48433079 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts @@ -47,40 +47,63 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); removeExtraFieldsFromTelemetryStats(stats); - expect(stats).to.eql({ - detection_rules: [ - [ - { - name: 'security:telemetry-detection-rules', - passed: true, - }, - ], + + expect(stats.detection_rules).to.eql([ + [ + { + name: 'security:telemetry-detection-rules', + passed: true, + }, ], - security_lists: [ - [ - { - name: 'security:telemetry-lists', - passed: true, - }, - ], + ]); + + expect(stats.security_lists).to.eql([ + [ + { + name: 'security:telemetry-lists', + passed: true, + }, ], - endpoints: [ - [ - { - name: 'security:endpoint-meta-telemetry', - passed: true, - }, - ], + ]); + + expect(stats.endpoints).to.eql([ + [ + { + name: 'security:endpoint-meta-telemetry', + passed: true, + }, ], - diagnostics: [ - [ - { - name: 'security:endpoint-diagnostics', - passed: true, - }, - ], + ]); + + expect(stats.diagnostics).to.eql([ + [ + { + name: 'security:endpoint-diagnostics', + passed: true, + }, ], - }); + ]); + + expect(stats.indices_metadata).to.be.an('array'); + const events = stats.indices_metadata as any[]; + expect(events).to.not.be.empty(); + + const eventTypes = events.map((e) => e.eventType); + expect(eventTypes).to.contain('telemetry_index_stats_event'); + expect(eventTypes).to.contain('telemetry_data_stream_event'); + + const indexStats = events.find((e) => e.eventType === 'telemetry_index_stats_event'); + expect(indexStats.eventData).to.have.keys( + 'index_name', + 'query_total', + 'query_time_in_millis', + 'docs_count', + 'docs_deleted', + 'docs_total_size_in_bytes' + ); + + const dataStreamStats = events.find((e) => e.eventType === 'telemetry_data_stream_event'); + expect(dataStreamStats.eventData).to.have.keys('datastream_name', 'indices'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts index 2d27b375684ba..b835832fcb550 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts @@ -5,19 +5,29 @@ * 2.0. */ -import { unset } from 'lodash'; - export const removeExtraFieldsFromTelemetryStats = (stats: any) => { - Object.entries(stats).forEach(([, value]: [unknown, any]) => { - value.forEach((entry: any, i: number) => { - entry.forEach((_e: any, j: number) => { - unset(value, `[${i}][${j}].time_executed_in_ms`); - unset(value, `[${i}][${j}].start_time`); - unset(value, `[${i}][${j}].end_time`); - unset(value, `[${i}][${j}].cluster_uuid`); - unset(value, `[${i}][${j}].cluster_name`); - unset(value, `[${i}][${j}].license`); - }); - }); - }); + removeExtraFields(stats, [ + 'time_executed_in_ms', + 'start_time', + 'end_time', + 'cluster_uuid', + 'cluster_name', + 'license', + ]); }; + +function removeExtraFields(obj: any, fields: string[]): void { + function traverseAndRemove(o: any): void { + if (typeof o !== 'object' || o === null) return; + + for (const key in o) { + if (fields.includes(key)) { + delete o[key]; + } else if (typeof o[key] === 'object') { + traverseAndRemove(o[key]); + } + } + } + + traverseAndRemove(obj); +} From 5b746f36f2a6b573eaa258ee857dace7312e9ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 11 Nov 2024 15:07:39 +0100 Subject: [PATCH 37/44] Update EBT schema --- .../server/lib/telemetry/async_sender.ts | 6 +- .../lib/telemetry/async_sender.types.ts | 4 +- .../lib/telemetry/event_based/events.ts | 348 ++++++++++-------- .../lib/telemetry/indices.metadata.types.ts | 15 + .../server/lib/telemetry/preview_sender.ts | 10 +- .../server/lib/telemetry/sender.ts | 8 +- .../lib/telemetry/tasks/indices.metadata.ts | 55 ++- .../task_based/all_types.ts | 13 +- .../telemetry/tasks/indices_metadata.ts | 52 +-- 9 files changed, 299 insertions(+), 212 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts index 014f1035b9e81..107ca457b2cfb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts @@ -8,7 +8,7 @@ import axios from 'axios'; import * as rx from 'rxjs'; import _, { cloneDeep } from 'lodash'; -import type { AnalyticsServiceSetup, EventType, Logger, LogMeta } from '@kbn/core/server'; +import type { AnalyticsServiceSetup, EventTypeOpts, Logger, LogMeta } from '@kbn/core/server'; import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; import type { ITelemetryReceiver } from './receiver'; @@ -205,11 +205,11 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { } } - public reportEBT(eventType: EventType, eventData: object) { + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { if (!this.analytics) { throw Error('analytics is unavailable'); } - this.analytics.reportEvent(eventType, eventData); + this.analytics.reportEvent(eventTypeOpts.eventType, eventData as object); } // internal methods diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts index e19cc4d7bf0ef..89ce55d102859 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts @@ -6,7 +6,7 @@ */ import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; -import type { AnalyticsServiceSetup, EventType } from '@kbn/core-analytics-server'; +import type { AnalyticsServiceSetup, EventTypeOpts } from '@kbn/core-analytics-server'; import { type TelemetryChannel } from './types'; import type { ITelemetryReceiver } from './receiver'; @@ -30,7 +30,7 @@ export interface IAsyncTelemetryEventsSender { simulateSend: (channel: TelemetryChannel, events: unknown[]) => string[]; updateQueueConfig: (channel: TelemetryChannel, config: QueueConfig) => void; updateDefaultQueueConfig: (config: QueueConfig) => void; - reportEBT: (eventType: EventType, eventData: object) => void; + reportEBT: (eventTypeOpts: EventTypeOpts, eventData: T) => void; } /** diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts index 77b86fd1f0e44..662704bc59bc9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -11,7 +11,7 @@ import type { ResponseActionsApiCommandNames, } from '../../../../common/endpoint/service/response_actions/constants'; import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics'; -import type { DataStream, IlmPolicy, IlmStats, IndexStats } from '../indices.metadata.types'; +import type { DataStreams, IlmPolicies, IlmsStats, IndicesStats } from '../indices.metadata.types'; export const RISK_SCORE_EXECUTION_SUCCESS_EVENT: EventTypeOpts<{ scoresWritten: number; @@ -272,195 +272,237 @@ export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ }, }; -export const TELEMETRY_DATA_STREAM_EVENT: EventTypeOpts = { +export const TELEMETRY_DATA_STREAM_EVENT: EventTypeOpts = { eventType: 'telemetry_data_stream_event', schema: { - datastream_name: { - type: 'keyword', - _meta: { description: 'Name of the data stream' }, - }, - indices: { + items: { type: 'array', items: { properties: { - index_name: { type: 'date', _meta: { description: 'Index name' } }, - ilm_policy: { type: 'date', _meta: { optional: true, description: 'ILM policy' } }, + datastream_name: { + type: 'keyword', + _meta: { description: 'Name of the data stream' }, + }, + indices: { + type: 'array', + items: { + properties: { + index_name: { type: 'date', _meta: { description: 'Index name' } }, + ilm_policy: { type: 'date', _meta: { optional: true, description: 'ILM policy' } }, + }, + }, + _meta: { optional: true, description: 'Indices associated with the data stream' }, + }, }, }, - _meta: { optional: true, description: 'Indices associated with the data stream' }, + _meta: { description: 'Datastreams' }, }, }, }; -export const TELEMETRY_INDEX_STATS_EVENT: EventTypeOpts = { +export const TELEMETRY_INDEX_STATS_EVENT: EventTypeOpts = { eventType: 'telemetry_index_stats_event', schema: { - index_name: { - type: 'keyword', - _meta: { description: 'The name of the index being monitored.' }, - }, - query_total: { - type: 'long', - _meta: { - optional: true, - description: 'The total number of search queries executed on the index.', - }, - }, - query_time_in_millis: { - type: 'long', - _meta: { - optional: true, - description: - 'The total time spent on query execution across all search requests, measured in milliseconds.', - }, - }, - docs_count: { - type: 'long', - _meta: { - optional: true, - description: 'The total number of documents currently stored in the index.', - }, - }, - docs_deleted: { - type: 'long', - _meta: { - optional: true, - description: 'The total number of documents that have been marked as deleted in the index.', - }, - }, - docs_total_size_in_bytes: { - type: 'long', - _meta: { - optional: true, - description: - 'The total size, in bytes, of all documents stored in the index, including storage overhead.', + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index being monitored.' }, + }, + query_total: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of search queries executed on the index.', + }, + }, + query_time_in_millis: { + type: 'long', + _meta: { + optional: true, + description: + 'The total time spent on query execution across all search requests, measured in milliseconds.', + }, + }, + docs_count: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of documents currently stored in the index.', + }, + }, + docs_deleted: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents that have been marked as deleted in the index.', + }, + }, + docs_total_size_in_bytes: { + type: 'long', + _meta: { + optional: true, + description: + 'The total size, in bytes, of all documents stored in the index, including storage overhead.', + }, + }, + }, }, + _meta: { description: 'Datastreams' }, }, }, }; -export const TELEMETRY_ILM_POLICY_EVENT: EventTypeOpts = { +export const TELEMETRY_ILM_POLICY_EVENT: EventTypeOpts = { eventType: 'telemetry_ilm_policy_event', schema: { - policy_name: { - type: 'keyword', - _meta: { description: 'The name of the ILM policy.' }, - }, - modified_date: { - type: 'date', - _meta: { description: 'The date when the ILM policy was last modified.' }, - }, - phases: { - properties: { - cold: { - properties: { - min_age: { - type: 'text', - _meta: { - description: 'The minimum age before the index transitions to the "cold" phase.', - }, - }, + items: { + type: 'array', + items: { + properties: { + policy_name: { + type: 'keyword', + _meta: { description: 'The name of the ILM policy.' }, }, - _meta: { - optional: true, - description: - 'Configuration settings for the "cold" phase of the ILM policy, applied when data is infrequently accessed.', + modified_date: { + type: 'date', + _meta: { description: 'The date when the ILM policy was last modified.' }, }, - }, - delete: { - properties: { - min_age: { - type: 'text', - _meta: { - description: 'The minimum age before the index transitions to the "delete" phase.', + phases: { + properties: { + cold: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "cold" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "cold" phase of the ILM policy, applied when data is infrequently accessed.', + }, }, - }, - }, - _meta: { - optional: true, - description: - 'Configuration settings for the "delete" phase of the ILM policy, specifying when the index should be removed.', - }, - }, - frozen: { - properties: { - min_age: { - type: 'text', - _meta: { - description: 'The minimum age before the index transitions to the "frozen" phase.', + delete: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "delete" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "delete" phase of the ILM policy, specifying when the index should be removed.', + }, }, - }, - }, - _meta: { - optional: true, - description: - 'Configuration settings for the "frozen" phase of the ILM policy, where data is fully searchable but stored with a reduced resource footprint.', - }, - }, - hot: { - properties: { - min_age: { - type: 'text', - _meta: { - description: 'The minimum age before the index transitions to the "hot" phase.', + frozen: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "frozen" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "frozen" phase of the ILM policy, where data is fully searchable but stored with a reduced resource footprint.', + }, }, - }, - }, - _meta: { - optional: true, - description: - 'Configuration settings for the "hot" phase of the ILM policy, applied to actively written and queried data.', - }, - }, - warm: { - properties: { - min_age: { - type: 'text', - _meta: { - description: 'The minimum age before the index transitions to the "warm" phase.', + hot: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "hot" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "hot" phase of the ILM policy, applied to actively written and queried data.', + }, + }, + warm: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "warm" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "warm" phase of the ILM policy, used for read-only data that is less frequently accessed.', + }, }, }, - }, - _meta: { - optional: true, - description: - 'Configuration settings for the "warm" phase of the ILM policy, used for read-only data that is less frequently accessed.', + _meta: { + description: + 'The different phases of the ILM policy that define how the index is managed over time.', + }, }, }, }, - _meta: { - description: - 'The different phases of the ILM policy that define how the index is managed over time.', - }, + _meta: { description: 'Datastreams' }, }, }, }; -export const TELEMETRY_ILM_STATS_EVENT: EventTypeOpts = { +export const TELEMETRY_ILM_STATS_EVENT: EventTypeOpts = { eventType: 'telemetry_ilm_stats_event', schema: { - index_name: { - type: 'keyword', - _meta: { description: 'The name of the index currently managed by the ILM policy.' }, - }, - phase: { - type: 'keyword', - _meta: { - optional: true, - description: - 'The current phase of the ILM policy that the index is in (e.g., hot, warm, cold, frozen, or delete).', - }, - }, - age: { - type: 'text', - _meta: { - optional: true, - description: 'The age of the index since its creation, indicating how long it has existed.', + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index currently managed by the ILM policy.' }, + }, + phase: { + type: 'keyword', + _meta: { + optional: true, + description: + 'The current phase of the ILM policy that the index is in (e.g., hot, warm, cold, frozen, or delete).', + }, + }, + age: { + type: 'text', + _meta: { + optional: true, + description: + 'The age of the index since its creation, indicating how long it has existed.', + }, + }, + policy_name: { + type: 'keyword', + _meta: { + optional: true, + description: 'The name of the ILM policy applied to this index.', + }, + }, + }, }, - }, - policy_name: { - type: 'keyword', - _meta: { optional: true, description: 'The name of the ILM policy applied to this index.' }, + _meta: { description: 'Datastreams' }, }, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts index 932015d9bc575..fe669c6693d2a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts @@ -8,6 +8,10 @@ import type { DateTime } from '@elastic/elasticsearch/lib/api/types'; import type { Nullable } from './types'; +export interface IlmPolicies { + items: IlmPolicy[]; +} + export interface IlmPolicy { policy_name: string; modified_date: DateTime; @@ -26,6 +30,10 @@ export interface IlmPhase { min_age: string; } +export interface IlmsStats { + items: IlmStats[]; +} + export interface IlmStats { index_name: string; phase?: string; @@ -33,6 +41,10 @@ export interface IlmStats { policy_name?: string; } +export interface IndicesStats { + items: IndexStats[]; +} + export interface IndexStats { index_name: string; query_total?: number; @@ -47,6 +59,9 @@ export interface Index { ilm_policy?: string; } +export interface DataStreams { + items: DataStream[]; +} export interface DataStream { datastream_name: string; indices?: Index[]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts index e304646816635..66161274fe45c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts @@ -7,7 +7,7 @@ import type { AxiosInstance, AxiosResponse } from 'axios'; import axios, { AxiosHeaders } from 'axios'; -import type { EventType, Logger } from '@kbn/core/server'; +import type { EventTypeOpts, Logger } from '@kbn/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; @@ -182,11 +182,11 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { this.composite.updateDefaultQueueConfig(config); } - public reportEBT(eventType: EventType, eventData: object): void { + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { this.ebtEventsSent.push({ - eventType, - eventData, + eventType: eventTypeOpts.eventType, + eventData: eventData as object, }); - this.composite.reportEBT(eventType, eventData); + this.composite.reportEBT(eventTypeOpts, eventData); } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 975e105f91e35..8e99a9e12981c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -9,7 +9,7 @@ import { cloneDeep } from 'lodash'; import { URL } from 'url'; import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import type { EventType, Logger, LogMeta } from '@kbn/core/server'; +import type { EventTypeOpts, Logger, LogMeta } from '@kbn/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; import type { AxiosInstance } from 'axios'; @@ -92,7 +92,7 @@ export interface ITelemetryEventsSender { /** * Reports EBT events */ - reportEBT: (eventType: EventType, eventData: object) => void; + reportEBT: (eventTypeOpts: EventTypeOpts, eventData: T) => void; } export class TelemetryEventsSender implements ITelemetryEventsSender { @@ -435,8 +435,8 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { this.getAsyncTelemetrySender().updateDefaultQueueConfig(config); } - public reportEBT(eventType: EventType, eventData: object): void { - this.getAsyncTelemetrySender().reportEBT(eventType, eventData); + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { + this.getAsyncTelemetrySender().reportEBT(eventTypeOpts, eventData); } private getAsyncTelemetrySender(): IAsyncTelemetryEventsSender { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 53ef9f09ce8ef..71a19f1e785bd 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -23,7 +23,13 @@ import { } from '../event_based/events'; import type { CommonPrefixesConfig } from '../collections_helpers'; import { telemetryConfiguration } from '../configuration'; -import type { DataStream } from '../indices.metadata.types'; +import type { + DataStream, + DataStreams, + IlmPolicies, + IlmsStats, + IndicesStats, +} from '../indices.metadata.types'; import { TelemetryCounter } from '../types'; const COUNTER_LABELS = ['security_solution', 'indices-metadata']; @@ -58,52 +64,63 @@ export function createTelemetryIndicesMetadataTaskConfig() { }; const publishDatastreamsStats = (stats: DataStream[]): number => { - let counter = 0; - for (const ds of stats) { - sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT.eventType, ds); - counter++; - } - log.info(`Sent data streams`, { count: counter } as LogMeta); - return counter; + const events: DataStreams = { + items: stats, + }; + sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT, events); + log.info(`Sent data streams`, { count: events.items.length } as LogMeta); + return events.items.length; }; const publishIndicesStats = async (indices: string[]): Promise => { - let counter = 0; + const indicesStats: IndicesStats = { + items: [], + }; + for await (const stat of receiver.getIndicesStats( indices.slice(0, taskConfig.indices_threshold), queryConfig )) { - sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT.eventType, stat); - counter++; + indicesStats.items.push(stat); } - log.info(`Sent indices stats`, { count: counter } as LogMeta); - return counter; + sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT, indicesStats); + log.info(`Sent indices stats`, { count: indicesStats.items.length } as LogMeta); + return indicesStats.items.length; }; const publishIlmStats = async (indices: string[]): Promise> => { const ilmNames = new Set(); + const ilmsStats: IlmsStats = { + items: [], + }; + for await (const stat of receiver.getIlmsStats(indices, queryConfig)) { if (stat.policy_name !== undefined) { ilmNames.add(stat.policy_name); - sender.reportEBT(TELEMETRY_ILM_STATS_EVENT.eventType, stat); + ilmsStats.items.push(stat); } } + + sender.reportEBT(TELEMETRY_ILM_STATS_EVENT, ilmsStats); log.info(`Sent ILM stats`, { count: ilmNames.size } as LogMeta); return ilmNames; }; const publishIlmPolicies = async (ilmNames: Set): Promise => { - let counter = 0; + const ilmPolicies: IlmPolicies = { + items: [], + }; + for await (const policy of receiver.getIlmsPolicies( Array.from(ilmNames.values()), queryConfig )) { - sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT.eventType, policy); - counter++; + ilmPolicies.items.push(policy); } - log.info(`Sent ILM policies`, { count: counter } as LogMeta); - return counter; + sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT, ilmPolicies); + log.info(`Sent ILM policies`, { count: ilmPolicies.items.length } as LogMeta); + return ilmPolicies.items.length; }; const incrementCounter = (type: TelemetryCounter, name: string, value: number) => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts index 2843a48433079..4b22a4b590621 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts @@ -86,14 +86,18 @@ export default ({ getService }: FtrProviderContext) => { expect(stats.indices_metadata).to.be.an('array'); const events = stats.indices_metadata as any[]; + expect(events).to.not.be.empty(); const eventTypes = events.map((e) => e.eventType); expect(eventTypes).to.contain('telemetry_index_stats_event'); expect(eventTypes).to.contain('telemetry_data_stream_event'); - const indexStats = events.find((e) => e.eventType === 'telemetry_index_stats_event'); - expect(indexStats.eventData).to.have.keys( + const indicesStats = events.find((e) => e.eventType === 'telemetry_index_stats_event'); + expect(indicesStats).to.be.ok(); + expect(indicesStats.eventData).to.be.ok(); + expect(indicesStats.eventData.items).to.not.be.empty(); + expect(indicesStats.eventData.items[0]).to.have.keys( 'index_name', 'query_total', 'query_time_in_millis', @@ -103,7 +107,10 @@ export default ({ getService }: FtrProviderContext) => { ); const dataStreamStats = events.find((e) => e.eventType === 'telemetry_data_stream_event'); - expect(dataStreamStats.eventData).to.have.keys('datastream_name', 'indices'); + expect(dataStreamStats).to.be.ok(); + expect(dataStreamStats.eventData).to.be.ok(); + expect(dataStreamStats.eventData.items).to.not.be.empty(); + expect(dataStreamStats.eventData.items[0]).to.have.keys('datastream_name', 'indices'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index c98cb80182375..c7d365baab767 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -54,18 +54,18 @@ export default ({ getService }: FtrProviderContext) => { eventTypes: [TELEMETRY_DATA_STREAM_EVENT.eventType], withTimeoutMs: 1000, fromTimestamp: new Date().toISOString(), - filters: { - 'properties.datastream_name': { - eq: dsName, - }, - }, }; await waitFor( async () => { - const eventCount = await ebtServer.getEventCount(opts); + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => result.filter((ev) => (ev as any).datastream_name === dsName)); const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + const eventCount = events.length; return hasRun && eventCount === 1; }, @@ -87,11 +87,17 @@ export default ({ getService }: FtrProviderContext) => { const regex = new RegExp(`^\.ds-${dsName}-\\d{4}.\\d{2}.\\d{2}-\\d{6}$`); await waitFor( async () => { - const events = await ebtServer.getEvents(Number.MAX_SAFE_INTEGER, opts); - const filtered = events.filter((ev) => regex.test(ev.properties.index_name as string)); + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => + result.filter((ev) => regex.test((ev as any).index_name as string)) + ); + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); - return hasRun && filtered.length === NUM_INDICES; + return hasRun && events.length === NUM_INDICES; }, 'waitForTaskToRun', logger @@ -120,19 +126,19 @@ export default ({ getService }: FtrProviderContext) => { eventTypes: [TELEMETRY_ILM_POLICY_EVENT.eventType], withTimeoutMs: 1000, fromTimestamp: new Date().toISOString(), - filters: { - 'properties.policy_name': { - eq: policyName, - }, - }, }; await waitFor( async () => { - const events = await ebtServer.getEventCount(opts); + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => result.filter((ev) => (ev as any).policy_name === policyName)); + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); - return hasRun && events === 1; + return hasRun && events.length === 1; }, 'waitForTaskToRun', logger @@ -146,19 +152,19 @@ export default ({ getService }: FtrProviderContext) => { eventTypes: [TELEMETRY_ILM_STATS_EVENT.eventType], withTimeoutMs: 1000, fromTimestamp: new Date().toISOString(), - filters: { - 'properties.policy_name': { - eq: policyName, - }, - }, }; await waitFor( async () => { - const events = await ebtServer.getEventCount(opts); + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => result.filter((ev) => (ev as any).policy_name === policyName)); + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); - return hasRun && events === NUM_INDICES; + return hasRun && events.length === NUM_INDICES; }, 'waitForTaskToRun', logger From c43ec7432d72717f99b2d4289d26bf03dd628e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 3 Dec 2024 14:44:51 +0100 Subject: [PATCH 38/44] Use alternative approach to paginate --- .../lib/telemetry/collections_helpers.test.ts | 33 ++++ .../lib/telemetry/collections_helpers.ts | 21 ++ .../server/lib/telemetry/receiver.ts | 182 +++++++++--------- .../lib/telemetry/tasks/indices.metadata.ts | 7 +- 4 files changed, 145 insertions(+), 98 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts index 1535344844df7..987c088d289fe 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts @@ -10,6 +10,7 @@ import { chunked, chunkedBy, findCommonPrefixes, + chunkStringsByMaxLength, } from './collections_helpers'; describe('telemetry.utils.chunked', () => { @@ -155,3 +156,35 @@ describe('telemetry.utils.findCommonPrefixes', () => { expect(output.map((v, _) => v.indexCount).reduce((acc, i) => acc + i, 0)).toBe(indices.length); }); }); + +describe('telemetry.utils.splitIndicesByNameLength', () => { + it('should chunk simple case', async () => { + const input = ['aa', 'b', 'ccc', 'ddd']; + const output = chunkStringsByMaxLength(input, 5); + expect(output).toEqual([['aa', 'b'], ['ccc'], ['ddd']]); + }); + + it('should chunk with remainder', async () => { + const input = ['aaa', 'b']; + const output = chunkStringsByMaxLength(input, 10); + expect(output).toEqual([['aaa', 'b']]); + }); + + it('should chunk with empty list', async () => { + const input: string[] = []; + const output = chunkStringsByMaxLength(input, 3); + expect(output).toEqual([]); + }); + + it('should chunk with single element smaller than max weight', async () => { + const input = ['aa']; + const output = chunkStringsByMaxLength(input, 3); + expect(output).toEqual([['aa']]); + }); + + it('should chunk with single element bigger than max weight', async () => { + const input = ['aaaa', 'bb']; + const output = chunkStringsByMaxLength(input, 4); + expect(output).toEqual([['aaaa'], ['bb']]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts index b3a68fe946a81..3d945667fd3cb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts @@ -192,3 +192,24 @@ export function findCommonPrefixes(indices: string[], config: CommonPrefixesConf return prefixes; } + +/** + * Splits an array of strings into chunks where the total length of strings in each chunk + * does not exceed the specified `maxLength`. + * + * @param strings - An array of strings to be chunked. + * @param maxLength - The maximum total length allowed for strings in each chunk. Defaults to 1024. + * @returns A two-dimensional array where each inner array is a chunk of strings. + * + * @example + * ```typescript + * const strings = ["hello", "world", "this", "is", "a", "test"]; + * const chunks = chunkStringsByMaxLength(strings, 10); + * console.log(chunks); + * // Output: [["hello", "world"], ["this", "is"], ["a", "test"]] + * ``` + */ +export function chunkStringsByMaxLength(strings: string[], maxLength: number = 3072): string[][] { + // plus 1 for the comma separator + return chunkedBy(strings, maxLength, (index) => index.length + 1); +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 8d9f91be622bb..5925c39469a1b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -101,7 +101,7 @@ import type { Index, IndexStats, } from './indices.metadata.types'; -import { type CommonPrefixesConfig, findCommonPrefixes } from './collections_helpers'; +import { type CommonPrefixesConfig, chunkStringsByMaxLength } from './collections_helpers'; export interface ITelemetryReceiver { start( @@ -1408,7 +1408,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { this.logger.l('Fetching indices stats', { indices } as LogMeta); - const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); + const groupedIndices = chunkStringsByMaxLength(indices); this.logger.l('Splitted indices into groups', { groups: groupedIndices.length, @@ -1416,37 +1416,35 @@ export class TelemetryReceiver implements ITelemetryReceiver { } as LogMeta); for (const group of groupedIndices) { - for (const name of group) { - const request: IndicesStatsRequest = { - index: `${name}*`, - level: 'indices', - metric: ['docs', 'search', 'store'], - expand_wildcards: ['open', 'hidden'], - filter_path: [ - 'indices.*.total.search.query_total', - 'indices.*.total.search.query_time_in_millis', - 'indices.*.total.docs.count', - 'indices.*.total.docs.deleted', - 'indices.*.total.store.size_in_bytes', - ], - }; - - try { - const response = await es.indices.stats(request); - for (const [indexName, stats] of Object.entries(response.indices ?? {})) { - yield { - index_name: indexName, - query_total: stats.total?.search?.query_total, - query_time_in_millis: stats.total?.search?.query_time_in_millis, - docs_count: stats.total?.docs?.count, - docs_deleted: stats.total?.docs?.deleted, - docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, - } as IndexStats; - } - } catch (error) { - this.logger.warn('Error fetching indices stats', { error_message: error } as LogMeta); - throw error; + const request: IndicesStatsRequest = { + index: group, + level: 'indices', + metric: ['docs', 'search', 'store'], + expand_wildcards: ['open', 'hidden'], + filter_path: [ + 'indices.*.total.search.query_total', + 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', + 'indices.*.total.docs.deleted', + 'indices.*.total.store.size_in_bytes', + ], + }; + + try { + const response = await es.indices.stats(request); + for (const [indexName, stats] of Object.entries(response.indices ?? {})) { + yield { + index_name: indexName, + query_total: stats.total?.search?.query_total, + query_time_in_millis: stats.total?.search?.query_time_in_millis, + docs_count: stats.total?.docs?.count, + docs_deleted: stats.total?.docs?.deleted, + docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + } as IndexStats; } + } catch (error) { + this.logger.warn('Error fetching indices stats', { error_message: error } as LogMeta); + throw error; } } } @@ -1454,38 +1452,37 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async *getIlmsStats(indices: string[], config: CommonPrefixesConfig) { const es = this.esClient(); - const groupedIndices = findCommonPrefixes(indices, config).map((v, _) => v.parts); + const groupedIndices = chunkStringsByMaxLength(indices); - this.logger.l('Splitted indices into groups', { + this.logger.l('Splitted ilms into groups', { groups: groupedIndices.length, indices: indices.length, } as LogMeta); for (const group of groupedIndices) { - for (const name of group) { - const request: IlmExplainLifecycleRequest = { - index: `${name}*`, - only_managed: false, - filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], - }; - - const data = await es.ilm.explainLifecycle(request); - - try { - for (const [indexName, stats] of Object.entries(data.indices ?? {})) { - const entry = { - index_name: indexName, - phase: ('phase' in stats && stats.phase) || undefined, - age: ('age' in stats && stats.age) || undefined, - policy_name: ('policy' in stats && stats.policy) || undefined, - } as IlmStats; - - yield entry; - } - } catch (error) { - this.logger.warn('Error fetching ilm stats', { error_message: error } as LogMeta); - throw error; + const indices = group.join(','); + const request: IlmExplainLifecycleRequest = { + index: indices, + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + const data = await es.ilm.explainLifecycle(request); + + try { + for (const [indexName, stats] of Object.entries(data.indices ?? {})) { + const entry = { + index_name: indexName, + phase: ('phase' in stats && stats.phase) || undefined, + age: ('age' in stats && stats.age) || undefined, + policy_name: ('policy' in stats && stats.policy) || undefined, + } as IlmStats; + + yield entry; } + } catch (error) { + this.logger.warn('Error fetching ilm stats', { error_message: error } as LogMeta); + throw error; } } } @@ -1503,49 +1500,48 @@ export class TelemetryReceiver implements ITelemetryReceiver { return value; }; - const groupedIlms = findCommonPrefixes(ilms, config).map((v, _) => v.parts); + const groupedIlms = chunkStringsByMaxLength(ilms); - this.logger.l(`Splitted ilms into groups`, { + this.logger.l('Splitted ilms into groups', { groups: groupedIlms.length, ilms: ilms.length, } as LogMeta); for (const group of groupedIlms) { - for (const name of group) { - this.logger.l('Fetching ilm policies', { name } as LogMeta); - const request: IlmGetLifecycleRequest = { - name: `${name}*`, - filter_path: [ - '*.policy.phases.cold.min_age', - '*.policy.phases.delete.min_age', - '*.policy.phases.frozen.min_age', - '*.policy.phases.hot.min_age', - '*.policy.phases.warm.min_age', - '*.modified_date', - ], - }; - - const response = await es.ilm.getLifecycle(request); - try { - for (const [policyName, stats] of Object.entries(response ?? {})) { - yield { - policy_name: policyName, - modified_date: stats.modified_date, - phases: { - cold: phase(stats.policy.phases.cold), - delete: phase(stats.policy.phases.delete), - frozen: phase(stats.policy.phases.frozen), - hot: phase(stats.policy.phases.hot), - warm: phase(stats.policy.phases.warm), - } as IlmPhases, - } as IlmPolicy; - } - } catch (error) { - this.logger.warn('Error fetching ilm policies', { - error_message: error.message, - } as LogMeta); - throw error; + const ilms = group.join(','); + this.logger.l('Fetching ilm policies', { ilms } as LogMeta); + const request: IlmGetLifecycleRequest = { + name: ilms, + filter_path: [ + '*.policy.phases.cold.min_age', + '*.policy.phases.delete.min_age', + '*.policy.phases.frozen.min_age', + '*.policy.phases.hot.min_age', + '*.policy.phases.warm.min_age', + '*.modified_date', + ], + }; + + const response = await es.ilm.getLifecycle(request); + try { + for (const [policyName, stats] of Object.entries(response ?? {})) { + yield { + policy_name: policyName, + modified_date: stats.modified_date, + phases: { + cold: phase(stats.policy.phases.cold), + delete: phase(stats.policy.phases.delete), + frozen: phase(stats.policy.phases.frozen), + hot: phase(stats.policy.phases.hot), + warm: phase(stats.policy.phases.warm), + } as IlmPhases, + } as IlmPolicy; } + } catch (error) { + this.logger.warn('Error fetching ilm policies', { + error_message: error.message, + } as LogMeta); + throw error; } } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 71a19f1e785bd..6649294a3114d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -77,10 +77,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { items: [], }; - for await (const stat of receiver.getIndicesStats( - indices.slice(0, taskConfig.indices_threshold), - queryConfig - )) { + for await (const stat of receiver.getIndicesStats(indices, queryConfig)) { indicesStats.items.push(stat); } sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT, indicesStats); @@ -119,7 +116,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { ilmPolicies.items.push(policy); } sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT, ilmPolicies); - log.info(`Sent ILM policies`, { count: ilmPolicies.items.length } as LogMeta); + log.info('Sent ILM policies', { count: ilmPolicies.items.length } as LogMeta); return ilmPolicies.items.length; }; From 02650b0e2ab920dac303a8e3fe0a3dd6c580dda0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 3 Dec 2024 18:25:01 +0100 Subject: [PATCH 39/44] Code style --- .../server/lib/telemetry/configuration.ts | 7 +++-- .../server/lib/telemetry/receiver.ts | 31 ++++++------------- .../lib/telemetry/tasks/indices.metadata.ts | 16 ++-------- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts index 40031574b7489..e9fe0641bb163 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts @@ -28,9 +28,10 @@ class TelemetryConfigurationDTO { private readonly DEFAULT_INDICES_METADATA_CONFIG = { indices_threshold: 15000, datastreams_threshold: 1000, - max_prefixes: 10, - max_group_size: 100, - min_group_size: 5, + + max_prefixes: 10, // @deprecated + max_group_size: 100, // @deprecated + min_group_size: 5, // @deprecated }; private _telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 5925c39469a1b..3788a51f208ef 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -101,7 +101,7 @@ import type { Index, IndexStats, } from './indices.metadata.types'; -import { type CommonPrefixesConfig, chunkStringsByMaxLength } from './collections_helpers'; +import { chunkStringsByMaxLength } from './collections_helpers'; export interface ITelemetryReceiver { start( @@ -256,18 +256,9 @@ export interface ITelemetryReceiver { getIndices(): Promise; getDataStreams(): Promise; - getIndicesStats( - indices: string[], - config: CommonPrefixesConfig - ): AsyncGenerator; - getIlmsStats( - indices: string[], - config: CommonPrefixesConfig - ): AsyncGenerator; - getIlmsPolicies( - ilms: string[], - config: CommonPrefixesConfig - ): AsyncGenerator; + getIndicesStats(indices: string[]): AsyncGenerator; + getIlmsStats(indices: string[]): AsyncGenerator; + getIlmsPolicies(ilms: string[]): AsyncGenerator; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -1403,7 +1394,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { }); } - public async *getIndicesStats(indices: string[], config: CommonPrefixesConfig) { + public async *getIndicesStats(indices: string[]) { const es = this.esClient(); this.logger.l('Fetching indices stats', { indices } as LogMeta); @@ -1449,7 +1440,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } } - public async *getIlmsStats(indices: string[], config: CommonPrefixesConfig) { + public async *getIlmsStats(indices: string[]) { const es = this.esClient(); const groupedIndices = chunkStringsByMaxLength(indices); @@ -1460,9 +1451,8 @@ export class TelemetryReceiver implements ITelemetryReceiver { } as LogMeta); for (const group of groupedIndices) { - const indices = group.join(','); const request: IlmExplainLifecycleRequest = { - index: indices, + index: group.join(','), only_managed: false, filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], }; @@ -1487,7 +1477,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } } - public async *getIlmsPolicies(ilms: string[], config: CommonPrefixesConfig) { + public async *getIlmsPolicies(ilms: string[]) { const es = this.esClient(); const phase = (obj: unknown): Nullable => { @@ -1508,10 +1498,9 @@ export class TelemetryReceiver implements ITelemetryReceiver { } as LogMeta); for (const group of groupedIlms) { - const ilms = group.join(','); - this.logger.l('Fetching ilm policies', { ilms } as LogMeta); + this.logger.l('Fetching ilm policies', { ilms: group } as LogMeta); const request: IlmGetLifecycleRequest = { - name: ilms, + name: group.join(','), filter_path: [ '*.policy.phases.cold.min_age', '*.policy.phases.delete.min_age', diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 6649294a3114d..8c90205fa890e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -21,7 +21,6 @@ import { TELEMETRY_ILM_STATS_EVENT, TELEMETRY_INDEX_STATS_EVENT, } from '../event_based/events'; -import type { CommonPrefixesConfig } from '../collections_helpers'; import { telemetryConfiguration } from '../configuration'; import type { DataStream, @@ -57,12 +56,6 @@ export function createTelemetryIndicesMetadataTaskConfig() { const taskConfig = telemetryConfiguration.indices_metadata_config; - const queryConfig: CommonPrefixesConfig = { - maxPrefixes: Number(taskConfig.max_prefixes), - maxGroupSize: Number(taskConfig.max_group_size), - minPrefixSize: Number(taskConfig.min_group_size), - }; - const publishDatastreamsStats = (stats: DataStream[]): number => { const events: DataStreams = { items: stats, @@ -77,7 +70,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { items: [], }; - for await (const stat of receiver.getIndicesStats(indices, queryConfig)) { + for await (const stat of receiver.getIndicesStats(indices)) { indicesStats.items.push(stat); } sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT, indicesStats); @@ -91,7 +84,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { items: [], }; - for await (const stat of receiver.getIlmsStats(indices, queryConfig)) { + for await (const stat of receiver.getIlmsStats(indices)) { if (stat.policy_name !== undefined) { ilmNames.add(stat.policy_name); ilmsStats.items.push(stat); @@ -109,10 +102,7 @@ export function createTelemetryIndicesMetadataTaskConfig() { items: [], }; - for await (const policy of receiver.getIlmsPolicies( - Array.from(ilmNames.values()), - queryConfig - )) { + for await (const policy of receiver.getIlmsPolicies(Array.from(ilmNames.values()))) { ilmPolicies.items.push(policy); } sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT, ilmPolicies); From ea57c6f995d0c4127f82e893a43bbef5387c88cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 11 Dec 2024 14:37:24 +0100 Subject: [PATCH 40/44] Add testing endpoint (revert this commit before merge) --- .../server/lib/telemetry/routes/index.ts | 60 +++++++++++++++++++ .../security_solution/server/routes/index.ts | 4 ++ 2 files changed, 64 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts new file mode 100644 index 0000000000000..5b43dd402f81c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { schema } from '@kbn/config-schema'; +import type { Logger, IRouter } from '@kbn/core/server'; +import type { ITelemetryReceiver } from '../receiver'; +import type { ITelemetryEventsSender } from '../sender'; +import { TaskMetricsService } from '../task_metrics'; +import { createTelemetryIndicesMetadataTaskConfig } from '../tasks/indices.metadata'; + +// TODO: just to test the POC, remove +export const getTriggerIndicesMetadataTaskRoute = ( + router: IRouter, + logger: Logger, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender +) => { + router.get( + { + path: '/api/trigger-indices-metadata-task', + options: { + tags: ['api'], + access: 'public', + summary: 'Trigger indices metadata task (for testing purposes)', + }, + validate: { + query: schema.object({ + maxPrefixes: schema.maybe(schema.number()), + maxGroupSize: schema.maybe(schema.number()), + }), + }, + }, + async (_context, request, response) => { + const taskMetricsService = new TaskMetricsService(logger, sender); + const task = createTelemetryIndicesMetadataTaskConfig(); + const timeStart = performance.now(); + + const { maxPrefixes, maxGroupSize } = request.query; + + logger.info( + `Triggering indices metadata task with pageSize: ${maxPrefixes} and dataStreamsLimit: ${maxGroupSize}` + ); + + const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { + last: `${maxPrefixes || 10}`, + current: `${maxGroupSize || 100}`, + }); + const elapsedTime = performance.now() - timeStart; + + return response.ok({ + body: { + message: `Task finished, it processed ${result} indices, took ${elapsedTime} ms to run `, + }, + }); + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index ca1cbb493311f..906397d4b6f91 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -7,6 +7,7 @@ import type { StartServicesAccessor, Logger, DocLinksServiceSetup } from '@kbn/core/server'; import type { IRuleDataClient, RuleDataPluginService } from '@kbn/rule-registry-plugin/server'; +import { getTriggerIndicesMetadataTaskRoute } from '../lib/telemetry/routes'; import type { EndpointAppContext } from '../endpoint/types'; import type { SecuritySolutionPluginRouter } from '../types'; @@ -160,4 +161,7 @@ export const initRoutes = ( getFleetManagedIndexTemplatesRoute(router); registerWorkflowInsightsRoutes(router, config, endpointContext); + + // TODO: just to test the PR, remove + getTriggerIndicesMetadataTaskRoute(router, logger, previewTelemetryReceiver, telemetrySender); }; From 18b06e7e932b5011306a1316a5851c61f9eb47f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 11 Dec 2024 17:26:15 +0100 Subject: [PATCH 41/44] Add parameters to the endpoint --- .../server/lib/telemetry/routes/index.ts | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts index 5b43dd402f81c..567ff06338e86 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts @@ -10,6 +10,7 @@ import type { ITelemetryReceiver } from '../receiver'; import type { ITelemetryEventsSender } from '../sender'; import { TaskMetricsService } from '../task_metrics'; import { createTelemetryIndicesMetadataTaskConfig } from '../tasks/indices.metadata'; +import { telemetryConfiguration } from '../configuration'; // TODO: just to test the POC, remove export const getTriggerIndicesMetadataTaskRoute = ( @@ -28,8 +29,8 @@ export const getTriggerIndicesMetadataTaskRoute = ( }, validate: { query: schema.object({ - maxPrefixes: schema.maybe(schema.number()), - maxGroupSize: schema.maybe(schema.number()), + datastreamsThreshold: schema.maybe(schema.number()), + indicesThreshold: schema.maybe(schema.number()), }), }, }, @@ -38,21 +39,27 @@ export const getTriggerIndicesMetadataTaskRoute = ( const task = createTelemetryIndicesMetadataTaskConfig(); const timeStart = performance.now(); - const { maxPrefixes, maxGroupSize } = request.query; + const taskConfig = telemetryConfiguration.indices_metadata_config; - logger.info( - `Triggering indices metadata task with pageSize: ${maxPrefixes} and dataStreamsLimit: ${maxGroupSize}` - ); + taskConfig.indices_threshold = request.query.indicesThreshold || 100; + taskConfig.datastreams_threshold = request.query.datastreamsThreshold || 100; + + const detail = `[pageSize: ${taskConfig.indices_threshold}, dataStreamsLimit: ${taskConfig.datastreams_threshold}]`; + + logger.info(`Triggering indices metadata task ${detail}`); const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { - last: `${maxPrefixes || 10}`, - current: `${maxGroupSize || 100}`, + last: '', + current: '', }); const elapsedTime = performance.now() - timeStart; return response.ok({ body: { - message: `Task finished, it processed ${result} indices, took ${elapsedTime} ms to run `, + message: `Task finished`, + indices: result, + execution_detail: detail, + elapsed_time: elapsedTime, }, }); } From ef230cb6bad84be146206e6e787c969fac31f39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 13 Dec 2024 14:03:30 +0100 Subject: [PATCH 42/44] Remove testing endpoint --- .../server/lib/telemetry/routes/index.ts | 67 ------------------- .../security_solution/server/routes/index.ts | 4 -- 2 files changed, 71 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts deleted file mode 100644 index 567ff06338e86..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/telemetry/routes/index.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { schema } from '@kbn/config-schema'; -import type { Logger, IRouter } from '@kbn/core/server'; -import type { ITelemetryReceiver } from '../receiver'; -import type { ITelemetryEventsSender } from '../sender'; -import { TaskMetricsService } from '../task_metrics'; -import { createTelemetryIndicesMetadataTaskConfig } from '../tasks/indices.metadata'; -import { telemetryConfiguration } from '../configuration'; - -// TODO: just to test the POC, remove -export const getTriggerIndicesMetadataTaskRoute = ( - router: IRouter, - logger: Logger, - receiver: ITelemetryReceiver, - sender: ITelemetryEventsSender -) => { - router.get( - { - path: '/api/trigger-indices-metadata-task', - options: { - tags: ['api'], - access: 'public', - summary: 'Trigger indices metadata task (for testing purposes)', - }, - validate: { - query: schema.object({ - datastreamsThreshold: schema.maybe(schema.number()), - indicesThreshold: schema.maybe(schema.number()), - }), - }, - }, - async (_context, request, response) => { - const taskMetricsService = new TaskMetricsService(logger, sender); - const task = createTelemetryIndicesMetadataTaskConfig(); - const timeStart = performance.now(); - - const taskConfig = telemetryConfiguration.indices_metadata_config; - - taskConfig.indices_threshold = request.query.indicesThreshold || 100; - taskConfig.datastreams_threshold = request.query.datastreamsThreshold || 100; - - const detail = `[pageSize: ${taskConfig.indices_threshold}, dataStreamsLimit: ${taskConfig.datastreams_threshold}]`; - - logger.info(`Triggering indices metadata task ${detail}`); - - const result = await task.runTask('id', logger, receiver, sender, taskMetricsService, { - last: '', - current: '', - }); - const elapsedTime = performance.now() - timeStart; - - return response.ok({ - body: { - message: `Task finished`, - indices: result, - execution_detail: detail, - elapsed_time: elapsedTime, - }, - }); - } - ); -}; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 906397d4b6f91..ca1cbb493311f 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -7,7 +7,6 @@ import type { StartServicesAccessor, Logger, DocLinksServiceSetup } from '@kbn/core/server'; import type { IRuleDataClient, RuleDataPluginService } from '@kbn/rule-registry-plugin/server'; -import { getTriggerIndicesMetadataTaskRoute } from '../lib/telemetry/routes'; import type { EndpointAppContext } from '../endpoint/types'; import type { SecuritySolutionPluginRouter } from '../types'; @@ -161,7 +160,4 @@ export const initRoutes = ( getFleetManagedIndexTemplatesRoute(router); registerWorkflowInsightsRoutes(router, config, endpointContext); - - // TODO: just to test the PR, remove - getTriggerIndicesMetadataTaskRoute(router, logger, previewTelemetryReceiver, telemetrySender); }; From 4cd915fef104878b2eabfd539a3db2a8b496850e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 13 Dec 2024 17:49:34 +0100 Subject: [PATCH 43/44] Update default indices threshold --- .../security_solution/server/lib/telemetry/configuration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts index e9fe0641bb163..963addde2afb4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts @@ -26,7 +26,7 @@ class TelemetryConfigurationDTO { num_docs_to_sample: 10, }; private readonly DEFAULT_INDICES_METADATA_CONFIG = { - indices_threshold: 15000, + indices_threshold: 10000, datastreams_threshold: 1000, max_prefixes: 10, // @deprecated From e6a9cbf8d3aa6aade193147633cac17a01d474ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 13 Dec 2024 17:50:47 +0100 Subject: [PATCH 44/44] Reduce logging noise --- .../security_solution/server/lib/telemetry/receiver.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 3788a51f208ef..db023a593f14b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -1397,7 +1397,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { public async *getIndicesStats(indices: string[]) { const es = this.esClient(); - this.logger.l('Fetching indices stats', { indices } as LogMeta); + this.logger.l('Fetching indices stats'); const groupedIndices = chunkStringsByMaxLength(indices); @@ -1498,7 +1498,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } as LogMeta); for (const group of groupedIlms) { - this.logger.l('Fetching ilm policies', { ilms: group } as LogMeta); + this.logger.l('Fetching ilm policies'); const request: IlmGetLifecycleRequest = { name: group.join(','), filter_path: [