diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts index 4c242c83afc67..9ffc0821ca5e1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -15,6 +15,7 @@ import type { DataStreams, IlmPolicies, IlmsStats, + IndexTemplatesStats, IndicesSettings, IndicesStats, } from '../indices.metadata.types'; @@ -622,6 +623,103 @@ export const TELEMETRY_ILM_STATS_EVENT: EventTypeOpts = { }, }; +export const TELEMETRY_INDEX_TEMPLATES_EVENT: EventTypeOpts = { + eventType: 'telemetry_index_templates_event', + schema: { + items: { + type: 'array', + items: { + properties: { + template_name: { + type: 'keyword', + _meta: { description: 'The name of the template.' }, + }, + index_mode: { + type: 'keyword', + _meta: { + optional: true, + description: 'The index mode.', + }, + }, + datastream: { + type: 'boolean', + _meta: { + description: 'Datastream dataset', + }, + }, + package_name: { + type: 'keyword', + _meta: { + optional: true, + description: 'The package name', + }, + }, + managed_by: { + type: 'keyword', + _meta: { + optional: true, + description: 'Managed by', + }, + }, + beat: { + type: 'keyword', + _meta: { + optional: true, + description: 'Shipper name', + }, + }, + is_managed: { + type: 'boolean', + _meta: { + optional: true, + description: 'Whether the template is managed', + }, + }, + composed_of: { + type: 'array', + items: { + type: 'keyword', + _meta: { + description: 'List of template components', + }, + }, + _meta: { description: '' }, + }, + source_enabled: { + type: 'boolean', + _meta: { + optional: true, + description: + 'The _source field contains the original JSON document body that was provided at index time', + }, + }, + source_includes: { + type: 'array', + items: { + type: 'keyword', + _meta: { + description: 'Fields included in _source, if enabled', + }, + }, + _meta: { description: '' }, + }, + source_excludes: { + type: 'array', + items: { + type: 'keyword', + _meta: { + description: '', + }, + }, + _meta: { description: 'Fields excludes from _source, if enabled' }, + }, + }, + }, + _meta: { description: 'Index templates info' }, + }, + }, +}; + export const TELEMETRY_NODE_INGEST_PIPELINES_STATS_EVENT: EventTypeOpts = { eventType: 'telemetry_node_ingest_pipelines_stats_event', @@ -1260,6 +1358,7 @@ export const events = [ TELEMETRY_ILM_STATS_EVENT, TELEMETRY_INDEX_SETTINGS_EVENT, TELEMETRY_INDEX_STATS_EVENT, + TELEMETRY_INDEX_TEMPLATES_EVENT, TELEMETRY_NODE_INGEST_PIPELINES_STATS_EVENT, SIEM_MIGRATIONS_MIGRATION_SUCCESS, SIEM_MIGRATIONS_MIGRATION_FAILURE, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts index 0257489d72503..645f5261391df 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts @@ -41,6 +41,24 @@ export interface IlmStats { policy_name?: string; } +export interface IndexTemplatesStats { + items: IndexTemplateInfo[]; +} + +export interface IndexTemplateInfo { + template_name: string; + index_mode: Nullable; + datastream: boolean; + package_name: Nullable; + managed_by: Nullable; + beat: Nullable; + is_managed: Nullable; + composed_of: string[]; + source_enabled: Nullable; + source_includes: string[]; + source_excludes: string[]; +} + export interface IndicesStats { items: IndexStats[]; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts index 504a41e5e6e67..9bf40c63486c0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -31,6 +31,7 @@ import type { IndicesGetRequest, NodesStatsRequest, Duration, + IndicesGetIndexTemplateRequest, } from '@elastic/elasticsearch/lib/api/types'; import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants'; import { @@ -101,6 +102,7 @@ import type { Index, IndexSettings, IndexStats, + IndexTemplateInfo, } from './indices.metadata.types'; import { chunkStringsByMaxLength } from './collections_helpers'; import type { @@ -265,6 +267,7 @@ export interface ITelemetryReceiver { getDataStreams(): Promise; getIndicesStats(indices: string[], chunkSize: number): AsyncGenerator; getIlmsStats(indices: string[], chunkSize: number): AsyncGenerator; + getIndexTemplatesStats(): Promise; getIlmsPolicies(ilms: string[], chunkSize: number): AsyncGenerator; getIngestPipelinesStats(timeout: Duration): Promise; @@ -1373,11 +1376,11 @@ export class TelemetryReceiver implements ITelemetryReceiver { name: '*', expand_wildcards: ['open', 'hidden'], filter_path: [ + 'data_streams.ilm_policy', 'data_streams.indices.ilm_policy', 'data_streams.indices.index_name', 'data_streams.name', - 'ilm_policy', - 'template', + 'data_streams.template', ], }; @@ -1509,6 +1512,54 @@ export class TelemetryReceiver implements ITelemetryReceiver { } } + public async getIndexTemplatesStats(): Promise { + const es = this.esClient(); + + this.logger.l('Fetching datstreams'); + + const request: IndicesGetIndexTemplateRequest = { + name: '*', + filter_path: [ + 'index_templates.name', + 'index_templates.index_template.template.settings.index.mode', + 'index_templates.index_template.data_stream', + 'index_templates.index_template._meta.package.name', + 'index_templates.index_template._meta.managed_by', + 'index_templates.index_template._meta.beat', + 'index_templates.index_template._meta.managed', + 'index_templates.index_template.composed_of', + 'index_templates.index_template.template.mappings._source.enabled', + 'index_templates.index_template.template.mappings._source.includes', + 'index_templates.index_template.template.mappings._source.excludes', + ], + }; + + return es.indices + .getIndexTemplate(request) + .then((response) => + response.index_templates.map((props) => { + const datastream = props.index_template?.data_stream !== undefined; + return { + template_name: props.name, + index_mode: props.index_template.template?.settings?.index?.mode, + package_name: props.index_template._meta?.package?.name, + datastream, + managed_by: props.index_template._meta?.managed_by, + beat: props.index_template._meta?.beat, + is_managed: props.index_template._meta?.managed, + composed_of: props.index_template.composed_of, + source_enabled: props.index_template.template?.mappings?._source?.enabled, + source_includes: props.index_template.template?.mappings?._source?.includes ?? [], + source_excludes: props.index_template.template?.mappings?._source?.excludes ?? [], + } as IndexTemplateInfo; + }) + ) + .catch((error) => { + this.logger.warn('Error fetching index templates', { error_message: error } as LogMeta); + throw error; + }); + } + public async *getIlmsPolicies(ilms: string[], chunkSize: number) { const es = this.esClient(); const safeChunkSize = Math.min(chunkSize, 3000); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts index 2f876aa37651c..7663f7b9570de 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -21,6 +21,7 @@ import { TELEMETRY_ILM_STATS_EVENT, TELEMETRY_INDEX_SETTINGS_EVENT, TELEMETRY_INDEX_STATS_EVENT, + TELEMETRY_INDEX_TEMPLATES_EVENT, } from '../event_based/events'; import { telemetryConfiguration } from '../configuration'; import type { @@ -29,6 +30,8 @@ import type { IlmPolicies, IlmsStats, IndexSettings, + IndexTemplateInfo, + IndexTemplatesStats, IndicesSettings, IndicesStats, } from '../indices.metadata.types'; @@ -81,6 +84,19 @@ export function createTelemetryIndicesMetadataTaskConfig() { return indicesStats.items.length; }; + const publishIndexTemplatesStats = async ( + indexTemplates: IndexTemplateInfo[] + ): Promise => { + const templateStats: IndexTemplatesStats = { + items: indexTemplates, + }; + + sender.reportEBT(TELEMETRY_INDEX_TEMPLATES_EVENT, templateStats); + log.info(`Sent index templates`, { count: indexTemplates.length } as LogMeta); + + return templateStats.items.length; + }; + const publishIndicesSettings = (settings: IndexSettings[]): number => { const indicesSettings: IndicesSettings = { items: settings, @@ -137,9 +153,10 @@ export function createTelemetryIndicesMetadataTaskConfig() { try { // 1. Get cluster stats and list of indices and datastreams - const [indicesSettings, dataStreams] = await Promise.all([ + const [indicesSettings, dataStreams, indexTemplates] = await Promise.all([ receiver.getIndices(), receiver.getDataStreams(), + receiver.getIndexTemplatesStats(), ]); const indices = indicesSettings.map((index) => index.index_name); @@ -194,11 +211,26 @@ export function createTelemetryIndicesMetadataTaskConfig() { return 0; }); + // 7. Publish index templates + const indexTemplatesCount: number = await publishIndexTemplatesStats( + indexTemplates.slice(0, taskConfig.indices_threshold) + ) + .then((count) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'index-templates', count); + return count; + }) + .catch((err) => { + log.warn(`Error getting index templates`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'index-templates', 1); + return 0; + }); + log.info(`Sent EBT events`, { datastreams: dsCount, indicesSettings: indicesSettingsCount, ilms: ilmNames.size, indices: indicesCount, + indexTemplates: indexTemplatesCount, policies: policyCount, } as LogMeta); diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts index cca7347090a96..ac160b856178a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -29,6 +29,7 @@ const TELEMETRY_ILM_STATS_EVENT = 'telemetry_ilm_stats_event'; const TELEMETRY_ILM_POLICY_EVENT = 'telemetry_ilm_policy_event'; const TELEMETRY_DATA_STREAM_EVENT = 'telemetry_data_stream_event'; const TELEMETRY_INDEX_SETTINGS_EVENT = 'telemetry_index_settings_event'; +const TELEMETRY_INDEX_TEMPLATES_EVENT = 'telemetry_index_templates_event'; export default ({ getService }: FtrProviderContext) => { const ebtServer = getService('kibana_ebt_server'); @@ -58,7 +59,18 @@ export default ({ getService }: FtrProviderContext) => { dsName, }); - expect(events.length).toEqual(1); + expect(events.length).toBeGreaterThanOrEqual(1); + }); + + it('should include `template` in data stream events when defined', async () => { + const events = await launchTaskAndWaitForEvents({ + eventTypes: [TELEMETRY_DATA_STREAM_EVENT], + dsName, + }); + + expect(events.length).toBeGreaterThanOrEqual(1); + const event = events[0] as any; + expect(event.template).toBeDefined(); }); it('should publish index stats events', async () => { @@ -88,13 +100,24 @@ export default ({ getService }: FtrProviderContext) => { await cleanupIngestPipelines(es); }); + it('should include `ilm_policy` in data stream events when defined', async () => { + const events = await launchTaskAndWaitForEvents({ + eventTypes: [TELEMETRY_DATA_STREAM_EVENT], + dsName, + }); + + expect(events.length).toBeGreaterThanOrEqual(1); + const event = events[0] as any; + expect(event.ilm_policy).toBeDefined(); + }); + it('should publish ilm policy events', async () => { const events = await launchTaskAndWaitForEvents({ eventTypes: [TELEMETRY_ILM_POLICY_EVENT], policyName, }); - expect(events.length).toEqual(1); + expect(events.length).toBeGreaterThanOrEqual(1); }); it('should publish ilm stats events', async () => { @@ -204,6 +227,14 @@ export default ({ getService }: FtrProviderContext) => { ); expect(events.filter((v) => v.final_pipeline === finalPipeline)).toHaveLength(NUM_INDICES); }); + + it('should publish index templates', async () => { + const events = await launchTaskAndWaitForEvents({ + eventTypes: [TELEMETRY_INDEX_TEMPLATES_EVENT], + }); + + expect(events.length).toBeGreaterThanOrEqual(1); + }); }); const indexRandomDocs = async (index: string, count: number) => {