[9.1] [Telemetry] Move indices metadata out of security solution (#212875)

.github/CODEOWNERS

@@ -905,6 +905,7 @@ x-pack/platform/plugins/private/global_search_providers @elastic/appex-sharedux
 x-pack/platform/plugins/private/graph @elastic/kibana-visualizations
 x-pack/platform/plugins/private/grokdebugger @elastic/kibana-management
 x-pack/platform/plugins/private/index_lifecycle_management @elastic/kibana-management
+x-pack/platform/plugins/private/indices_metadata @elastic/security-solution
 x-pack/platform/plugins/private/intercepts @elastic/appex-sharedux
 x-pack/platform/plugins/private/license_api_guard @elastic/kibana-management
 x-pack/platform/plugins/private/logstash @elastic/logstash

docs/extend/plugin-list.md

@@ -151,6 +151,7 @@ mapped_pages:
 | [grokdebugger](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/private/grokdebugger/README.md) | This plugin helps users define Grok patterns, which are particularly useful for ingesting logs. |
 | [indexLifecycleManagement](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/private/index_lifecycle_management/README.md) | You can test that the Frozen badge, phase filtering, and lifecycle information is surfaced in Index Management by running this series of requests in Console: |
 | [indexManagement](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/index_management/README.md) | This service is exposed from the Index Management setup contract and can be used to add content to the indices list and the index details page. |
+| [indicesMetadata](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/private/indices_metadata/README.md) | Plugin for managing and retrieving metadata about indices in Kibana. This plugin collects and processes metadata from Elasticsearch indices, data streams, ILM policies, and index templates. |
 | [inference](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/inference/README.md) | The inference plugin is a central place to handle all interactions with the Elasticsearch Inference API and external LLM APIs. Its goals are: |
 | [inferenceEndpoint](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/inference_endpoint/README.md) | A Kibana plugin |
 | [infra](https://github.com/elastic/kibana/blob/main/x-pack/solutions/observability/plugins/infra/README.md) | This is the home of the infra plugin, which aims to provide a solution for the infrastructure monitoring use-case within Kibana. |

package.json

@@ -598,6 +598,7 @@
     "@kbn/index-management-plugin": "link:x-pack/platform/plugins/shared/index_management",
     "@kbn/index-management-shared-types": "link:x-pack/platform/packages/shared/index-management/index_management_shared_types",
     "@kbn/index-patterns-test-plugin": "link:src/platform/test/plugin_functional/plugins/index_patterns",
+    "@kbn/indices-metadata-plugin": "link:x-pack/platform/plugins/private/indices_metadata",
     "@kbn/inference-common": "link:x-pack/platform/packages/shared/ai-infra/inference-common",
     "@kbn/inference-endpoint-plugin": "link:x-pack/platform/plugins/shared/inference_endpoint",
     "@kbn/inference-endpoint-ui-common": "link:x-pack/platform/packages/shared/kbn-inference-endpoint-ui-common",

tsconfig.base.json

@@ -1106,6 +1106,8 @@
       "@kbn/index-management-shared-types/*": ["x-pack/platform/packages/shared/index-management/index_management_shared_types/*"],
       "@kbn/index-patterns-test-plugin": ["src/platform/test/plugin_functional/plugins/index_patterns"],
       "@kbn/index-patterns-test-plugin/*": ["src/platform/test/plugin_functional/plugins/index_patterns/*"],
+      "@kbn/indices-metadata-plugin": ["x-pack/platform/plugins/private/indices_metadata"],
+      "@kbn/indices-metadata-plugin/*": ["x-pack/platform/plugins/private/indices_metadata/*"],
       "@kbn/inference-cli": ["x-pack/platform/packages/shared/kbn-inference-cli"],
       "@kbn/inference-cli/*": ["x-pack/platform/packages/shared/kbn-inference-cli/*"],
       "@kbn/inference-common": ["x-pack/platform/packages/shared/ai-infra/inference-common"],
@@ -2324,4 +2326,4 @@
       "@kbn/ambient-storybook-types"
     ]
   }
-}
+}

x-pack/platform/plugins/private/indices_metadata/README.md

@@ -0,0 +1,66 @@
+# Indices Metadata Plugin
+
+Plugin for managing and retrieving metadata about indices in
+Kibana. This plugin collects and processes metadata from Elasticsearch indices,
+data streams, ILM policies, and index templates.
+
+## Overview
+
+The Indices Metadata plugin is owned by the Security Solution team and provides:
+
+1. **Automated metadata collection** — Runs scheduled tasks to gather indices
+   metadata
+2. **Analytics integration** — Sends telemetry events for indices statistics
+   and configurations
+3. **Data stream monitoring** — Tracks data streams and their associated
+   metadata
+
+## Features
+
+- **Index Statistics Collection**: Gathers comprehensive statistics about indices
+- **Data Stream Monitoring**: Tracks data streams and their metadata
+- **ILM Policy Tracking**: Monitors Index Lifecycle Management policies and statistics
+- **Index Template Management**: Collects information about index templates
+- **Settings Monitoring**: Retrieves and tracks index settings
+- **Task Scheduling**: Runs collection tasks every 24 hours via Task Manager
+
+## Configuration
+
+The plugin's configuration prefix is `xpack.indicesMetadata`
+
+### Plugin Configuration Options
+
+- `enabled`: Whether the plugin is enabled (default: `true`)
+- `cdn.url` — URL for artifact downloads
+- `cdn.publicKey` — Public key for artifact verification
+- `cdn.requestTimeout` — Timeout for CDN requests
+
+### Configuration Example
+
+`kibana.yml`
+
+```yaml
+xpack.indicesMetadata:
+  enabled: false
+  cdn:
+    url: 'https://artifacts.elastic.co'
+    publicKey: '...'
+    requestTimeout: 30000
+```
+
+## Dependencies
+
+- **Required Plugins**: `taskManager`
+- **Owner**: `@elastic/security-solution`
+
+## Development
+
+This plugin is server-side only and depends on:
+
+- Task Manager for scheduled metadata collection
+- Analytics service for telemetry
+- Elasticsearch client for metadata retrieval
+
+See the [Kibana contributing
+guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for
+instructions setting up your development environment.

x-pack/platform/plugins/private/indices_metadata/jest.config.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/x-pack/platform/plugins/private/indices_metadata'],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/platform/plugins/private/indices_metadata',
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/platform/plugins/private/indices_metadata/{common,public,server}/**/*.{ts,tsx}',
+  ],
+  globals: {
+    Uint8Array: Uint8Array,
+  },
+};

x-pack/platform/plugins/private/indices_metadata/kibana.jsonc

@@ -0,0 +1,25 @@
+{
+  "type": "plugin",
+  "id": "@kbn/indices-metadata-plugin",
+  "owner": [
+    "@elastic/security-solution"
+  ],
+  "group": "platform",
+  "visibility": "private",
+  "description": "Indices metadata plugin for managing and retrieving metadata about indices in Kibana.",
+  "plugin": {
+    "id": "indicesMetadata",
+    "server": true,
+    "browser": false,
+    "configPath": [
+      "xpack",
+      "indicesMetadata"
+    ],
+    "requiredPlugins": [
+      "taskManager"
+    ],
+    "requiredBundles": [],
+    "optionalPlugins": [],
+    "extraPublicDirs": []
+  }
+}

x-pack/platform/plugins/private/indices_metadata/server/config.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+import { PluginConfigDescriptor } from '@kbn/core/server';
+
+const pluginConfigSchema = schema.object({
+  enabled: schema.boolean({ defaultValue: true }),
+  cdn: schema.maybe(
+    schema.object({
+      url: schema.maybe(schema.string()),
+      publicKey: schema.maybe(schema.string()),
+      requestTimeout: schema.maybe(schema.number()),
+    })
+  ),
+});
+
+export type PluginConfig = TypeOf<typeof pluginConfigSchema>;
+
+export const config: PluginConfigDescriptor<PluginConfig> = {
+  schema: pluginConfigSchema,
+};

x-pack/platform/plugins/private/indices_metadata/server/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PluginInitializerContext } from '@kbn/core/server';
+import { IndicesMetadataPlugin } from './plugin';
+
+export { config } from './config';
+export async function plugin(context: PluginInitializerContext) {
+  return new IndicesMetadataPlugin(context);
+}

x-pack/platform/plugins/private/indices_metadata/server/lib/constants.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IndicesMetadataConfiguration } from './services/indices_metadata.types';
+import type { CdnConfig } from './services/artifact.types';
+
+export const DEFAULT_CDN_CONFIG: CdnConfig = {
+  url: 'https://artifacts.security.elastic.co',
+  pubKey: `
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA6AB2sJ5M1ImN/bkQ7Te6
+uI7vMXjN2yupEmh2rYz4gNzWS351d4JOhuQH3nzxfKdayHgusP/Kq2MXVqALH8Ru
+Yu2AF08GdvYlQXPgEVI+tB/riekwU7PXZHdA1dY5/mEZ8SUSM25kcDJ3vTCzFTlL
+gl2RNAdkR80d9nhvNSWlhWMwr8coQkr6NmujVU/Wa0w0EXbN1arjcG4qzbOCaR+b
+cgQ9LRUoFfK9w+JJHDNjOI7rOmaIDA6Ep4oeDLy5AcGCE8bNmQzxZhRW7NvlNUGS
+NTgU0CZTatVsL9AyP15W3k635Cpmy2SMPX+d/CFgvr8QPxtqdrz3q9iOeU3a1LMY
+gDcFVmSzn5zieQEPfo/FcQID/gnCmkX0ADVMf1Q20ew66H7UCOejGaerbFZXYnTz
+5AgQBWF2taOSSE7gDjGAHereeKp+1PR+tCkoDZIrPEjo0V6+KaTMuYS3oZj1/RZN
+oTjQrdfeDj02mEIL+XkcWKAp03PYlWylVwgTMa178DDVuTWtS5lZL8j5LijlH9+6
+xH8o++ghwfxp6ENLKDZPV5IvHHG7Vth9HScoPTQWQ+s8Bt26QENPUV2AbyxbJykY
+mJfTDke3bEemHZzRbAmwiQ7VpJjJ4OfLGRy8Pp2AHo8kYIvWyM5+aLMxcxUaYdA9
+5SxoDOgcDBA4lLb6XFLYiDUCAwEAAQ==
+-----END PUBLIC KEY-----
+`,
+  requestTimeout: 10_000,
+};
+
+export const DEFAULT_INDICES_METADATA_CONFIGURATION: IndicesMetadataConfiguration = {
+  datastreams_threshold: 0,
+  ilm_policy_query_size: 0,
+  ilm_stats_query_size: 0,
+  index_query_size: 0,
+  indices_settings_threshold: 0,
+  indices_threshold: 0,
+};